1075 files changed, 36657 insertions, 26184 deletions

diff --git a/compiler/rustc_abi/src/layout.rs b/compiler/rustc_abi/src/layout.rs
index 996fd5bbe..525247226 100644
--- a/compiler/rustc_abi/src/layout.rs
+++ b/compiler/rustc_abi/src/layout.rs
@@ -14,7 +14,7 @@ use crate::{
 pub trait LayoutCalculator {
     type TargetDataLayoutRef: Borrow<TargetDataLayout>;
 
-    fn delay_bug(&self, txt: String);
+    fn delayed_bug(&self, txt: String);
     fn current_data_layout(&self) -> Self::TargetDataLayoutRef;
 
     fn scalar_pair<FieldIdx: Idx, VariantIdx: Idx>(
@@ -111,8 +111,8 @@ pub trait LayoutCalculator {
                             alt_tail_space,
                             layout.fields.count(),
                             prefer_alt_layout,
-                            format_field_niches(&layout, &fields, &dl),
-                            format_field_niches(&alt_layout, &fields, &dl),
+                            format_field_niches(layout, fields, dl),
+                            format_field_niches(&alt_layout, fields, dl),
                         );
 
                         if prefer_alt_layout {
@@ -382,7 +382,7 @@ pub trait LayoutCalculator {
                             *offset += this_offset;
                         }
                     }
-                    _ => {
+                    FieldsShape::Primitive | FieldsShape::Array { .. } | FieldsShape::Union(..) => {
                         panic!("Layout of fields should be Arbitrary for variants")
                     }
                 }
@@ -600,7 +600,9 @@ pub trait LayoutCalculator {
                             variant.size = new_ity_size;
                         }
                     }
-                    _ => panic!(),
+                    FieldsShape::Primitive | FieldsShape::Array { .. } | FieldsShape::Union(..) => {
+                        panic!("encountered a non-arbitrary layout during enum layout")
+                    }
                 }
             }
         }
@@ -628,7 +630,7 @@ pub trait LayoutCalculator {
             let mut common_prim_initialized_in_all_variants = true;
             for (field_layouts, layout_variant) in iter::zip(variants, &layout_variants) {
                 let FieldsShape::Arbitrary { ref offsets, .. } = layout_variant.fields else {
-                    panic!();
+                    panic!("encountered a non-arbitrary layout during enum layout");
                 };
                 // We skip *all* ZST here and later check if we are good in terms of alignment.
                 // This lets us handle some cases involving aligned ZST.
@@ -681,7 +683,7 @@ pub trait LayoutCalculator {
                         assert_eq!(memory_index.raw, [0, 1]);
                         offsets
                     }
-                    _ => panic!(),
+                    _ => panic!("encountered a non-arbitrary layout during enum layout"),
                 };
                 if pair_offsets[FieldIdx::new(0)] == Size::ZERO
                     && pair_offsets[FieldIdx::new(1)] == *offset
@@ -758,7 +760,9 @@ pub trait LayoutCalculator {
             Variants::Multiple { tag, tag_encoding, tag_field, .. } => {
                 Variants::Multiple { tag, tag_encoding, tag_field, variants: best_layout.variants }
             }
-            _ => panic!(),
+            Variants::Single { .. } => {
+                panic!("encountered a single-variant enum during multi-variant layout")
+            }
         };
         Some(best_layout.layout)
     }
@@ -792,7 +796,7 @@ pub trait LayoutCalculator {
         let only_variant = &variants[VariantIdx::new(0)];
         for field in only_variant {
             if field.is_unsized() {
-                self.delay_bug("unsized field in union".to_string());
+                self.delayed_bug("unsized field in union".to_string());
             }
 
             align = align.max(field.align);
@@ -1025,7 +1029,7 @@ fn univariant<
     // At the bottom of this function, we invert `inverse_memory_index` to
     // produce `memory_index` (see `invert_mapping`).
     let mut sized = true;
-    let mut offsets = IndexVec::from_elem(Size::ZERO, &fields);
+    let mut offsets = IndexVec::from_elem(Size::ZERO, fields);
     let mut offset = Size::ZERO;
     let mut largest_niche = None;
     let mut largest_niche_available = 0;
@@ -1038,7 +1042,7 @@ fn univariant<
     for &i in &inverse_memory_index {
         let field = &fields[i];
         if !sized {
-            this.delay_bug(format!(
+            this.delayed_bug(format!(
                 "univariant: field #{} comes after unsized field",
                 offsets.len(),
             ));
@@ -1154,7 +1158,11 @@ fn univariant<
                                 assert_eq!(memory_index.raw, [0, 1]);
                                 offsets
                             }
-                            _ => panic!(),
+                            FieldsShape::Primitive
+                            | FieldsShape::Array { .. }
+                            | FieldsShape::Union(..) => {
+                                panic!("encountered a non-arbitrary layout during enum layout")
+                            }
                         };
                         if offsets[i] == pair_offsets[FieldIdx::new(0)]
                             && offsets[j] == pair_offsets[FieldIdx::new(1)]
diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs
index 09a87cf8e..eb42803f9 100644
--- a/compiler/rustc_abi/src/lib.rs
+++ b/compiler/rustc_abi/src/lib.rs
@@ -1,7 +1,7 @@
 #![cfg_attr(feature = "nightly", feature(step_trait))]
 #![cfg_attr(feature = "nightly", allow(internal_features))]
-#![cfg_attr(all(not(bootstrap), feature = "nightly"), doc(rust_logo))]
-#![cfg_attr(all(not(bootstrap), feature = "nightly"), feature(rustdoc_internals))]
+#![cfg_attr(feature = "nightly", doc(rust_logo))]
+#![cfg_attr(feature = "nightly", feature(rustdoc_internals))]
 
 use std::fmt;
 use std::num::{NonZeroUsize, ParseIntError};
diff --git a/compiler/rustc_arena/src/lib.rs b/compiler/rustc_arena/src/lib.rs
index a53fd4ae9..621516af9 100644
--- a/compiler/rustc_arena/src/lib.rs
+++ b/compiler/rustc_arena/src/lib.rs
@@ -11,8 +11,8 @@
     html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/",
     test(no_crate_inject, attr(deny(warnings)))
 )]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
+#![doc(rust_logo)]
+#![feature(rustdoc_internals)]
 #![feature(core_intrinsics)]
 #![feature(dropck_eyepatch)]
 #![feature(new_uninit)]
@@ -197,23 +197,24 @@ impl<T> TypedArena<T> {
         start_ptr
     }
 
+    /// Allocates the elements of this iterator into a contiguous slice in the `TypedArena`.
+    ///
+    /// Note: for reasons of reentrancy and panic safety we collect into a `SmallVec<[_; 8]>` before
+    /// storing the elements in the arena.
     #[inline]
     pub fn alloc_from_iter<I: IntoIterator<Item = T>>(&self, iter: I) -> &mut [T] {
-        // This implementation is entirely separate to
-        // `DroplessIterator::alloc_from_iter`, even though conceptually they
-        // are the same.
+        // Despite the similarlty with `DroplessArena`, we cannot reuse their fast case. The reason
+        // is subtle: these arenas are reentrant. In other words, `iter` may very well be holding a
+        // reference to `self` and adding elements to the arena during iteration.
         //
-        // `DroplessIterator` (in the fast case) writes elements from the
-        // iterator one at a time into the allocated memory. That's easy
-        // because the elements don't implement `Drop`. But for `TypedArena`
-        // they do implement `Drop`, which means that if the iterator panics we
-        // could end up with some allocated-but-uninitialized elements, which
-        // will then cause UB in `TypedArena::drop`.
+        // For this reason, if we pre-allocated any space for the elements of this iterator, we'd
+        // have to track that some uninitialized elements are followed by some initialized elements,
+        // else we might accidentally drop uninitialized memory if something panics or if the
+        // iterator doesn't fill all the length we expected.
         //
-        // Instead we use an approach where any iterator panic will occur
-        // before the memory is allocated. This function is much less hot than
-        // `DroplessArena::alloc_from_iter`, so it doesn't need to be
-        // hyper-optimized.
+        // So we collect all the elements beforehand, which takes care of reentrancy and panic
+        // safety. This function is much less hot than `DroplessArena::alloc_from_iter`, so it
+        // doesn't need to be hyper-optimized.
         assert!(mem::size_of::<T>() != 0);
 
         let mut vec: SmallVec<[_; 8]> = iter.into_iter().collect();
@@ -483,10 +484,25 @@ impl DroplessArena {
         }
     }
 
+    /// Allocates a string slice that is copied into the `DroplessArena`, returning a
+    /// reference to it. Will panic if passed an empty string.
+    ///
+    /// Panics:
+    ///
+    ///  - Zero-length string
+    #[inline]
+    pub fn alloc_str(&self, string: &str) -> &str {
+        let slice = self.alloc_slice(string.as_bytes());
+
+        // SAFETY: the result has a copy of the same valid UTF-8 bytes.
+        unsafe { std::str::from_utf8_unchecked(slice) }
+    }
+
     /// # Safety
     ///
-    /// The caller must ensure that `mem` is valid for writes up to
-    /// `size_of::<T>() * len`.
+    /// The caller must ensure that `mem` is valid for writes up to `size_of::<T>() * len`, and that
+    /// that memory stays allocated and not shared for the lifetime of `self`. This must hold even
+    /// if `iter.next()` allocates onto `self`.
     #[inline]
     unsafe fn write_from_iter<T, I: Iterator<Item = T>>(
         &self,
@@ -516,6 +532,8 @@ impl DroplessArena {
 
     #[inline]
     pub fn alloc_from_iter<T, I: IntoIterator<Item = T>>(&self, iter: I) -> &mut [T] {
+        // Warning: this function is reentrant: `iter` could hold a reference to `&self` and
+        // allocate additional elements while we're iterating.
         let iter = iter.into_iter();
         assert!(mem::size_of::<T>() != 0);
         assert!(!mem::needs_drop::<T>());
@@ -524,7 +542,7 @@ impl DroplessArena {
 
         match size_hint {
             (min, Some(max)) if min == max => {
-                // We know the exact number of elements the iterator will produce here
+                // We know the exact number of elements the iterator expects to produce here.
                 let len = min;
 
                 if len == 0 {
@@ -532,10 +550,15 @@ impl DroplessArena {
                 }
 
                 let mem = self.alloc_raw(Layout::array::<T>(len).unwrap()) as *mut T;
+                // SAFETY: `write_from_iter` doesn't touch `self`. It only touches the slice we just
+                // reserved. If the iterator panics or doesn't output `len` elements, this will
+                // leave some unallocated slots in the arena, which is fine because we do not call
+                // `drop`.
                 unsafe { self.write_from_iter(iter, len, mem) }
             }
             (_, _) => {
                 outline(move || -> &mut [T] {
+                    // Takes care of reentrancy.
                     let mut vec: SmallVec<[_; 8]> = iter.collect();
                     if vec.is_empty() {
                         return &mut [];
@@ -646,6 +669,14 @@ pub macro declare_arena([$($a:tt $name:ident: $ty:ty,)*]) {
             self.dropless.alloc_slice(value)
         }
 
+        #[inline]
+        pub fn alloc_str(&self, string: &str) -> &str {
+            if string.is_empty() {
+                return "";
+            }
+            self.dropless.alloc_str(string)
+        }
+
         #[allow(clippy::mut_from_ref)]
         pub fn alloc_from_iter<T: ArenaAllocatable<'tcx, C>, C>(
             &self,
diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs
index c85ff6f5c..a121b5a9b 100644
--- a/compiler/rustc_ast/src/ast.rs
+++ b/compiler/rustc_ast/src/ast.rs
@@ -301,7 +301,7 @@ pub enum TraitBoundModifier {
     Maybe,
 
     /// `~const Trait`
-    MaybeConst,
+    MaybeConst(Span),
 
     /// `~const !Trait`
     //
@@ -317,8 +317,7 @@ pub enum TraitBoundModifier {
 impl TraitBoundModifier {
     pub fn to_constness(self) -> Const {
         match self {
-            // FIXME(effects) span
-            Self::MaybeConst => Const::Yes(DUMMY_SP),
+            Self::MaybeConst(span) => Const::Yes(span),
             _ => Const::No,
         }
     }
@@ -646,6 +645,7 @@ impl Pat {
             // These patterns do not contain subpatterns, skip.
             PatKind::Wild
             | PatKind::Rest
+            | PatKind::Never
             | PatKind::Lit(_)
             | PatKind::Range(..)
             | PatKind::Ident(..)
@@ -658,6 +658,37 @@ impl Pat {
     pub fn is_rest(&self) -> bool {
         matches!(self.kind, PatKind::Rest)
     }
+
+    /// Whether this could be a never pattern, taking into account that a macro invocation can
+    /// return a never pattern. Used to inform errors during parsing.
+    pub fn could_be_never_pattern(&self) -> bool {
+        let mut could_be_never_pattern = false;
+        self.walk(&mut |pat| match &pat.kind {
+            PatKind::Never | PatKind::MacCall(_) => {
+                could_be_never_pattern = true;
+                false
+            }
+            PatKind::Or(s) => {
+                could_be_never_pattern = s.iter().all(|p| p.could_be_never_pattern());
+                false
+            }
+            _ => true,
+        });
+        could_be_never_pattern
+    }
+
+    /// Whether this contains a `!` pattern. This in particular means that a feature gate error will
+    /// be raised if the feature is off. Used to avoid gating the feature twice.
+    pub fn contains_never_pattern(&self) -> bool {
+        let mut contains_never_pattern = false;
+        self.walk(&mut |pat| {
+            if matches!(pat.kind, PatKind::Never) {
+                contains_never_pattern = true;
+            }
+            true
+        });
+        contains_never_pattern
+    }
 }
 
 /// A single field in a struct pattern.
@@ -796,6 +827,9 @@ pub enum PatKind {
     /// only one rest pattern may occur in the pattern sequences.
     Rest,
 
+    // A never pattern `!`
+    Never,
+
     /// Parentheses in patterns used for grouping (i.e., `(PAT)`).
     Paren(P<Pat>),
 
@@ -818,7 +852,7 @@ pub enum BorrowKind {
     Raw,
 }
 
-#[derive(Clone, PartialEq, Encodable, Decodable, Debug, Copy)]
+#[derive(Clone, Copy, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
 pub enum BinOpKind {
     /// The `+` operator (addition)
     Add,
@@ -859,9 +893,9 @@ pub enum BinOpKind {
 }
 
 impl BinOpKind {
-    pub fn to_string(&self) -> &'static str {
+    pub fn as_str(&self) -> &'static str {
         use BinOpKind::*;
-        match *self {
+        match self {
             Add => "+",
             Sub => "-",
             Mul => "*",
@@ -882,19 +916,25 @@ impl BinOpKind {
             Gt => ">",
         }
     }
-    pub fn lazy(&self) -> bool {
+
+    pub fn is_lazy(&self) -> bool {
         matches!(self, BinOpKind::And | BinOpKind::Or)
     }
 
     pub fn is_comparison(&self) -> bool {
         use BinOpKind::*;
-        // Note for developers: please keep this as is;
+        // Note for developers: please keep this match exhaustive;
         // we want compilation to fail if another variant is added.
         match *self {
             Eq | Lt | Le | Ne | Gt | Ge => true,
             And | Or | Add | Sub | Mul | Div | Rem | BitXor | BitAnd | BitOr | Shl | Shr => false,
         }
     }
+
+    /// Returns `true` if the binary operator takes its arguments by value.
+    pub fn is_by_value(self) -> bool {
+        !self.is_comparison()
+    }
 }
 
 pub type BinOp = Spanned<BinOpKind>;
@@ -902,7 +942,7 @@ pub type BinOp = Spanned<BinOpKind>;
 /// Unary operator.
 ///
 /// Note that `&data` is not an operator, it's an `AddrOf` expression.
-#[derive(Clone, Encodable, Decodable, Debug, Copy)]
+#[derive(Clone, Copy, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
 pub enum UnOp {
     /// The `*` operator for dereferencing
     Deref,
@@ -913,13 +953,18 @@ pub enum UnOp {
 }
 
 impl UnOp {
-    pub fn to_string(op: UnOp) -> &'static str {
-        match op {
+    pub fn as_str(&self) -> &'static str {
+        match self {
             UnOp::Deref => "*",
             UnOp::Not => "!",
             UnOp::Neg => "-",
         }
     }
+
+    /// Returns `true` if the unary operator takes its argument by value.
+    pub fn is_by_value(self) -> bool {
+        matches!(self, Self::Neg | Self::Not)
+    }
 }
 
 /// A statement
@@ -1066,8 +1111,8 @@ pub struct Arm {
     pub pat: P<Pat>,
     /// Match arm guard, e.g. `n > 10` in `match foo { n if n > 10 => {}, _ => {} }`
     pub guard: Option<P<Expr>>,
-    /// Match arm body.
-    pub body: P<Expr>,
+    /// Match arm body. Omitted if the pattern is a never pattern.
+    pub body: Option<P<Expr>>,
     pub span: Span,
     pub id: NodeId,
     pub is_placeholder: bool,
@@ -1297,7 +1342,7 @@ pub struct Closure {
     pub binder: ClosureBinder,
     pub capture_clause: CaptureBy,
     pub constness: Const,
-    pub asyncness: Async,
+    pub coroutine_kind: Option<CoroutineKind>,
     pub movability: Movability,
     pub fn_decl: P<FnDecl>,
     pub body: P<Expr>,
@@ -1502,6 +1547,7 @@ pub enum ExprKind {
 pub enum GenBlockKind {
     Async,
     Gen,
+    AsyncGen,
 }
 
 impl fmt::Display for GenBlockKind {
@@ -1515,6 +1561,7 @@ impl GenBlockKind {
         match self {
             GenBlockKind::Async => "async",
             GenBlockKind::Gen => "gen",
+            GenBlockKind::AsyncGen => "async gen",
         }
     }
 }
@@ -2237,6 +2284,18 @@ pub enum InlineAsmOperand {
     },
 }
 
+impl InlineAsmOperand {
+    pub fn reg(&self) -> Option<&InlineAsmRegOrRegClass> {
+        match self {
+            Self::In { reg, .. }
+            | Self::Out { reg, .. }
+            | Self::InOut { reg, .. }
+            | Self::SplitInOut { reg, .. } => Some(reg),
+            Self::Const { .. } | Self::Sym { .. } => None,
+        }
+    }
+}
+
 /// Inline assembly.
 ///
 /// E.g., `asm!("NOP");`.
@@ -2380,28 +2439,47 @@ pub enum Unsafe {
     No,
 }
 
+/// Describes what kind of coroutine markers, if any, a function has.
+///
+/// Coroutine markers are things that cause the function to generate a coroutine, such as `async`,
+/// which makes the function return `impl Future`, or `gen`, which makes the function return `impl
+/// Iterator`.
 #[derive(Copy, Clone, Encodable, Decodable, Debug)]
-pub enum Async {
-    Yes { span: Span, closure_id: NodeId, return_impl_trait_id: NodeId },
-    No,
-}
-
-#[derive(Copy, Clone, Encodable, Decodable, Debug)]
-pub enum Gen {
-    Yes { span: Span, closure_id: NodeId, return_impl_trait_id: NodeId },
-    No,
+pub enum CoroutineKind {
+    /// `async`, which returns an `impl Future`
+    Async { span: Span, closure_id: NodeId, return_impl_trait_id: NodeId },
+    /// `gen`, which returns an `impl Iterator`
+    Gen { span: Span, closure_id: NodeId, return_impl_trait_id: NodeId },
+    /// `async gen`, which returns an `impl AsyncIterator`
+    AsyncGen { span: Span, closure_id: NodeId, return_impl_trait_id: NodeId },
 }
 
-impl Async {
+impl CoroutineKind {
     pub fn is_async(self) -> bool {
-        matches!(self, Async::Yes { .. })
+        matches!(self, CoroutineKind::Async { .. })
+    }
+
+    pub fn is_gen(self) -> bool {
+        matches!(self, CoroutineKind::Gen { .. })
+    }
+
+    pub fn closure_id(self) -> NodeId {
+        match self {
+            CoroutineKind::Async { closure_id, .. }
+            | CoroutineKind::Gen { closure_id, .. }
+            | CoroutineKind::AsyncGen { closure_id, .. } => closure_id,
+        }
     }
 
-    /// In this case this is an `async` return, the `NodeId` for the generated `impl Trait` item.
-    pub fn opt_return_id(self) -> Option<(NodeId, Span)> {
+    /// In this case this is an `async` or `gen` return, the `NodeId` for the generated `impl Trait`
+    /// item.
+    pub fn return_id(self) -> (NodeId, Span) {
         match self {
-            Async::Yes { return_impl_trait_id, span, .. } => Some((return_impl_trait_id, span)),
-            Async::No => None,
+            CoroutineKind::Async { return_impl_trait_id, span, .. }
+            | CoroutineKind::Gen { return_impl_trait_id, span, .. }
+            | CoroutineKind::AsyncGen { return_impl_trait_id, span, .. } => {
+                (return_impl_trait_id, span)
+            }
         }
     }
 }
@@ -2574,7 +2652,7 @@ pub enum AttrStyle {
 }
 
 rustc_index::newtype_index! {
-    #[custom_encodable]
+    #[orderable]
     #[debug_format = "AttrId({})"]
     pub struct AttrId {}
 }
@@ -2710,7 +2788,11 @@ pub enum VariantData {
     /// Struct variant.
     ///
     /// E.g., `Bar { .. }` as in `enum Foo { Bar { .. } }`.
-    Struct(ThinVec<FieldDef>, bool),
+    Struct {
+        fields: ThinVec<FieldDef>,
+        // FIXME: investigate making this a `Option<ErrorGuaranteed>`
+        recovered: bool,
+    },
     /// Tuple variant.
     ///
     /// E.g., `Bar(..)` as in `enum Foo { Bar(..) }`.
@@ -2725,7 +2807,7 @@ impl VariantData {
     /// Return the fields of this variant.
     pub fn fields(&self) -> &[FieldDef] {
         match self {
-            VariantData::Struct(fields, ..) | VariantData::Tuple(fields, _) => fields,
+            VariantData::Struct { fields, .. } | VariantData::Tuple(fields, _) => fields,
             _ => &[],
         }
     }
@@ -2733,7 +2815,7 @@ impl VariantData {
     /// Return the `NodeId` of this variant's constructor, if it has one.
     pub fn ctor_node_id(&self) -> Option<NodeId> {
         match *self {
-            VariantData::Struct(..) => None,
+            VariantData::Struct { .. } => None,
             VariantData::Tuple(_, id) | VariantData::Unit(id) => Some(id),
         }
     }
@@ -2767,6 +2849,28 @@ impl Item {
     pub fn span_with_attributes(&self) -> Span {
         self.attrs.iter().fold(self.span, |acc, attr| acc.to(attr.span))
     }
+
+    pub fn opt_generics(&self) -> Option<&Generics> {
+        match &self.kind {
+            ItemKind::ExternCrate(_)
+            | ItemKind::Use(_)
+            | ItemKind::Mod(_, _)
+            | ItemKind::ForeignMod(_)
+            | ItemKind::GlobalAsm(_)
+            | ItemKind::MacCall(_)
+            | ItemKind::MacroDef(_) => None,
+            ItemKind::Static(_) => None,
+            ItemKind::Const(i) => Some(&i.generics),
+            ItemKind::Fn(i) => Some(&i.generics),
+            ItemKind::TyAlias(i) => Some(&i.generics),
+            ItemKind::TraitAlias(generics, _)
+            | ItemKind::Enum(_, generics)
+            | ItemKind::Struct(_, generics)
+            | ItemKind::Union(_, generics) => Some(&generics),
+            ItemKind::Trait(i) => Some(&i.generics),
+            ItemKind::Impl(i) => Some(&i.generics),
+        }
+    }
 }
 
 /// `extern` qualifier on a function item or function type.
@@ -2805,8 +2909,8 @@ impl Extern {
 pub struct FnHeader {
     /// The `unsafe` keyword, if any
     pub unsafety: Unsafe,
-    /// The `async` keyword, if any
-    pub asyncness: Async,
+    /// Whether this is `async`, `gen`, or nothing.
+    pub coroutine_kind: Option<CoroutineKind>,
     /// The `const` keyword, if any
     pub constness: Const,
     /// The `extern` keyword and corresponding ABI string, if any
@@ -2816,9 +2920,9 @@ pub struct FnHeader {
 impl FnHeader {
     /// Does this function header have any qualifiers or is it empty?
     pub fn has_qualifiers(&self) -> bool {
-        let Self { unsafety, asyncness, constness, ext } = self;
+        let Self { unsafety, coroutine_kind, constness, ext } = self;
         matches!(unsafety, Unsafe::Yes(_))
-            || asyncness.is_async()
+            || coroutine_kind.is_some()
             || matches!(constness, Const::Yes(_))
             || !matches!(ext, Extern::None)
     }
@@ -2828,7 +2932,7 @@ impl Default for FnHeader {
     fn default() -> FnHeader {
         FnHeader {
             unsafety: Unsafe::No,
-            asyncness: Async::No,
+            coroutine_kind: None,
             constness: Const::No,
             ext: Extern::None,
         }
@@ -3151,11 +3255,11 @@ mod size_asserts {
     static_assert_size!(Block, 32);
     static_assert_size!(Expr, 72);
     static_assert_size!(ExprKind, 40);
-    static_assert_size!(Fn, 152);
+    static_assert_size!(Fn, 160);
     static_assert_size!(ForeignItem, 96);
     static_assert_size!(ForeignItemKind, 24);
     static_assert_size!(GenericArg, 24);
-    static_assert_size!(GenericBound, 56);
+    static_assert_size!(GenericBound, 64);
     static_assert_size!(Generics, 40);
     static_assert_size!(Impl, 136);
     static_assert_size!(Item, 136);
diff --git a/compiler/rustc_ast/src/attr/mod.rs b/compiler/rustc_ast/src/attr/mod.rs
index be7d1b207..98138cedb 100644
--- a/compiler/rustc_ast/src/attr/mod.rs
+++ b/compiler/rustc_ast/src/attr/mod.rs
@@ -342,7 +342,7 @@ impl MetaItem {
                 let span = span.with_hi(segments.last().unwrap().ident.span.hi());
                 Path { span, segments, tokens: None }
             }
-            Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. }, _)) => match &**nt {
+            Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. }, _)) => match &nt.0 {
                 token::Nonterminal::NtMeta(item) => return item.meta(item.path.span),
                 token::Nonterminal::NtPath(path) => (**path).clone(),
                 _ => return None,
@@ -387,11 +387,11 @@ impl MetaItemKind {
         tokens: &mut impl Iterator<Item = &'a TokenTree>,
     ) -> Option<MetaItemKind> {
         match tokens.next() {
-            Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => {
+            Some(TokenTree::Delimited(.., Delimiter::Invisible, inner_tokens)) => {
                 MetaItemKind::name_value_from_tokens(&mut inner_tokens.trees())
             }
             Some(TokenTree::Token(token, _)) => {
-                MetaItemLit::from_token(&token).map(MetaItemKind::NameValue)
+                MetaItemLit::from_token(token).map(MetaItemKind::NameValue)
             }
             _ => None,
         }
@@ -401,7 +401,7 @@ impl MetaItemKind {
         tokens: &mut iter::Peekable<impl Iterator<Item = &'a TokenTree>>,
     ) -> Option<MetaItemKind> {
         match tokens.peek() {
-            Some(TokenTree::Delimited(_, Delimiter::Parenthesis, inner_tokens)) => {
+            Some(TokenTree::Delimited(.., Delimiter::Parenthesis, inner_tokens)) => {
                 let inner_tokens = inner_tokens.clone();
                 tokens.next();
                 MetaItemKind::list_from_tokens(inner_tokens).map(MetaItemKind::List)
@@ -524,7 +524,7 @@ impl NestedMetaItem {
                 tokens.next();
                 return Some(NestedMetaItem::Lit(lit));
             }
-            Some(TokenTree::Delimited(_, Delimiter::Invisible, inner_tokens)) => {
+            Some(TokenTree::Delimited(.., Delimiter::Invisible, inner_tokens)) => {
                 tokens.next();
                 return NestedMetaItem::from_tokens(&mut inner_tokens.trees().peekable());
             }
diff --git a/compiler/rustc_ast/src/lib.rs b/compiler/rustc_ast/src/lib.rs
index c1f6ad6a2..7e713a49a 100644
--- a/compiler/rustc_ast/src/lib.rs
+++ b/compiler/rustc_ast/src/lib.rs
@@ -8,9 +8,9 @@
     html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/",
     test(attr(deny(warnings)))
 )]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
-#![cfg_attr(not(bootstrap), allow(internal_features))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
+#![doc(rust_logo)]
+#![allow(internal_features)]
+#![feature(rustdoc_internals)]
 #![feature(associated_type_bounds)]
 #![feature(box_patterns)]
 #![feature(const_trait_impl)]
@@ -59,9 +59,7 @@ use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 /// Requirements for a `StableHashingContext` to be used in this crate.
 /// This is a hack to allow using the `HashStable_Generic` derive macro
 /// instead of implementing everything in `rustc_middle`.
-pub trait HashStableContext:
-    rustc_type_ir::HashStableContext + rustc_span::HashStableContext
-{
+pub trait HashStableContext: rustc_span::HashStableContext {
     fn hash_attr(&mut self, _: &ast::Attribute, hasher: &mut StableHasher);
 }
 
diff --git a/compiler/rustc_ast/src/mut_visit.rs b/compiler/rustc_ast/src/mut_visit.rs
index 7c0a78253..557ae02a8 100644
--- a/compiler/rustc_ast/src/mut_visit.rs
+++ b/compiler/rustc_ast/src/mut_visit.rs
@@ -7,10 +7,10 @@
 //! a `MutVisitor` renaming item names in a module will miss all of those
 //! that are created by the expansion of a macro.
 
+use crate::ast::*;
 use crate::ptr::P;
 use crate::token::{self, Token};
 use crate::tokenstream::*;
-use crate::{ast::*, StaticItem};
 
 use rustc_data_structures::flat_map_in_place::FlatMapInPlace;
 use rustc_data_structures::stack::ensure_sufficient_stack;
@@ -121,8 +121,8 @@ pub trait MutVisitor: Sized {
         noop_visit_fn_decl(d, self);
     }
 
-    fn visit_asyncness(&mut self, a: &mut Async) {
-        noop_visit_asyncness(a, self);
+    fn visit_coroutine_kind(&mut self, a: &mut CoroutineKind) {
+        noop_visit_coroutine_kind(a, self);
     }
 
     fn visit_closure_binder(&mut self, b: &mut ClosureBinder) {
@@ -453,7 +453,7 @@ pub fn noop_flat_map_arm<T: MutVisitor>(mut arm: Arm, vis: &mut T) -> SmallVec<[
     vis.visit_id(id);
     vis.visit_pat(pat);
     visit_opt(guard, |guard| vis.visit_expr(guard));
-    vis.visit_expr(body);
+    visit_opt(body, |body| vis.visit_expr(body));
     vis.visit_span(span);
     smallvec![arm]
 }
@@ -682,7 +682,7 @@ pub fn visit_attr_tt<T: MutVisitor>(tt: &mut AttrTokenTree, vis: &mut T) {
         AttrTokenTree::Token(token, _) => {
             visit_token(token, vis);
         }
-        AttrTokenTree::Delimited(DelimSpan { open, close }, _delim, tts) => {
+        AttrTokenTree::Delimited(DelimSpan { open, close }, _spacing, _delim, tts) => {
             vis.visit_span(open);
             vis.visit_span(close);
             visit_attr_tts(tts, vis);
@@ -709,7 +709,7 @@ pub fn visit_tt<T: MutVisitor>(tt: &mut TokenTree, vis: &mut T) {
         TokenTree::Token(token, _) => {
             visit_token(token, vis);
         }
-        TokenTree::Delimited(DelimSpan { open, close }, _delim, tts) => {
+        TokenTree::Delimited(DelimSpan { open, close }, _spacing, _delim, tts) => {
             vis.visit_span(open);
             vis.visit_span(close);
             visit_tts(tts, vis);
@@ -764,7 +764,10 @@ pub fn visit_token<T: MutVisitor>(t: &mut Token, vis: &mut T) {
             return; // Avoid visiting the span for the second time.
         }
         token::Interpolated(nt) => {
-            visit_nonterminal(Lrc::make_mut(nt), vis);
+            let nt = Lrc::make_mut(nt);
+            let (nt, sp) = (&mut nt.0, &mut nt.1);
+            vis.visit_span(sp);
+            visit_nonterminal(nt, vis);
         }
         _ => {}
     }
@@ -868,13 +871,15 @@ pub fn noop_visit_closure_binder<T: MutVisitor>(binder: &mut ClosureBinder, vis:
     }
 }
 
-pub fn noop_visit_asyncness<T: MutVisitor>(asyncness: &mut Async, vis: &mut T) {
-    match asyncness {
-        Async::Yes { span: _, closure_id, return_impl_trait_id } => {
+pub fn noop_visit_coroutine_kind<T: MutVisitor>(coroutine_kind: &mut CoroutineKind, vis: &mut T) {
+    match coroutine_kind {
+        CoroutineKind::Async { span, closure_id, return_impl_trait_id }
+        | CoroutineKind::Gen { span, closure_id, return_impl_trait_id }
+        | CoroutineKind::AsyncGen { span, closure_id, return_impl_trait_id } => {
+            vis.visit_span(span);
             vis.visit_id(closure_id);
             vis.visit_id(return_impl_trait_id);
         }
-        Async::No => {}
     }
 }
 
@@ -971,7 +976,7 @@ pub fn noop_visit_where_predicate<T: MutVisitor>(pred: &mut WherePredicate, vis:
 
 pub fn noop_visit_variant_data<T: MutVisitor>(vdata: &mut VariantData, vis: &mut T) {
     match vdata {
-        VariantData::Struct(fields, ..) => {
+        VariantData::Struct { fields, .. } => {
             fields.flat_map_in_place(|field| vis.flat_map_field_def(field));
         }
         VariantData::Tuple(fields, id) => {
@@ -1167,9 +1172,9 @@ fn visit_const_item<T: MutVisitor>(
 }
 
 pub fn noop_visit_fn_header<T: MutVisitor>(header: &mut FnHeader, vis: &mut T) {
-    let FnHeader { unsafety, asyncness, constness, ext: _ } = header;
+    let FnHeader { unsafety, coroutine_kind, constness, ext: _ } = header;
     visit_constness(constness, vis);
-    vis.visit_asyncness(asyncness);
+    coroutine_kind.as_mut().map(|coroutine_kind| vis.visit_coroutine_kind(coroutine_kind));
     visit_unsafety(unsafety, vis);
 }
 
@@ -1246,7 +1251,7 @@ pub fn noop_visit_pat<T: MutVisitor>(pat: &mut P<Pat>, vis: &mut T) {
     let Pat { id, kind, span, tokens } = pat.deref_mut();
     vis.visit_id(id);
     match kind {
-        PatKind::Wild | PatKind::Rest => {}
+        PatKind::Wild | PatKind::Rest | PatKind::Never => {}
         PatKind::Ident(_binding_mode, ident, sub) => {
             vis.visit_ident(ident);
             visit_opt(sub, |sub| vis.visit_pat(sub));
@@ -1403,7 +1408,7 @@ pub fn noop_visit_expr<T: MutVisitor>(
             binder,
             capture_clause,
             constness,
-            asyncness,
+            coroutine_kind,
             movability: _,
             fn_decl,
             body,
@@ -1412,7 +1417,7 @@ pub fn noop_visit_expr<T: MutVisitor>(
         }) => {
             vis.visit_closure_binder(binder);
             visit_constness(constness, vis);
-            vis.visit_asyncness(asyncness);
+            coroutine_kind.as_mut().map(|coroutine_kind| vis.visit_coroutine_kind(coroutine_kind));
             vis.visit_capture_by(capture_clause);
             vis.visit_fn_decl(fn_decl);
             vis.visit_expr(body);
diff --git a/compiler/rustc_ast/src/node_id.rs b/compiler/rustc_ast/src/node_id.rs
index d16741757..1cd244953 100644
--- a/compiler/rustc_ast/src/node_id.rs
+++ b/compiler/rustc_ast/src/node_id.rs
@@ -8,6 +8,8 @@ rustc_index::newtype_index! {
     /// This is later turned into [`DefId`] and `HirId` for the HIR.
     ///
     /// [`DefId`]: rustc_span::def_id::DefId
+    #[encodable]
+    #[orderable]
     #[debug_format = "NodeId({})"]
     pub struct NodeId {
         /// The [`NodeId`] used to represent the root of the crate.
diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs
index 914c97a14..b0cd2ec98 100644
--- a/compiler/rustc_ast/src/token.rs
+++ b/compiler/rustc_ast/src/token.rs
@@ -13,7 +13,7 @@ use rustc_macros::HashStable_Generic;
 use rustc_span::symbol::{kw, sym};
 #[allow(hidden_glob_reexports)]
 use rustc_span::symbol::{Ident, Symbol};
-use rustc_span::{self, edition::Edition, Span, DUMMY_SP};
+use rustc_span::{edition::Edition, Span, DUMMY_SP};
 use std::borrow::Cow;
 use std::fmt;
 
@@ -110,7 +110,7 @@ impl Lit {
             Ident(name, false) if name.is_bool_lit() => Some(Lit::new(Bool, name, None)),
             Literal(token_lit) => Some(token_lit),
             Interpolated(ref nt)
-                if let NtExpr(expr) | NtLiteral(expr) = &**nt
+                if let NtExpr(expr) | NtLiteral(expr) = &nt.0
                     && let ast::ExprKind::Lit(token_lit) = expr.kind =>
             {
                 Some(token_lit)
@@ -238,9 +238,9 @@ pub enum TokenKind {
     EqEq,
     /// `!=`
     Ne,
-    /// `>`
-    Ge,
     /// `>=`
+    Ge,
+    /// `>`
     Gt,
     /// `&&`
     AndAnd,
@@ -314,7 +314,7 @@ pub enum TokenKind {
     /// - It prevents `Token` from implementing `Copy`.
     /// It adds complexity and likely slows things down. Please don't add new
     /// occurrences of this token kind!
-    Interpolated(Lrc<Nonterminal>),
+    Interpolated(Lrc<(Nonterminal, Span)>),
 
     /// A doc comment token.
     /// `Symbol` is the doc comment's data excluding its "quotes" (`///`, `/**`, etc)
@@ -388,7 +388,8 @@ impl TokenKind {
         match *self {
             Comma => Some(vec![Dot, Lt, Semi]),
             Semi => Some(vec![Colon, Comma]),
-            FatArrow => Some(vec![Eq, RArrow]),
+            Colon => Some(vec![Semi]),
+            FatArrow => Some(vec![Eq, RArrow, Ge, Gt]),
             _ => None,
         }
     }
@@ -421,7 +422,7 @@ impl Token {
     /// if they keep spans or perform edition checks.
     pub fn uninterpolated_span(&self) -> Span {
         match &self.kind {
-            Interpolated(nt) => nt.span(),
+            Interpolated(nt) => nt.0.use_span(),
             _ => self.span,
         }
     }
@@ -464,7 +465,7 @@ impl Token {
             ModSep                            | // global path
             Lifetime(..)                      | // labeled loop
             Pound                             => true, // expression attributes
-            Interpolated(ref nt) => matches!(**nt, NtLiteral(..) |
+            Interpolated(ref nt) => matches!(&nt.0, NtLiteral(..) |
                 NtExpr(..)    |
                 NtBlock(..)   |
                 NtPath(..)),
@@ -488,7 +489,7 @@ impl Token {
             | DotDot | DotDotDot | DotDotEq      // ranges
             | Lt | BinOp(Shl)                    // associated path
             | ModSep                    => true, // global path
-            Interpolated(ref nt) => matches!(**nt, NtLiteral(..) |
+            Interpolated(ref nt) => matches!(&nt.0, NtLiteral(..) |
                 NtPat(..)     |
                 NtBlock(..)   |
                 NtPath(..)),
@@ -511,7 +512,7 @@ impl Token {
             Lifetime(..)                | // lifetime bound in trait object
             Lt | BinOp(Shl)             | // associated path
             ModSep                      => true, // global path
-            Interpolated(ref nt) => matches!(**nt, NtTy(..) | NtPath(..)),
+            Interpolated(ref nt) => matches!(&nt.0, NtTy(..) | NtPath(..)),
             // For anonymous structs or unions, which only appear in specific positions
             // (type of struct fields or union fields), we don't consider them as regular types
             _ => false,
@@ -522,7 +523,7 @@ impl Token {
     pub fn can_begin_const_arg(&self) -> bool {
         match self.kind {
             OpenDelim(Delimiter::Brace) => true,
-            Interpolated(ref nt) => matches!(**nt, NtExpr(..) | NtBlock(..) | NtLiteral(..)),
+            Interpolated(ref nt) => matches!(&nt.0, NtExpr(..) | NtBlock(..) | NtLiteral(..)),
             _ => self.can_begin_literal_maybe_minus(),
         }
     }
@@ -576,7 +577,7 @@ impl Token {
         match self.uninterpolate().kind {
             Literal(..) | BinOp(Minus) => true,
             Ident(name, false) if name.is_bool_lit() => true,
-            Interpolated(ref nt) => match &**nt {
+            Interpolated(ref nt) => match &nt.0 {
                 NtLiteral(_) => true,
                 NtExpr(e) => match &e.kind {
                     ast::ExprKind::Lit(_) => true,
@@ -597,9 +598,9 @@ impl Token {
     /// otherwise returns the original token.
     pub fn uninterpolate(&self) -> Cow<'_, Token> {
         match &self.kind {
-            Interpolated(nt) => match **nt {
+            Interpolated(nt) => match &nt.0 {
                 NtIdent(ident, is_raw) => {
-                    Cow::Owned(Token::new(Ident(ident.name, is_raw), ident.span))
+                    Cow::Owned(Token::new(Ident(ident.name, *is_raw), ident.span))
                 }
                 NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)),
                 _ => Cow::Borrowed(self),
@@ -614,8 +615,8 @@ impl Token {
         // We avoid using `Token::uninterpolate` here because it's slow.
         match &self.kind {
             &Ident(name, is_raw) => Some((Ident::new(name, self.span), is_raw)),
-            Interpolated(nt) => match **nt {
-                NtIdent(ident, is_raw) => Some((ident, is_raw)),
+            Interpolated(nt) => match &nt.0 {
+                NtIdent(ident, is_raw) => Some((*ident, *is_raw)),
                 _ => None,
             },
             _ => None,
@@ -628,8 +629,8 @@ impl Token {
         // We avoid using `Token::uninterpolate` here because it's slow.
         match &self.kind {
             &Lifetime(name) => Some(Ident::new(name, self.span)),
-            Interpolated(nt) => match **nt {
-                NtLifetime(ident) => Some(ident),
+            Interpolated(nt) => match &nt.0 {
+                NtLifetime(ident) => Some(*ident),
                 _ => None,
             },
             _ => None,
@@ -655,7 +656,7 @@ impl Token {
     /// Returns `true` if the token is an interpolated path.
     fn is_path(&self) -> bool {
         if let Interpolated(nt) = &self.kind
-            && let NtPath(..) = **nt
+            && let NtPath(..) = &nt.0
         {
             return true;
         }
@@ -668,7 +669,7 @@ impl Token {
     /// (which happens while parsing the result of macro expansion)?
     pub fn is_whole_expr(&self) -> bool {
         if let Interpolated(nt) = &self.kind
-            && let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtBlock(_) = **nt
+            && let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtBlock(_) = &nt.0
         {
             return true;
         }
@@ -679,7 +680,7 @@ impl Token {
     /// Is the token an interpolated block (`$b:block`)?
     pub fn is_whole_block(&self) -> bool {
         if let Interpolated(nt) = &self.kind
-            && let NtBlock(..) = **nt
+            && let NtBlock(..) = &nt.0
         {
             return true;
         }
@@ -755,6 +756,11 @@ impl Token {
         )
     }
 
+    /// Returns `true` if the token is the integer literal.
+    pub fn is_integer_lit(&self) -> bool {
+        matches!(self.kind, Literal(Lit { kind: LitKind::Integer, .. }))
+    }
+
     /// Returns `true` if the token is a non-raw identifier for which `pred` holds.
     pub fn is_non_raw_ident_where(&self, pred: impl FnOnce(Ident) -> bool) -> bool {
         match self.ident() {
@@ -927,7 +933,7 @@ impl fmt::Display for NonterminalKind {
 }
 
 impl Nonterminal {
-    pub fn span(&self) -> Span {
+    pub fn use_span(&self) -> Span {
         match self {
             NtItem(item) => item.span,
             NtBlock(block) => block.span,
@@ -941,6 +947,23 @@ impl Nonterminal {
             NtVis(vis) => vis.span,
         }
     }
+
+    pub fn descr(&self) -> &'static str {
+        match self {
+            NtItem(..) => "item",
+            NtBlock(..) => "block",
+            NtStmt(..) => "statement",
+            NtPat(..) => "pattern",
+            NtExpr(..) => "expression",
+            NtLiteral(..) => "literal",
+            NtTy(..) => "type",
+            NtIdent(..) => "identifier",
+            NtLifetime(..) => "lifetime",
+            NtMeta(..) => "attribute",
+            NtPath(..) => "path",
+            NtVis(..) => "visibility",
+        }
+    }
 }
 
 impl PartialEq for Nonterminal {
diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs
index 23b8f9c12..4c0c49658 100644
--- a/compiler/rustc_ast/src/tokenstream.rs
+++ b/compiler/rustc_ast/src/tokenstream.rs
@@ -46,7 +46,7 @@ pub enum TokenTree {
     /// delimiters are implicitly represented by `Delimited`.
     Token(Token, Spacing),
     /// A delimited sequence of token trees.
-    Delimited(DelimSpan, Delimiter, TokenStream),
+    Delimited(DelimSpan, DelimSpacing, Delimiter, TokenStream),
 }
 
 // Ensure all fields of `TokenTree` are `DynSend` and `DynSync`.
@@ -62,11 +62,11 @@ where
 }
 
 impl TokenTree {
-    /// Checks if this `TokenTree` is equal to the other, regardless of span information.
+    /// Checks if this `TokenTree` is equal to the other, regardless of span/spacing information.
     pub fn eq_unspanned(&self, other: &TokenTree) -> bool {
         match (self, other) {
             (TokenTree::Token(token, _), TokenTree::Token(token2, _)) => token.kind == token2.kind,
-            (TokenTree::Delimited(_, delim, tts), TokenTree::Delimited(_, delim2, tts2)) => {
+            (TokenTree::Delimited(.., delim, tts), TokenTree::Delimited(.., delim2, tts2)) => {
                 delim == delim2 && tts.eq_unspanned(tts2)
             }
             _ => false,
@@ -99,6 +99,11 @@ impl TokenTree {
         TokenTree::Token(Token::new(kind, span), Spacing::Joint)
     }
 
+    /// Create a `TokenTree::Token` with joint-hidden spacing.
+    pub fn token_joint_hidden(kind: TokenKind, span: Span) -> TokenTree {
+        TokenTree::Token(Token::new(kind, span), Spacing::JointHidden)
+    }
+
     pub fn uninterpolate(&self) -> Cow<'_, TokenTree> {
         match self {
             TokenTree::Token(token, spacing) => match token.uninterpolate() {
@@ -183,7 +188,7 @@ pub struct AttrTokenStream(pub Lrc<Vec<AttrTokenTree>>);
 #[derive(Clone, Debug, Encodable, Decodable)]
 pub enum AttrTokenTree {
     Token(Token, Spacing),
-    Delimited(DelimSpan, Delimiter, AttrTokenStream),
+    Delimited(DelimSpan, DelimSpacing, Delimiter, AttrTokenStream),
     /// Stores the attributes for an attribute target,
     /// along with the tokens for that attribute target.
     /// See `AttributesData` for more information
@@ -208,9 +213,14 @@ impl AttrTokenStream {
                 AttrTokenTree::Token(inner, spacing) => {
                     smallvec![TokenTree::Token(inner.clone(), *spacing)].into_iter()
                 }
-                AttrTokenTree::Delimited(span, delim, stream) => {
-                    smallvec![TokenTree::Delimited(*span, *delim, stream.to_tokenstream()),]
-                        .into_iter()
+                AttrTokenTree::Delimited(span, spacing, delim, stream) => {
+                    smallvec![TokenTree::Delimited(
+                        *span,
+                        *spacing,
+                        *delim,
+                        stream.to_tokenstream()
+                    ),]
+                    .into_iter()
                 }
                 AttrTokenTree::Attributes(data) => {
                     let idx = data
@@ -230,7 +240,7 @@ impl AttrTokenStream {
                         let mut found = false;
                         // Check the last two trees (to account for a trailing semi)
                         for tree in target_tokens.iter_mut().rev().take(2) {
-                            if let TokenTree::Delimited(span, delim, delim_tokens) = tree {
+                            if let TokenTree::Delimited(span, spacing, delim, delim_tokens) = tree {
                                 // Inner attributes are only supported on extern blocks, functions,
                                 // impls, and modules. All of these have their inner attributes
                                 // placed at the beginning of the rightmost outermost braced group:
@@ -250,7 +260,7 @@ impl AttrTokenStream {
                                     stream.push_stream(inner_attr.tokens());
                                 }
                                 stream.push_stream(delim_tokens.clone());
-                                *tree = TokenTree::Delimited(*span, *delim, stream);
+                                *tree = TokenTree::Delimited(*span, *spacing, *delim, stream);
                                 found = true;
                                 break;
                             }
@@ -303,21 +313,64 @@ pub struct AttributesData {
 #[derive(Clone, Debug, Default, Encodable, Decodable)]
 pub struct TokenStream(pub(crate) Lrc<Vec<TokenTree>>);
 
-/// Similar to `proc_macro::Spacing`, but for tokens.
-///
-/// Note that all `ast::TokenTree::Token` instances have a `Spacing`, but when
-/// we convert to `proc_macro::TokenTree` for proc macros only `Punct`
-/// `TokenTree`s have a `proc_macro::Spacing`.
+/// Indicates whether a token can join with the following token to form a
+/// compound token. Used for conversions to `proc_macro::Spacing`. Also used to
+/// guide pretty-printing, which is where the `JointHidden` value (which isn't
+/// part of `proc_macro::Spacing`) comes in useful.
 #[derive(Clone, Copy, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
 pub enum Spacing {
-    /// The token is not immediately followed by an operator token (as
-    /// determined by `Token::is_op`). E.g. a `+` token is `Alone` in `+ =`,
-    /// `+/*foo*/=`, `+ident`, and `+()`.
+    /// The token cannot join with the following token to form a compound
+    /// token.
+    ///
+    /// In token streams parsed from source code, the compiler will use `Alone`
+    /// for any token immediately followed by whitespace, a non-doc comment, or
+    /// EOF.
+    ///
+    /// When constructing token streams within the compiler, use this for each
+    /// token that (a) should be pretty-printed with a space after it, or (b)
+    /// is the last token in the stream. (In the latter case the choice of
+    /// spacing doesn't matter because it is never used for the last token. We
+    /// arbitrarily use `Alone`.)
+    ///
+    /// Converts to `proc_macro::Spacing::Alone`, and
+    /// `proc_macro::Spacing::Alone` converts back to this.
     Alone,
 
-    /// The token is immediately followed by an operator token. E.g. a `+`
-    /// token is `Joint` in `+=` and `++`.
+    /// The token can join with the following token to form a compound token.
+    ///
+    /// In token streams parsed from source code, the compiler will use `Joint`
+    /// for any token immediately followed by punctuation (as determined by
+    /// `Token::is_punct`).
+    ///
+    /// When constructing token streams within the compiler, use this for each
+    /// token that (a) should be pretty-printed without a space after it, and
+    /// (b) is followed by a punctuation token.
+    ///
+    /// Converts to `proc_macro::Spacing::Joint`, and
+    /// `proc_macro::Spacing::Joint` converts back to this.
     Joint,
+
+    /// The token can join with the following token to form a compound token,
+    /// but this will not be visible at the proc macro level. (This is what the
+    /// `Hidden` means; see below.)
+    ///
+    /// In token streams parsed from source code, the compiler will use
+    /// `JointHidden` for any token immediately followed by anything not
+    /// covered by the `Alone` and `Joint` cases: an identifier, lifetime,
+    /// literal, delimiter, doc comment.
+    ///
+    /// When constructing token streams, use this for each token that (a)
+    /// should be pretty-printed without a space after it, and (b) is followed
+    /// by a non-punctuation token.
+    ///
+    /// Converts to `proc_macro::Spacing::Alone`, but
+    /// `proc_macro::Spacing::Alone` converts back to `token::Spacing::Alone`.
+    /// Because of that, pretty-printing of `TokenStream`s produced by proc
+    /// macros is unavoidably uglier (with more whitespace between tokens) than
+    /// pretty-printing of `TokenStream`'s produced by other means (i.e. parsed
+    /// source code, internally constructed token streams, and token streams
+    /// produced by declarative macros).
+    JointHidden,
 }
 
 impl TokenStream {
@@ -421,21 +474,14 @@ impl TokenStream {
         self
     }
 
-    /// Create a token stream containing a single token with alone spacing.
+    /// Create a token stream containing a single token with alone spacing. The
+    /// spacing used for the final token in a constructed stream doesn't matter
+    /// because it's never used. In practice we arbitrarily use
+    /// `Spacing::Alone`.
     pub fn token_alone(kind: TokenKind, span: Span) -> TokenStream {
         TokenStream::new(vec![TokenTree::token_alone(kind, span)])
     }
 
-    /// Create a token stream containing a single token with joint spacing.
-    pub fn token_joint(kind: TokenKind, span: Span) -> TokenStream {
-        TokenStream::new(vec![TokenTree::token_joint(kind, span)])
-    }
-
-    /// Create a token stream containing a single `Delimited`.
-    pub fn delimited(span: DelimSpan, delim: Delimiter, tts: TokenStream) -> TokenStream {
-        TokenStream::new(vec![TokenTree::Delimited(span, delim, tts)])
-    }
-
     pub fn from_ast(node: &(impl HasAttrs + HasSpan + HasTokens + fmt::Debug)) -> TokenStream {
         let Some(tokens) = node.tokens() else {
             panic!("missing tokens for node at {:?}: {:?}", node.span(), node);
@@ -477,13 +523,14 @@ impl TokenStream {
 
     fn flatten_token(token: &Token, spacing: Spacing) -> TokenTree {
         match &token.kind {
-            token::Interpolated(nt) if let token::NtIdent(ident, is_raw) = **nt => {
+            token::Interpolated(nt) if let token::NtIdent(ident, is_raw) = nt.0 => {
                 TokenTree::Token(Token::new(token::Ident(ident.name, is_raw), ident.span), spacing)
             }
             token::Interpolated(nt) => TokenTree::Delimited(
                 DelimSpan::from_single(token.span),
+                DelimSpacing::new(Spacing::JointHidden, spacing),
                 Delimiter::Invisible,
-                TokenStream::from_nonterminal_ast(nt).flattened(),
+                TokenStream::from_nonterminal_ast(&nt.0).flattened(),
             ),
             _ => TokenTree::Token(token.clone(), spacing),
         }
@@ -492,8 +539,8 @@ impl TokenStream {
     fn flatten_token_tree(tree: &TokenTree) -> TokenTree {
         match tree {
             TokenTree::Token(token, spacing) => TokenStream::flatten_token(token, *spacing),
-            TokenTree::Delimited(span, delim, tts) => {
-                TokenTree::Delimited(*span, *delim, tts.flattened())
+            TokenTree::Delimited(span, spacing, delim, tts) => {
+                TokenTree::Delimited(*span, *spacing, *delim, tts.flattened())
             }
         }
     }
@@ -503,7 +550,7 @@ impl TokenStream {
         fn can_skip(stream: &TokenStream) -> bool {
             stream.trees().all(|tree| match tree {
                 TokenTree::Token(token, _) => !matches!(token.kind, token::Interpolated(_)),
-                TokenTree::Delimited(_, _, inner) => can_skip(inner),
+                TokenTree::Delimited(.., inner) => can_skip(inner),
             })
         }
 
@@ -517,7 +564,7 @@ impl TokenStream {
     // If `vec` is not empty, try to glue `tt` onto its last token. The return
     // value indicates if gluing took place.
     fn try_glue_to_last(vec: &mut Vec<TokenTree>, tt: &TokenTree) -> bool {
-        if let Some(TokenTree::Token(last_tok, Spacing::Joint)) = vec.last()
+        if let Some(TokenTree::Token(last_tok, Spacing::Joint | Spacing::JointHidden)) = vec.last()
             && let TokenTree::Token(tok, spacing) = tt
             && let Some(glued_tok) = last_tok.glue(tok)
         {
@@ -592,9 +639,10 @@ impl TokenStream {
 
                     &TokenTree::Token(..) => i += 1,
 
-                    &TokenTree::Delimited(sp, delim, ref delim_stream) => {
+                    &TokenTree::Delimited(sp, spacing, delim, ref delim_stream) => {
                         if let Some(desugared_delim_stream) = desugar_inner(delim_stream.clone()) {
-                            let new_tt = TokenTree::Delimited(sp, delim, desugared_delim_stream);
+                            let new_tt =
+                                TokenTree::Delimited(sp, spacing, delim, desugared_delim_stream);
                             Lrc::make_mut(&mut stream.0)[i] = new_tt;
                             modified = true;
                         }
@@ -622,10 +670,11 @@ impl TokenStream {
                 num_of_hashes = cmp::max(num_of_hashes, count);
             }
 
-            // `/// foo` becomes `doc = r"foo"`.
+            // `/// foo` becomes `[doc = r"foo"]`.
             let delim_span = DelimSpan::from_single(span);
             let body = TokenTree::Delimited(
                 delim_span,
+                DelimSpacing::new(Spacing::JointHidden, Spacing::Alone),
                 Delimiter::Bracket,
                 [
                     TokenTree::token_alone(token::Ident(sym::doc, false), span),
@@ -641,7 +690,7 @@ impl TokenStream {
 
             if attr_style == AttrStyle::Inner {
                 vec![
-                    TokenTree::token_alone(token::Pound, span),
+                    TokenTree::token_joint(token::Pound, span),
                     TokenTree::token_alone(token::Not, span),
                     body,
                 ]
@@ -738,6 +787,18 @@ impl DelimSpan {
     }
 }
 
+#[derive(Copy, Clone, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
+pub struct DelimSpacing {
+    pub open: Spacing,
+    pub close: Spacing,
+}
+
+impl DelimSpacing {
+    pub fn new(open: Spacing, close: Spacing) -> DelimSpacing {
+        DelimSpacing { open, close }
+    }
+}
+
 // Some types are used a lot. Make sure they don't unintentionally get bigger.
 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 mod size_asserts {
diff --git a/compiler/rustc_ast/src/util/classify.rs b/compiler/rustc_ast/src/util/classify.rs
index 821fca665..4dece0797 100644
--- a/compiler/rustc_ast/src/util/classify.rs
+++ b/compiler/rustc_ast/src/util/classify.rs
@@ -40,15 +40,44 @@ pub fn expr_trailing_brace(mut expr: &ast::Expr) -> Option<&ast::Expr> {
             | Range(_, Some(e), _)
             | Ret(Some(e))
             | Unary(_, e)
-            | Yield(Some(e)) => {
+            | Yield(Some(e))
+            | Yeet(Some(e))
+            | Become(e) => {
                 expr = e;
             }
             Closure(closure) => {
                 expr = &closure.body;
             }
             Gen(..) | Block(..) | ForLoop(..) | If(..) | Loop(..) | Match(..) | Struct(..)
-            | TryBlock(..) | While(..) => break Some(expr),
-            _ => break None,
+            | TryBlock(..) | While(..) | ConstBlock(_) => break Some(expr),
+
+            // FIXME: These can end in `}`, but changing these would break stable code.
+            InlineAsm(_) | OffsetOf(_, _) | MacCall(_) | IncludedBytes(_) | FormatArgs(_) => {
+                break None;
+            }
+
+            Break(_, None)
+            | Range(_, None, _)
+            | Ret(None)
+            | Yield(None)
+            | Array(_)
+            | Call(_, _)
+            | MethodCall(_)
+            | Tup(_)
+            | Lit(_)
+            | Cast(_, _)
+            | Type(_, _)
+            | Await(_, _)
+            | Field(_, _)
+            | Index(_, _, _)
+            | Underscore
+            | Path(_, _)
+            | Continue(_)
+            | Repeat(_, _)
+            | Paren(_)
+            | Try(_)
+            | Yeet(None)
+            | Err => break None,
         }
     }
 }
diff --git a/compiler/rustc_ast/src/util/literal.rs b/compiler/rustc_ast/src/util/literal.rs
index 50eb92125..92b9adf1d 100644
--- a/compiler/rustc_ast/src/util/literal.rs
+++ b/compiler/rustc_ast/src/util/literal.rs
@@ -77,6 +77,8 @@ impl LitKind {
                 // new symbol because the string in the LitKind is different to the
                 // string in the token.
                 let s = symbol.as_str();
+                // Vanilla strings are so common we optimize for the common case where no chars
+                // requiring special behaviour are present.
                 let symbol = if s.contains(['\\', '\r']) {
                     let mut buf = String::with_capacity(s.len());
                     let mut error = Ok(());
@@ -104,27 +106,20 @@ impl LitKind {
                 LitKind::Str(symbol, ast::StrStyle::Cooked)
             }
             token::StrRaw(n) => {
-                // Ditto.
-                let s = symbol.as_str();
-                let symbol =
-                    if s.contains('\r') {
-                        let mut buf = String::with_capacity(s.len());
-                        let mut error = Ok(());
-                        unescape_literal(s, Mode::RawStr, &mut |_, unescaped_char| {
-                            match unescaped_char {
-                                Ok(c) => buf.push(c),
-                                Err(err) => {
-                                    if err.is_fatal() {
-                                        error = Err(LitError::LexerError);
-                                    }
-                                }
+                // Raw strings have no escapes, so we only need to check for invalid chars, and we
+                // can reuse the symbol on success.
+                let mut error = Ok(());
+                unescape_literal(symbol.as_str(), Mode::RawStr, &mut |_, unescaped_char| {
+                    match unescaped_char {
+                        Ok(_) => {}
+                        Err(err) => {
+                            if err.is_fatal() {
+                                error = Err(LitError::LexerError);
                             }
-                        });
-                        error?;
-                        Symbol::intern(&buf)
-                    } else {
-                        symbol
-                    };
+                        }
+                    }
+                });
+                error?;
                 LitKind::Str(symbol, ast::StrStyle::Raw(n))
             }
             token::ByteStr => {
@@ -143,25 +138,19 @@ impl LitKind {
                 LitKind::ByteStr(buf.into(), StrStyle::Cooked)
             }
             token::ByteStrRaw(n) => {
+                // Raw strings have no escapes, so we only need to check for invalid chars, and we
+                // can convert the symbol directly to a `Lrc<u8>` on success.
                 let s = symbol.as_str();
-                let bytes = if s.contains('\r') {
-                    let mut buf = Vec::with_capacity(s.len());
-                    let mut error = Ok(());
-                    unescape_literal(s, Mode::RawByteStr, &mut |_, c| match c {
-                        Ok(c) => buf.push(byte_from_char(c)),
-                        Err(err) => {
-                            if err.is_fatal() {
-                                error = Err(LitError::LexerError);
-                            }
+                let mut error = Ok(());
+                unescape_literal(s, Mode::RawByteStr, &mut |_, c| match c {
+                    Ok(_) => {}
+                    Err(err) => {
+                        if err.is_fatal() {
+                            error = Err(LitError::LexerError);
                         }
-                    });
-                    error?;
-                    buf
-                } else {
-                    symbol.to_string().into_bytes()
-                };
-
-                LitKind::ByteStr(bytes.into(), StrStyle::Raw(n))
+                    }
+                });
+                LitKind::ByteStr(s.to_owned().into_bytes().into(), StrStyle::Raw(n))
             }
             token::CStr => {
                 let s = symbol.as_str();
@@ -172,7 +161,6 @@ impl LitKind {
                         error = Err(LitError::NulInCStr(span));
                     }
                     Ok(CStrUnit::Byte(b)) => buf.push(b),
-                    Ok(CStrUnit::Char(c)) if c.len_utf8() == 1 => buf.push(c as u8),
                     Ok(CStrUnit::Char(c)) => {
                         buf.extend_from_slice(c.encode_utf8(&mut [0; 4]).as_bytes())
                     }
@@ -187,18 +175,15 @@ impl LitKind {
                 LitKind::CStr(buf.into(), StrStyle::Cooked)
             }
             token::CStrRaw(n) => {
+                // Raw strings have no escapes, so we only need to check for invalid chars, and we
+                // can convert the symbol directly to a `Lrc<u8>` on success.
                 let s = symbol.as_str();
-                let mut buf = Vec::with_capacity(s.len());
                 let mut error = Ok(());
                 unescape_c_string(s, Mode::RawCStr, &mut |span, c| match c {
                     Ok(CStrUnit::Byte(0) | CStrUnit::Char('\0')) => {
                         error = Err(LitError::NulInCStr(span));
                     }
-                    Ok(CStrUnit::Byte(b)) => buf.push(b),
-                    Ok(CStrUnit::Char(c)) if c.len_utf8() == 1 => buf.push(c as u8),
-                    Ok(CStrUnit::Char(c)) => {
-                        buf.extend_from_slice(c.encode_utf8(&mut [0; 4]).as_bytes())
-                    }
+                    Ok(_) => {}
                     Err(err) => {
                         if err.is_fatal() {
                             error = Err(LitError::LexerError);
@@ -206,6 +191,7 @@ impl LitKind {
                     }
                 });
                 error?;
+                let mut buf = s.to_owned().into_bytes();
                 buf.push(0);
                 LitKind::CStr(buf.into(), StrStyle::Raw(n))
             }
diff --git a/compiler/rustc_ast/src/visit.rs b/compiler/rustc_ast/src/visit.rs
index 1caa39e2d..27f1b84f3 100644
--- a/compiler/rustc_ast/src/visit.rs
+++ b/compiler/rustc_ast/src/visit.rs
@@ -13,7 +13,7 @@
 //! instance, a walker looking for item names in a module will miss all of
 //! those that are created by the expansion of a macro.
 
-use crate::{ast::*, StaticItem};
+use crate::ast::*;
 
 use rustc_span::symbol::Ident;
 use rustc_span::Span;
@@ -559,7 +559,7 @@ pub fn walk_pat<'a, V: Visitor<'a>>(visitor: &mut V, pattern: &'a Pat) {
             walk_list!(visitor, visit_expr, lower_bound);
             walk_list!(visitor, visit_expr, upper_bound);
         }
-        PatKind::Wild | PatKind::Rest => {}
+        PatKind::Wild | PatKind::Rest | PatKind::Never => {}
         PatKind::Tuple(elems) | PatKind::Slice(elems) | PatKind::Or(elems) => {
             walk_list!(visitor, visit_pat, elems);
         }
@@ -861,7 +861,7 @@ pub fn walk_expr<'a, V: Visitor<'a>>(visitor: &mut V, expression: &'a Expr) {
         ExprKind::Closure(box Closure {
             binder,
             capture_clause,
-            asyncness: _,
+            coroutine_kind: _,
             constness: _,
             movability: _,
             fn_decl,
@@ -951,7 +951,7 @@ pub fn walk_param<'a, V: Visitor<'a>>(visitor: &mut V, param: &'a Param) {
 pub fn walk_arm<'a, V: Visitor<'a>>(visitor: &mut V, arm: &'a Arm) {
     visitor.visit_pat(&arm.pat);
     walk_list!(visitor, visit_expr, &arm.guard);
-    visitor.visit_expr(&arm.body);
+    walk_list!(visitor, visit_expr, &arm.body);
     walk_list!(visitor, visit_attribute, &arm.attrs);
 }
 
diff --git a/compiler/rustc_ast_lowering/messages.ftl b/compiler/rustc_ast_lowering/messages.ftl
index 91591a716..6bde4f2d8 100644
--- a/compiler/rustc_ast_lowering/messages.ftl
+++ b/compiler/rustc_ast_lowering/messages.ftl
@@ -91,6 +91,10 @@ ast_lowering_invalid_register =
 ast_lowering_invalid_register_class =
     invalid register class `{$reg_class}`: {$error}
 
+ast_lowering_match_arm_with_no_body =
+    `match` arm with no body
+    .suggestion = add a body after the pattern
+
 ast_lowering_misplaced_assoc_ty_binding =
     associated type bounds are only allowed in where clauses and function signatures, not in {$position}
 
@@ -104,6 +108,15 @@ ast_lowering_misplaced_impl_trait =
 ast_lowering_misplaced_relax_trait_bound =
     `?Trait` bounds are only permitted at the point where a type parameter is declared
 
+ast_lowering_never_pattern_with_body =
+    a never pattern is always unreachable
+    .label = this will never be executed
+    .suggestion = remove this expression
+
+ast_lowering_never_pattern_with_guard =
+    a guard on a never pattern will never be run
+    .suggestion = remove this guard
+
 ast_lowering_not_supported_for_lifetime_binder_async_closure =
     `for<...>` binders on `async` closures are not currently supported
 
diff --git a/compiler/rustc_ast_lowering/src/asm.rs b/compiler/rustc_ast_lowering/src/asm.rs
index a1e626996..4c81983c2 100644
--- a/compiler/rustc_ast_lowering/src/asm.rs
+++ b/compiler/rustc_ast_lowering/src/asm.rs
@@ -14,8 +14,8 @@ use rustc_ast::*;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexMap};
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
-use rustc_hir::definitions::DefPathData;
 use rustc_session::parse::feature_err;
+use rustc_span::symbol::kw;
 use rustc_span::{sym, Span};
 use rustc_target::asm;
 use std::collections::hash_map::Entry;
@@ -227,7 +227,8 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                             self.create_def(
                                 parent_def_id.def_id,
                                 node_id,
-                                DefPathData::AnonConst,
+                                kw::Empty,
+                                DefKind::AnonConst,
                                 *op_sp,
                             );
                             let anon_const = AnonConst { id: node_id, value: P(expr) };
@@ -335,67 +336,81 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                         hir::InlineAsmOperand::Const { .. }
                         | hir::InlineAsmOperand::SymFn { .. }
                         | hir::InlineAsmOperand::SymStatic { .. } => {
-                            unreachable!()
+                            unreachable!("{op:?} is not a register operand");
                         }
                     };
 
                     // Flag to output the error only once per operand
                     let mut skip = false;
-                    reg.overlapping_regs(|r| {
-                        let mut check = |used_regs: &mut FxHashMap<asm::InlineAsmReg, usize>,
-                                         input| {
-                            match used_regs.entry(r) {
-                                Entry::Occupied(o) => {
-                                    if skip {
-                                        return;
-                                    }
-                                    skip = true;
 
-                                    let idx2 = *o.get();
-                                    let (ref op2, op_sp2) = operands[idx2];
-                                    let Some(asm::InlineAsmRegOrRegClass::Reg(reg2)) = op2.reg()
-                                    else {
-                                        unreachable!();
-                                    };
+                    let mut check = |used_regs: &mut FxHashMap<asm::InlineAsmReg, usize>,
+                                     input,
+                                     r: asm::InlineAsmReg| {
+                        match used_regs.entry(r) {
+                            Entry::Occupied(o) => {
+                                if skip {
+                                    return;
+                                }
+                                skip = true;
 
-                                    let in_out = match (op, op2) {
-                                        (
-                                            hir::InlineAsmOperand::In { .. },
-                                            hir::InlineAsmOperand::Out { late, .. },
-                                        )
-                                        | (
-                                            hir::InlineAsmOperand::Out { late, .. },
-                                            hir::InlineAsmOperand::In { .. },
-                                        ) => {
-                                            assert!(!*late);
-                                            let out_op_sp = if input { op_sp2 } else { op_sp };
-                                            Some(out_op_sp)
-                                        }
-                                        _ => None,
-                                    };
+                                let idx2 = *o.get();
+                                let (ref op2, op_sp2) = operands[idx2];
 
-                                    sess.emit_err(RegisterConflict {
-                                        op_span1: op_sp,
-                                        op_span2: op_sp2,
-                                        reg1_name: reg.name(),
-                                        reg2_name: reg2.name(),
-                                        in_out,
-                                    });
-                                }
-                                Entry::Vacant(v) => {
-                                    if r == reg {
-                                        v.insert(idx);
+                                let in_out = match (op, op2) {
+                                    (
+                                        hir::InlineAsmOperand::In { .. },
+                                        hir::InlineAsmOperand::Out { late, .. },
+                                    )
+                                    | (
+                                        hir::InlineAsmOperand::Out { late, .. },
+                                        hir::InlineAsmOperand::In { .. },
+                                    ) => {
+                                        assert!(!*late);
+                                        let out_op_sp = if input { op_sp2 } else { op_sp };
+                                        Some(out_op_sp)
+                                    }
+                                    _ => None,
+                                };
+                                let reg_str = |idx| -> &str {
+                                    // HIR asm doesn't preserve the original alias string of the explicit register,
+                                    // so we have to retrieve it from AST
+                                    let (op, _): &(InlineAsmOperand, Span) = &asm.operands[idx];
+                                    if let Some(ast::InlineAsmRegOrRegClass::Reg(reg_sym)) =
+                                        op.reg()
+                                    {
+                                        reg_sym.as_str()
+                                    } else {
+                                        unreachable!("{op:?} is not a register operand");
                                     }
+                                };
+
+                                sess.emit_err(RegisterConflict {
+                                    op_span1: op_sp,
+                                    op_span2: op_sp2,
+                                    reg1_name: reg_str(idx),
+                                    reg2_name: reg_str(idx2),
+                                    in_out,
+                                });
+                            }
+                            Entry::Vacant(v) => {
+                                if r == reg {
+                                    v.insert(idx);
                                 }
                             }
-                        };
+                        }
+                    };
+                    let mut overlapping_with = vec![];
+                    reg.overlapping_regs(|r| {
+                        overlapping_with.push(r);
+                    });
+                    for r in overlapping_with {
                         if input {
-                            check(&mut used_input_regs, true);
+                            check(&mut used_input_regs, true, r);
                         }
                         if output {
-                            check(&mut used_output_regs, false);
+                            check(&mut used_output_regs, false, r);
                         }
-                    });
+                    }
                 }
             }
         }
@@ -410,12 +425,12 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                     continue;
                 }
 
-                let mut output_used = false;
+                let mut overlapping_with = vec![];
                 clobber.overlapping_regs(|reg| {
-                    if used_output_regs.contains_key(&reg) {
-                        output_used = true;
-                    }
+                    overlapping_with.push(reg);
                 });
+                let output_used =
+                    overlapping_with.iter().any(|reg| used_output_regs.contains_key(&reg));
 
                 if !output_used {
                     operands.push((
diff --git a/compiler/rustc_ast_lowering/src/errors.rs b/compiler/rustc_ast_lowering/src/errors.rs
index 6e1a9eff5..11bb55971 100644
--- a/compiler/rustc_ast_lowering/src/errors.rs
+++ b/compiler/rustc_ast_lowering/src/errors.rs
@@ -340,6 +340,32 @@ pub struct NotSupportedForLifetimeBinderAsyncClosure {
     pub span: Span,
 }
 
+#[derive(Diagnostic)]
+#[diag(ast_lowering_match_arm_with_no_body)]
+pub struct MatchArmWithNoBody {
+    #[primary_span]
+    pub span: Span,
+    #[suggestion(code = " => todo!(),", applicability = "has-placeholders")]
+    pub suggestion: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_lowering_never_pattern_with_body)]
+pub struct NeverPatternWithBody {
+    #[primary_span]
+    #[label]
+    #[suggestion(code = "", applicability = "maybe-incorrect")]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(ast_lowering_never_pattern_with_guard)]
+pub struct NeverPatternWithGuard {
+    #[primary_span]
+    #[suggestion(code = "", applicability = "maybe-incorrect")]
+    pub span: Span,
+}
+
 #[derive(Diagnostic, Clone, Copy)]
 #[diag(ast_lowering_arbitrary_expression_in_pattern)]
 pub struct ArbitraryExpressionInPattern {
diff --git a/compiler/rustc_ast_lowering/src/expr.rs b/compiler/rustc_ast_lowering/src/expr.rs
index 0fff9a6be..704f124db 100644
--- a/compiler/rustc_ast_lowering/src/expr.rs
+++ b/compiler/rustc_ast_lowering/src/expr.rs
@@ -1,22 +1,22 @@
 use super::errors::{
     AsyncCoroutinesNotSupported, AsyncNonMoveClosureNotSupported, AwaitOnlyInAsyncFnAndBlocks,
     BaseExpressionDoubleDot, ClosureCannotBeStatic, CoroutineTooManyParameters,
-    FunctionalRecordUpdateDestructuringAssignment, InclusiveRangeWithNoEnd,
-    NotSupportedForLifetimeBinderAsyncClosure, UnderscoreExprLhsAssign,
+    FunctionalRecordUpdateDestructuringAssignment, InclusiveRangeWithNoEnd, MatchArmWithNoBody,
+    NeverPatternWithBody, NeverPatternWithGuard, NotSupportedForLifetimeBinderAsyncClosure,
+    UnderscoreExprLhsAssign,
 };
 use super::ResolverAstLoweringExt;
 use super::{ImplTraitContext, LoweringContext, ParamMode, ParenthesizedGenericArgs};
 use crate::{FnDeclKind, ImplTraitPosition};
-use rustc_ast::attr;
 use rustc_ast::ptr::P as AstP;
 use rustc_ast::*;
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_hir as hir;
-use rustc_hir::def::Res;
-use rustc_hir::definitions::DefPathData;
+use rustc_hir::def::{DefKind, Res};
+use rustc_middle::span_bug;
 use rustc_session::errors::report_lit_error;
 use rustc_span::source_map::{respan, Spanned};
-use rustc_span::symbol::{sym, Ident, Symbol};
+use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::DUMMY_SP;
 use rustc_span::{DesugaringKind, Span};
 use thin_vec::{thin_vec, ThinVec};
@@ -42,8 +42,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     }
                     // Merge attributes into the inner expression.
                     if !e.attrs.is_empty() {
-                        let old_attrs =
-                            self.attrs.get(&ex.hir_id.local_id).map(|la| *la).unwrap_or(&[]);
+                        let old_attrs = self.attrs.get(&ex.hir_id.local_id).copied().unwrap_or(&[]);
                         self.attrs.insert(
                             ex.hir_id.local_id,
                             &*self.arena.alloc_from_iter(
@@ -73,7 +72,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
             let kind = match &e.kind {
                 ExprKind::Array(exprs) => hir::ExprKind::Array(self.lower_exprs(exprs)),
                 ExprKind::ConstBlock(c) => {
-                    let c = self.with_new_scopes(|this| hir::ConstBlock {
+                    let c = self.with_new_scopes(c.value.span, |this| hir::ConstBlock {
                         def_id: this.local_def_id(c.id),
                         hir_id: this.lower_node_id(c.id),
                         body: this.lower_const_body(c.value.span, Some(&c.value)),
@@ -190,46 +189,43 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     None,
                     e.span,
                     hir::CoroutineSource::Block,
-                    |this| this.with_new_scopes(|this| this.lower_block_expr(block)),
+                    |this| this.with_new_scopes(e.span, |this| this.lower_block_expr(block)),
                 ),
                 ExprKind::Await(expr, await_kw_span) => self.lower_expr_await(*await_kw_span, expr),
                 ExprKind::Closure(box Closure {
                     binder,
                     capture_clause,
                     constness,
-                    asyncness,
+                    coroutine_kind,
                     movability,
                     fn_decl,
                     body,
                     fn_decl_span,
                     fn_arg_span,
-                }) => {
-                    if let Async::Yes { closure_id, .. } = asyncness {
-                        self.lower_expr_async_closure(
-                            binder,
-                            *capture_clause,
-                            e.id,
-                            hir_id,
-                            *closure_id,
-                            fn_decl,
-                            body,
-                            *fn_decl_span,
-                            *fn_arg_span,
-                        )
-                    } else {
-                        self.lower_expr_closure(
-                            binder,
-                            *capture_clause,
-                            e.id,
-                            *constness,
-                            *movability,
-                            fn_decl,
-                            body,
-                            *fn_decl_span,
-                            *fn_arg_span,
-                        )
-                    }
-                }
+                }) => match coroutine_kind {
+                    Some(coroutine_kind) => self.lower_expr_coroutine_closure(
+                        binder,
+                        *capture_clause,
+                        e.id,
+                        hir_id,
+                        *coroutine_kind,
+                        fn_decl,
+                        body,
+                        *fn_decl_span,
+                        *fn_arg_span,
+                    ),
+                    None => self.lower_expr_closure(
+                        binder,
+                        *capture_clause,
+                        e.id,
+                        *constness,
+                        *movability,
+                        fn_decl,
+                        body,
+                        *fn_decl_span,
+                        *fn_arg_span,
+                    ),
+                },
                 ExprKind::Block(blk, opt_label) => {
                     let opt_label = self.lower_label(*opt_label);
                     hir::ExprKind::Block(self.lower_block(blk, opt_label.is_some()), opt_label)
@@ -324,11 +320,20 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     None,
                     e.span,
                     hir::CoroutineSource::Block,
-                    |this| this.with_new_scopes(|this| this.lower_block_expr(block)),
+                    |this| this.with_new_scopes(e.span, |this| this.lower_block_expr(block)),
                 ),
+                ExprKind::Gen(capture_clause, block, GenBlockKind::AsyncGen) => self
+                    .make_async_gen_expr(
+                        *capture_clause,
+                        e.id,
+                        None,
+                        e.span,
+                        hir::CoroutineSource::Block,
+                        |this| this.with_new_scopes(e.span, |this| this.lower_block_expr(block)),
+                    ),
                 ExprKind::Yield(opt_expr) => self.lower_expr_yield(e.span, opt_expr.as_deref()),
                 ExprKind::Err => hir::ExprKind::Err(
-                    self.tcx.sess.delay_span_bug(e.span, "lowered ExprKind::Err"),
+                    self.tcx.sess.span_delayed_bug(e.span, "lowered ExprKind::Err"),
                 ),
                 ExprKind::Try(sub_expr) => self.lower_expr_try(e.span, sub_expr),
 
@@ -351,30 +356,8 @@ impl<'hir> LoweringContext<'_, 'hir> {
         }
     }
 
-    fn lower_binop(&mut self, b: BinOp) -> hir::BinOp {
-        Spanned {
-            node: match b.node {
-                BinOpKind::Add => hir::BinOpKind::Add,
-                BinOpKind::Sub => hir::BinOpKind::Sub,
-                BinOpKind::Mul => hir::BinOpKind::Mul,
-                BinOpKind::Div => hir::BinOpKind::Div,
-                BinOpKind::Rem => hir::BinOpKind::Rem,
-                BinOpKind::And => hir::BinOpKind::And,
-                BinOpKind::Or => hir::BinOpKind::Or,
-                BinOpKind::BitXor => hir::BinOpKind::BitXor,
-                BinOpKind::BitAnd => hir::BinOpKind::BitAnd,
-                BinOpKind::BitOr => hir::BinOpKind::BitOr,
-                BinOpKind::Shl => hir::BinOpKind::Shl,
-                BinOpKind::Shr => hir::BinOpKind::Shr,
-                BinOpKind::Eq => hir::BinOpKind::Eq,
-                BinOpKind::Lt => hir::BinOpKind::Lt,
-                BinOpKind::Le => hir::BinOpKind::Le,
-                BinOpKind::Ne => hir::BinOpKind::Ne,
-                BinOpKind::Ge => hir::BinOpKind::Ge,
-                BinOpKind::Gt => hir::BinOpKind::Gt,
-            },
-            span: self.lower_span(b.span),
-        }
+    fn lower_binop(&mut self, b: BinOp) -> BinOp {
+        Spanned { node: b.node, span: self.lower_span(b.span) }
     }
 
     fn lower_legacy_const_generics(
@@ -396,7 +379,13 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 let node_id = self.next_node_id();
 
                 // Add a definition for the in-band const def.
-                self.create_def(parent_def_id.def_id, node_id, DefPathData::AnonConst, f.span);
+                self.create_def(
+                    parent_def_id.def_id,
+                    node_id,
+                    kw::Empty,
+                    DefKind::AnonConst,
+                    f.span,
+                );
 
                 let anon_const = AnonConst { id: node_id, value: arg };
                 generic_args.push(AngleBracketedArg::Arg(GenericArg::Const(anon_const)));
@@ -525,7 +514,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     this.mark_span_with_reason(
                         DesugaringKind::TryBlock,
                         expr.span,
-                        this.allow_try_trait.clone(),
+                        Some(this.allow_try_trait.clone()),
                     ),
                     expr,
                 )
@@ -533,7 +522,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 let try_span = this.mark_span_with_reason(
                     DesugaringKind::TryBlock,
                     this.tcx.sess.source_map().end_point(body.span),
-                    this.allow_try_trait.clone(),
+                    Some(this.allow_try_trait.clone()),
                 );
 
                 (try_span, this.expr_unit(try_span))
@@ -561,13 +550,13 @@ impl<'hir> LoweringContext<'_, 'hir> {
         expr: &'hir hir::Expr<'hir>,
         overall_span: Span,
     ) -> &'hir hir::Expr<'hir> {
-        let constructor = self.arena.alloc(self.expr_lang_item_path(method_span, lang_item, None));
+        let constructor = self.arena.alloc(self.expr_lang_item_path(method_span, lang_item));
         self.expr_call(overall_span, constructor, std::slice::from_ref(expr))
     }
 
     fn lower_arm(&mut self, arm: &Arm) -> hir::Arm<'hir> {
         let pat = self.lower_pat(&arm.pat);
-        let guard = arm.guard.as_ref().map(|cond| {
+        let mut guard = arm.guard.as_ref().map(|cond| {
             if let ExprKind::Let(pat, scrutinee, span, is_recovered) = &cond.kind {
                 hir::Guard::IfLet(self.arena.alloc(hir::Let {
                     hir_id: self.next_id(),
@@ -582,14 +571,46 @@ impl<'hir> LoweringContext<'_, 'hir> {
             }
         });
         let hir_id = self.next_id();
+        let span = self.lower_span(arm.span);
         self.lower_attrs(hir_id, &arm.attrs);
-        hir::Arm {
-            hir_id,
-            pat,
-            guard,
-            body: self.lower_expr(&arm.body),
-            span: self.lower_span(arm.span),
-        }
+        let is_never_pattern = pat.is_never_pattern();
+        let body = if let Some(body) = &arm.body
+            && !is_never_pattern
+        {
+            self.lower_expr(body)
+        } else {
+            // Either `body.is_none()` or `is_never_pattern` here.
+            if !is_never_pattern {
+                if self.tcx.features().never_patterns {
+                    // If the feature is off we already emitted the error after parsing.
+                    let suggestion = span.shrink_to_hi();
+                    self.tcx.sess.emit_err(MatchArmWithNoBody { span, suggestion });
+                }
+            } else if let Some(body) = &arm.body {
+                self.tcx.sess.emit_err(NeverPatternWithBody { span: body.span });
+                guard = None;
+            } else if let Some(g) = &arm.guard {
+                self.tcx.sess.emit_err(NeverPatternWithGuard { span: g.span });
+                guard = None;
+            }
+
+            // We add a fake `loop {}` arm body so that it typecks to `!`.
+            // FIXME(never_patterns): Desugar into a call to `unreachable_unchecked`.
+            let block = self.arena.alloc(hir::Block {
+                stmts: &[],
+                expr: None,
+                hir_id: self.next_id(),
+                rules: hir::BlockCheckMode::DefaultBlock,
+                span,
+                targeted_by_break: false,
+            });
+            self.arena.alloc(hir::Expr {
+                hir_id: self.next_id(),
+                kind: hir::ExprKind::Loop(block, None, hir::LoopSource::Loop, span),
+                span,
+            })
+        };
+        hir::Arm { hir_id, pat, guard, body, span }
     }
 
     /// Lower an `async` construct to a coroutine that implements `Future`.
@@ -613,9 +634,12 @@ impl<'hir> LoweringContext<'_, 'hir> {
         let output = ret_ty.unwrap_or_else(|| hir::FnRetTy::DefaultReturn(self.lower_span(span)));
 
         // Resume argument type: `ResumeTy`
-        let unstable_span =
-            self.mark_span_with_reason(DesugaringKind::Async, span, self.allow_gen_future.clone());
-        let resume_ty = hir::QPath::LangItem(hir::LangItem::ResumeTy, unstable_span, None);
+        let unstable_span = self.mark_span_with_reason(
+            DesugaringKind::Async,
+            self.lower_span(span),
+            Some(self.allow_gen_future.clone()),
+        );
+        let resume_ty = hir::QPath::LangItem(hir::LangItem::ResumeTy, unstable_span);
         let input_ty = hir::Ty {
             hir_id: self.next_id(),
             kind: hir::TyKind::Path(resume_ty),
@@ -721,6 +745,87 @@ impl<'hir> LoweringContext<'_, 'hir> {
         }))
     }
 
+    /// Lower a `async gen` construct to a generator that implements `AsyncIterator`.
+    ///
+    /// This results in:
+    ///
+    /// ```text
+    /// static move? |_task_context| -> () {
+    ///     <body>
+    /// }
+    /// ```
+    pub(super) fn make_async_gen_expr(
+        &mut self,
+        capture_clause: CaptureBy,
+        closure_node_id: NodeId,
+        _yield_ty: Option<hir::FnRetTy<'hir>>,
+        span: Span,
+        async_coroutine_source: hir::CoroutineSource,
+        body: impl FnOnce(&mut Self) -> hir::Expr<'hir>,
+    ) -> hir::ExprKind<'hir> {
+        let output = hir::FnRetTy::DefaultReturn(self.lower_span(span));
+
+        // Resume argument type: `ResumeTy`
+        let unstable_span = self.mark_span_with_reason(
+            DesugaringKind::Async,
+            self.lower_span(span),
+            Some(self.allow_gen_future.clone()),
+        );
+        let resume_ty = hir::QPath::LangItem(hir::LangItem::ResumeTy, unstable_span);
+        let input_ty = hir::Ty {
+            hir_id: self.next_id(),
+            kind: hir::TyKind::Path(resume_ty),
+            span: unstable_span,
+        };
+
+        // The closure/coroutine `FnDecl` takes a single (resume) argument of type `input_ty`.
+        let fn_decl = self.arena.alloc(hir::FnDecl {
+            inputs: arena_vec![self; input_ty],
+            output,
+            c_variadic: false,
+            implicit_self: hir::ImplicitSelfKind::None,
+            lifetime_elision_allowed: false,
+        });
+
+        // Lower the argument pattern/ident. The ident is used again in the `.await` lowering.
+        let (pat, task_context_hid) = self.pat_ident_binding_mode(
+            span,
+            Ident::with_dummy_span(sym::_task_context),
+            hir::BindingAnnotation::MUT,
+        );
+        let param = hir::Param {
+            hir_id: self.next_id(),
+            pat,
+            ty_span: self.lower_span(span),
+            span: self.lower_span(span),
+        };
+        let params = arena_vec![self; param];
+
+        let body = self.lower_body(move |this| {
+            this.coroutine_kind = Some(hir::CoroutineKind::AsyncGen(async_coroutine_source));
+
+            let old_ctx = this.task_context;
+            this.task_context = Some(task_context_hid);
+            let res = body(this);
+            this.task_context = old_ctx;
+            (params, res)
+        });
+
+        // `static |_task_context| -> <ret_ty> { body }`:
+        hir::ExprKind::Closure(self.arena.alloc(hir::Closure {
+            def_id: self.local_def_id(closure_node_id),
+            binder: hir::ClosureBinder::Default,
+            capture_clause,
+            bound_generic_params: &[],
+            fn_decl,
+            body,
+            fn_decl_span: self.lower_span(span),
+            fn_arg_span: None,
+            movability: Some(hir::Movability::Static),
+            constness: hir::Constness::NotConst,
+        }))
+    }
+
     /// Forwards a possible `#[track_caller]` annotation from `outer_hir_id` to
     /// `inner_hir_id` in case the `async_fn_track_caller` feature is enabled.
     pub(super) fn maybe_forward_track_caller(
@@ -736,7 +841,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
             let unstable_span = self.mark_span_with_reason(
                 DesugaringKind::Async,
                 span,
-                self.allow_gen_future.clone(),
+                Some(self.allow_gen_future.clone()),
             );
             self.lower_attrs(
                 inner_hir_id,
@@ -770,20 +875,23 @@ impl<'hir> LoweringContext<'_, 'hir> {
     /// ```
     fn lower_expr_await(&mut self, await_kw_span: Span, expr: &Expr) -> hir::ExprKind<'hir> {
         let full_span = expr.span.to(await_kw_span);
-        match self.coroutine_kind {
-            Some(hir::CoroutineKind::Async(_)) => {}
+
+        let is_async_gen = match self.coroutine_kind {
+            Some(hir::CoroutineKind::Async(_)) => false,
+            Some(hir::CoroutineKind::AsyncGen(_)) => true,
             Some(hir::CoroutineKind::Coroutine) | Some(hir::CoroutineKind::Gen(_)) | None => {
-                self.tcx.sess.emit_err(AwaitOnlyInAsyncFnAndBlocks {
+                return hir::ExprKind::Err(self.tcx.sess.emit_err(AwaitOnlyInAsyncFnAndBlocks {
                     await_kw_span,
                     item_span: self.current_item,
-                });
+                }));
             }
-        }
+        };
+
         let span = self.mark_span_with_reason(DesugaringKind::Await, await_kw_span, None);
         let gen_future_span = self.mark_span_with_reason(
             DesugaringKind::Await,
             full_span,
-            self.allow_gen_future.clone(),
+            Some(self.allow_gen_future.clone()),
         );
         let expr = self.lower_expr_mut(expr);
         let expr_hir_id = expr.hir_id;
@@ -792,8 +900,11 @@ impl<'hir> LoweringContext<'_, 'hir> {
         // debuggers and debugger extensions expect it to be called `__awaitee`. They use
         // this name to identify what is being awaited by a suspended async functions.
         let awaitee_ident = Ident::with_dummy_span(sym::__awaitee);
-        let (awaitee_pat, awaitee_pat_hid) =
-            self.pat_ident_binding_mode(span, awaitee_ident, hir::BindingAnnotation::MUT);
+        let (awaitee_pat, awaitee_pat_hid) = self.pat_ident_binding_mode(
+            gen_future_span,
+            awaitee_ident,
+            hir::BindingAnnotation::MUT,
+        );
 
         let task_context_ident = Ident::with_dummy_span(sym::_task_context);
 
@@ -806,29 +917,27 @@ impl<'hir> LoweringContext<'_, 'hir> {
         let poll_expr = {
             let awaitee = self.expr_ident(span, awaitee_ident, awaitee_pat_hid);
             let ref_mut_awaitee = self.expr_mut_addr_of(span, awaitee);
-            let task_context = if let Some(task_context_hid) = self.task_context {
-                self.expr_ident_mut(span, task_context_ident, task_context_hid)
-            } else {
-                // Use of `await` outside of an async context, we cannot use `task_context` here.
-                self.expr_err(span, self.tcx.sess.delay_span_bug(span, "no task_context hir id"))
+
+            let Some(task_context_hid) = self.task_context else {
+                unreachable!("use of `await` outside of an async context.");
             };
+
+            let task_context = self.expr_ident_mut(span, task_context_ident, task_context_hid);
+
             let new_unchecked = self.expr_call_lang_item_fn_mut(
                 span,
                 hir::LangItem::PinNewUnchecked,
                 arena_vec![self; ref_mut_awaitee],
-                Some(expr_hir_id),
             );
             let get_context = self.expr_call_lang_item_fn_mut(
                 gen_future_span,
                 hir::LangItem::GetContext,
                 arena_vec![self; task_context],
-                Some(expr_hir_id),
             );
             let call = self.expr_call_lang_item_fn(
                 span,
                 hir::LangItem::FuturePoll,
                 arena_vec![self; new_unchecked, get_context],
-                Some(expr_hir_id),
             );
             self.arena.alloc(self.expr_unsafe(call))
         };
@@ -841,12 +950,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
             let (x_pat, x_pat_hid) = self.pat_ident(gen_future_span, x_ident);
             let x_expr = self.expr_ident(gen_future_span, x_ident, x_pat_hid);
             let ready_field = self.single_pat_field(gen_future_span, x_pat);
-            let ready_pat = self.pat_lang_item_variant(
-                span,
-                hir::LangItem::PollReady,
-                ready_field,
-                Some(expr_hir_id),
-            );
+            let ready_pat = self.pat_lang_item_variant(span, hir::LangItem::PollReady, ready_field);
             let break_x = self.with_loop_scope(loop_node_id, move |this| {
                 let expr_break =
                     hir::ExprKind::Break(this.lower_loop_destination(None), Some(x_expr));
@@ -857,12 +961,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
 
         // `::std::task::Poll::Pending => {}`
         let pending_arm = {
-            let pending_pat = self.pat_lang_item_variant(
-                span,
-                hir::LangItem::PollPending,
-                &[],
-                Some(expr_hir_id),
-            );
+            let pending_pat = self.pat_lang_item_variant(span, hir::LangItem::PollPending, &[]);
             let empty_block = self.expr_block_empty(span);
             self.arm(pending_pat, empty_block)
         };
@@ -877,25 +976,30 @@ impl<'hir> LoweringContext<'_, 'hir> {
             self.stmt_expr(span, match_expr)
         };
 
-        // task_context = yield ();
+        // Depending on `async` of `async gen`:
+        // async     - task_context = yield ();
+        // async gen - task_context = yield ASYNC_GEN_PENDING;
         let yield_stmt = {
-            let unit = self.expr_unit(span);
+            let yielded = if is_async_gen {
+                self.arena.alloc(self.expr_lang_item_path(span, hir::LangItem::AsyncGenPending))
+            } else {
+                self.expr_unit(span)
+            };
+
             let yield_expr = self.expr(
                 span,
-                hir::ExprKind::Yield(unit, hir::YieldSource::Await { expr: Some(expr_hir_id) }),
+                hir::ExprKind::Yield(yielded, hir::YieldSource::Await { expr: Some(expr_hir_id) }),
             );
             let yield_expr = self.arena.alloc(yield_expr);
 
-            if let Some(task_context_hid) = self.task_context {
-                let lhs = self.expr_ident(span, task_context_ident, task_context_hid);
-                let assign =
-                    self.expr(span, hir::ExprKind::Assign(lhs, yield_expr, self.lower_span(span)));
-                self.stmt_expr(span, assign)
-            } else {
-                // Use of `await` outside of an async context. Return `yield_expr` so that we can
-                // proceed with type checking.
-                self.stmt(span, hir::StmtKind::Semi(yield_expr))
-            }
+            let Some(task_context_hid) = self.task_context else {
+                unreachable!("use of `await` outside of an async context.");
+            };
+
+            let lhs = self.expr_ident(span, task_context_ident, task_context_hid);
+            let assign =
+                self.expr(span, hir::ExprKind::Assign(lhs, yield_expr, self.lower_span(span)));
+            self.stmt_expr(span, assign)
         };
 
         let loop_block = self.block_all(span, arena_vec![self; inner_match_stmt, yield_stmt], None);
@@ -920,7 +1024,6 @@ impl<'hir> LoweringContext<'_, 'hir> {
             span,
             hir::LangItem::IntoFutureIntoFuture,
             arena_vec![self; expr],
-            Some(expr_hir_id),
         );
 
         // match <into_future_expr> {
@@ -947,9 +1050,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
     ) -> hir::ExprKind<'hir> {
         let (binder_clause, generic_params) = self.lower_closure_binder(binder);
 
-        let (body_id, coroutine_option) = self.with_new_scopes(move |this| {
-            let prev = this.current_item;
-            this.current_item = Some(fn_decl_span);
+        let (body_id, coroutine_option) = self.with_new_scopes(fn_decl_span, move |this| {
             let mut coroutine_kind = None;
             let body_id = this.lower_fn_body(decl, |this| {
                 let e = this.lower_expr_mut(body);
@@ -957,8 +1058,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 e
             });
             let coroutine_option =
-                this.coroutine_movability_for_fn(&decl, fn_decl_span, coroutine_kind, movability);
-            this.current_item = prev;
+                this.coroutine_movability_for_fn(decl, fn_decl_span, coroutine_kind, movability);
             (body_id, coroutine_option)
         });
 
@@ -996,7 +1096,11 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 }
                 Some(movability)
             }
-            Some(hir::CoroutineKind::Gen(_)) | Some(hir::CoroutineKind::Async(_)) => {
+            Some(
+                hir::CoroutineKind::Gen(_)
+                | hir::CoroutineKind::Async(_)
+                | hir::CoroutineKind::AsyncGen(_),
+            ) => {
                 panic!("non-`async`/`gen` closure body turned `async`/`gen` during lowering");
             }
             None => {
@@ -1023,18 +1127,22 @@ impl<'hir> LoweringContext<'_, 'hir> {
         (binder, params)
     }
 
-    fn lower_expr_async_closure(
+    fn lower_expr_coroutine_closure(
         &mut self,
         binder: &ClosureBinder,
         capture_clause: CaptureBy,
         closure_id: NodeId,
         closure_hir_id: hir::HirId,
-        inner_closure_id: NodeId,
+        coroutine_kind: CoroutineKind,
         decl: &FnDecl,
         body: &Expr,
         fn_decl_span: Span,
         fn_arg_span: Span,
     ) -> hir::ExprKind<'hir> {
+        let CoroutineKind::Async { closure_id: inner_closure_id, .. } = coroutine_kind else {
+            span_bug!(fn_decl_span, "`async gen` and `gen` closures are not supported, yet");
+        };
+
         if let &ClosureBinder::For { span, .. } = binder {
             self.tcx.sess.emit_err(NotSupportedForLifetimeBinderAsyncClosure { span });
         }
@@ -1044,7 +1152,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         let outer_decl =
             FnDecl { inputs: decl.inputs.clone(), output: FnRetTy::Default(fn_decl_span) };
 
-        let body = self.with_new_scopes(|this| {
+        let body = self.with_new_scopes(fn_decl_span, |this| {
             // FIXME(cramertj): allow `async` non-`move` closures with arguments.
             if capture_clause == CaptureBy::Ref && !decl.inputs.is_empty() {
                 this.tcx.sess.emit_err(AsyncNonMoveClosureNotSupported { fn_decl_span });
@@ -1055,7 +1163,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
             let body_id = this.lower_fn_body(&outer_decl, |this| {
                 let async_ret_ty = if let FnRetTy::Ty(ty) = &decl.output {
                     let itctx = ImplTraitContext::Disallowed(ImplTraitPosition::AsyncBlock);
-                    Some(hir::FnRetTy::Return(this.lower_ty(&ty, &itctx)))
+                    Some(hir::FnRetTy::Return(this.lower_ty(ty, &itctx)))
                 } else {
                     None
                 };
@@ -1066,7 +1174,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     async_ret_ty,
                     body.span,
                     hir::CoroutineSource::Closure,
-                    |this| this.with_new_scopes(|this| this.lower_expr_mut(body)),
+                    |this| this.with_new_scopes(fn_decl_span, |this| this.lower_expr_mut(body)),
                 );
                 let hir_id = this.lower_node_id(inner_closure_id);
                 this.maybe_forward_track_caller(body.span, closure_hir_id, hir_id);
@@ -1113,6 +1221,8 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 | ExprKind::Struct(..)
                 | ExprKind::Tup(..)
                 | ExprKind::Underscore => false,
+                // Check for unit struct constructor.
+                ExprKind::Path(..) => lower_ctx.extract_unit_struct_path(lhs).is_none(),
                 // Check for tuple struct constructor.
                 ExprKind::Call(callee, ..) => lower_ctx.extract_tuple_struct_path(callee).is_none(),
                 ExprKind::Paren(e) => {
@@ -1149,12 +1259,10 @@ impl<'hir> LoweringContext<'_, 'hir> {
         );
 
         // `a = lhs1; b = lhs2;`.
-        let stmts = self
-            .arena
-            .alloc_from_iter(std::iter::once(destructure_let).chain(assignments.into_iter()));
+        let stmts = self.arena.alloc_from_iter(std::iter::once(destructure_let).chain(assignments));
 
         // Wrap everything in a block.
-        hir::ExprKind::Block(&self.block_all(whole_span, stmts, None), None)
+        hir::ExprKind::Block(self.block_all(whole_span, stmts, None), None)
     }
 
     /// If the given expression is a path to a tuple struct, returns that path.
@@ -1377,8 +1485,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
     fn lower_expr_range_closed(&mut self, span: Span, e1: &Expr, e2: &Expr) -> hir::ExprKind<'hir> {
         let e1 = self.lower_expr_mut(e1);
         let e2 = self.lower_expr_mut(e2);
-        let fn_path =
-            hir::QPath::LangItem(hir::LangItem::RangeInclusiveNew, self.lower_span(span), None);
+        let fn_path = hir::QPath::LangItem(hir::LangItem::RangeInclusiveNew, self.lower_span(span));
         let fn_expr = self.arena.alloc(self.expr(span, hir::ExprKind::Path(fn_path)));
         hir::ExprKind::Call(fn_expr, arena_vec![self; e1, e2])
     }
@@ -1411,7 +1518,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         let fields = self.arena.alloc_from_iter(
             e1.iter().map(|e| (sym::start, e)).chain(e2.iter().map(|e| (sym::end, e))).map(
                 |(s, e)| {
-                    let expr = self.lower_expr(&e);
+                    let expr = self.lower_expr(e);
                     let ident = Ident::new(s, self.lower_span(e.span));
                     self.expr_field(ident, expr, e.span)
                 },
@@ -1419,7 +1526,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         );
 
         hir::ExprKind::Struct(
-            self.arena.alloc(hir::QPath::LangItem(lang_item, self.lower_span(span), None)),
+            self.arena.alloc(hir::QPath::LangItem(lang_item, self.lower_span(span))),
             fields,
             None,
         )
@@ -1504,10 +1611,13 @@ impl<'hir> LoweringContext<'_, 'hir> {
     }
 
     fn lower_expr_yield(&mut self, span: Span, opt_expr: Option<&Expr>) -> hir::ExprKind<'hir> {
-        match self.coroutine_kind {
-            Some(hir::CoroutineKind::Gen(_)) => {}
+        let is_async_gen = match self.coroutine_kind {
+            Some(hir::CoroutineKind::Gen(_)) => false,
+            Some(hir::CoroutineKind::AsyncGen(_)) => true,
             Some(hir::CoroutineKind::Async(_)) => {
-                self.tcx.sess.emit_err(AsyncCoroutinesNotSupported { span });
+                return hir::ExprKind::Err(
+                    self.tcx.sess.emit_err(AsyncCoroutinesNotSupported { span }),
+                );
             }
             Some(hir::CoroutineKind::Coroutine) | None => {
                 if !self.tcx.features().coroutines {
@@ -1519,14 +1629,37 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     )
                     .emit();
                 }
-                self.coroutine_kind = Some(hir::CoroutineKind::Coroutine)
+                self.coroutine_kind = Some(hir::CoroutineKind::Coroutine);
+                false
             }
-        }
+        };
 
-        let expr =
+        let yielded =
             opt_expr.as_ref().map(|x| self.lower_expr(x)).unwrap_or_else(|| self.expr_unit(span));
 
-        hir::ExprKind::Yield(expr, hir::YieldSource::Yield)
+        if is_async_gen {
+            // `yield $expr` is transformed into `task_context = yield async_gen_ready($expr)`.
+            // This ensures that we store our resumed `ResumeContext` correctly, and also that
+            // the apparent value of the `yield` expression is `()`.
+            let wrapped_yielded = self.expr_call_lang_item_fn(
+                span,
+                hir::LangItem::AsyncGenReady,
+                std::slice::from_ref(yielded),
+            );
+            let yield_expr = self.arena.alloc(
+                self.expr(span, hir::ExprKind::Yield(wrapped_yielded, hir::YieldSource::Yield)),
+            );
+
+            let Some(task_context_hid) = self.task_context else {
+                unreachable!("use of `await` outside of an async context.");
+            };
+            let task_context_ident = Ident::with_dummy_span(sym::_task_context);
+            let lhs = self.expr_ident(span, task_context_ident, task_context_hid);
+
+            hir::ExprKind::Assign(lhs, yield_expr, self.lower_span(span))
+        } else {
+            hir::ExprKind::Yield(yielded, hir::YieldSource::Yield)
+        }
     }
 
     /// Desugar `ExprForLoop` from: `[opt_ident]: for <pat> in <head> <body>` into:
@@ -1588,7 +1721,6 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 head_span,
                 hir::LangItem::IteratorNext,
                 arena_vec![self; ref_mut_iter],
-                None,
             );
             let arms = arena_vec![self; none_arm, some_arm];
 
@@ -1617,7 +1749,6 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 head_span,
                 hir::LangItem::IntoIterIntoIter,
                 arena_vec![self; head],
-                None,
             )
         };
 
@@ -1655,13 +1786,13 @@ impl<'hir> LoweringContext<'_, 'hir> {
         let unstable_span = self.mark_span_with_reason(
             DesugaringKind::QuestionMark,
             span,
-            self.allow_try_trait.clone(),
+            Some(self.allow_try_trait.clone()),
         );
         let try_span = self.tcx.sess.source_map().end_point(span);
         let try_span = self.mark_span_with_reason(
             DesugaringKind::QuestionMark,
             try_span,
-            self.allow_try_trait.clone(),
+            Some(self.allow_try_trait.clone()),
         );
 
         // `Try::branch(<expr>)`
@@ -1673,7 +1804,6 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 unstable_span,
                 hir::LangItem::TryTraitBranch,
                 arena_vec![self; sub_expr],
-                None,
             )
         };
 
@@ -1755,7 +1885,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         let unstable_span = self.mark_span_with_reason(
             DesugaringKind::YeetExpr,
             span,
-            self.allow_try_trait.clone(),
+            Some(self.allow_try_trait.clone()),
         );
 
         let from_yeet_expr = self.wrap_in_try_constructor(
@@ -1878,9 +2008,8 @@ impl<'hir> LoweringContext<'_, 'hir> {
         span: Span,
         lang_item: hir::LangItem,
         args: &'hir [hir::Expr<'hir>],
-        hir_id: Option<hir::HirId>,
     ) -> hir::Expr<'hir> {
-        let path = self.arena.alloc(self.expr_lang_item_path(span, lang_item, hir_id));
+        let path = self.arena.alloc(self.expr_lang_item_path(span, lang_item));
         self.expr_call_mut(span, path, args)
     }
 
@@ -1889,21 +2018,12 @@ impl<'hir> LoweringContext<'_, 'hir> {
         span: Span,
         lang_item: hir::LangItem,
         args: &'hir [hir::Expr<'hir>],
-        hir_id: Option<hir::HirId>,
     ) -> &'hir hir::Expr<'hir> {
-        self.arena.alloc(self.expr_call_lang_item_fn_mut(span, lang_item, args, hir_id))
+        self.arena.alloc(self.expr_call_lang_item_fn_mut(span, lang_item, args))
     }
 
-    fn expr_lang_item_path(
-        &mut self,
-        span: Span,
-        lang_item: hir::LangItem,
-        hir_id: Option<hir::HirId>,
-    ) -> hir::Expr<'hir> {
-        self.expr(
-            span,
-            hir::ExprKind::Path(hir::QPath::LangItem(lang_item, self.lower_span(span), hir_id)),
-        )
+    fn expr_lang_item_path(&mut self, span: Span, lang_item: hir::LangItem) -> hir::Expr<'hir> {
+        self.expr(span, hir::ExprKind::Path(hir::QPath::LangItem(lang_item, self.lower_span(span))))
     }
 
     /// `<LangItem>::name`
@@ -1916,7 +2036,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         let path = hir::ExprKind::Path(hir::QPath::TypeRelative(
             self.arena.alloc(self.ty(
                 span,
-                hir::TyKind::Path(hir::QPath::LangItem(lang_item, self.lower_span(span), None)),
+                hir::TyKind::Path(hir::QPath::LangItem(lang_item, self.lower_span(span))),
             )),
             self.arena.alloc(hir::PathSegment::new(
                 Ident::new(name, span),
diff --git a/compiler/rustc_ast_lowering/src/format.rs b/compiler/rustc_ast_lowering/src/format.rs
index c7d0719e7..6a82005c4 100644
--- a/compiler/rustc_ast_lowering/src/format.rs
+++ b/compiler/rustc_ast_lowering/src/format.rs
@@ -1,6 +1,6 @@
 use super::LoweringContext;
 use rustc_ast as ast;
-use rustc_ast::visit::{self, Visitor};
+use rustc_ast::visit::Visitor;
 use rustc_ast::*;
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_hir as hir;
@@ -267,7 +267,7 @@ fn make_count<'hir>(
                 ctx.expr(
                     sp,
                     hir::ExprKind::Err(
-                        ctx.tcx.sess.delay_span_bug(sp, "lowered bad format_args count"),
+                        ctx.tcx.sess.span_delayed_bug(sp, "lowered bad format_args count"),
                     ),
                 )
             }
@@ -306,7 +306,7 @@ fn make_format_spec<'hir>(
         }
         Err(_) => ctx.expr(
             sp,
-            hir::ExprKind::Err(ctx.tcx.sess.delay_span_bug(sp, "lowered bad format_args count")),
+            hir::ExprKind::Err(ctx.tcx.sess.span_delayed_bug(sp, "lowered bad format_args count")),
         ),
     };
     let &FormatOptions {
@@ -338,8 +338,8 @@ fn make_format_spec<'hir>(
         | ((debug_hex == Some(FormatDebugHex::Lower)) as u32) << 4
         | ((debug_hex == Some(FormatDebugHex::Upper)) as u32) << 5;
     let flags = ctx.expr_u32(sp, flags);
-    let precision = make_count(ctx, sp, &precision, argmap);
-    let width = make_count(ctx, sp, &width, argmap);
+    let precision = make_count(ctx, sp, precision, argmap);
+    let width = make_count(ctx, sp, width, argmap);
     let format_placeholder_new = ctx.arena.alloc(ctx.expr_lang_item_type_relative(
         sp,
         hir::LangItem::FormatPlaceholder,
diff --git a/compiler/rustc_ast_lowering/src/index.rs b/compiler/rustc_ast_lowering/src/index.rs
index eff362f3f..993ddf00e 100644
--- a/compiler/rustc_ast_lowering/src/index.rs
+++ b/compiler/rustc_ast_lowering/src/index.rs
@@ -1,8 +1,7 @@
-use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sorted_map::SortedMap;
 use rustc_hir as hir;
-use rustc_hir::def_id::LocalDefId;
-use rustc_hir::intravisit::{self, Visitor};
+use rustc_hir::def_id::{LocalDefId, LocalDefIdMap};
+use rustc_hir::intravisit::Visitor;
 use rustc_hir::*;
 use rustc_index::{Idx, IndexVec};
 use rustc_middle::span_bug;
@@ -10,14 +9,14 @@ use rustc_middle::ty::TyCtxt;
 use rustc_span::{Span, DUMMY_SP};
 
 /// A visitor that walks over the HIR and collects `Node`s into a HIR map.
-pub(super) struct NodeCollector<'a, 'hir> {
+struct NodeCollector<'a, 'hir> {
     tcx: TyCtxt<'hir>,
 
     bodies: &'a SortedMap<ItemLocalId, &'hir Body<'hir>>,
 
     /// Outputs
     nodes: IndexVec<ItemLocalId, Option<ParentedNode<'hir>>>,
-    parenting: FxHashMap<LocalDefId, ItemLocalId>,
+    parenting: LocalDefIdMap<ItemLocalId>,
 
     /// The parent of this node
     parent_node: hir::ItemLocalId,
@@ -30,7 +29,7 @@ pub(super) fn index_hir<'hir>(
     tcx: TyCtxt<'hir>,
     item: hir::OwnerNode<'hir>,
     bodies: &SortedMap<ItemLocalId, &'hir Body<'hir>>,
-) -> (IndexVec<ItemLocalId, Option<ParentedNode<'hir>>>, FxHashMap<LocalDefId, ItemLocalId>) {
+) -> (IndexVec<ItemLocalId, Option<ParentedNode<'hir>>>, LocalDefIdMap<ItemLocalId>) {
     let mut nodes = IndexVec::new();
     // This node's parent should never be accessed: the owner's parent is computed by the
     // hir_owner_parent query. Make it invalid (= ItemLocalId::MAX) to force an ICE whenever it is
@@ -42,12 +41,12 @@ pub(super) fn index_hir<'hir>(
         parent_node: ItemLocalId::new(0),
         nodes,
         bodies,
-        parenting: FxHashMap::default(),
+        parenting: Default::default(),
     };
 
     match item {
         OwnerNode::Crate(citem) => {
-            collector.visit_mod(&citem, citem.spans.inner_span, hir::CRATE_HIR_ID)
+            collector.visit_mod(citem, citem.spans.inner_span, hir::CRATE_HIR_ID)
         }
         OwnerNode::Item(item) => collector.visit_item(item),
         OwnerNode::TraitItem(item) => collector.visit_trait_item(item),
@@ -89,7 +88,7 @@ impl<'a, 'hir> NodeCollector<'a, 'hir> {
             }
         }
 
-        self.nodes.insert(hir_id.local_id, ParentedNode { parent: self.parent_node, node: node });
+        self.nodes.insert(hir_id.local_id, ParentedNode { parent: self.parent_node, node });
     }
 
     fn with_parent<F: FnOnce(&mut Self)>(&mut self, parent_node_id: HirId, f: F) {
diff --git a/compiler/rustc_ast_lowering/src/item.rs b/compiler/rustc_ast_lowering/src/item.rs
index 9a70e6d7c..5bddbe5f4 100644
--- a/compiler/rustc_ast_lowering/src/item.rs
+++ b/compiler/rustc_ast_lowering/src/item.rs
@@ -3,11 +3,9 @@ use super::ResolverAstLoweringExt;
 use super::{AstOwner, ImplTraitContext, ImplTraitPosition};
 use super::{FnDeclKind, LoweringContext, ParamMode};
 
-use hir::definitions::DefPathData;
 use rustc_ast::ptr::P;
 use rustc_ast::visit::AssocCtxt;
 use rustc_ast::*;
-use rustc_data_structures::sorted_map::SortedMap;
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
@@ -55,42 +53,7 @@ impl<'a, 'hir> ItemLowerer<'a, 'hir> {
         owner: NodeId,
         f: impl FnOnce(&mut LoweringContext<'_, 'hir>) -> hir::OwnerNode<'hir>,
     ) {
-        let allow_gen_future = Some(if self.tcx.features().async_fn_track_caller {
-            [sym::gen_future, sym::closure_track_caller][..].into()
-        } else {
-            [sym::gen_future][..].into()
-        });
-        let mut lctx = LoweringContext {
-            // Pseudo-globals.
-            tcx: self.tcx,
-            resolver: self.resolver,
-            arena: self.tcx.hir_arena,
-
-            // HirId handling.
-            bodies: Vec::new(),
-            attrs: SortedMap::default(),
-            children: Vec::default(),
-            current_hir_id_owner: hir::CRATE_OWNER_ID,
-            item_local_id_counter: hir::ItemLocalId::new(0),
-            node_id_to_local_id: Default::default(),
-            trait_map: Default::default(),
-
-            // Lowering state.
-            catch_scope: None,
-            loop_scope: None,
-            is_in_loop_condition: false,
-            is_in_trait_impl: false,
-            is_in_dyn_type: false,
-            coroutine_kind: None,
-            task_context: None,
-            current_item: None,
-            impl_trait_defs: Vec::new(),
-            impl_trait_bounds: Vec::new(),
-            allow_try_trait: Some([sym::try_trait_v2, sym::yeet_desugar_details][..].into()),
-            allow_gen_future,
-            generics_def_id_map: Default::default(),
-            host_param_id: None,
-        };
+        let mut lctx = LoweringContext::new(self.tcx, self.resolver);
         lctx.with_hir_id_owner(owner, |lctx| f(lctx));
 
         for (def_id, info) in lctx.children {
@@ -136,39 +99,9 @@ impl<'a, 'hir> ItemLowerer<'a, 'hir> {
 
     fn lower_assoc_item(&mut self, item: &AssocItem, ctxt: AssocCtxt) {
         let def_id = self.resolver.node_id_to_def_id[&item.id];
-
         let parent_id = self.tcx.local_parent(def_id);
         let parent_hir = self.lower_node(parent_id).unwrap();
-        self.with_lctx(item.id, |lctx| {
-            // Evaluate with the lifetimes in `params` in-scope.
-            // This is used to track which lifetimes have already been defined,
-            // and which need to be replicated when lowering an async fn.
-
-            match parent_hir.node().expect_item().kind {
-                hir::ItemKind::Impl(impl_) => {
-                    lctx.is_in_trait_impl = impl_.of_trait.is_some();
-                }
-                hir::ItemKind::Trait(_, _, generics, _, _) if lctx.tcx.features().effects => {
-                    lctx.host_param_id = generics
-                        .params
-                        .iter()
-                        .find(|param| {
-                            parent_hir
-                                .attrs
-                                .get(param.hir_id.local_id)
-                                .iter()
-                                .any(|attr| attr.has_name(sym::rustc_host))
-                        })
-                        .map(|param| param.def_id);
-                }
-                _ => {}
-            }
-
-            match ctxt {
-                AssocCtxt::Trait => hir::OwnerNode::TraitItem(lctx.lower_trait_item(item)),
-                AssocCtxt::Impl => hir::OwnerNode::ImplItem(lctx.lower_impl_item(item)),
-            }
-        })
+        self.with_lctx(item.id, |lctx| lctx.lower_assoc_item(item, ctxt, parent_hir))
     }
 
     fn lower_foreign_item(&mut self, item: &ForeignItem) {
@@ -268,27 +201,30 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 body,
                 ..
             }) => {
-                self.with_new_scopes(|this| {
-                    this.current_item = Some(ident.span);
-
+                self.with_new_scopes(ident.span, |this| {
                     // Note: we don't need to change the return type from `T` to
                     // `impl Future<Output = T>` here because lower_body
                     // only cares about the input argument patterns in the function
                     // declaration (decl), not the return types.
-                    let asyncness = header.asyncness;
-                    let body_id = this.lower_maybe_async_body(
+                    let coroutine_kind = header.coroutine_kind;
+                    let body_id = this.lower_maybe_coroutine_body(
                         span,
                         hir_id,
-                        &decl,
-                        asyncness,
+                        decl,
+                        coroutine_kind,
                         body.as_deref(),
                     );
 
                     let itctx = ImplTraitContext::Universal;
                     let (generics, decl) =
                         this.lower_generics(generics, header.constness, id, &itctx, |this| {
-                            let ret_id = asyncness.opt_return_id();
-                            this.lower_fn_decl(&decl, id, *fn_sig_span, FnDeclKind::Fn, ret_id)
+                            this.lower_fn_decl(
+                                decl,
+                                id,
+                                *fn_sig_span,
+                                FnDeclKind::Fn,
+                                coroutine_kind,
+                            )
                         });
                     let sig = hir::FnSig {
                         decl,
@@ -329,7 +265,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     &ImplTraitContext::Disallowed(ImplTraitPosition::Generic),
                     |this| match ty {
                         None => {
-                            let guar = this.tcx.sess.delay_span_bug(
+                            let guar = this.tcx.sess.span_delayed_bug(
                                 span,
                                 "expected to lower type alias type, but it was missing",
                             );
@@ -485,7 +421,14 @@ impl<'hir> LoweringContext<'_, 'hir> {
             }
             ItemKind::MacroDef(MacroDef { body, macro_rules }) => {
                 let body = P(self.lower_delim_args(body));
-                let macro_kind = self.resolver.decl_macro_kind(self.local_def_id(id));
+                let def_id = self.local_def_id(id);
+                let def_kind = self.tcx.def_kind(def_id);
+                let DefKind::Macro(macro_kind) = def_kind else {
+                    unreachable!(
+                        "expected DefKind::Macro for macro item, found {}",
+                        def_kind.descr(def_id.to_def_id())
+                    );
+                };
                 let macro_def = self.arena.alloc(ast::MacroDef { body, macro_rules: *macro_rules });
                 hir::ItemKind::Macro(macro_def, macro_kind)
             }
@@ -614,6 +557,41 @@ impl<'hir> LoweringContext<'_, 'hir> {
         }
     }
 
+    fn lower_assoc_item(
+        &mut self,
+        item: &AssocItem,
+        ctxt: AssocCtxt,
+        parent_hir: &'hir hir::OwnerInfo<'hir>,
+    ) -> hir::OwnerNode<'hir> {
+        // Evaluate with the lifetimes in `params` in-scope.
+        // This is used to track which lifetimes have already been defined,
+        // and which need to be replicated when lowering an async fn.
+
+        match parent_hir.node().expect_item().kind {
+            hir::ItemKind::Impl(impl_) => {
+                self.is_in_trait_impl = impl_.of_trait.is_some();
+            }
+            hir::ItemKind::Trait(_, _, generics, _, _) if self.tcx.features().effects => {
+                self.host_param_id = generics
+                    .params
+                    .iter()
+                    .find(|param| {
+                        matches!(
+                            param.kind,
+                            hir::GenericParamKind::Const { is_host_effect: true, .. }
+                        )
+                    })
+                    .map(|param| param.def_id);
+            }
+            _ => {}
+        }
+
+        match ctxt {
+            AssocCtxt::Trait => hir::OwnerNode::TraitItem(self.lower_trait_item(item)),
+            AssocCtxt::Impl => hir::OwnerNode::ImplItem(self.lower_impl_item(item)),
+        }
+    }
+
     fn lower_foreign_item(&mut self, i: &ForeignItem) -> &'hir hir::ForeignItem<'hir> {
         let hir_id = self.lower_node_id(i.id);
         let owner_id = hir_id.expect_owner();
@@ -683,11 +661,12 @@ impl<'hir> LoweringContext<'_, 'hir> {
         vdata: &VariantData,
     ) -> hir::VariantData<'hir> {
         match vdata {
-            VariantData::Struct(fields, recovered) => hir::VariantData::Struct(
-                self.arena
+            VariantData::Struct { fields, recovered } => hir::VariantData::Struct {
+                fields: self
+                    .arena
                     .alloc_from_iter(fields.iter().enumerate().map(|f| self.lower_field_def(f))),
-                *recovered,
-            ),
+                recovered: *recovered,
+            },
             VariantData::Tuple(fields, id) => {
                 let ctor_id = self.lower_node_id(*id);
                 self.alias_attrs(ctor_id, parent_id);
@@ -744,7 +723,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         let (generics, kind, has_default) = match &i.kind {
             AssocItemKind::Const(box ConstItem { generics, ty, expr, .. }) => {
                 let (generics, kind) = self.lower_generics(
-                    &generics,
+                    generics,
                     Const::No,
                     i.id,
                     &ImplTraitContext::Disallowed(ImplTraitPosition::Generic),
@@ -761,27 +740,30 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 (generics, kind, expr.is_some())
             }
             AssocItemKind::Fn(box Fn { sig, generics, body: None, .. }) => {
-                let asyncness = sig.header.asyncness;
                 let names = self.lower_fn_params_to_names(&sig.decl);
                 let (generics, sig) = self.lower_method_sig(
                     generics,
                     sig,
                     i.id,
                     FnDeclKind::Trait,
-                    asyncness.opt_return_id(),
+                    sig.header.coroutine_kind,
                 );
                 (generics, hir::TraitItemKind::Fn(sig, hir::TraitFn::Required(names)), false)
             }
             AssocItemKind::Fn(box Fn { sig, generics, body: Some(body), .. }) => {
-                let asyncness = sig.header.asyncness;
-                let body_id =
-                    self.lower_maybe_async_body(i.span, hir_id, &sig.decl, asyncness, Some(&body));
+                let body_id = self.lower_maybe_coroutine_body(
+                    i.span,
+                    hir_id,
+                    &sig.decl,
+                    sig.header.coroutine_kind,
+                    Some(body),
+                );
                 let (generics, sig) = self.lower_method_sig(
                     generics,
                     sig,
                     i.id,
                     FnDeclKind::Trait,
-                    asyncness.opt_return_id(),
+                    sig.header.coroutine_kind,
                 );
                 (generics, hir::TraitItemKind::Fn(sig, hir::TraitFn::Provided(body_id)), true)
             }
@@ -857,7 +839,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
 
         let (generics, kind) = match &i.kind {
             AssocItemKind::Const(box ConstItem { generics, ty, expr, .. }) => self.lower_generics(
-                &generics,
+                generics,
                 Const::No,
                 i.id,
                 &ImplTraitContext::Disallowed(ImplTraitPosition::Generic),
@@ -870,13 +852,11 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 },
             ),
             AssocItemKind::Fn(box Fn { sig, generics, body, .. }) => {
-                self.current_item = Some(i.span);
-                let asyncness = sig.header.asyncness;
-                let body_id = self.lower_maybe_async_body(
+                let body_id = self.lower_maybe_coroutine_body(
                     i.span,
                     hir_id,
                     &sig.decl,
-                    asyncness,
+                    sig.header.coroutine_kind,
                     body.as_deref(),
                 );
                 let (generics, sig) = self.lower_method_sig(
@@ -884,7 +864,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     sig,
                     i.id,
                     if self.is_in_trait_impl { FnDeclKind::Impl } else { FnDeclKind::Inherent },
-                    asyncness.opt_return_id(),
+                    sig.header.coroutine_kind,
                 );
 
                 (generics, hir::ImplItemKind::Fn(sig, body_id))
@@ -899,7 +879,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     &ImplTraitContext::Disallowed(ImplTraitPosition::Generic),
                     |this| match ty {
                         None => {
-                            let guar = this.tcx.sess.delay_span_bug(
+                            let guar = this.tcx.sess.span_delayed_bug(
                                 i.span,
                                 "expected to lower associated type, but it was missing",
                             );
@@ -1032,7 +1012,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
     fn lower_block_expr_opt(&mut self, span: Span, block: Option<&Block>) -> hir::Expr<'hir> {
         match block {
             Some(block) => self.lower_block_expr(block),
-            None => self.expr_err(span, self.tcx.sess.delay_span_bug(span, "no block")),
+            None => self.expr_err(span, self.tcx.sess.span_delayed_bug(span, "no block")),
         }
     }
 
@@ -1042,24 +1022,26 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 &[],
                 match expr {
                     Some(expr) => this.lower_expr_mut(expr),
-                    None => this.expr_err(span, this.tcx.sess.delay_span_bug(span, "no block")),
+                    None => this.expr_err(span, this.tcx.sess.span_delayed_bug(span, "no block")),
                 },
             )
         })
     }
 
-    fn lower_maybe_async_body(
+    /// Takes what may be the body of an `async fn` or a `gen fn` and wraps it in an `async {}` or
+    /// `gen {}` block as appropriate.
+    fn lower_maybe_coroutine_body(
         &mut self,
         span: Span,
         fn_id: hir::HirId,
         decl: &FnDecl,
-        asyncness: Async,
+        coroutine_kind: Option<CoroutineKind>,
         body: Option<&Block>,
     ) -> hir::BodyId {
-        let (closure_id, body) = match (asyncness, body) {
-            (Async::Yes { closure_id, .. }, Some(body)) => (closure_id, body),
-            _ => return self.lower_fn_body_block(span, decl, body),
+        let (Some(coroutine_kind), Some(body)) = (coroutine_kind, body) else {
+            return self.lower_fn_body_block(span, decl, body);
         };
+        let closure_id = coroutine_kind.closure_id();
 
         self.lower_body(|this| {
             let mut parameters: Vec<hir::Param<'_>> = Vec::new();
@@ -1200,44 +1182,63 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 parameters.push(new_parameter);
             }
 
-            let async_expr = this.make_async_expr(
-                CaptureBy::Value { move_kw: rustc_span::DUMMY_SP },
-                closure_id,
-                None,
-                body.span,
-                hir::CoroutineSource::Fn,
-                |this| {
-                    // Create a block from the user's function body:
-                    let user_body = this.lower_block_expr(body);
+            let mkbody = |this: &mut LoweringContext<'_, 'hir>| {
+                // Create a block from the user's function body:
+                let user_body = this.lower_block_expr(body);
 
-                    // Transform into `drop-temps { <user-body> }`, an expression:
-                    let desugared_span =
-                        this.mark_span_with_reason(DesugaringKind::Async, user_body.span, None);
-                    let user_body =
-                        this.expr_drop_temps(desugared_span, this.arena.alloc(user_body));
+                // Transform into `drop-temps { <user-body> }`, an expression:
+                let desugared_span =
+                    this.mark_span_with_reason(DesugaringKind::Async, user_body.span, None);
+                let user_body = this.expr_drop_temps(desugared_span, this.arena.alloc(user_body));
 
-                    // As noted above, create the final block like
-                    //
-                    // ```
-                    // {
-                    //   let $param_pattern = $raw_param;
-                    //   ...
-                    //   drop-temps { <user-body> }
-                    // }
-                    // ```
-                    let body = this.block_all(
-                        desugared_span,
-                        this.arena.alloc_from_iter(statements),
-                        Some(user_body),
-                    );
+                // As noted above, create the final block like
+                //
+                // ```
+                // {
+                //   let $param_pattern = $raw_param;
+                //   ...
+                //   drop-temps { <user-body> }
+                // }
+                // ```
+                let body = this.block_all(
+                    desugared_span,
+                    this.arena.alloc_from_iter(statements),
+                    Some(user_body),
+                );
 
-                    this.expr_block(body)
-                },
-            );
+                this.expr_block(body)
+            };
+            // FIXME(gen_blocks): Consider unifying the `make_*_expr` functions.
+            let coroutine_expr = match coroutine_kind {
+                CoroutineKind::Async { .. } => this.make_async_expr(
+                    CaptureBy::Value { move_kw: rustc_span::DUMMY_SP },
+                    closure_id,
+                    None,
+                    body.span,
+                    hir::CoroutineSource::Fn,
+                    mkbody,
+                ),
+                CoroutineKind::Gen { .. } => this.make_gen_expr(
+                    CaptureBy::Value { move_kw: rustc_span::DUMMY_SP },
+                    closure_id,
+                    None,
+                    body.span,
+                    hir::CoroutineSource::Fn,
+                    mkbody,
+                ),
+                CoroutineKind::AsyncGen { .. } => this.make_async_gen_expr(
+                    CaptureBy::Value { move_kw: rustc_span::DUMMY_SP },
+                    closure_id,
+                    None,
+                    body.span,
+                    hir::CoroutineSource::Fn,
+                    mkbody,
+                ),
+            };
 
             let hir_id = this.lower_node_id(closure_id);
             this.maybe_forward_track_caller(body.span, fn_id, hir_id);
-            let expr = hir::Expr { hir_id, kind: async_expr, span: this.lower_span(body.span) };
+            let expr = hir::Expr { hir_id, kind: coroutine_expr, span: this.lower_span(body.span) };
 
             (this.arena.alloc_from_iter(parameters), expr)
         })
@@ -1249,21 +1250,26 @@ impl<'hir> LoweringContext<'_, 'hir> {
         sig: &FnSig,
         id: NodeId,
         kind: FnDeclKind,
-        is_async: Option<(NodeId, Span)>,
+        coroutine_kind: Option<CoroutineKind>,
     ) -> (&'hir hir::Generics<'hir>, hir::FnSig<'hir>) {
         let header = self.lower_fn_header(sig.header);
         let itctx = ImplTraitContext::Universal;
         let (generics, decl) =
             self.lower_generics(generics, sig.header.constness, id, &itctx, |this| {
-                this.lower_fn_decl(&sig.decl, id, sig.span, kind, is_async)
+                this.lower_fn_decl(&sig.decl, id, sig.span, kind, coroutine_kind)
             });
         (generics, hir::FnSig { header, decl, span: self.lower_span(sig.span) })
     }
 
     fn lower_fn_header(&mut self, h: FnHeader) -> hir::FnHeader {
+        let asyncness = if let Some(CoroutineKind::Async { span, .. }) = h.coroutine_kind {
+            hir::IsAsync::Async(span)
+        } else {
+            hir::IsAsync::NotAsync
+        };
         hir::FnHeader {
             unsafety: self.lower_unsafety(h.unsafety),
-            asyncness: self.lower_asyncness(h.asyncness),
+            asyncness: asyncness,
             constness: self.lower_constness(h.constness),
             abi: self.lower_extern(h.ext),
         }
@@ -1305,13 +1311,6 @@ impl<'hir> LoweringContext<'_, 'hir> {
         });
     }
 
-    fn lower_asyncness(&mut self, a: Async) -> hir::IsAsync {
-        match a {
-            Async::Yes { span, .. } => hir::IsAsync::Async(span),
-            Async::No => hir::IsAsync::NotAsync,
-        }
-    }
-
     pub(super) fn lower_constness(&mut self, c: Const) -> hir::Constness {
         match c {
             Const::Yes(_) => hir::Constness::Const,
@@ -1389,26 +1388,18 @@ impl<'hir> LoweringContext<'_, 'hir> {
         let host_param_parts = if let Const::Yes(span) = constness
             && self.tcx.features().effects
         {
-            if let Some(param) =
-                generics.params.iter().find(|x| x.attrs.iter().any(|x| x.has_name(sym::rustc_host)))
-            {
-                // user has manually specified a `rustc_host` param, in this case, we set
-                // the param id so that lowering logic can use that. But we don't create
-                // another host param, so this gives `None`.
-                self.host_param_id = Some(self.local_def_id(param.id));
-                None
-            } else {
-                let param_node_id = self.next_node_id();
-                let hir_id = self.next_id();
-                let def_id = self.create_def(
-                    self.local_def_id(parent_node_id),
-                    param_node_id,
-                    DefPathData::TypeNs(sym::host),
-                    span,
-                );
-                self.host_param_id = Some(def_id);
-                Some((span, hir_id, def_id))
-            }
+            let span = self.lower_span(span);
+            let param_node_id = self.next_node_id();
+            let hir_id = self.next_id();
+            let def_id = self.create_def(
+                self.local_def_id(parent_node_id),
+                param_node_id,
+                sym::host,
+                DefKind::ConstParam,
+                span,
+            );
+            self.host_param_id = Some(def_id);
+            Some((span, hir_id, def_id))
         } else {
             None
         };
@@ -1462,8 +1453,8 @@ impl<'hir> LoweringContext<'_, 'hir> {
 
         if let Some((span, hir_id, def_id)) = host_param_parts {
             let const_node_id = self.next_node_id();
-            let anon_const: LocalDefId =
-                self.create_def(def_id, const_node_id, DefPathData::AnonConst, span);
+            let anon_const =
+                self.create_def(def_id, const_node_id, kw::Empty, DefKind::AnonConst, span);
 
             let const_id = self.next_id();
             let const_expr_id = self.next_id();
@@ -1472,19 +1463,6 @@ impl<'hir> LoweringContext<'_, 'hir> {
             self.children.push((def_id, hir::MaybeOwner::NonOwner(hir_id)));
             self.children.push((anon_const, hir::MaybeOwner::NonOwner(const_id)));
 
-            let attr_id = self.tcx.sess.parse_sess.attr_id_generator.mk_attr_id();
-
-            let attrs = self.arena.alloc_from_iter([Attribute {
-                kind: AttrKind::Normal(P(NormalAttr::from_ident(Ident::new(
-                    sym::rustc_host,
-                    span,
-                )))),
-                span,
-                id: attr_id,
-                style: AttrStyle::Outer,
-            }]);
-            self.attrs.insert(hir_id.local_id, attrs);
-
             let const_body = self.lower_body(|this| {
                 (
                     &[],
@@ -1526,6 +1504,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                         hir_id: const_id,
                         body: const_body,
                     }),
+                    is_host_effect: true,
                 },
                 colon_span: None,
                 pure_wrt_drop: false,
diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs
index a88493acf..47b929816 100644
--- a/compiler/rustc_ast_lowering/src/lib.rs
+++ b/compiler/rustc_ast_lowering/src/lib.rs
@@ -30,12 +30,11 @@
 //! get confused if the spans from leaf AST nodes occur in multiple places
 //! in the HIR, especially for multiple identifiers.
 
-#![cfg_attr(not(bootstrap), allow(internal_features))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
+#![allow(internal_features)]
+#![feature(rustdoc_internals)]
+#![doc(rust_logo)]
 #![feature(box_patterns)]
 #![feature(let_chains)]
-#![feature(never_type)]
 #![recursion_limit = "256"]
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
@@ -45,32 +44,24 @@ extern crate tracing;
 
 use crate::errors::{AssocTyParentheses, AssocTyParenthesesSub, MisplacedImplTrait};
 
+use rustc_ast::node_id::NodeMap;
 use rustc_ast::ptr::P;
-use rustc_ast::visit;
 use rustc_ast::{self as ast, *};
 use rustc_ast_pretty::pprust;
 use rustc_data_structures::captures::Captures;
 use rustc_data_structures::fingerprint::Fingerprint;
-use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sorted_map::SortedMap;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::sync::Lrc;
-use rustc_errors::{
-    DiagnosticArgFromDisplay, DiagnosticMessage, Handler, StashKey, SubdiagnosticMessage,
-};
-use rustc_fluent_macro::fluent_messages;
+use rustc_errors::{DiagnosticArgFromDisplay, StashKey};
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, LifetimeRes, Namespace, PartialRes, PerNS, Res};
-use rustc_hir::def_id::{LocalDefId, CRATE_DEF_ID, LOCAL_CRATE};
-use rustc_hir::definitions::DefPathData;
-use rustc_hir::{ConstArg, GenericArg, ItemLocalId, ParamName, TraitCandidate};
+use rustc_hir::def_id::{LocalDefId, LocalDefIdMap, CRATE_DEF_ID, LOCAL_CRATE};
+use rustc_hir::{ConstArg, GenericArg, ItemLocalMap, ParamName, TraitCandidate};
 use rustc_index::{Idx, IndexSlice, IndexVec};
-use rustc_middle::{
-    span_bug,
-    ty::{ResolverAstLowering, TyCtxt},
-};
+use rustc_middle::span_bug;
+use rustc_middle::ty::{ResolverAstLowering, TyCtxt};
 use rustc_session::parse::{add_feature_diagnostics, feature_err};
-use rustc_span::hygiene::MacroKind;
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::{DesugaringKind, Span, DUMMY_SP};
 use smallvec::SmallVec;
@@ -94,7 +85,7 @@ mod lifetime_collector;
 mod pat;
 mod path;
 
-fluent_messages! { "../messages.ftl" }
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
 
 struct LoweringContext<'a, 'hir> {
     tcx: TyCtxt<'hir>,
@@ -128,26 +119,70 @@ struct LoweringContext<'a, 'hir> {
 
     current_hir_id_owner: hir::OwnerId,
     item_local_id_counter: hir::ItemLocalId,
-    trait_map: FxHashMap<ItemLocalId, Box<[TraitCandidate]>>,
+    trait_map: ItemLocalMap<Box<[TraitCandidate]>>,
 
     impl_trait_defs: Vec<hir::GenericParam<'hir>>,
     impl_trait_bounds: Vec<hir::WherePredicate<'hir>>,
 
     /// NodeIds that are lowered inside the current HIR owner.
-    node_id_to_local_id: FxHashMap<NodeId, hir::ItemLocalId>,
+    node_id_to_local_id: NodeMap<hir::ItemLocalId>,
 
-    allow_try_trait: Option<Lrc<[Symbol]>>,
-    allow_gen_future: Option<Lrc<[Symbol]>>,
+    allow_try_trait: Lrc<[Symbol]>,
+    allow_gen_future: Lrc<[Symbol]>,
+    allow_async_iterator: Lrc<[Symbol]>,
 
     /// Mapping from generics `def_id`s to TAIT generics `def_id`s.
     /// For each captured lifetime (e.g., 'a), we create a new lifetime parameter that is a generic
     /// defined on the TAIT, so we have type Foo<'a1> = ... and we establish a mapping in this
     /// field from the original parameter 'a to the new parameter 'a1.
-    generics_def_id_map: Vec<FxHashMap<LocalDefId, LocalDefId>>,
+    generics_def_id_map: Vec<LocalDefIdMap<LocalDefId>>,
 
     host_param_id: Option<LocalDefId>,
 }
 
+impl<'a, 'hir> LoweringContext<'a, 'hir> {
+    fn new(tcx: TyCtxt<'hir>, resolver: &'a mut ResolverAstLowering) -> Self {
+        Self {
+            // Pseudo-globals.
+            tcx,
+            resolver: resolver,
+            arena: tcx.hir_arena,
+
+            // HirId handling.
+            bodies: Vec::new(),
+            attrs: SortedMap::default(),
+            children: Vec::default(),
+            current_hir_id_owner: hir::CRATE_OWNER_ID,
+            item_local_id_counter: hir::ItemLocalId::new(0),
+            node_id_to_local_id: Default::default(),
+            trait_map: Default::default(),
+
+            // Lowering state.
+            catch_scope: None,
+            loop_scope: None,
+            is_in_loop_condition: false,
+            is_in_trait_impl: false,
+            is_in_dyn_type: false,
+            coroutine_kind: None,
+            task_context: None,
+            current_item: None,
+            impl_trait_defs: Vec::new(),
+            impl_trait_bounds: Vec::new(),
+            allow_try_trait: [sym::try_trait_v2, sym::yeet_desugar_details].into(),
+            allow_gen_future: if tcx.features().async_fn_track_caller {
+                [sym::gen_future, sym::closure_track_caller].into()
+            } else {
+                [sym::gen_future].into()
+            },
+            // FIXME(gen_blocks): how does `closure_track_caller`/`async_fn_track_caller`
+            // interact with `gen`/`async gen` blocks
+            allow_async_iterator: [sym::gen_future, sym::async_iterator].into(),
+            generics_def_id_map: Default::default(),
+            host_param_id: None,
+        }
+    }
+}
+
 trait ResolverAstLoweringExt {
     fn legacy_const_generic_args(&self, expr: &Expr) -> Option<Vec<usize>>;
     fn get_partial_res(&self, id: NodeId) -> Option<PartialRes>;
@@ -156,7 +191,6 @@ trait ResolverAstLoweringExt {
     fn get_lifetime_res(&self, id: NodeId) -> Option<LifetimeRes>;
     fn take_extra_lifetime_params(&mut self, id: NodeId) -> Vec<(Ident, NodeId, LifetimeRes)>;
     fn remap_extra_lifetime_params(&mut self, from: NodeId, to: NodeId);
-    fn decl_macro_kind(&self, def_id: LocalDefId) -> MacroKind;
 }
 
 impl ResolverAstLoweringExt for ResolverAstLowering {
@@ -220,10 +254,6 @@ impl ResolverAstLoweringExt for ResolverAstLowering {
         let lifetimes = self.extra_lifetime_params_map.remove(&from).unwrap_or_default();
         self.extra_lifetime_params_map.insert(to, lifetimes);
     }
-
-    fn decl_macro_kind(&self, def_id: LocalDefId) -> MacroKind {
-        self.builtin_macro_kinds.get(&def_id).copied().unwrap_or(MacroKind::Bang)
-    }
 }
 
 /// Context of `impl Trait` in code, which determines whether it is allowed in an HIR subtree,
@@ -350,7 +380,7 @@ enum AstOwner<'a> {
 }
 
 fn index_crate<'a>(
-    node_id_to_def_id: &FxHashMap<NodeId, LocalDefId>,
+    node_id_to_def_id: &NodeMap<LocalDefId>,
     krate: &'a Crate,
 ) -> IndexVec<LocalDefId, AstOwner<'a>> {
     let mut indexer = Indexer { node_id_to_def_id, index: IndexVec::new() };
@@ -360,7 +390,7 @@ fn index_crate<'a>(
     return indexer.index;
 
     struct Indexer<'s, 'a> {
-        node_id_to_def_id: &'s FxHashMap<NodeId, LocalDefId>,
+        node_id_to_def_id: &'s NodeMap<LocalDefId>,
         index: IndexVec<LocalDefId, AstOwner<'a>>,
     }
 
@@ -421,6 +451,7 @@ pub fn lower_to_hir(tcx: TyCtxt<'_>, (): ()) -> hir::Crate<'_> {
     tcx.ensure_with_value().output_filenames(());
     tcx.ensure_with_value().early_lint_checks(());
     tcx.ensure_with_value().debugger_visualizers(LOCAL_CRATE);
+    tcx.ensure_with_value().get_lang_items(());
     let (mut resolver, krate) = tcx.resolver_for_lowering(()).steal();
 
     let ast_index = index_crate(&resolver.node_id_to_def_id, &krate);
@@ -443,11 +474,6 @@ pub fn lower_to_hir(tcx: TyCtxt<'_>, (): ()) -> hir::Crate<'_> {
     drop(ast_index);
     sess.time("drop_ast", || drop(krate));
 
-    // Discard hygiene data, which isn't required after lowering to HIR.
-    if !sess.opts.unstable_opts.keep_hygiene_data {
-        rustc_span::hygiene::clear_syntax_context_map();
-    }
-
     // Don't hash unless necessary, because it's expensive.
     let opt_hir_hash =
         if tcx.needs_crate_hash() { Some(compute_hir_hash(tcx, &owners)) } else { None };
@@ -474,19 +500,20 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         &mut self,
         parent: LocalDefId,
         node_id: ast::NodeId,
-        data: DefPathData,
+        name: Symbol,
+        def_kind: DefKind,
         span: Span,
     ) -> LocalDefId {
         debug_assert_ne!(node_id, ast::DUMMY_NODE_ID);
         assert!(
             self.opt_local_def_id(node_id).is_none(),
-            "adding a def'n for node-id {:?} and data {:?} but a previous def'n exists: {:?}",
+            "adding a def'n for node-id {:?} and def kind {:?} but a previous def'n exists: {:?}",
             node_id,
-            data,
+            def_kind,
             self.tcx.hir().def_key(self.local_def_id(node_id)),
         );
 
-        let def_id = self.tcx.at(span).create_def(parent, data).def_id();
+        let def_id = self.tcx.at(span).create_def(parent, name, def_kind).def_id();
 
         debug!("create_def: def_id_to_node_id[{:?}] <-> {:?}", def_id, node_id);
         self.resolver.node_id_to_def_id.insert(node_id, def_id);
@@ -504,7 +531,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
     /// Given the id of some node in the AST, finds the `LocalDefId` associated with it by the name
     /// resolver (if any).
     fn orig_opt_local_def_id(&self, node: NodeId) -> Option<LocalDefId> {
-        self.resolver.node_id_to_def_id.get(&node).map(|local_def_id| *local_def_id)
+        self.resolver.node_id_to_def_id.get(&node).copied()
     }
 
     /// Given the id of some node in the AST, finds the `LocalDefId` associated with it by the name
@@ -547,7 +574,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         self.generics_def_id_map
             .iter()
             .rev()
-            .find_map(|map| map.get(&local_def_id).map(|local_def_id| *local_def_id))
+            .find_map(|map| map.get(&local_def_id).copied())
             .unwrap_or(local_def_id)
     }
 
@@ -615,7 +642,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
     /// `'a` declared on the TAIT, instead of the function.
     fn with_remapping<R>(
         &mut self,
-        remap: FxHashMap<LocalDefId, LocalDefId>,
+        remap: LocalDefIdMap<LocalDefId>,
         f: impl FnOnce(&mut Self) -> R,
     ) -> R {
         self.generics_def_id_map.push(remap);
@@ -737,8 +764,26 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         self.resolver.get_import_res(id).present_items()
     }
 
-    fn diagnostic(&self) -> &Handler {
-        self.tcx.sess.diagnostic()
+    fn make_lang_item_path(
+        &mut self,
+        lang_item: hir::LangItem,
+        span: Span,
+        args: Option<&'hir hir::GenericArgs<'hir>>,
+    ) -> &'hir hir::Path<'hir> {
+        let def_id = self.tcx.require_lang_item(lang_item, Some(span));
+        let def_kind = self.tcx.def_kind(def_id);
+        let res = Res::Def(def_kind, def_id);
+        self.arena.alloc(hir::Path {
+            span,
+            res,
+            segments: self.arena.alloc_from_iter([hir::PathSegment {
+                ident: Ident::new(lang_item.name(), span),
+                hir_id: self.next_id(),
+                res,
+                args,
+                infer_args: false,
+            }]),
+        })
     }
 
     /// Reuses the span but adds information like the kind of the desugaring and features that are
@@ -787,7 +832,8 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                 let _def_id = self.create_def(
                     self.current_hir_id_owner.def_id,
                     param,
-                    DefPathData::LifetimeNs(kw::UnderscoreLifetime),
+                    kw::UnderscoreLifetime,
+                    DefKind::LifetimeParam,
                     ident.span,
                 );
                 debug!(?_def_id);
@@ -851,7 +897,10 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         result
     }
 
-    fn with_new_scopes<T>(&mut self, f: impl FnOnce(&mut Self) -> T) -> T {
+    fn with_new_scopes<T>(&mut self, scope_span: Span, f: impl FnOnce(&mut Self) -> T) -> T {
+        let current_item = self.current_item;
+        self.current_item = Some(scope_span);
+
         let was_in_loop_condition = self.is_in_loop_condition;
         self.is_in_loop_condition = false;
 
@@ -863,6 +912,8 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
 
         self.is_in_loop_condition = was_in_loop_condition;
 
+        self.current_item = current_item;
+
         ret
     }
 
@@ -1162,7 +1213,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         itctx: &ImplTraitContext,
     ) -> hir::GenericArg<'hir> {
         match arg {
-            ast::GenericArg::Lifetime(lt) => GenericArg::Lifetime(self.lower_lifetime(&lt)),
+            ast::GenericArg::Lifetime(lt) => GenericArg::Lifetime(self.lower_lifetime(lt)),
             ast::GenericArg::Type(ty) => {
                 match &ty.kind {
                     TyKind::Infer if self.tcx.features().generic_arg_infer => {
@@ -1199,7 +1250,8 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                                 let def_id = self.create_def(
                                     parent_def_id.def_id,
                                     node_id,
-                                    DefPathData::AnonConst,
+                                    kw::Empty,
+                                    DefKind::AnonConst,
                                     span,
                                 );
 
@@ -1211,7 +1263,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                                     tokens: None,
                                 };
 
-                                let ct = self.with_new_scopes(|this| hir::AnonConst {
+                                let ct = self.with_new_scopes(span, |this| hir::AnonConst {
                                     def_id,
                                     hir_id: this.lower_node_id(node_id),
                                     body: this.lower_const_body(path_expr.span, Some(&path_expr)),
@@ -1226,10 +1278,10 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                     }
                     _ => {}
                 }
-                GenericArg::Type(self.lower_ty(&ty, itctx))
+                GenericArg::Type(self.lower_ty(ty, itctx))
             }
             ast::GenericArg::Const(ct) => GenericArg::Const(ConstArg {
-                value: self.lower_anon_const(&ct),
+                value: self.lower_anon_const(ct),
                 span: self.lower_span(ct.value.span),
                 is_desugared_from_effects: false,
             }),
@@ -1272,7 +1324,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                 let lifetime_bound = this.elided_dyn_bound(t.span);
                 (bounds, lifetime_bound)
             });
-            let kind = hir::TyKind::TraitObject(bounds, &lifetime_bound, TraitObjectSyntax::None);
+            let kind = hir::TyKind::TraitObject(bounds, lifetime_bound, TraitObjectSyntax::None);
             return hir::Ty { kind, span: self.lower_span(t.span), hir_id: self.next_id() };
         }
 
@@ -1293,7 +1345,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         let kind = match &t.kind {
             TyKind::Infer => hir::TyKind::Infer,
             TyKind::Err => {
-                hir::TyKind::Err(self.tcx.sess.delay_span_bug(t.span, "TyKind::Err lowered"))
+                hir::TyKind::Err(self.tcx.sess.span_delayed_bug(t.span, "TyKind::Err lowered"))
             }
             // FIXME(unnamed_fields): IMPLEMENTATION IN PROGRESS
             #[allow(rustc::untranslatable_diagnostic)]
@@ -1374,7 +1426,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                             GenericBound::Trait(
                                 ty,
                                 modifier @ (TraitBoundModifier::None
-                                | TraitBoundModifier::MaybeConst
+                                | TraitBoundModifier::MaybeConst(_)
                                 | TraitBoundModifier::Negative),
                             ) => {
                                 Some(this.lower_poly_trait_ref(ty, itctx, modifier.to_constness()))
@@ -1436,7 +1488,8 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                         self.create_def(
                             self.current_hir_id_owner.def_id,
                             *def_node_id,
-                            DefPathData::TypeNs(ident.name),
+                            ident.name,
+                            DefKind::TyParam,
                             span,
                         );
                         let (param, bounds, path) = self.lower_universal_param_and_bounds(
@@ -1476,7 +1529,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             }
             TyKind::MacCall(_) => panic!("`TyKind::MacCall` should have been expanded by now"),
             TyKind::CVarArgs => {
-                let guar = self.tcx.sess.delay_span_bug(
+                let guar = self.tcx.sess.span_delayed_bug(
                     t.span,
                     "`TyKind::CVarArgs` should have been handled elsewhere",
                 );
@@ -1542,8 +1595,11 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                 Vec::new()
             }
             hir::OpaqueTyOrigin::FnReturn(..) => {
-                if let FnDeclKind::Impl | FnDeclKind::Trait =
-                    fn_kind.expect("expected RPITs to be lowered with a FnKind")
+                if matches!(
+                    fn_kind.expect("expected RPITs to be lowered with a FnKind"),
+                    FnDeclKind::Impl | FnDeclKind::Trait
+                ) || self.tcx.features().lifetime_capture_rules_2024
+                    || span.at_least_rust_2024()
                 {
                     // return-position impl trait in trait was decided to capture all
                     // in-scope lifetimes, which we collect for all opaques during resolution.
@@ -1556,7 +1612,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                     // in fn return position, like the `fn test<'a>() -> impl Debug + 'a`
                     // example, we only need to duplicate lifetimes that appear in the
                     // bounds, since those are the only ones that are captured by the opaque.
-                    lifetime_collector::lifetimes_in_bounds(&self.resolver, bounds)
+                    lifetime_collector::lifetimes_in_bounds(self.resolver, bounds)
                 }
             }
             hir::OpaqueTyOrigin::AsyncFn(..) => {
@@ -1589,14 +1645,15 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         let opaque_ty_def_id = self.create_def(
             self.current_hir_id_owner.def_id,
             opaque_ty_node_id,
-            DefPathData::ImplTrait,
+            kw::Empty,
+            DefKind::OpaqueTy,
             opaque_ty_span,
         );
         debug!(?opaque_ty_def_id);
 
         // Map from captured (old) lifetime to synthetic (new) lifetime.
         // Used to resolve lifetimes in the bounds of the opaque.
-        let mut captured_to_synthesized_mapping = FxHashMap::default();
+        let mut captured_to_synthesized_mapping = LocalDefIdMap::default();
         // List of (early-bound) synthetic lifetimes that are owned by the opaque.
         // This is used to create the `hir::Generics` owned by the opaque.
         let mut synthesized_lifetime_definitions = vec![];
@@ -1618,7 +1675,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                     } else {
                         self.tcx
                             .sess
-                            .delay_span_bug(lifetime.ident.span, "no def-id for fresh lifetime");
+                            .span_delayed_bug(lifetime.ident.span, "no def-id for fresh lifetime");
                         continue;
                     }
                 }
@@ -1643,8 +1700,9 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                 let duplicated_lifetime_def_id = self.create_def(
                     opaque_ty_def_id,
                     duplicated_lifetime_node_id,
-                    DefPathData::LifetimeNs(lifetime.ident.name),
-                    lifetime.ident.span,
+                    lifetime.ident.name,
+                    DefKind::LifetimeParam,
+                    self.lower_span(lifetime.ident.span),
                 );
                 captured_to_synthesized_mapping.insert(old_def_id, duplicated_lifetime_def_id);
                 // FIXME: Instead of doing this, we could move this whole loop
@@ -1653,7 +1711,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                 synthesized_lifetime_definitions.push((
                     duplicated_lifetime_node_id,
                     duplicated_lifetime_def_id,
-                    lifetime.ident,
+                    self.lower_ident(lifetime.ident),
                 ));
 
                 // Now make an arg that we can use for the generic params of the opaque tykind.
@@ -1747,13 +1805,15 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         }))
     }
 
-    // Lowers a function declaration.
-    //
-    // `decl`: the unlowered (AST) function declaration.
-    // `fn_node_id`: `impl Trait` arguments are lowered into generic parameters on the given `NodeId`.
-    // `make_ret_async`: if `Some`, converts `-> T` into `-> impl Future<Output = T>` in the
-    //      return type. This is used for `async fn` declarations. The `NodeId` is the ID of the
-    //      return type `impl Trait` item, and the `Span` points to the `async` keyword.
+    /// Lowers a function declaration.
+    ///
+    /// `decl`: the unlowered (AST) function declaration.
+    ///
+    /// `fn_node_id`: `impl Trait` arguments are lowered into generic parameters on the given
+    /// `NodeId`.
+    ///
+    /// `transform_return_type`: if `Some`, applies some conversion to the return type, such as is
+    /// needed for `async fn` and `gen fn`. See [`CoroutineKind`] for more details.
     #[instrument(level = "debug", skip(self))]
     fn lower_fn_decl(
         &mut self,
@@ -1761,7 +1821,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         fn_node_id: NodeId,
         fn_span: Span,
         kind: FnDeclKind,
-        make_ret_async: Option<(NodeId, Span)>,
+        coro: Option<CoroutineKind>,
     ) -> &'hir hir::FnDecl<'hir> {
         let c_variadic = decl.c_variadic();
 
@@ -1790,11 +1850,12 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             self.lower_ty_direct(&param.ty, &itctx)
         }));
 
-        let output = if let Some((ret_id, _span)) = make_ret_async {
-            let fn_def_id = self.local_def_id(fn_node_id);
-            self.lower_async_fn_ret_ty(&decl.output, fn_def_id, ret_id, kind, fn_span)
-        } else {
-            match &decl.output {
+        let output = match coro {
+            Some(coro) => {
+                let fn_def_id = self.local_def_id(fn_node_id);
+                self.lower_coroutine_fn_ret_ty(&decl.output, fn_def_id, coro, kind, fn_span)
+            }
+            None => match &decl.output {
                 FnRetTy::Ty(ty) => {
                     let context = if kind.return_impl_trait_allowed() {
                         let fn_def_id = self.local_def_id(fn_node_id);
@@ -1818,7 +1879,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                     hir::FnRetTy::Return(self.lower_ty(ty, &context))
                 }
                 FnRetTy::Default(span) => hir::FnRetTy::DefaultReturn(self.lower_span(*span)),
-            }
+            },
         };
 
         self.arena.alloc(hir::FnDecl {
@@ -1857,16 +1918,26 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
     // `fn_node_id`: `NodeId` of the parent function (used to create child impl trait definition)
     // `opaque_ty_node_id`: `NodeId` of the opaque `impl Trait` type that should be created
     #[instrument(level = "debug", skip(self))]
-    fn lower_async_fn_ret_ty(
+    fn lower_coroutine_fn_ret_ty(
         &mut self,
         output: &FnRetTy,
         fn_def_id: LocalDefId,
-        opaque_ty_node_id: NodeId,
+        coro: CoroutineKind,
         fn_kind: FnDeclKind,
         fn_span: Span,
     ) -> hir::FnRetTy<'hir> {
         let span = self.lower_span(fn_span);
-        let opaque_ty_span = self.mark_span_with_reason(DesugaringKind::Async, span, None);
+
+        let (opaque_ty_node_id, allowed_features) = match coro {
+            CoroutineKind::Async { return_impl_trait_id, .. } => (return_impl_trait_id, None),
+            CoroutineKind::Gen { return_impl_trait_id, .. } => (return_impl_trait_id, None),
+            CoroutineKind::AsyncGen { return_impl_trait_id, .. } => {
+                (return_impl_trait_id, Some(self.allow_async_iterator.clone()))
+            }
+        };
+
+        let opaque_ty_span =
+            self.mark_span_with_reason(DesugaringKind::Async, span, allowed_features);
 
         let captured_lifetimes: Vec<_> = self
             .resolver
@@ -1883,15 +1954,16 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             span,
             opaque_ty_span,
             |this| {
-                let future_bound = this.lower_async_fn_output_type_to_future_bound(
+                let bound = this.lower_coroutine_fn_output_type_to_bound(
                     output,
-                    span,
+                    coro,
+                    opaque_ty_span,
                     ImplTraitContext::ReturnPositionOpaqueTy {
                         origin: hir::OpaqueTyOrigin::FnReturn(fn_def_id),
                         fn_kind,
                     },
                 );
-                arena_vec![this; future_bound]
+                arena_vec![this; bound]
             },
         );
 
@@ -1900,10 +1972,11 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
     }
 
     /// Transforms `-> T` into `Future<Output = T>`.
-    fn lower_async_fn_output_type_to_future_bound(
+    fn lower_coroutine_fn_output_type_to_bound(
         &mut self,
         output: &FnRetTy,
-        span: Span,
+        coro: CoroutineKind,
+        opaque_ty_span: Span,
         nested_impl_trait_context: ImplTraitContext,
     ) -> hir::GenericBound<'hir> {
         // Compute the `T` in `Future<Output = T>` from the return type.
@@ -1917,20 +1990,34 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             FnRetTy::Default(ret_ty_span) => self.arena.alloc(self.ty_tup(*ret_ty_span, &[])),
         };
 
-        // "<Output = T>"
-        let future_args = self.arena.alloc(hir::GenericArgs {
+        // "<$assoc_ty_name = T>"
+        let (assoc_ty_name, trait_lang_item) = match coro {
+            CoroutineKind::Async { .. } => (sym::Output, hir::LangItem::Future),
+            CoroutineKind::Gen { .. } => (sym::Item, hir::LangItem::Iterator),
+            CoroutineKind::AsyncGen { .. } => (sym::Item, hir::LangItem::AsyncIterator),
+        };
+
+        let bound_args = self.arena.alloc(hir::GenericArgs {
             args: &[],
-            bindings: arena_vec![self; self.output_ty_binding(span, output_ty)],
+            bindings: arena_vec![self; self.assoc_ty_binding(assoc_ty_name, opaque_ty_span, output_ty)],
             parenthesized: hir::GenericArgsParentheses::No,
             span_ext: DUMMY_SP,
         });
 
-        hir::GenericBound::LangItemTrait(
-            // ::std::future::Future<future_params>
-            hir::LangItem::Future,
-            self.lower_span(span),
-            self.next_id(),
-            future_args,
+        hir::GenericBound::Trait(
+            hir::PolyTraitRef {
+                bound_generic_params: &[],
+                trait_ref: hir::TraitRef {
+                    path: self.make_lang_item_path(
+                        trait_lang_item,
+                        opaque_ty_span,
+                        Some(bound_args),
+                    ),
+                    hir_ref_id: self.next_id(),
+                },
+                span: opaque_ty_span,
+            },
+            hir::TraitBoundModifier::None,
         )
     }
 
@@ -2072,14 +2159,12 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                 (hir::ParamName::Plain(self.lower_ident(param.ident)), kind)
             }
             GenericParamKind::Const { ty, kw_span: _, default } => {
-                let ty = self.lower_ty(
-                    &ty,
-                    &ImplTraitContext::Disallowed(ImplTraitPosition::GenericDefault),
-                );
+                let ty = self
+                    .lower_ty(ty, &ImplTraitContext::Disallowed(ImplTraitPosition::GenericDefault));
                 let default = default.as_ref().map(|def| self.lower_anon_const(def));
                 (
                     hir::ParamName::Plain(self.lower_ident(param.ident)),
-                    hir::GenericParamKind::Const { ty, default },
+                    hir::GenericParamKind::Const { ty, default, is_host_effect: false },
                 )
             }
         }
@@ -2200,7 +2285,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         match c.value.kind {
             ExprKind::Underscore => {
                 if self.tcx.features().generic_arg_infer {
-                    hir::ArrayLen::Infer(self.lower_node_id(c.id), c.value.span)
+                    hir::ArrayLen::Infer(self.lower_node_id(c.id), self.lower_span(c.value.span))
                 } else {
                     feature_err(
                         &self.tcx.sess.parse_sess,
@@ -2217,7 +2302,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
     }
 
     fn lower_anon_const(&mut self, c: &AnonConst) -> hir::AnonConst {
-        self.with_new_scopes(|this| hir::AnonConst {
+        self.with_new_scopes(c.value.span, |this| hir::AnonConst {
             def_id: this.local_def_id(c.id),
             hir_id: this.lower_node_id(c.id),
             body: this.lower_const_body(c.value.span, Some(&c.value)),
@@ -2234,7 +2319,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
     fn lower_trait_bound_modifier(&mut self, f: TraitBoundModifier) -> hir::TraitBoundModifier {
         match f {
             TraitBoundModifier::None => hir::TraitBoundModifier::None,
-            TraitBoundModifier::MaybeConst => hir::TraitBoundModifier::MaybeConst,
+            TraitBoundModifier::MaybeConst(_) => hir::TraitBoundModifier::MaybeConst,
 
             TraitBoundModifier::Negative => {
                 if self.tcx.features().negative_bounds {
@@ -2311,21 +2396,21 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
 
     fn pat_cf_continue(&mut self, span: Span, pat: &'hir hir::Pat<'hir>) -> &'hir hir::Pat<'hir> {
         let field = self.single_pat_field(span, pat);
-        self.pat_lang_item_variant(span, hir::LangItem::ControlFlowContinue, field, None)
+        self.pat_lang_item_variant(span, hir::LangItem::ControlFlowContinue, field)
     }
 
     fn pat_cf_break(&mut self, span: Span, pat: &'hir hir::Pat<'hir>) -> &'hir hir::Pat<'hir> {
         let field = self.single_pat_field(span, pat);
-        self.pat_lang_item_variant(span, hir::LangItem::ControlFlowBreak, field, None)
+        self.pat_lang_item_variant(span, hir::LangItem::ControlFlowBreak, field)
     }
 
     fn pat_some(&mut self, span: Span, pat: &'hir hir::Pat<'hir>) -> &'hir hir::Pat<'hir> {
         let field = self.single_pat_field(span, pat);
-        self.pat_lang_item_variant(span, hir::LangItem::OptionSome, field, None)
+        self.pat_lang_item_variant(span, hir::LangItem::OptionSome, field)
     }
 
     fn pat_none(&mut self, span: Span) -> &'hir hir::Pat<'hir> {
-        self.pat_lang_item_variant(span, hir::LangItem::OptionNone, &[], None)
+        self.pat_lang_item_variant(span, hir::LangItem::OptionNone, &[])
     }
 
     fn single_pat_field(
@@ -2348,9 +2433,8 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         span: Span,
         lang_item: hir::LangItem,
         fields: &'hir [hir::PatField<'hir>],
-        hir_id: Option<hir::HirId>,
     ) -> &'hir hir::Pat<'hir> {
-        let qpath = hir::QPath::LangItem(lang_item, self.lower_span(span), hir_id);
+        let qpath = hir::QPath::LangItem(lang_item, self.lower_span(span));
         self.pat(span, hir::PatKind::Struct(qpath, fields, false))
     }
 
@@ -2482,9 +2566,10 @@ impl<'hir> GenericArgsCtor<'hir> {
         let hir_id = lcx.next_id();
 
         let Some(host_param_id) = lcx.host_param_id else {
-            lcx.tcx
-                .sess
-                .delay_span_bug(span, "no host param id for call in const yet no errors reported");
+            lcx.tcx.sess.span_delayed_bug(
+                span,
+                "no host param id for call in const yet no errors reported",
+            );
             return;
         };
 
@@ -2507,17 +2592,14 @@ impl<'hir> GenericArgsCtor<'hir> {
             })
         });
 
-        let attr_id = lcx.tcx.sess.parse_sess.attr_id_generator.mk_attr_id();
-        let attr = lcx.arena.alloc(Attribute {
-            kind: AttrKind::Normal(P(NormalAttr::from_ident(Ident::new(sym::rustc_host, span)))),
+        let def_id = lcx.create_def(
+            lcx.current_hir_id_owner.def_id,
+            id,
+            kw::Empty,
+            DefKind::AnonConst,
             span,
-            id: attr_id,
-            style: AttrStyle::Outer,
-        });
-        lcx.attrs.insert(hir_id.local_id, std::slice::from_ref(attr));
+        );
 
-        let def_id =
-            lcx.create_def(lcx.current_hir_id_owner.def_id, id, DefPathData::AnonConst, span);
         lcx.children.push((def_id, hir::MaybeOwner::NonOwner(hir_id)));
         self.args.push(hir::GenericArg::Const(hir::ConstArg {
             value: hir::AnonConst { def_id, hir_id, body },
diff --git a/compiler/rustc_ast_lowering/src/lifetime_collector.rs b/compiler/rustc_ast_lowering/src/lifetime_collector.rs
index d66bba517..4b1c057cd 100644
--- a/compiler/rustc_ast_lowering/src/lifetime_collector.rs
+++ b/compiler/rustc_ast_lowering/src/lifetime_collector.rs
@@ -108,7 +108,7 @@ impl<'ast> Visitor<'ast> for LifetimeCollectVisitor<'ast> {
     }
 }
 
-pub fn lifetimes_in_bounds(
+pub(crate) fn lifetimes_in_bounds(
     resolver: &ResolverAstLowering,
     bounds: &GenericBounds,
 ) -> Vec<Lifetime> {
diff --git a/compiler/rustc_ast_lowering/src/pat.rs b/compiler/rustc_ast_lowering/src/pat.rs
index a30f264bc..017314ee4 100644
--- a/compiler/rustc_ast_lowering/src/pat.rs
+++ b/compiler/rustc_ast_lowering/src/pat.rs
@@ -18,12 +18,13 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         self.arena.alloc(self.lower_pat_mut(pattern))
     }
 
-    pub(crate) fn lower_pat_mut(&mut self, mut pattern: &Pat) -> hir::Pat<'hir> {
+    fn lower_pat_mut(&mut self, mut pattern: &Pat) -> hir::Pat<'hir> {
         ensure_sufficient_stack(|| {
             // loop here to avoid recursion
             let node = loop {
                 match &pattern.kind {
                     PatKind::Wild => break hir::PatKind::Wild,
+                    PatKind::Never => break hir::PatKind::Never,
                     PatKind::Ident(binding_mode, ident, sub) => {
                         let lower_sub = |this: &mut Self| sub.as_ref().map(|s| this.lower_pat(s));
                         break self.lower_pat_ident(pattern, *binding_mode, *ident, lower_sub);
diff --git a/compiler/rustc_ast_lowering/src/path.rs b/compiler/rustc_ast_lowering/src/path.rs
index 899f92a99..5ceeb72f2 100644
--- a/compiler/rustc_ast_lowering/src/path.rs
+++ b/compiler/rustc_ast_lowering/src/path.rs
@@ -9,6 +9,7 @@ use rustc_ast::{self as ast, *};
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, PartialRes, Res};
 use rustc_hir::GenericArg;
+use rustc_middle::span_bug;
 use rustc_span::symbol::{kw, sym, Ident};
 use rustc_span::{BytePos, Span, DUMMY_SP};
 
@@ -139,7 +140,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
 
         // We should've returned in the for loop above.
 
-        self.diagnostic().span_bug(
+        self.tcx.sess.dcx().span_bug(
             p.span,
             format!(
                 "lower_qpath: no final extension segment in {}..{}",
@@ -285,7 +286,9 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         let (start, end) = match self.resolver.get_lifetime_res(segment_id) {
             Some(LifetimeRes::ElidedAnchor { start, end }) => (start, end),
             None => return,
-            Some(_) => panic!(),
+            Some(res) => {
+                span_bug!(path_span, "expected an elided lifetime to insert. found {res:?}")
+            }
         };
         let expected_lifetimes = end.as_usize() - start.as_usize();
         debug!(expected_lifetimes);
@@ -372,10 +375,10 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             // ```
             FnRetTy::Ty(ty) if matches!(itctx, ImplTraitContext::ReturnPositionOpaqueTy { .. }) => {
                 if self.tcx.features().impl_trait_in_fn_trait_return {
-                    self.lower_ty(&ty, itctx)
+                    self.lower_ty(ty, itctx)
                 } else {
                     self.lower_ty(
-                        &ty,
+                        ty,
                         &ImplTraitContext::FeatureGated(
                             ImplTraitPosition::FnTraitReturn,
                             sym::impl_trait_in_fn_trait_return,
@@ -384,12 +387,12 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                 }
             }
             FnRetTy::Ty(ty) => {
-                self.lower_ty(&ty, &ImplTraitContext::Disallowed(ImplTraitPosition::FnTraitReturn))
+                self.lower_ty(ty, &ImplTraitContext::Disallowed(ImplTraitPosition::FnTraitReturn))
             }
             FnRetTy::Default(_) => self.arena.alloc(self.ty_tup(*span, &[])),
         };
         let args = smallvec![GenericArg::Type(self.arena.alloc(self.ty_tup(*inputs_span, inputs)))];
-        let binding = self.output_ty_binding(output_ty.span, output_ty);
+        let binding = self.assoc_ty_binding(sym::Output, output_ty.span, output_ty);
         (
             GenericArgsCtor {
                 args,
@@ -401,13 +404,14 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
         )
     }
 
-    /// An associated type binding `Output = $ty`.
-    pub(crate) fn output_ty_binding(
+    /// An associated type binding `$assoc_ty_name = $ty`.
+    pub(crate) fn assoc_ty_binding(
         &mut self,
+        assoc_ty_name: rustc_span::Symbol,
         span: Span,
         ty: &'hir hir::Ty<'hir>,
     ) -> hir::TypeBinding<'hir> {
-        let ident = Ident::with_dummy_span(hir::FN_OUTPUT_NAME);
+        let ident = Ident::with_dummy_span(assoc_ty_name);
         let kind = hir::TypeBindingKind::Equality { term: ty.into() };
         let args = arena_vec![self;];
         let bindings = arena_vec![self;];
diff --git a/compiler/rustc_ast_passes/Cargo.toml b/compiler/rustc_ast_passes/Cargo.toml
index 0001394c8..99e79f65f 100644
--- a/compiler/rustc_ast_passes/Cargo.toml
+++ b/compiler/rustc_ast_passes/Cargo.toml
@@ -5,7 +5,7 @@ edition = "2021"
 
 [dependencies]
 # tidy-alphabetical-start
-itertools = "0.10.1"
+itertools = "0.11"
 rustc_ast = { path = "../rustc_ast" }
 rustc_ast_pretty = { path = "../rustc_ast_pretty" }
 rustc_attr = { path = "../rustc_attr" }
diff --git a/compiler/rustc_ast_passes/messages.ftl b/compiler/rustc_ast_passes/messages.ftl
index d22bae816..790b58313 100644
--- a/compiler/rustc_ast_passes/messages.ftl
+++ b/compiler/rustc_ast_passes/messages.ftl
@@ -174,6 +174,10 @@ ast_passes_item_underscore = `{$kind}` items in this context need a name
 ast_passes_keyword_lifetime =
     lifetimes cannot use keyword names
 
+ast_passes_match_arm_with_no_body =
+    `match` arm with no body
+    .suggestion = add a body after the pattern
+
 ast_passes_module_nonascii = trying to load file for module `{$name}` with non-ascii identifier name
     .help = consider using the `#[path]` attribute to specify filesystem path
 
@@ -218,9 +222,13 @@ ast_passes_static_without_body =
     .suggestion = provide a definition for the static
 
 ast_passes_tilde_const_disallowed = `~const` is not allowed here
-    .trait = trait objects cannot have `~const` trait bounds
     .closure = closures cannot have `~const` trait bounds
     .function = this function is not `const`, so it cannot have `~const` trait bounds
+    .trait = this trait is not a `#[const_trait]`, so it cannot have `~const` trait bounds
+    .trait_impl = this impl is not `const`, so it cannot have `~const` trait bounds
+    .impl = inherent impls cannot have `~const` trait bounds
+    .object = trait objects cannot have `~const` trait bounds
+    .item = this item cannot have `~const` trait bounds
 
 ast_passes_trait_fn_const =
     functions in traits cannot be declared const
diff --git a/compiler/rustc_ast_passes/src/ast_validation.rs b/compiler/rustc_ast_passes/src/ast_validation.rs
index 3d0513c89..887cb434a 100644
--- a/compiler/rustc_ast_passes/src/ast_validation.rs
+++ b/compiler/rustc_ast_passes/src/ast_validation.rs
@@ -8,9 +8,9 @@
 
 use itertools::{Either, Itertools};
 use rustc_ast::ptr::P;
-use rustc_ast::visit::{self, AssocCtxt, BoundKind, FnCtxt, FnKind, Visitor};
+use rustc_ast::visit::{AssocCtxt, BoundKind, FnCtxt, FnKind, Visitor};
+use rustc_ast::walk_list;
 use rustc_ast::*;
-use rustc_ast::{walk_list, StaticItem};
 use rustc_ast_pretty::pprust::{self, State};
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_feature::Features;
@@ -40,6 +40,10 @@ enum SelfSemantic {
 enum DisallowTildeConstContext<'a> {
     TraitObject,
     Fn(FnKind<'a>),
+    Trait(Span),
+    TraitImpl(Span),
+    Impl(Span),
+    Item,
 }
 
 struct AstValidator<'a> {
@@ -110,18 +114,6 @@ impl<'a> AstValidator<'a> {
         self.disallow_tilde_const = old;
     }
 
-    fn with_tilde_const_allowed(&mut self, f: impl FnOnce(&mut Self)) {
-        self.with_tilde_const(None, f)
-    }
-
-    fn with_banned_tilde_const(
-        &mut self,
-        ctx: DisallowTildeConstContext<'a>,
-        f: impl FnOnce(&mut Self),
-    ) {
-        self.with_tilde_const(Some(ctx), f)
-    }
-
     fn check_type_alias_where_clause_location(
         &mut self,
         ty_alias: &TyAlias,
@@ -173,7 +165,7 @@ impl<'a> AstValidator<'a> {
                 self.with_impl_trait(Some(t.span), |this| visit::walk_ty(this, t))
             }
             TyKind::TraitObject(..) => self
-                .with_banned_tilde_const(DisallowTildeConstContext::TraitObject, |this| {
+                .with_tilde_const(Some(DisallowTildeConstContext::TraitObject), |this| {
                     visit::walk_ty(this, t)
                 }),
             TyKind::Path(qself, path) => {
@@ -229,8 +221,8 @@ impl<'a> AstValidator<'a> {
         }
     }
 
-    fn err_handler(&self) -> &rustc_errors::Handler {
-        &self.session.diagnostic()
+    fn dcx(&self) -> &rustc_errors::DiagCtxt {
+        self.session.dcx()
     }
 
     fn check_lifetime(&self, ident: Ident) {
@@ -278,7 +270,7 @@ impl<'a> AstValidator<'a> {
         ) {
             return;
         }
-        self.err_handler().emit_err(errors::InvalidUnnamedFieldTy { span, ty_span: ty.span });
+        self.dcx().emit_err(errors::InvalidUnnamedFieldTy { span, ty_span: ty.span });
     }
 
     fn deny_anon_struct_or_union(&self, ty: &Ty) {
@@ -287,15 +279,14 @@ impl<'a> AstValidator<'a> {
             TyKind::AnonUnion(..) => "union",
             _ => return,
         };
-        self.err_handler()
-            .emit_err(errors::AnonStructOrUnionNotAllowed { struct_or_union, span: ty.span });
+        self.dcx().emit_err(errors::AnonStructOrUnionNotAllowed { struct_or_union, span: ty.span });
     }
 
     fn deny_unnamed_field(&self, field: &FieldDef) {
         if let Some(ident) = field.ident
             && ident.name == kw::Underscore
         {
-            self.err_handler()
+            self.dcx()
                 .emit_err(errors::InvalidUnnamedField { span: field.span, ident_span: ident.span });
         }
     }
@@ -401,7 +392,7 @@ impl<'a> AstValidator<'a> {
             [b0] => b0.span(),
             [b0, .., bl] => b0.span().to(bl.span()),
         };
-        self.err_handler().emit_err(errors::BoundInContext { span, ctx });
+        self.dcx().emit_err(errors::BoundInContext { span, ctx });
     }
 
     fn check_foreign_ty_genericless(
@@ -411,7 +402,7 @@ impl<'a> AstValidator<'a> {
         after_where_clause: &TyAliasWhereClause,
     ) {
         let cannot_have = |span, descr, remove_descr| {
-            self.err_handler().emit_err(errors::ExternTypesCannotHave {
+            self.dcx().emit_err(errors::ExternTypesCannotHave {
                 span,
                 descr,
                 remove_descr,
@@ -437,7 +428,7 @@ impl<'a> AstValidator<'a> {
         let Some(body) = body else {
             return;
         };
-        self.err_handler().emit_err(errors::BodyInExtern {
+        self.dcx().emit_err(errors::BodyInExtern {
             span: ident.span,
             body,
             block: self.current_extern_span(),
@@ -450,7 +441,7 @@ impl<'a> AstValidator<'a> {
         let Some(body) = body else {
             return;
         };
-        self.err_handler().emit_err(errors::FnBodyInExtern {
+        self.dcx().emit_err(errors::FnBodyInExtern {
             span: ident.span,
             body: body.span,
             block: self.current_extern_span(),
@@ -464,7 +455,7 @@ impl<'a> AstValidator<'a> {
     /// An `fn` in `extern { ... }` cannot have qualifiers, e.g. `async fn`.
     fn check_foreign_fn_headerless(&self, ident: Ident, span: Span, header: FnHeader) {
         if header.has_qualifiers() {
-            self.err_handler().emit_err(errors::FnQualifierInExtern {
+            self.dcx().emit_err(errors::FnQualifierInExtern {
                 span: ident.span,
                 block: self.current_extern_span(),
                 sugg_span: span.until(ident.span.shrink_to_lo()),
@@ -475,7 +466,7 @@ impl<'a> AstValidator<'a> {
     /// An item in `extern { ... }` cannot use non-ascii identifier.
     fn check_foreign_item_ascii_only(&self, ident: Ident) {
         if !ident.as_str().is_ascii() {
-            self.err_handler().emit_err(errors::ExternItemAscii {
+            self.dcx().emit_err(errors::ExternItemAscii {
                 span: ident.span,
                 block: self.current_extern_span(),
             });
@@ -504,7 +495,7 @@ impl<'a> AstValidator<'a> {
             if let Const::Yes(const_span) = header.constness {
                 let mut spans = variadic_spans.clone();
                 spans.push(const_span);
-                self.err_handler().emit_err(errors::ConstAndCVariadic {
+                self.dcx().emit_err(errors::ConstAndCVariadic {
                     spans,
                     const_span,
                     variadic_spans: variadic_spans.clone(),
@@ -526,14 +517,14 @@ impl<'a> AstValidator<'a> {
             _ => {}
         };
 
-        self.err_handler().emit_err(errors::BadCVariadic { span: variadic_spans });
+        self.dcx().emit_err(errors::BadCVariadic { span: variadic_spans });
     }
 
     fn check_item_named(&self, ident: Ident, kind: &str) {
         if ident.name != kw::Underscore {
             return;
         }
-        self.err_handler().emit_err(errors::ItemUnderscore { span: ident.span, kind });
+        self.dcx().emit_err(errors::ItemUnderscore { span: ident.span, kind });
     }
 
     fn check_nomangle_item_asciionly(&self, ident: Ident, item_span: Span) {
@@ -624,14 +615,14 @@ impl<'a> AstValidator<'a> {
         let args_len = arg_spans.len();
         let constraint_len = constraint_spans.len();
         // ...and then error:
-        self.err_handler().emit_err(errors::ArgsBeforeConstraint {
+        self.dcx().emit_err(errors::ArgsBeforeConstraint {
             arg_spans: arg_spans.clone(),
             constraints: constraint_spans[0],
             args: *arg_spans.iter().last().unwrap(),
             data: data.span,
             constraint_spans: errors::EmptyLabelManySpans(constraint_spans),
             arg_spans2: errors::EmptyLabelManySpans(arg_spans),
-            suggestion: self.correct_generic_order_suggestion(&data),
+            suggestion: self.correct_generic_order_suggestion(data),
             constraint_len,
             args_len,
         });
@@ -676,7 +667,7 @@ impl<'a> AstValidator<'a> {
                 }
 
                 if !bounds.iter().any(|b| matches!(b, GenericBound::Trait(..))) {
-                    self.err_handler().emit_err(errors::AtLeastOneTrait { span: ty.span });
+                    self.dcx().emit_err(errors::AtLeastOneTrait { span: ty.span });
                 }
             }
             _ => {}
@@ -706,7 +697,7 @@ impl<'a> AstValidator<'a> {
 /// Checks that generic parameters are in the correct order,
 /// which is lifetimes, then types and then consts. (`<'a, T, const N: usize>`)
 fn validate_generic_param_order(
-    handler: &rustc_errors::Handler,
+    dcx: &rustc_errors::DiagCtxt,
     generics: &[GenericParam],
     span: Span,
 ) {
@@ -747,7 +738,7 @@ fn validate_generic_param_order(
 
             if !bounds.is_empty() {
                 ordered_params += ": ";
-                ordered_params += &pprust::bounds_to_string(&bounds);
+                ordered_params += &pprust::bounds_to_string(bounds);
             }
 
             match kind {
@@ -769,7 +760,7 @@ fn validate_generic_param_order(
         ordered_params += ">";
 
         for (param_ord, (max_param, spans)) in &out_of_order {
-            handler.emit_err(errors::OutOfOrderParams {
+            dcx.emit_err(errors::OutOfOrderParams {
                 spans: spans.clone(),
                 sugg_span: span,
                 param_ord,
@@ -832,7 +823,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                         errors::VisibilityNotPermittedNote::TraitImpl,
                     );
                     if let TyKind::Err = self_ty.kind {
-                        this.err_handler().emit_err(errors::ObsoleteAuto { span: item.span });
+                        this.dcx().emit_err(errors::ObsoleteAuto { span: item.span });
                     }
                     if let (&Unsafe::Yes(span), &ImplPolarity::Negative(sp)) = (unsafety, polarity)
                     {
@@ -845,11 +836,9 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
 
                     this.visit_vis(&item.vis);
                     this.visit_ident(item.ident);
-                    if let Const::Yes(_) = constness {
-                        this.with_tilde_const_allowed(|this| this.visit_generics(generics));
-                    } else {
-                        this.visit_generics(generics);
-                    }
+                    let disallowed = matches!(constness, Const::No)
+                        .then(|| DisallowTildeConstContext::TraitImpl(item.span));
+                    this.with_tilde_const(disallowed, |this| this.visit_generics(generics));
                     this.visit_trait_ref(t);
                     this.visit_ty(self_ty);
 
@@ -863,10 +852,10 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                 polarity,
                 defaultness,
                 constness,
-                generics: _,
+                generics,
                 of_trait: None,
                 self_ty,
-                items: _,
+                items,
             }) => {
                 let error =
                     |annotation_span, annotation, only_trait: bool| errors::InherentImplCannot {
@@ -882,7 +871,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                     errors::VisibilityNotPermittedNote::IndividualImplItems,
                 );
                 if let &Unsafe::Yes(span) = unsafety {
-                    self.err_handler().emit_err(errors::InherentImplCannotUnsafe {
+                    self.dcx().emit_err(errors::InherentImplCannotUnsafe {
                         span: self_ty.span,
                         annotation_span: span,
                         annotation: "unsafe",
@@ -890,14 +879,24 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                     });
                 }
                 if let &ImplPolarity::Negative(span) = polarity {
-                    self.err_handler().emit_err(error(span, "negative", false));
+                    self.dcx().emit_err(error(span, "negative", false));
                 }
                 if let &Defaultness::Default(def_span) = defaultness {
-                    self.err_handler().emit_err(error(def_span, "`default`", true));
+                    self.dcx().emit_err(error(def_span, "`default`", true));
                 }
                 if let &Const::Yes(span) = constness {
-                    self.err_handler().emit_err(error(span, "`const`", true));
+                    self.dcx().emit_err(error(span, "`const`", true));
                 }
+
+                self.visit_vis(&item.vis);
+                self.visit_ident(item.ident);
+                self.with_tilde_const(Some(DisallowTildeConstContext::Impl(item.span)), |this| {
+                    this.visit_generics(generics)
+                });
+                self.visit_ty(self_ty);
+                walk_list!(self, visit_assoc_item, items, AssocCtxt::Impl);
+                walk_list!(self, visit_attribute, &item.attrs);
+                return; // Avoid visiting again.
             }
             ItemKind::Fn(box Fn { defaultness, sig, generics, body }) => {
                 self.check_defaultness(item.span, *defaultness);
@@ -940,7 +939,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                     errors::VisibilityNotPermittedNote::IndividualForeignItems,
                 );
                 if let &Unsafe::Yes(span) = unsafety {
-                    self.err_handler().emit_err(errors::UnsafeItem { span, kind: "extern block" });
+                    self.dcx().emit_err(errors::UnsafeItem { span, kind: "extern block" });
                 }
                 if abi.is_none() {
                     self.maybe_lint_missing_abi(item.span, item.id);
@@ -978,8 +977,10 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                     // context for the supertraits.
                     this.visit_vis(&item.vis);
                     this.visit_ident(item.ident);
-                    this.visit_generics(generics);
-                    this.with_tilde_const_allowed(|this| {
+                    let disallowed =
+                        (!is_const_trait).then(|| DisallowTildeConstContext::Trait(item.span));
+                    this.with_tilde_const(disallowed, |this| {
+                        this.visit_generics(generics);
                         walk_list!(this, visit_param_bound, bounds, BoundKind::SuperTraits)
                     });
                     walk_list!(this, visit_assoc_item, items, AssocCtxt::Trait);
@@ -989,7 +990,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
             }
             ItemKind::Mod(unsafety, mod_kind) => {
                 if let &Unsafe::Yes(span) = unsafety {
-                    self.err_handler().emit_err(errors::UnsafeItem { span, kind: "module" });
+                    self.dcx().emit_err(errors::UnsafeItem { span, kind: "module" });
                 }
                 // Ensure that `path` attributes on modules are recorded as used (cf. issue #35584).
                 if !matches!(mod_kind, ModKind::Loaded(_, Inline::Yes, _))
@@ -999,16 +1000,11 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                 }
             }
             ItemKind::Struct(vdata, generics) => match vdata {
-                // Duplicating the `Visitor` logic allows catching all cases
-                // of `Anonymous(Struct, Union)` outside of a field struct or union.
-                //
-                // Inside `visit_ty` the validator catches every `Anonymous(Struct, Union)` it
-                // encounters, and only on `ItemKind::Struct` and `ItemKind::Union`
-                // it uses `visit_ty_common`, which doesn't contain that specific check.
-                VariantData::Struct(fields, ..) => {
+                VariantData::Struct { fields, .. } => {
                     self.visit_vis(&item.vis);
                     self.visit_ident(item.ident);
                     self.visit_generics(generics);
+                    // Permit `Anon{Struct,Union}` as field type.
                     walk_list!(self, visit_struct_field_def, fields);
                     walk_list!(self, visit_attribute, &item.attrs);
                     return;
@@ -1017,13 +1013,14 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
             },
             ItemKind::Union(vdata, generics) => {
                 if vdata.fields().is_empty() {
-                    self.err_handler().emit_err(errors::FieldlessUnion { span: item.span });
+                    self.dcx().emit_err(errors::FieldlessUnion { span: item.span });
                 }
                 match vdata {
-                    VariantData::Struct(fields, ..) => {
+                    VariantData::Struct { fields, .. } => {
                         self.visit_vis(&item.vis);
                         self.visit_ident(item.ident);
                         self.visit_generics(generics);
+                        // Permit `Anon{Struct,Union}` as field type.
                         walk_list!(self, visit_struct_field_def, fields);
                         walk_list!(self, visit_attribute, &item.attrs);
                         return;
@@ -1031,12 +1028,14 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                     _ => {}
                 }
             }
-            ItemKind::Const(box ConstItem { defaultness, expr: None, .. }) => {
+            ItemKind::Const(box ConstItem { defaultness, expr, .. }) => {
                 self.check_defaultness(item.span, *defaultness);
-                self.session.emit_err(errors::ConstWithoutBody {
-                    span: item.span,
-                    replace_span: self.ending_semi_or_hi(item.span),
-                });
+                if expr.is_none() {
+                    self.session.emit_err(errors::ConstWithoutBody {
+                        span: item.span,
+                        replace_span: self.ending_semi_or_hi(item.span),
+                    });
+                }
             }
             ItemKind::Static(box StaticItem { expr: None, .. }) => {
                 self.session.emit_err(errors::StaticWithoutBody {
@@ -1058,10 +1057,10 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
 
                 if self.features.lazy_type_alias {
                     if let Err(err) = self.check_type_alias_where_clause_location(ty_alias) {
-                        self.err_handler().emit_err(err);
+                        self.dcx().emit_err(err);
                     }
                 } else if where_clauses.1.0 {
-                    self.err_handler().emit_err(errors::WhereClauseAfterTypeAlias {
+                    self.dcx().emit_err(errors::WhereClauseAfterTypeAlias {
                         span: where_clauses.1.1,
                         help: self.session.is_nightly_build().then_some(()),
                     });
@@ -1146,14 +1145,14 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                 }
                 GenericParamKind::Type { .. } | GenericParamKind::Const { .. } => {
                     if let Some(span) = prev_param_default {
-                        self.err_handler().emit_err(errors::GenericDefaultTrailing { span });
+                        self.dcx().emit_err(errors::GenericDefaultTrailing { span });
                         break;
                     }
                 }
             }
         }
 
-        validate_generic_param_order(self.err_handler(), &generics.params, generics.span);
+        validate_generic_param_order(self.dcx(), &generics.params, generics.span);
 
         for predicate in &generics.where_clause.predicates {
             if let WherePredicate::EqPredicate(predicate) = predicate {
@@ -1174,7 +1173,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                             match bound {
                                 GenericBound::Trait(t, _) => {
                                     if !t.bound_generic_params.is_empty() {
-                                        self.err_handler()
+                                        self.dcx()
                                             .emit_err(errors::NestedLifetimes { span: t.span });
                                     }
                                 }
@@ -1200,39 +1199,50 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
         if let GenericBound::Trait(poly, modify) = bound {
             match (ctxt, modify) {
                 (BoundKind::SuperTraits, TraitBoundModifier::Maybe) => {
-                    self.err_handler().emit_err(errors::OptionalTraitSupertrait {
+                    self.dcx().emit_err(errors::OptionalTraitSupertrait {
                         span: poly.span,
                         path_str: pprust::path_to_string(&poly.trait_ref.path),
                     });
                 }
                 (BoundKind::TraitObject, TraitBoundModifier::Maybe) => {
-                    self.err_handler().emit_err(errors::OptionalTraitObject { span: poly.span });
+                    self.dcx().emit_err(errors::OptionalTraitObject { span: poly.span });
                 }
-                (_, TraitBoundModifier::MaybeConst)
+                (_, &TraitBoundModifier::MaybeConst(span))
                     if let Some(reason) = &self.disallow_tilde_const =>
                 {
                     let reason = match reason {
-                        DisallowTildeConstContext::TraitObject => {
-                            errors::TildeConstReason::TraitObject
-                        }
                         DisallowTildeConstContext::Fn(FnKind::Closure(..)) => {
                             errors::TildeConstReason::Closure
                         }
                         DisallowTildeConstContext::Fn(FnKind::Fn(_, ident, ..)) => {
                             errors::TildeConstReason::Function { ident: ident.span }
                         }
+                        &DisallowTildeConstContext::Trait(span) => {
+                            errors::TildeConstReason::Trait { span }
+                        }
+                        &DisallowTildeConstContext::TraitImpl(span) => {
+                            errors::TildeConstReason::TraitImpl { span }
+                        }
+                        &DisallowTildeConstContext::Impl(span) => {
+                            // FIXME(effects): Consider providing a help message or even a structured
+                            // suggestion for moving such bounds to the assoc const fns if available.
+                            errors::TildeConstReason::Impl { span }
+                        }
+                        DisallowTildeConstContext::TraitObject => {
+                            errors::TildeConstReason::TraitObject
+                        }
+                        DisallowTildeConstContext::Item => errors::TildeConstReason::Item,
                     };
-                    self.err_handler()
-                        .emit_err(errors::TildeConstDisallowed { span: bound.span(), reason });
+                    self.dcx().emit_err(errors::TildeConstDisallowed { span, reason });
                 }
                 (_, TraitBoundModifier::MaybeConstMaybe) => {
-                    self.err_handler().emit_err(errors::OptionalConstExclusive {
+                    self.dcx().emit_err(errors::OptionalConstExclusive {
                         span: bound.span(),
                         modifier: "?",
                     });
                 }
                 (_, TraitBoundModifier::MaybeConstNegative) => {
-                    self.err_handler().emit_err(errors::OptionalConstExclusive {
+                    self.dcx().emit_err(errors::OptionalConstExclusive {
                         span: bound.span(),
                         modifier: "!",
                     });
@@ -1248,7 +1258,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
         {
             for arg in &args.args {
                 if let ast::AngleBracketedArg::Constraint(constraint) = arg {
-                    self.err_handler()
+                    self.dcx()
                         .emit_err(errors::ConstraintOnNegativeBound { span: constraint.span });
                 }
             }
@@ -1267,14 +1277,20 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
 
         self.check_c_variadic_type(fk);
 
-        // Functions cannot both be `const async`
+        // Functions cannot both be `const async` or `const gen`
         if let Some(&FnHeader {
             constness: Const::Yes(cspan),
-            asyncness: Async::Yes { span: aspan, .. },
+            coroutine_kind: Some(coroutine_kind),
             ..
         }) = fk.header()
         {
-            self.err_handler().emit_err(errors::ConstAndAsync {
+            let aspan = match coroutine_kind {
+                CoroutineKind::Async { span: aspan, .. }
+                | CoroutineKind::Gen { span: aspan, .. }
+                | CoroutineKind::AsyncGen { span: aspan, .. } => aspan,
+            };
+            // FIXME(gen_blocks): Report a different error for `const gen`
+            self.dcx().emit_err(errors::ConstAndAsync {
                 spans: vec![cspan, aspan],
                 cspan,
                 aspan,
@@ -1314,10 +1330,8 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                     }
                 } else {
                     match ctxt {
-                        FnCtxt::Foreign => {
-                            self.err_handler().emit_err(errors::PatternInForeign { span })
-                        }
-                        _ => self.err_handler().emit_err(errors::PatternInBodiless { span }),
+                        FnCtxt::Foreign => self.dcx().emit_err(errors::PatternInForeign { span }),
+                        _ => self.dcx().emit_err(errors::PatternInBodiless { span }),
                     };
                 }
             });
@@ -1328,7 +1342,6 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                 || matches!(fk.ctxt(), Some(FnCtxt::Assoc(_)) if self.in_const_trait_or_impl);
 
         let disallowed = (!tilde_const_allowed).then(|| DisallowTildeConstContext::Fn(fk));
-
         self.with_tilde_const(disallowed, |this| visit::walk_fn(this, fk));
     }
 
@@ -1397,18 +1410,6 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
         }
 
         match &item.kind {
-            AssocItemKind::Type(box TyAlias { generics, bounds, ty, .. })
-                if ctxt == AssocCtxt::Trait =>
-            {
-                self.visit_vis(&item.vis);
-                self.visit_ident(item.ident);
-                walk_list!(self, visit_attribute, &item.attrs);
-                self.with_tilde_const_allowed(|this| {
-                    this.visit_generics(generics);
-                    walk_list!(this, visit_param_bound, bounds, BoundKind::Bound);
-                });
-                walk_list!(self, visit_ty, ty);
-            }
             AssocItemKind::Fn(box Fn { sig, generics, body, .. })
                 if self.in_const_trait_or_impl
                     || ctxt == AssocCtxt::Trait
@@ -1461,9 +1462,7 @@ fn deny_equality_constraints(
                     id: rustc_ast::node_id::DUMMY_NODE_ID,
                     ident: *ident,
                     gen_args,
-                    kind: AssocConstraintKind::Equality {
-                        term: predicate.rhs_ty.clone().into(),
-                    },
+                    kind: AssocConstraintKind::Equality { term: predicate.rhs_ty.clone().into() },
                     span: ident.span,
                 });
                 // Add `<Bar = RhsTy>` to `Foo`.
@@ -1476,11 +1475,7 @@ fn deny_equality_constraints(
                     },
                     empty_args => {
                         *empty_args = Some(
-                            AngleBracketedArgs {
-                                span: ident.span,
-                                args: thin_vec![arg],
-                            }
-                            .into(),
+                            AngleBracketedArgs { span: ident.span, args: thin_vec![arg] }.into(),
                         );
                     }
                 }
@@ -1535,7 +1530,7 @@ fn deny_equality_constraints(
             }
         }
     }
-    this.err_handler().emit_err(err);
+    this.dcx().emit_err(err);
 }
 
 pub fn check_crate(
@@ -1552,7 +1547,7 @@ pub fn check_crate(
         in_const_trait_or_impl: false,
         has_proc_macro_decls: false,
         outer_impl_trait: None,
-        disallow_tilde_const: None,
+        disallow_tilde_const: Some(DisallowTildeConstContext::Item),
         is_impl_trait_banned: false,
         lint_buffer: lints,
     };
diff --git a/compiler/rustc_ast_passes/src/errors.rs b/compiler/rustc_ast_passes/src/errors.rs
index d14b62d6b..4283fc7c0 100644
--- a/compiler/rustc_ast_passes/src/errors.rs
+++ b/compiler/rustc_ast_passes/src/errors.rs
@@ -551,8 +551,6 @@ pub struct TildeConstDisallowed {
 
 #[derive(Subdiagnostic)]
 pub enum TildeConstReason {
-    #[note(ast_passes_trait)]
-    TraitObject,
     #[note(ast_passes_closure)]
     Closure,
     #[note(ast_passes_function)]
@@ -560,6 +558,25 @@ pub enum TildeConstReason {
         #[primary_span]
         ident: Span,
     },
+    #[note(ast_passes_trait)]
+    Trait {
+        #[primary_span]
+        span: Span,
+    },
+    #[note(ast_passes_trait_impl)]
+    TraitImpl {
+        #[primary_span]
+        span: Span,
+    },
+    #[note(ast_passes_impl)]
+    Impl {
+        #[primary_span]
+        span: Span,
+    },
+    #[note(ast_passes_object)]
+    TraitObject,
+    #[note(ast_passes_item)]
+    Item,
 }
 
 #[derive(Diagnostic)]
@@ -746,3 +763,12 @@ pub struct AnonStructOrUnionNotAllowed {
     pub span: Span,
     pub struct_or_union: &'static str,
 }
+
+#[derive(Diagnostic)]
+#[diag(ast_passes_match_arm_with_no_body)]
+pub struct MatchArmWithNoBody {
+    #[primary_span]
+    pub span: Span,
+    #[suggestion(code = " => todo!(),", applicability = "has-placeholders")]
+    pub suggestion: Span,
+}
diff --git a/compiler/rustc_ast_passes/src/feature_gate.rs b/compiler/rustc_ast_passes/src/feature_gate.rs
index e1cf0a258..6900d3392 100644
--- a/compiler/rustc_ast_passes/src/feature_gate.rs
+++ b/compiler/rustc_ast_passes/src/feature_gate.rs
@@ -88,7 +88,7 @@ impl<'a> PostExpansionVisitor<'a> {
             }
         }
 
-        match abi::is_enabled(&self.features, span, symbol_unescaped.as_str()) {
+        match abi::is_enabled(self.features, span, symbol_unescaped.as_str()) {
             Ok(()) => (),
             Err(abi::AbiDisabled::Unstable { feature, explain }) => {
                 feature_err_issue(
@@ -102,7 +102,7 @@ impl<'a> PostExpansionVisitor<'a> {
             }
             Err(abi::AbiDisabled::Unrecognized) => {
                 if self.sess.opts.pretty.map_or(true, |ppm| ppm.needs_hir()) {
-                    self.sess.parse_sess.span_diagnostic.delay_span_bug(
+                    self.sess.dcx().span_delayed_bug(
                         span,
                         format!(
                             "unrecognized ABI not caught in lowering: {}",
@@ -182,7 +182,7 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
             ..
         }) = attr_info
         {
-            gate_alt!(self, has_feature(&self.features), *name, attr.span, *descr);
+            gate_alt!(self, has_feature(self.features), *name, attr.span, *descr);
         }
         // Check unstable flavors of the `#[doc]` attribute.
         if attr.has_name(sym::doc) {
@@ -300,7 +300,7 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
             }
 
             ast::ItemKind::TyAlias(box ast::TyAlias { ty: Some(ty), .. }) => {
-                self.check_impl_trait(&ty, false)
+                self.check_impl_trait(ty, false)
             }
 
             _ => {}
@@ -556,6 +556,34 @@ pub fn check_crate(krate: &ast::Crate, sess: &Session, features: &Features) {
     gate_all!(generic_const_items, "generic const items are experimental");
     gate_all!(unnamed_fields, "unnamed fields are not yet fully implemented");
 
+    if !visitor.features.never_patterns {
+        if let Some(spans) = spans.get(&sym::never_patterns) {
+            for &span in spans {
+                if span.allows_unstable(sym::never_patterns) {
+                    continue;
+                }
+                let sm = sess.source_map();
+                // We gate two types of spans: the span of a `!` pattern, and the span of a
+                // match arm without a body. For the latter we want to give the user a normal
+                // error.
+                if let Ok(snippet) = sm.span_to_snippet(span)
+                    && snippet == "!"
+                {
+                    feature_err(
+                        &sess.parse_sess,
+                        sym::never_patterns,
+                        span,
+                        "`!` patterns are experimental",
+                    )
+                    .emit();
+                } else {
+                    let suggestion = span.shrink_to_hi();
+                    sess.emit_err(errors::MatchArmWithNoBody { span, suggestion });
+                }
+            }
+        }
+    }
+
     if !visitor.features.negative_bounds {
         for &span in spans.get(&sym::negative_bounds).iter().copied().flatten() {
             sess.emit_err(errors::NegativeBoundUnsupported { span });
@@ -627,7 +655,7 @@ fn maybe_stage_features(sess: &Session, features: &Features, krate: &ast::Crate)
             if all_stable {
                 err.sugg = Some(attr.span);
             }
-            sess.parse_sess.span_diagnostic.emit_err(err);
+            sess.dcx().emit_err(err);
         }
     }
 }
diff --git a/compiler/rustc_ast_passes/src/lib.rs b/compiler/rustc_ast_passes/src/lib.rs
index 5147e672f..ba0918337 100644
--- a/compiler/rustc_ast_passes/src/lib.rs
+++ b/compiler/rustc_ast_passes/src/lib.rs
@@ -4,9 +4,9 @@
 //!
 //! The crate also contains other misc AST visitors, e.g. `node_count` and `show_span`.
 
-#![cfg_attr(not(bootstrap), allow(internal_features))]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
+#![allow(internal_features)]
+#![doc(rust_logo)]
+#![feature(rustdoc_internals)]
 #![feature(box_patterns)]
 #![feature(if_let_guard)]
 #![feature(iter_is_partitioned)]
@@ -15,13 +15,10 @@
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
 
-use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
-use rustc_fluent_macro::fluent_messages;
-
 pub mod ast_validation;
 mod errors;
 pub mod feature_gate;
 pub mod node_count;
 pub mod show_span;
 
-fluent_messages! { "../messages.ftl" }
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
diff --git a/compiler/rustc_ast_passes/src/show_span.rs b/compiler/rustc_ast_passes/src/show_span.rs
index 280cf3284..9882f1d23 100644
--- a/compiler/rustc_ast_passes/src/show_span.rs
+++ b/compiler/rustc_ast_passes/src/show_span.rs
@@ -31,37 +31,37 @@ impl FromStr for Mode {
 }
 
 struct ShowSpanVisitor<'a> {
-    span_diagnostic: &'a rustc_errors::Handler,
+    dcx: &'a rustc_errors::DiagCtxt,
     mode: Mode,
 }
 
 impl<'a> Visitor<'a> for ShowSpanVisitor<'a> {
     fn visit_expr(&mut self, e: &'a ast::Expr) {
         if let Mode::Expression = self.mode {
-            self.span_diagnostic.emit_warning(errors::ShowSpan { span: e.span, msg: "expression" });
+            self.dcx.emit_warning(errors::ShowSpan { span: e.span, msg: "expression" });
         }
         visit::walk_expr(self, e);
     }
 
     fn visit_pat(&mut self, p: &'a ast::Pat) {
         if let Mode::Pattern = self.mode {
-            self.span_diagnostic.emit_warning(errors::ShowSpan { span: p.span, msg: "pattern" });
+            self.dcx.emit_warning(errors::ShowSpan { span: p.span, msg: "pattern" });
         }
         visit::walk_pat(self, p);
     }
 
     fn visit_ty(&mut self, t: &'a ast::Ty) {
         if let Mode::Type = self.mode {
-            self.span_diagnostic.emit_warning(errors::ShowSpan { span: t.span, msg: "type" });
+            self.dcx.emit_warning(errors::ShowSpan { span: t.span, msg: "type" });
         }
         visit::walk_ty(self, t);
     }
 }
 
-pub fn run(span_diagnostic: &rustc_errors::Handler, mode: &str, krate: &ast::Crate) {
+pub fn run(dcx: &rustc_errors::DiagCtxt, mode: &str, krate: &ast::Crate) {
     let Ok(mode) = mode.parse() else {
         return;
     };
-    let mut v = ShowSpanVisitor { span_diagnostic, mode };
+    let mut v = ShowSpanVisitor { dcx, mode };
     visit::walk_crate(&mut v, krate);
 }
diff --git a/compiler/rustc_ast_pretty/Cargo.toml b/compiler/rustc_ast_pretty/Cargo.toml
index af1524c8b..12a08f065 100644
--- a/compiler/rustc_ast_pretty/Cargo.toml
+++ b/compiler/rustc_ast_pretty/Cargo.toml
@@ -5,6 +5,7 @@ edition = "2021"
 
 [dependencies]
 # tidy-alphabetical-start
+itertools = "0.11"
 rustc_ast = { path = "../rustc_ast" }
 rustc_span = { path = "../rustc_span" }
 thin-vec = "0.2.12"
diff --git a/compiler/rustc_ast_pretty/src/lib.rs b/compiler/rustc_ast_pretty/src/lib.rs
index 475bdb023..670f2a458 100644
--- a/compiler/rustc_ast_pretty/src/lib.rs
+++ b/compiler/rustc_ast_pretty/src/lib.rs
@@ -1,11 +1,9 @@
-#![cfg_attr(not(bootstrap), allow(internal_features))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
+#![allow(internal_features)]
+#![feature(rustdoc_internals)]
+#![doc(rust_logo)]
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
-#![feature(associated_type_bounds)]
 #![feature(box_patterns)]
-#![feature(with_negative_coherence)]
 #![recursion_limit = "256"]
 
 mod helpers;
diff --git a/compiler/rustc_ast_pretty/src/pp.rs b/compiler/rustc_ast_pretty/src/pp.rs
index 7ab8c3eab..96f5eff68 100644
--- a/compiler/rustc_ast_pretty/src/pp.rs
+++ b/compiler/rustc_ast_pretty/src/pp.rs
@@ -165,20 +165,20 @@ enum IndentStyle {
 }
 
 #[derive(Clone, Copy, Default, PartialEq)]
-pub struct BreakToken {
+pub(crate) struct BreakToken {
     offset: isize,
     blank_space: isize,
     pre_break: Option<char>,
 }
 
 #[derive(Clone, Copy, PartialEq)]
-pub struct BeginToken {
+pub(crate) struct BeginToken {
     indent: IndentStyle,
     breaks: Breaks,
 }
 
-#[derive(Clone, PartialEq)]
-pub enum Token {
+#[derive(PartialEq)]
+pub(crate) enum Token {
     // In practice a string token contains either a `&'static str` or a
     // `String`. `Cow` is overkill for this because we never modify the data,
     // but it's more convenient than rolling our own more specialized type.
@@ -229,7 +229,6 @@ pub struct Printer {
     last_printed: Option<Token>,
 }
 
-#[derive(Clone)]
 struct BufEntry {
     token: Token,
     size: isize,
@@ -251,16 +250,16 @@ impl Printer {
         }
     }
 
-    pub fn last_token(&self) -> Option<&Token> {
+    pub(crate) fn last_token(&self) -> Option<&Token> {
         self.last_token_still_buffered().or_else(|| self.last_printed.as_ref())
     }
 
-    pub fn last_token_still_buffered(&self) -> Option<&Token> {
+    pub(crate) fn last_token_still_buffered(&self) -> Option<&Token> {
         self.buf.last().map(|last| &last.token)
     }
 
     /// Be very careful with this!
-    pub fn replace_last_token_still_buffered(&mut self, token: Token) {
+    pub(crate) fn replace_last_token_still_buffered(&mut self, token: Token) {
         self.buf.last_mut().unwrap().token = token;
     }
 
@@ -314,7 +313,7 @@ impl Printer {
         }
     }
 
-    pub fn offset(&mut self, offset: isize) {
+    pub(crate) fn offset(&mut self, offset: isize) {
         if let Some(BufEntry { token: Token::Break(token), .. }) = &mut self.buf.last_mut() {
             token.offset += offset;
         }
diff --git a/compiler/rustc_ast_pretty/src/pp/convenience.rs b/compiler/rustc_ast_pretty/src/pp/convenience.rs
index 93310dd45..c4c4fdce7 100644
--- a/compiler/rustc_ast_pretty/src/pp/convenience.rs
+++ b/compiler/rustc_ast_pretty/src/pp/convenience.rs
@@ -66,7 +66,7 @@ impl Printer {
         }
     }
 
-    pub fn hardbreak_tok_offset(off: isize) -> Token {
+    pub(crate) fn hardbreak_tok_offset(off: isize) -> Token {
         Token::Break(BreakToken {
             offset: off,
             blank_space: SIZE_INFINITY,
diff --git a/compiler/rustc_ast_pretty/src/pprust/state.rs b/compiler/rustc_ast_pretty/src/pprust/state.rs
index 48421ff71..d6c15ec35 100644
--- a/compiler/rustc_ast_pretty/src/pprust/state.rs
+++ b/compiler/rustc_ast_pretty/src/pprust/state.rs
@@ -1,13 +1,17 @@
-mod delimited;
+//! AST pretty printing.
+//!
+//! Note that HIR pretty printing is layered on top of this crate.
+
 mod expr;
 mod item;
 
 use crate::pp::Breaks::{Consistent, Inconsistent};
 use crate::pp::{self, Breaks};
+use crate::pprust::state::expr::FixupContext;
 use rustc_ast::attr::AttrIdGenerator;
 use rustc_ast::ptr::P;
 use rustc_ast::token::{self, BinOpToken, CommentKind, Delimiter, Nonterminal, Token, TokenKind};
-use rustc_ast::tokenstream::{TokenStream, TokenTree};
+use rustc_ast::tokenstream::{Spacing, TokenStream, TokenTree};
 use rustc_ast::util::classify;
 use rustc_ast::util::comments::{gather_comments, Comment, CommentStyle};
 use rustc_ast::util::parser;
@@ -23,8 +27,6 @@ use rustc_span::{BytePos, FileName, Span, DUMMY_SP};
 use std::borrow::Cow;
 use thin_vec::ThinVec;
 
-pub use self::delimited::IterDelimited;
-
 pub enum MacHeader<'a> {
     Path(&'a ast::Path),
     Keyword(&'static str),
@@ -46,8 +48,7 @@ pub trait PpAnn {
     fn post(&self, _state: &mut State<'_>, _node: AnnNode<'_>) {}
 }
 
-#[derive(Copy, Clone)]
-pub struct NoAnn;
+struct NoAnn;
 
 impl PpAnn for NoAnn {}
 
@@ -64,11 +65,11 @@ impl<'a> Comments<'a> {
     }
 
     // FIXME: This shouldn't probably clone lmao
-    pub fn next(&self) -> Option<Comment> {
+    fn next(&self) -> Option<Comment> {
         self.comments.get(self.current).cloned()
     }
 
-    pub fn trailing_comment(
+    fn trailing_comment(
         &self,
         span: rustc_span::Span,
         next_pos: Option<BytePos>,
@@ -95,7 +96,7 @@ pub struct State<'a> {
     ann: &'a (dyn PpAnn + 'a),
 }
 
-pub(crate) const INDENT_UNIT: isize = 4;
+const INDENT_UNIT: isize = 4;
 
 /// Requires you to pass an input filename and reader so that
 /// it can scan the input text for comments to copy forward.
@@ -151,7 +152,7 @@ pub fn print_crate<'a>(
 /// Note: some old proc macros parse pretty-printed output, so changes here can
 /// break old code. For example:
 /// - #63896: `#[allow(unused,` must be printed rather than `#[allow(unused ,`
-/// - #73345: `#[allow(unused)] must be printed rather than `# [allow(unused)]
+/// - #73345: `#[allow(unused)]` must be printed rather than `# [allow(unused)]`
 ///
 fn space_between(tt1: &TokenTree, tt2: &TokenTree) -> bool {
     use token::*;
@@ -183,10 +184,10 @@ fn space_between(tt1: &TokenTree, tt2: &TokenTree) -> bool {
         //
         // FIXME: Incorrect cases:
         // - Let: `let(a, b) = (1, 2)`
-        (Tok(Token { kind: Ident(..), .. }, _), Del(_, Parenthesis, _)) => false,
+        (Tok(Token { kind: Ident(..), .. }, _), Del(_, _, Parenthesis, _)) => false,
 
         // `#` + `[`: `#[attr]`
-        (Tok(Token { kind: Pound, .. }, _), Del(_, Bracket, _)) => false,
+        (Tok(Token { kind: Pound, .. }, _), Del(_, _, Bracket, _)) => false,
 
         _ => true,
     }
@@ -220,7 +221,7 @@ fn doc_comment_to_string(
     }
 }
 
-pub fn literal_to_string(lit: token::Lit) -> String {
+fn literal_to_string(lit: token::Lit) -> String {
     let token::Lit { kind, symbol, suffix } = lit;
     let mut out = match kind {
         token::Byte => format!("b'{symbol}'"),
@@ -260,11 +261,17 @@ impl std::ops::DerefMut for State<'_> {
     }
 }
 
+/// This trait is used for both AST and HIR pretty-printing.
 pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::DerefMut {
     fn comments(&mut self) -> &mut Option<Comments<'a>>;
-    fn print_ident(&mut self, ident: Ident);
+    fn ann_post(&mut self, ident: Ident);
     fn print_generic_args(&mut self, args: &ast::GenericArgs, colons_before_params: bool);
 
+    fn print_ident(&mut self, ident: Ident) {
+        self.word(IdentPrinter::for_ast_ident(ident, ident.is_raw_guess()).to_string());
+        self.ann_post(ident)
+    }
+
     fn strsep<T, F>(
         &mut self,
         sep: &'static str,
@@ -401,15 +408,6 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
         }
     }
 
-    fn print_meta_item_lit(&mut self, lit: &ast::MetaItemLit) {
-        self.print_token_literal(lit.as_token_lit(), lit.span)
-    }
-
-    fn print_token_literal(&mut self, token_lit: token::Lit, span: Span) {
-        self.maybe_print_comment(span.lo());
-        self.word(token_lit.to_string())
-    }
-
     fn print_string(&mut self, st: &str, style: ast::StrStyle) {
         let st = match style {
             ast::StrStyle::Cooked => format!("\"{}\"", st.escape_debug()),
@@ -420,30 +418,14 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
         self.word(st)
     }
 
-    fn print_symbol(&mut self, sym: Symbol, style: ast::StrStyle) {
-        self.print_string(sym.as_str(), style);
-    }
-
     fn print_inner_attributes(&mut self, attrs: &[ast::Attribute]) -> bool {
         self.print_either_attributes(attrs, ast::AttrStyle::Inner, false, true)
     }
 
-    fn print_inner_attributes_no_trailing_hardbreak(&mut self, attrs: &[ast::Attribute]) -> bool {
-        self.print_either_attributes(attrs, ast::AttrStyle::Inner, false, false)
-    }
-
     fn print_outer_attributes(&mut self, attrs: &[ast::Attribute]) -> bool {
         self.print_either_attributes(attrs, ast::AttrStyle::Outer, false, true)
     }
 
-    fn print_inner_attributes_inline(&mut self, attrs: &[ast::Attribute]) -> bool {
-        self.print_either_attributes(attrs, ast::AttrStyle::Inner, true, true)
-    }
-
-    fn print_outer_attributes_inline(&mut self, attrs: &[ast::Attribute]) -> bool {
-        self.print_either_attributes(attrs, ast::AttrStyle::Outer, true, true)
-    }
-
     fn print_either_attributes(
         &mut self,
         attrs: &[ast::Attribute],
@@ -467,10 +449,6 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
         printed
     }
 
-    fn print_attribute(&mut self, attr: &ast::Attribute) {
-        self.print_attribute_inline(attr, false)
-    }
-
     fn print_attribute_inline(&mut self, attr: &ast::Attribute, is_inline: bool) {
         if !is_inline {
             self.hardbreak_if_not_bol();
@@ -525,33 +503,6 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
         self.end();
     }
 
-    fn print_meta_list_item(&mut self, item: &ast::NestedMetaItem) {
-        match item {
-            ast::NestedMetaItem::MetaItem(mi) => self.print_meta_item(mi),
-            ast::NestedMetaItem::Lit(lit) => self.print_meta_item_lit(lit),
-        }
-    }
-
-    fn print_meta_item(&mut self, item: &ast::MetaItem) {
-        self.ibox(INDENT_UNIT);
-        match &item.kind {
-            ast::MetaItemKind::Word => self.print_path(&item.path, false, 0),
-            ast::MetaItemKind::NameValue(value) => {
-                self.print_path(&item.path, false, 0);
-                self.space();
-                self.word_space("=");
-                self.print_meta_item_lit(value);
-            }
-            ast::MetaItemKind::List(items) => {
-                self.print_path(&item.path, false, 0);
-                self.popen();
-                self.commasep(Consistent, items, |s, i| s.print_meta_list_item(i));
-                self.pclose();
-            }
-        }
-        self.end();
-    }
-
     /// This doesn't deserve to be called "pretty" printing, but it should be
     /// meaning-preserving. A quick hack that might help would be to look at the
     /// spans embedded in the TTs to decide where to put spaces and newlines.
@@ -559,16 +510,17 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
     /// appropriate macro, transcribe back into the grammar we just parsed from,
     /// and then pretty-print the resulting AST nodes (so, e.g., we print
     /// expression arguments as expressions). It can be done! I think.
-    fn print_tt(&mut self, tt: &TokenTree, convert_dollar_crate: bool) {
+    fn print_tt(&mut self, tt: &TokenTree, convert_dollar_crate: bool) -> Spacing {
         match tt {
-            TokenTree::Token(token, _) => {
+            TokenTree::Token(token, spacing) => {
                 let token_str = self.token_to_string_ext(token, convert_dollar_crate);
                 self.word(token_str);
                 if let token::DocComment(..) = token.kind {
                     self.hardbreak()
                 }
+                *spacing
             }
-            TokenTree::Delimited(dspan, delim, tts) => {
+            TokenTree::Delimited(dspan, spacing, delim, tts) => {
                 self.print_mac_common(
                     None,
                     false,
@@ -578,6 +530,7 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
                     convert_dollar_crate,
                     dspan.entire(),
                 );
+                spacing.close
             }
         }
     }
@@ -585,9 +538,20 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
     fn print_tts(&mut self, tts: &TokenStream, convert_dollar_crate: bool) {
         let mut iter = tts.trees().peekable();
         while let Some(tt) = iter.next() {
-            self.print_tt(tt, convert_dollar_crate);
+            let spacing = self.print_tt(tt, convert_dollar_crate);
             if let Some(next) = iter.peek() {
-                if space_between(tt, next) {
+                // Should we print a space after `tt`? There are two guiding
+                // factors.
+                // - `spacing` is the more important and accurate one. Most
+                //   tokens have good spacing information, and
+                //   `Joint`/`JointHidden` get used a lot.
+                // - `space_between` is the backup. Code produced by proc
+                //   macros has worse spacing information, with no
+                //   `JointHidden` usage and too much `Alone` usage, which
+                //   would result in over-spaced output such as
+                //   `( x () , y . z )`. `space_between` avoids some of the
+                //   excess whitespace.
+                if spacing == Spacing::Alone && space_between(tt, next) {
                     self.space();
                 }
             }
@@ -825,7 +789,7 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
             }
             token::Eof => "<eof>".into(),
 
-            token::Interpolated(ref nt) => self.nonterminal_to_string(nt).into(),
+            token::Interpolated(ref nt) => self.nonterminal_to_string(&nt.0).into(),
         }
     }
 
@@ -843,37 +807,18 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
         Self::to_string(|s| s.print_type(ty))
     }
 
-    fn bounds_to_string(&self, bounds: &[ast::GenericBound]) -> String {
-        Self::to_string(|s| s.print_type_bounds(bounds))
-    }
-
-    fn where_bound_predicate_to_string(
-        &self,
-        where_bound_predicate: &ast::WhereBoundPredicate,
-    ) -> String {
-        Self::to_string(|s| s.print_where_bound_predicate(where_bound_predicate))
-    }
-
     fn pat_to_string(&self, pat: &ast::Pat) -> String {
         Self::to_string(|s| s.print_pat(pat))
     }
 
     fn expr_to_string(&self, e: &ast::Expr) -> String {
-        Self::to_string(|s| s.print_expr(e))
+        Self::to_string(|s| s.print_expr(e, FixupContext::default()))
     }
 
     fn meta_item_lit_to_string(&self, lit: &ast::MetaItemLit) -> String {
         Self::to_string(|s| s.print_meta_item_lit(lit))
     }
 
-    fn tt_to_string(&self, tt: &TokenTree) -> String {
-        Self::to_string(|s| s.print_tt(tt, false))
-    }
-
-    fn tts_to_string(&self, tokens: &TokenStream) -> String {
-        Self::to_string(|s| s.print_tts(tokens, false))
-    }
-
     fn stmt_to_string(&self, stmt: &ast::Stmt) -> String {
         Self::to_string(|s| s.print_stmt(stmt))
     }
@@ -882,26 +827,10 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
         Self::to_string(|s| s.print_item(i))
     }
 
-    fn assoc_item_to_string(&self, i: &ast::AssocItem) -> String {
-        Self::to_string(|s| s.print_assoc_item(i))
-    }
-
-    fn foreign_item_to_string(&self, i: &ast::ForeignItem) -> String {
-        Self::to_string(|s| s.print_foreign_item(i))
-    }
-
-    fn generic_params_to_string(&self, generic_params: &[ast::GenericParam]) -> String {
-        Self::to_string(|s| s.print_generic_params(generic_params))
-    }
-
     fn path_to_string(&self, p: &ast::Path) -> String {
         Self::to_string(|s| s.print_path(p, false, 0))
     }
 
-    fn path_segment_to_string(&self, p: &ast::PathSegment) -> String {
-        Self::to_string(|s| s.print_path_segment(p, false))
-    }
-
     fn vis_to_string(&self, v: &ast::Visibility) -> String {
         Self::to_string(|s| s.print_visibility(v))
     }
@@ -916,22 +845,10 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
         })
     }
 
-    fn meta_list_item_to_string(&self, li: &ast::NestedMetaItem) -> String {
-        Self::to_string(|s| s.print_meta_list_item(li))
-    }
-
     fn attr_item_to_string(&self, ai: &ast::AttrItem) -> String {
         Self::to_string(|s| s.print_attr_item(ai, ai.path.span))
     }
 
-    fn attribute_to_string(&self, attr: &ast::Attribute) -> String {
-        Self::to_string(|s| s.print_attribute(attr))
-    }
-
-    fn param_to_string(&self, arg: &ast::Param) -> String {
-        Self::to_string(|s| s.print_param(arg, false))
-    }
-
     fn to_string(f: impl FnOnce(&mut State<'_>)) -> String {
         let mut printer = State::new();
         f(&mut printer);
@@ -944,9 +861,8 @@ impl<'a> PrintState<'a> for State<'a> {
         &mut self.comments
     }
 
-    fn print_ident(&mut self, ident: Ident) {
-        self.word(IdentPrinter::for_ast_ident(ident, ident.is_raw_guess()).to_string());
-        self.ann.post(self, AnnNode::Ident(&ident))
+    fn ann_post(&mut self, ident: Ident) {
+        self.ann.post(self, AnnNode::Ident(&ident));
     }
 
     fn print_generic_args(&mut self, args: &ast::GenericArgs, colons_before_params: bool) {
@@ -979,13 +895,8 @@ impl<'a> State<'a> {
         State { s: pp::Printer::new(), comments: None, ann: &NoAnn }
     }
 
-    pub(crate) fn commasep_cmnt<T, F, G>(
-        &mut self,
-        b: Breaks,
-        elts: &[T],
-        mut op: F,
-        mut get_span: G,
-    ) where
+    fn commasep_cmnt<T, F, G>(&mut self, b: Breaks, elts: &[T], mut op: F, mut get_span: G)
+    where
         F: FnMut(&mut State<'_>, &T),
         G: FnMut(&T) -> rustc_span::Span,
     {
@@ -1005,8 +916,8 @@ impl<'a> State<'a> {
         self.end();
     }
 
-    pub(crate) fn commasep_exprs(&mut self, b: Breaks, exprs: &[P<ast::Expr>]) {
-        self.commasep_cmnt(b, exprs, |s, e| s.print_expr(e), |e| e.span)
+    fn commasep_exprs(&mut self, b: Breaks, exprs: &[P<ast::Expr>]) {
+        self.commasep_cmnt(b, exprs, |s, e| s.print_expr(e, FixupContext::default()), |e| e.span)
     }
 
     pub fn print_opt_lifetime(&mut self, lifetime: &Option<ast::Lifetime>) {
@@ -1043,7 +954,7 @@ impl<'a> State<'a> {
         match generic_arg {
             GenericArg::Lifetime(lt) => self.print_lifetime(*lt),
             GenericArg::Type(ty) => self.print_type(ty),
-            GenericArg::Const(ct) => self.print_expr(&ct.value),
+            GenericArg::Const(ct) => self.print_expr(&ct.value, FixupContext::default()),
         }
     }
 
@@ -1078,11 +989,11 @@ impl<'a> State<'a> {
             }
             ast::TyKind::AnonStruct(fields) => {
                 self.head("struct");
-                self.print_record_struct_body(&fields, ty.span);
+                self.print_record_struct_body(fields, ty.span);
             }
             ast::TyKind::AnonUnion(fields) => {
                 self.head("union");
-                self.print_record_struct_body(&fields, ty.span);
+                self.print_record_struct_body(fields, ty.span);
             }
             ast::TyKind::Paren(typ) => {
                 self.popen();
@@ -1110,12 +1021,12 @@ impl<'a> State<'a> {
                 self.word("[");
                 self.print_type(ty);
                 self.word("; ");
-                self.print_expr(&length.value);
+                self.print_expr(&length.value, FixupContext::default());
                 self.word("]");
             }
             ast::TyKind::Typeof(e) => {
                 self.word("typeof(");
-                self.print_expr(&e.value);
+                self.print_expr(&e.value, FixupContext::default());
                 self.word(")");
             }
             ast::TyKind::Infer => {
@@ -1156,7 +1067,7 @@ impl<'a> State<'a> {
         self.print_trait_ref(&t.trait_ref)
     }
 
-    pub(crate) fn print_stmt(&mut self, st: &ast::Stmt) {
+    fn print_stmt(&mut self, st: &ast::Stmt) {
         self.maybe_print_comment(st.span.lo());
         match &st.kind {
             ast::StmtKind::Local(loc) => {
@@ -1171,7 +1082,7 @@ impl<'a> State<'a> {
                 if let Some((init, els)) = loc.kind.init_else_opt() {
                     self.nbsp();
                     self.word_space("=");
-                    self.print_expr(init);
+                    self.print_expr(init, FixupContext::default());
                     if let Some(els) = els {
                         self.cbox(INDENT_UNIT);
                         self.ibox(INDENT_UNIT);
@@ -1185,14 +1096,14 @@ impl<'a> State<'a> {
             ast::StmtKind::Item(item) => self.print_item(item),
             ast::StmtKind::Expr(expr) => {
                 self.space_if_not_bol();
-                self.print_expr_outer_attr_style(expr, false);
+                self.print_expr_outer_attr_style(expr, false, FixupContext::default());
                 if classify::expr_requires_semi_to_be_stmt(expr) {
                     self.word(";");
                 }
             }
             ast::StmtKind::Semi(expr) => {
                 self.space_if_not_bol();
-                self.print_expr_outer_attr_style(expr, false);
+                self.print_expr_outer_attr_style(expr, false, FixupContext::default());
                 self.word(";");
             }
             ast::StmtKind::Empty => {
@@ -1211,19 +1122,19 @@ impl<'a> State<'a> {
         self.maybe_print_trailing_comment(st.span, None)
     }
 
-    pub(crate) fn print_block(&mut self, blk: &ast::Block) {
+    fn print_block(&mut self, blk: &ast::Block) {
         self.print_block_with_attrs(blk, &[])
     }
 
-    pub(crate) fn print_block_unclosed_indent(&mut self, blk: &ast::Block) {
+    fn print_block_unclosed_indent(&mut self, blk: &ast::Block) {
         self.print_block_maybe_unclosed(blk, &[], false)
     }
 
-    pub(crate) fn print_block_with_attrs(&mut self, blk: &ast::Block, attrs: &[ast::Attribute]) {
+    fn print_block_with_attrs(&mut self, blk: &ast::Block, attrs: &[ast::Attribute]) {
         self.print_block_maybe_unclosed(blk, attrs, true)
     }
 
-    pub(crate) fn print_block_maybe_unclosed(
+    fn print_block_maybe_unclosed(
         &mut self,
         blk: &ast::Block,
         attrs: &[ast::Attribute],
@@ -1244,7 +1155,7 @@ impl<'a> State<'a> {
                 ast::StmtKind::Expr(expr) if i == blk.stmts.len() - 1 => {
                     self.maybe_print_comment(st.span.lo());
                     self.space_if_not_bol();
-                    self.print_expr_outer_attr_style(expr, false);
+                    self.print_expr_outer_attr_style(expr, false, FixupContext::default());
                     self.maybe_print_trailing_comment(expr.span, Some(blk.span.hi()));
                 }
                 _ => self.print_stmt(st),
@@ -1257,16 +1168,44 @@ impl<'a> State<'a> {
     }
 
     /// Print a `let pat = expr` expression.
-    pub(crate) fn print_let(&mut self, pat: &ast::Pat, expr: &ast::Expr) {
+    ///
+    /// Parentheses are inserted surrounding `expr` if a round-trip through the
+    /// parser would otherwise work out the wrong way in a condition position.
+    ///
+    /// For example each of the following would mean the wrong thing without
+    /// parentheses.
+    ///
+    /// ```ignore (illustrative)
+    /// if let _ = (Struct {}) {}
+    ///
+    /// if let _ = (true && false) {}
+    /// ```
+    ///
+    /// In a match guard, the second case still requires parens, but the first
+    /// case no longer does because anything until `=>` is considered part of
+    /// the match guard expression. Parsing of the expression is not terminated
+    /// by `{` in that position.
+    ///
+    /// ```ignore (illustrative)
+    /// match () {
+    ///     () if let _ = Struct {} => {}
+    ///     () if let _ = (true && false) => {}
+    /// }
+    /// ```
+    fn print_let(&mut self, pat: &ast::Pat, expr: &ast::Expr, fixup: FixupContext) {
         self.word("let ");
         self.print_pat(pat);
         self.space();
         self.word_space("=");
-        let npals = || parser::needs_par_as_let_scrutinee(expr.precedence().order());
-        self.print_expr_cond_paren(expr, Self::cond_needs_par(expr) || npals())
+        self.print_expr_cond_paren(
+            expr,
+            fixup.parenthesize_exterior_struct_lit && parser::contains_exterior_struct_lit(expr)
+                || parser::needs_par_as_let_scrutinee(expr.precedence().order()),
+            FixupContext::default(),
+        );
     }
 
-    pub(crate) fn print_mac(&mut self, m: &ast::MacCall) {
+    fn print_mac(&mut self, m: &ast::MacCall) {
         self.print_mac_common(
             Some(MacHeader::Path(&m.path)),
             true,
@@ -1310,7 +1249,7 @@ impl<'a> State<'a> {
                         print_reg_or_class(s, reg);
                         s.pclose();
                         s.space();
-                        s.print_expr(expr);
+                        s.print_expr(expr, FixupContext::default());
                     }
                     InlineAsmOperand::Out { reg, late, expr } => {
                         s.word(if *late { "lateout" } else { "out" });
@@ -1319,7 +1258,7 @@ impl<'a> State<'a> {
                         s.pclose();
                         s.space();
                         match expr {
-                            Some(expr) => s.print_expr(expr),
+                            Some(expr) => s.print_expr(expr, FixupContext::default()),
                             None => s.word("_"),
                         }
                     }
@@ -1329,7 +1268,7 @@ impl<'a> State<'a> {
                         print_reg_or_class(s, reg);
                         s.pclose();
                         s.space();
-                        s.print_expr(expr);
+                        s.print_expr(expr, FixupContext::default());
                     }
                     InlineAsmOperand::SplitInOut { reg, late, in_expr, out_expr } => {
                         s.word(if *late { "inlateout" } else { "inout" });
@@ -1337,18 +1276,18 @@ impl<'a> State<'a> {
                         print_reg_or_class(s, reg);
                         s.pclose();
                         s.space();
-                        s.print_expr(in_expr);
+                        s.print_expr(in_expr, FixupContext::default());
                         s.space();
                         s.word_space("=>");
                         match out_expr {
-                            Some(out_expr) => s.print_expr(out_expr),
+                            Some(out_expr) => s.print_expr(out_expr, FixupContext::default()),
                             None => s.word("_"),
                         }
                     }
                     InlineAsmOperand::Const { anon_const } => {
                         s.word("const");
                         s.space();
-                        s.print_expr(&anon_const.value);
+                        s.print_expr(&anon_const.value, FixupContext::default());
                     }
                     InlineAsmOperand::Sym { sym } => {
                         s.word("sym");
@@ -1407,7 +1346,7 @@ impl<'a> State<'a> {
         self.pclose();
     }
 
-    pub(crate) fn print_local_decl(&mut self, loc: &ast::Local) {
+    fn print_local_decl(&mut self, loc: &ast::Local) {
         self.print_pat(&loc.pat);
         if let Some(ty) = &loc.ty {
             self.word_space(":");
@@ -1415,7 +1354,7 @@ impl<'a> State<'a> {
         }
     }
 
-    pub(crate) fn print_name(&mut self, name: Symbol) {
+    fn print_name(&mut self, name: Symbol) {
         self.word(name.to_string());
         self.ann.post(self, AnnNode::Name(&name))
     }
@@ -1439,13 +1378,14 @@ impl<'a> State<'a> {
         }
     }
 
-    pub(crate) fn print_pat(&mut self, pat: &ast::Pat) {
+    fn print_pat(&mut self, pat: &ast::Pat) {
         self.maybe_print_comment(pat.span.lo());
         self.ann.pre(self, AnnNode::Pat(pat));
         /* Pat isn't normalized, but the beauty of it
         is that it doesn't matter */
         match &pat.kind {
             PatKind::Wild => self.word("_"),
+            PatKind::Never => self.word("!"),
             PatKind::Ident(BindingAnnotation(by_ref, mutbl), ident, sub) => {
                 if *by_ref == ByRef::Yes {
                     self.word_nbsp("ref");
@@ -1541,10 +1481,10 @@ impl<'a> State<'a> {
                     self.print_pat(inner);
                 }
             }
-            PatKind::Lit(e) => self.print_expr(e),
+            PatKind::Lit(e) => self.print_expr(e, FixupContext::default()),
             PatKind::Range(begin, end, Spanned { node: end_kind, .. }) => {
                 if let Some(e) = begin {
-                    self.print_expr(e);
+                    self.print_expr(e, FixupContext::default());
                 }
                 match end_kind {
                     RangeEnd::Included(RangeSyntax::DotDotDot) => self.word("..."),
@@ -1552,7 +1492,7 @@ impl<'a> State<'a> {
                     RangeEnd::Excluded => self.word(".."),
                 }
                 if let Some(e) = end {
-                    self.print_expr(e);
+                    self.print_expr(e, FixupContext::default());
                 }
             }
             PatKind::Slice(elts) => {
@@ -1592,9 +1532,18 @@ impl<'a> State<'a> {
         }
     }
 
-    pub(crate) fn print_asyncness(&mut self, asyncness: ast::Async) {
-        if asyncness.is_async() {
-            self.word_nbsp("async");
+    fn print_coroutine_kind(&mut self, coroutine_kind: ast::CoroutineKind) {
+        match coroutine_kind {
+            ast::CoroutineKind::Gen { .. } => {
+                self.word_nbsp("gen");
+            }
+            ast::CoroutineKind::Async { .. } => {
+                self.word_nbsp("async");
+            }
+            ast::CoroutineKind::AsyncGen { .. } => {
+                self.word_nbsp("async");
+                self.word_nbsp("gen");
+            }
         }
     }
 
@@ -1618,7 +1567,7 @@ impl<'a> State<'a> {
                         TraitBoundModifier::Maybe => {
                             self.word("?");
                         }
-                        TraitBoundModifier::MaybeConst => {
+                        TraitBoundModifier::MaybeConst(_) => {
                             self.word_space("~const");
                         }
                         TraitBoundModifier::MaybeConstNegative => {
@@ -1637,23 +1586,25 @@ impl<'a> State<'a> {
         }
     }
 
-    pub(crate) fn print_lifetime(&mut self, lifetime: ast::Lifetime) {
+    fn print_lifetime(&mut self, lifetime: ast::Lifetime) {
         self.print_name(lifetime.ident.name)
     }
 
-    pub(crate) fn print_lifetime_bounds(&mut self, bounds: &ast::GenericBounds) {
+    fn print_lifetime_bounds(&mut self, bounds: &ast::GenericBounds) {
         for (i, bound) in bounds.iter().enumerate() {
             if i != 0 {
                 self.word(" + ");
             }
             match bound {
                 ast::GenericBound::Outlives(lt) => self.print_lifetime(*lt),
-                _ => panic!(),
+                _ => {
+                    panic!("expected a lifetime bound, found a trait bound")
+                }
             }
         }
     }
 
-    pub(crate) fn print_generic_params(&mut self, generic_params: &[ast::GenericParam]) {
+    fn print_generic_params(&mut self, generic_params: &[ast::GenericParam]) {
         if generic_params.is_empty() {
             return;
         }
@@ -1697,7 +1648,7 @@ impl<'a> State<'a> {
                     if let Some(default) = default {
                         s.space();
                         s.word_space("=");
-                        s.print_expr(&default.value);
+                        s.print_expr(&default.value, FixupContext::default());
                     }
                 }
             }
@@ -1717,12 +1668,12 @@ impl<'a> State<'a> {
         }
     }
 
-    pub(crate) fn print_mt(&mut self, mt: &ast::MutTy, print_const: bool) {
+    fn print_mt(&mut self, mt: &ast::MutTy, print_const: bool) {
         self.print_mutability(mt.mutbl, print_const);
         self.print_type(&mt.ty)
     }
 
-    pub(crate) fn print_param(&mut self, input: &ast::Param, is_closure: bool) {
+    fn print_param(&mut self, input: &ast::Param, is_closure: bool) {
         self.ibox(INDENT_UNIT);
 
         self.print_outer_attributes_inline(&input.attrs);
@@ -1750,7 +1701,7 @@ impl<'a> State<'a> {
         self.end();
     }
 
-    pub(crate) fn print_fn_ret_ty(&mut self, fn_ret_ty: &ast::FnRetTy) {
+    fn print_fn_ret_ty(&mut self, fn_ret_ty: &ast::FnRetTy) {
         if let ast::FnRetTy::Ty(ty) = fn_ret_ty {
             self.space_if_not_bol();
             self.ibox(INDENT_UNIT);
@@ -1761,7 +1712,7 @@ impl<'a> State<'a> {
         }
     }
 
-    pub(crate) fn print_ty_fn(
+    fn print_ty_fn(
         &mut self,
         ext: ast::Extern,
         unsafety: ast::Unsafe,
@@ -1785,9 +1736,9 @@ impl<'a> State<'a> {
         self.end();
     }
 
-    pub(crate) fn print_fn_header_info(&mut self, header: ast::FnHeader) {
+    fn print_fn_header_info(&mut self, header: ast::FnHeader) {
         self.print_constness(header.constness);
-        self.print_asyncness(header.asyncness);
+        header.coroutine_kind.map(|coroutine_kind| self.print_coroutine_kind(coroutine_kind));
         self.print_unsafety(header.unsafety);
 
         match header.ext {
@@ -1805,24 +1756,109 @@ impl<'a> State<'a> {
         self.word("fn")
     }
 
-    pub(crate) fn print_unsafety(&mut self, s: ast::Unsafe) {
+    fn print_unsafety(&mut self, s: ast::Unsafe) {
         match s {
             ast::Unsafe::No => {}
             ast::Unsafe::Yes(_) => self.word_nbsp("unsafe"),
         }
     }
 
-    pub(crate) fn print_constness(&mut self, s: ast::Const) {
+    fn print_constness(&mut self, s: ast::Const) {
         match s {
             ast::Const::No => {}
             ast::Const::Yes(_) => self.word_nbsp("const"),
         }
     }
 
-    pub(crate) fn print_is_auto(&mut self, s: ast::IsAuto) {
+    fn print_is_auto(&mut self, s: ast::IsAuto) {
         match s {
             ast::IsAuto::Yes => self.word_nbsp("auto"),
             ast::IsAuto::No => {}
         }
     }
+
+    fn print_meta_item_lit(&mut self, lit: &ast::MetaItemLit) {
+        self.print_token_literal(lit.as_token_lit(), lit.span)
+    }
+
+    fn print_token_literal(&mut self, token_lit: token::Lit, span: Span) {
+        self.maybe_print_comment(span.lo());
+        self.word(token_lit.to_string())
+    }
+
+    fn print_symbol(&mut self, sym: Symbol, style: ast::StrStyle) {
+        self.print_string(sym.as_str(), style);
+    }
+
+    fn print_inner_attributes_no_trailing_hardbreak(&mut self, attrs: &[ast::Attribute]) -> bool {
+        self.print_either_attributes(attrs, ast::AttrStyle::Inner, false, false)
+    }
+
+    fn print_outer_attributes_inline(&mut self, attrs: &[ast::Attribute]) -> bool {
+        self.print_either_attributes(attrs, ast::AttrStyle::Outer, true, true)
+    }
+
+    fn print_attribute(&mut self, attr: &ast::Attribute) {
+        self.print_attribute_inline(attr, false)
+    }
+
+    fn print_meta_list_item(&mut self, item: &ast::NestedMetaItem) {
+        match item {
+            ast::NestedMetaItem::MetaItem(mi) => self.print_meta_item(mi),
+            ast::NestedMetaItem::Lit(lit) => self.print_meta_item_lit(lit),
+        }
+    }
+
+    fn print_meta_item(&mut self, item: &ast::MetaItem) {
+        self.ibox(INDENT_UNIT);
+        match &item.kind {
+            ast::MetaItemKind::Word => self.print_path(&item.path, false, 0),
+            ast::MetaItemKind::NameValue(value) => {
+                self.print_path(&item.path, false, 0);
+                self.space();
+                self.word_space("=");
+                self.print_meta_item_lit(value);
+            }
+            ast::MetaItemKind::List(items) => {
+                self.print_path(&item.path, false, 0);
+                self.popen();
+                self.commasep(Consistent, items, |s, i| s.print_meta_list_item(i));
+                self.pclose();
+            }
+        }
+        self.end();
+    }
+
+    pub(crate) fn bounds_to_string(&self, bounds: &[ast::GenericBound]) -> String {
+        Self::to_string(|s| s.print_type_bounds(bounds))
+    }
+
+    pub(crate) fn where_bound_predicate_to_string(
+        &self,
+        where_bound_predicate: &ast::WhereBoundPredicate,
+    ) -> String {
+        Self::to_string(|s| s.print_where_bound_predicate(where_bound_predicate))
+    }
+
+    pub(crate) fn tt_to_string(&self, tt: &TokenTree) -> String {
+        Self::to_string(|s| {
+            s.print_tt(tt, false);
+        })
+    }
+
+    pub(crate) fn tts_to_string(&self, tokens: &TokenStream) -> String {
+        Self::to_string(|s| s.print_tts(tokens, false))
+    }
+
+    pub(crate) fn path_segment_to_string(&self, p: &ast::PathSegment) -> String {
+        Self::to_string(|s| s.print_path_segment(p, false))
+    }
+
+    pub(crate) fn meta_list_item_to_string(&self, li: &ast::NestedMetaItem) -> String {
+        Self::to_string(|s| s.print_meta_list_item(li))
+    }
+
+    pub(crate) fn attribute_to_string(&self, attr: &ast::Attribute) -> String {
+        Self::to_string(|s| s.print_attribute(attr))
+    }
 }
diff --git a/compiler/rustc_ast_pretty/src/pprust/state/delimited.rs b/compiler/rustc_ast_pretty/src/pprust/state/delimited.rs
deleted file mode 100644
index fe0640baa..000000000
--- a/compiler/rustc_ast_pretty/src/pprust/state/delimited.rs
+++ /dev/null
@@ -1,41 +0,0 @@
-use std::iter::Peekable;
-use std::mem;
-use std::ops::Deref;
-
-pub struct Delimited<I: Iterator> {
-    is_first: bool,
-    iter: Peekable<I>,
-}
-
-pub trait IterDelimited: Iterator + Sized {
-    fn delimited(self) -> Delimited<Self> {
-        Delimited { is_first: true, iter: self.peekable() }
-    }
-}
-
-impl<I: Iterator> IterDelimited for I {}
-
-pub struct IteratorItem<T> {
-    value: T,
-    pub is_first: bool,
-    pub is_last: bool,
-}
-
-impl<I: Iterator> Iterator for Delimited<I> {
-    type Item = IteratorItem<I::Item>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        let value = self.iter.next()?;
-        let is_first = mem::replace(&mut self.is_first, false);
-        let is_last = self.iter.peek().is_none();
-        Some(IteratorItem { value, is_first, is_last })
-    }
-}
-
-impl<T> Deref for IteratorItem<T> {
-    type Target = T;
-
-    fn deref(&self) -> &Self::Target {
-        &self.value
-    }
-}
diff --git a/compiler/rustc_ast_pretty/src/pprust/state/expr.rs b/compiler/rustc_ast_pretty/src/pprust/state/expr.rs
index edbc35003..f5ffcddb8 100644
--- a/compiler/rustc_ast_pretty/src/pprust/state/expr.rs
+++ b/compiler/rustc_ast_pretty/src/pprust/state/expr.rs
@@ -1,6 +1,6 @@
 use crate::pp::Breaks::Inconsistent;
-use crate::pprust::state::{AnnNode, IterDelimited, PrintState, State, INDENT_UNIT};
-
+use crate::pprust::state::{AnnNode, PrintState, State, INDENT_UNIT};
+use itertools::{Itertools, Position};
 use rustc_ast::ptr::P;
 use rustc_ast::token;
 use rustc_ast::util::literal::escape_byte_str_symbol;
@@ -12,6 +12,19 @@ use rustc_ast::{
 };
 use std::fmt::Write;
 
+#[derive(Copy, Clone, Debug)]
+pub(crate) struct FixupContext {
+    pub parenthesize_exterior_struct_lit: bool,
+}
+
+/// The default amount of fixing is minimal fixing. Fixups should be turned on
+/// in a targetted fashion where needed.
+impl Default for FixupContext {
+    fn default() -> Self {
+        FixupContext { parenthesize_exterior_struct_lit: false }
+    }
+}
+
 impl<'a> State<'a> {
     fn print_else(&mut self, els: Option<&ast::Expr>) {
         if let Some(_else) = els {
@@ -55,21 +68,22 @@ impl<'a> State<'a> {
         self.pclose()
     }
 
-    fn print_expr_maybe_paren(&mut self, expr: &ast::Expr, prec: i8) {
-        self.print_expr_cond_paren(expr, expr.precedence().order() < prec)
+    fn print_expr_maybe_paren(&mut self, expr: &ast::Expr, prec: i8, fixup: FixupContext) {
+        self.print_expr_cond_paren(expr, expr.precedence().order() < prec, fixup);
     }
 
     /// Prints an expr using syntax that's acceptable in a condition position, such as the `cond` in
     /// `if cond { ... }`.
     fn print_expr_as_cond(&mut self, expr: &ast::Expr) {
-        self.print_expr_cond_paren(expr, Self::cond_needs_par(expr))
+        let fixup = FixupContext { parenthesize_exterior_struct_lit: true };
+        self.print_expr_cond_paren(expr, Self::cond_needs_par(expr), fixup)
     }
 
     /// Does `expr` need parentheses when printed in a condition position?
     ///
     /// These cases need parens due to the parse error observed in #26461: `if return {}`
     /// parses as the erroneous construct `if (return {})`, not `if (return) {}`.
-    pub(super) fn cond_needs_par(expr: &ast::Expr) -> bool {
+    fn cond_needs_par(expr: &ast::Expr) -> bool {
         match expr.kind {
             ast::ExprKind::Break(..)
             | ast::ExprKind::Closure(..)
@@ -80,11 +94,32 @@ impl<'a> State<'a> {
     }
 
     /// Prints `expr` or `(expr)` when `needs_par` holds.
-    pub(super) fn print_expr_cond_paren(&mut self, expr: &ast::Expr, needs_par: bool) {
+    pub(super) fn print_expr_cond_paren(
+        &mut self,
+        expr: &ast::Expr,
+        needs_par: bool,
+        fixup: FixupContext,
+    ) {
         if needs_par {
             self.popen();
         }
-        self.print_expr(expr);
+
+        // If we are surrounding the whole cond in parentheses, such as:
+        //
+        //     if (return Struct {}) {}
+        //
+        // then there is no need for parenthesizing the individual struct
+        // expressions within. On the other hand if the whole cond is not
+        // parenthesized, then print_expr must parenthesize exterior struct
+        // literals.
+        //
+        //     if x == (Struct {}) {}
+        //
+        let fixup = FixupContext {
+            parenthesize_exterior_struct_lit: fixup.parenthesize_exterior_struct_lit && !needs_par,
+        };
+        self.print_expr(expr, fixup);
+
         if needs_par {
             self.pclose();
         }
@@ -111,7 +146,7 @@ impl<'a> State<'a> {
             self.ibox(0);
             self.print_block_with_attrs(block, attrs);
         } else {
-            self.print_expr(&expr.value);
+            self.print_expr(&expr.value, FixupContext::default());
         }
         self.end();
     }
@@ -119,9 +154,9 @@ impl<'a> State<'a> {
     fn print_expr_repeat(&mut self, element: &ast::Expr, count: &ast::AnonConst) {
         self.ibox(INDENT_UNIT);
         self.word("[");
-        self.print_expr(element);
+        self.print_expr(element, FixupContext::default());
         self.word_space(";");
-        self.print_expr(&count.value);
+        self.print_expr(&count.value, FixupContext::default());
         self.word("]");
         self.end();
     }
@@ -149,18 +184,20 @@ impl<'a> State<'a> {
             return;
         }
         self.cbox(0);
-        for field in fields.iter().delimited() {
+        for (pos, field) in fields.iter().with_position() {
+            let is_first = matches!(pos, Position::First | Position::Only);
+            let is_last = matches!(pos, Position::Last | Position::Only);
             self.maybe_print_comment(field.span.hi());
             self.print_outer_attributes(&field.attrs);
-            if field.is_first {
+            if is_first {
                 self.space_if_not_bol();
             }
             if !field.is_shorthand {
                 self.print_ident(field.ident);
                 self.word_nbsp(":");
             }
-            self.print_expr(&field.expr);
-            if !field.is_last || has_rest {
+            self.print_expr(&field.expr, FixupContext::default());
+            if !is_last || has_rest {
                 self.word_space(",");
             } else {
                 self.trailing_comma_or_space();
@@ -172,7 +209,7 @@ impl<'a> State<'a> {
             }
             self.word("..");
             if let ast::StructRest::Base(expr) = rest {
-                self.print_expr(expr);
+                self.print_expr(expr, FixupContext::default());
             }
             self.space();
         }
@@ -190,13 +227,13 @@ impl<'a> State<'a> {
         self.pclose()
     }
 
-    fn print_expr_call(&mut self, func: &ast::Expr, args: &[P<ast::Expr>]) {
+    fn print_expr_call(&mut self, func: &ast::Expr, args: &[P<ast::Expr>], fixup: FixupContext) {
         let prec = match func.kind {
             ast::ExprKind::Field(..) => parser::PREC_FORCE_PAREN,
             _ => parser::PREC_POSTFIX,
         };
 
-        self.print_expr_maybe_paren(func, prec);
+        self.print_expr_maybe_paren(func, prec, fixup);
         self.print_call_post(args)
     }
 
@@ -205,8 +242,9 @@ impl<'a> State<'a> {
         segment: &ast::PathSegment,
         receiver: &ast::Expr,
         base_args: &[P<ast::Expr>],
+        fixup: FixupContext,
     ) {
-        self.print_expr_maybe_paren(receiver, parser::PREC_POSTFIX);
+        self.print_expr_maybe_paren(receiver, parser::PREC_POSTFIX, fixup);
         self.word(".");
         self.print_ident(segment.ident);
         if let Some(args) = &segment.args {
@@ -215,7 +253,13 @@ impl<'a> State<'a> {
         self.print_call_post(base_args)
     }
 
-    fn print_expr_binary(&mut self, op: ast::BinOp, lhs: &ast::Expr, rhs: &ast::Expr) {
+    fn print_expr_binary(
+        &mut self,
+        op: ast::BinOp,
+        lhs: &ast::Expr,
+        rhs: &ast::Expr,
+        fixup: FixupContext,
+    ) {
         let assoc_op = AssocOp::from_ast_binop(op.node);
         let prec = assoc_op.precedence() as i8;
         let fixity = assoc_op.fixity();
@@ -251,15 +295,15 @@ impl<'a> State<'a> {
             _ => left_prec,
         };
 
-        self.print_expr_maybe_paren(lhs, left_prec);
+        self.print_expr_maybe_paren(lhs, left_prec, fixup);
         self.space();
-        self.word_space(op.node.to_string());
-        self.print_expr_maybe_paren(rhs, right_prec)
+        self.word_space(op.node.as_str());
+        self.print_expr_maybe_paren(rhs, right_prec, fixup)
     }
 
-    fn print_expr_unary(&mut self, op: ast::UnOp, expr: &ast::Expr) {
-        self.word(ast::UnOp::to_string(op));
-        self.print_expr_maybe_paren(expr, parser::PREC_PREFIX)
+    fn print_expr_unary(&mut self, op: ast::UnOp, expr: &ast::Expr, fixup: FixupContext) {
+        self.word(op.as_str());
+        self.print_expr_maybe_paren(expr, parser::PREC_PREFIX, fixup)
     }
 
     fn print_expr_addr_of(
@@ -267,6 +311,7 @@ impl<'a> State<'a> {
         kind: ast::BorrowKind,
         mutability: ast::Mutability,
         expr: &ast::Expr,
+        fixup: FixupContext,
     ) {
         self.word("&");
         match kind {
@@ -276,14 +321,19 @@ impl<'a> State<'a> {
                 self.print_mutability(mutability, true);
             }
         }
-        self.print_expr_maybe_paren(expr, parser::PREC_PREFIX)
+        self.print_expr_maybe_paren(expr, parser::PREC_PREFIX, fixup)
     }
 
-    pub fn print_expr(&mut self, expr: &ast::Expr) {
-        self.print_expr_outer_attr_style(expr, true)
+    pub(super) fn print_expr(&mut self, expr: &ast::Expr, fixup: FixupContext) {
+        self.print_expr_outer_attr_style(expr, true, fixup)
     }
 
-    pub(super) fn print_expr_outer_attr_style(&mut self, expr: &ast::Expr, is_inline: bool) {
+    pub(super) fn print_expr_outer_attr_style(
+        &mut self,
+        expr: &ast::Expr,
+        is_inline: bool,
+        fixup: FixupContext,
+    ) {
         self.maybe_print_comment(expr.span.lo());
 
         let attrs = &expr.attrs;
@@ -312,19 +362,19 @@ impl<'a> State<'a> {
                 self.print_expr_tup(exprs);
             }
             ast::ExprKind::Call(func, args) => {
-                self.print_expr_call(func, args);
+                self.print_expr_call(func, args, fixup);
             }
             ast::ExprKind::MethodCall(box ast::MethodCall { seg, receiver, args, .. }) => {
-                self.print_expr_method_call(seg, receiver, args);
+                self.print_expr_method_call(seg, receiver, args, fixup);
             }
             ast::ExprKind::Binary(op, lhs, rhs) => {
-                self.print_expr_binary(*op, lhs, rhs);
+                self.print_expr_binary(*op, lhs, rhs, fixup);
             }
             ast::ExprKind::Unary(op, expr) => {
-                self.print_expr_unary(*op, expr);
+                self.print_expr_unary(*op, expr, fixup);
             }
             ast::ExprKind::AddrOf(k, m, expr) => {
-                self.print_expr_addr_of(*k, *m, expr);
+                self.print_expr_addr_of(*k, *m, expr, fixup);
             }
             ast::ExprKind::Lit(token_lit) => {
                 self.print_token_literal(*token_lit, expr.span);
@@ -335,7 +385,7 @@ impl<'a> State<'a> {
             }
             ast::ExprKind::Cast(expr, ty) => {
                 let prec = AssocOp::As.precedence() as i8;
-                self.print_expr_maybe_paren(expr, prec);
+                self.print_expr_maybe_paren(expr, prec, fixup);
                 self.space();
                 self.word_space("as");
                 self.print_type(ty);
@@ -343,7 +393,7 @@ impl<'a> State<'a> {
             ast::ExprKind::Type(expr, ty) => {
                 self.word("type_ascribe!(");
                 self.ibox(0);
-                self.print_expr(expr);
+                self.print_expr(expr, FixupContext::default());
 
                 self.word(",");
                 self.space_if_not_bol();
@@ -353,7 +403,7 @@ impl<'a> State<'a> {
                 self.word(")");
             }
             ast::ExprKind::Let(pat, scrutinee, _, _) => {
-                self.print_let(pat, scrutinee);
+                self.print_let(pat, scrutinee, fixup);
             }
             ast::ExprKind::If(test, blk, elseopt) => self.print_if(test, blk, elseopt.as_deref()),
             ast::ExprKind::While(test, blk, opt_label) => {
@@ -411,7 +461,7 @@ impl<'a> State<'a> {
                 binder,
                 capture_clause,
                 constness,
-                asyncness,
+                coroutine_kind,
                 movability,
                 fn_decl,
                 body,
@@ -421,12 +471,12 @@ impl<'a> State<'a> {
                 self.print_closure_binder(binder);
                 self.print_constness(*constness);
                 self.print_movability(*movability);
-                self.print_asyncness(*asyncness);
+                coroutine_kind.map(|coroutine_kind| self.print_coroutine_kind(coroutine_kind));
                 self.print_capture_clause(*capture_clause);
 
                 self.print_fn_params_and_ret(fn_decl, true);
                 self.space();
-                self.print_expr(body);
+                self.print_expr(body, FixupContext::default());
                 self.end(); // need to close a box
 
                 // a box will be closed by print_expr, but we didn't want an overall
@@ -454,33 +504,33 @@ impl<'a> State<'a> {
                 self.print_block_with_attrs(blk, attrs);
             }
             ast::ExprKind::Await(expr, _) => {
-                self.print_expr_maybe_paren(expr, parser::PREC_POSTFIX);
+                self.print_expr_maybe_paren(expr, parser::PREC_POSTFIX, fixup);
                 self.word(".await");
             }
             ast::ExprKind::Assign(lhs, rhs, _) => {
                 let prec = AssocOp::Assign.precedence() as i8;
-                self.print_expr_maybe_paren(lhs, prec + 1);
+                self.print_expr_maybe_paren(lhs, prec + 1, fixup);
                 self.space();
                 self.word_space("=");
-                self.print_expr_maybe_paren(rhs, prec);
+                self.print_expr_maybe_paren(rhs, prec, fixup);
             }
             ast::ExprKind::AssignOp(op, lhs, rhs) => {
                 let prec = AssocOp::Assign.precedence() as i8;
-                self.print_expr_maybe_paren(lhs, prec + 1);
+                self.print_expr_maybe_paren(lhs, prec + 1, fixup);
                 self.space();
-                self.word(op.node.to_string());
+                self.word(op.node.as_str());
                 self.word_space("=");
-                self.print_expr_maybe_paren(rhs, prec);
+                self.print_expr_maybe_paren(rhs, prec, fixup);
             }
             ast::ExprKind::Field(expr, ident) => {
-                self.print_expr_maybe_paren(expr, parser::PREC_POSTFIX);
+                self.print_expr_maybe_paren(expr, parser::PREC_POSTFIX, fixup);
                 self.word(".");
                 self.print_ident(*ident);
             }
             ast::ExprKind::Index(expr, index, _) => {
-                self.print_expr_maybe_paren(expr, parser::PREC_POSTFIX);
+                self.print_expr_maybe_paren(expr, parser::PREC_POSTFIX, fixup);
                 self.word("[");
-                self.print_expr(index);
+                self.print_expr(index, FixupContext::default());
                 self.word("]");
             }
             ast::ExprKind::Range(start, end, limits) => {
@@ -490,14 +540,14 @@ impl<'a> State<'a> {
                 // a "normal" binop gets parenthesized. (`LOr` is the lowest-precedence binop.)
                 let fake_prec = AssocOp::LOr.precedence() as i8;
                 if let Some(e) = start {
-                    self.print_expr_maybe_paren(e, fake_prec);
+                    self.print_expr_maybe_paren(e, fake_prec, fixup);
                 }
                 match limits {
                     ast::RangeLimits::HalfOpen => self.word(".."),
                     ast::RangeLimits::Closed => self.word("..="),
                 }
                 if let Some(e) = end {
-                    self.print_expr_maybe_paren(e, fake_prec);
+                    self.print_expr_maybe_paren(e, fake_prec, fixup);
                 }
             }
             ast::ExprKind::Underscore => self.word("_"),
@@ -511,7 +561,7 @@ impl<'a> State<'a> {
                 }
                 if let Some(expr) = opt_expr {
                     self.space();
-                    self.print_expr_maybe_paren(expr, parser::PREC_JUMP);
+                    self.print_expr_maybe_paren(expr, parser::PREC_JUMP, fixup);
                 }
             }
             ast::ExprKind::Continue(opt_label) => {
@@ -525,7 +575,7 @@ impl<'a> State<'a> {
                 self.word("return");
                 if let Some(expr) = result {
                     self.word(" ");
-                    self.print_expr_maybe_paren(expr, parser::PREC_JUMP);
+                    self.print_expr_maybe_paren(expr, parser::PREC_JUMP, fixup);
                 }
             }
             ast::ExprKind::Yeet(result) => {
@@ -534,13 +584,13 @@ impl<'a> State<'a> {
                 self.word("yeet");
                 if let Some(expr) = result {
                     self.word(" ");
-                    self.print_expr_maybe_paren(expr, parser::PREC_JUMP);
+                    self.print_expr_maybe_paren(expr, parser::PREC_JUMP, fixup);
                 }
             }
             ast::ExprKind::Become(result) => {
                 self.word("become");
                 self.word(" ");
-                self.print_expr_maybe_paren(result, parser::PREC_JUMP);
+                self.print_expr_maybe_paren(result, parser::PREC_JUMP, fixup);
             }
             ast::ExprKind::InlineAsm(a) => {
                 // FIXME: This should have its own syntax, distinct from a macro invocation.
@@ -555,7 +605,7 @@ impl<'a> State<'a> {
                 self.word(reconstruct_format_args_template_string(&fmt.template));
                 for arg in fmt.arguments.all_args() {
                     self.word_space(",");
-                    self.print_expr(&arg.expr);
+                    self.print_expr(&arg.expr, FixupContext::default());
                 }
                 self.end();
                 self.pclose();
@@ -582,7 +632,7 @@ impl<'a> State<'a> {
             ast::ExprKind::MacCall(m) => self.print_mac(m),
             ast::ExprKind::Paren(e) => {
                 self.popen();
-                self.print_expr(e);
+                self.print_expr(e, FixupContext::default());
                 self.pclose();
             }
             ast::ExprKind::Yield(e) => {
@@ -590,11 +640,11 @@ impl<'a> State<'a> {
 
                 if let Some(expr) = e {
                     self.space();
-                    self.print_expr_maybe_paren(expr, parser::PREC_JUMP);
+                    self.print_expr_maybe_paren(expr, parser::PREC_JUMP, fixup);
                 }
             }
             ast::ExprKind::Try(e) => {
-                self.print_expr_maybe_paren(e, parser::PREC_POSTFIX);
+                self.print_expr_maybe_paren(e, parser::PREC_POSTFIX, fixup);
                 self.word("?")
             }
             ast::ExprKind::TryBlock(blk) => {
@@ -626,31 +676,36 @@ impl<'a> State<'a> {
         self.space();
         if let Some(e) = &arm.guard {
             self.word_space("if");
-            self.print_expr(e);
+            self.print_expr(e, FixupContext::default());
             self.space();
         }
-        self.word_space("=>");
 
-        match &arm.body.kind {
-            ast::ExprKind::Block(blk, opt_label) => {
-                if let Some(label) = opt_label {
-                    self.print_ident(label.ident);
-                    self.word_space(":");
-                }
+        if let Some(body) = &arm.body {
+            self.word_space("=>");
+
+            match &body.kind {
+                ast::ExprKind::Block(blk, opt_label) => {
+                    if let Some(label) = opt_label {
+                        self.print_ident(label.ident);
+                        self.word_space(":");
+                    }
 
-                // The block will close the pattern's ibox.
-                self.print_block_unclosed_indent(blk);
+                    // The block will close the pattern's ibox.
+                    self.print_block_unclosed_indent(blk);
 
-                // If it is a user-provided unsafe block, print a comma after it.
-                if let BlockCheckMode::Unsafe(ast::UserProvided) = blk.rules {
+                    // If it is a user-provided unsafe block, print a comma after it.
+                    if let BlockCheckMode::Unsafe(ast::UserProvided) = blk.rules {
+                        self.word(",");
+                    }
+                }
+                _ => {
+                    self.end(); // Close the ibox for the pattern.
+                    self.print_expr(body, FixupContext::default());
                     self.word(",");
                 }
             }
-            _ => {
-                self.end(); // Close the ibox for the pattern.
-                self.print_expr(&arm.body);
-                self.word(",");
-            }
+        } else {
+            self.word(",");
         }
         self.end(); // Close enclosing cbox.
     }
@@ -679,7 +734,7 @@ impl<'a> State<'a> {
     }
 }
 
-pub fn reconstruct_format_args_template_string(pieces: &[FormatArgsPiece]) -> String {
+fn reconstruct_format_args_template_string(pieces: &[FormatArgsPiece]) -> String {
     let mut template = "\"".to_string();
     for piece in pieces {
         match piece {
diff --git a/compiler/rustc_ast_pretty/src/pprust/state/item.rs b/compiler/rustc_ast_pretty/src/pprust/state/item.rs
index 3393f034b..405ccc722 100644
--- a/compiler/rustc_ast_pretty/src/pprust/state/item.rs
+++ b/compiler/rustc_ast_pretty/src/pprust/state/item.rs
@@ -1,8 +1,9 @@
 use crate::pp::Breaks::Inconsistent;
-use crate::pprust::state::delimited::IterDelimited;
+use crate::pprust::state::expr::FixupContext;
 use crate::pprust::state::{AnnNode, PrintState, State, INDENT_UNIT};
 
 use ast::StaticItem;
+use itertools::{Itertools, Position};
 use rustc_ast as ast;
 use rustc_ast::GenericBound;
 use rustc_ast::ModKind;
@@ -20,7 +21,7 @@ impl<'a> State<'a> {
         }
     }
 
-    pub(crate) fn print_foreign_item(&mut self, item: &ast::ForeignItem) {
+    fn print_foreign_item(&mut self, item: &ast::ForeignItem) {
         let ast::Item { id, span, ident, ref attrs, ref kind, ref vis, tokens: _ } = *item;
         self.ann.pre(self, AnnNode::SubItem(id));
         self.hardbreak_if_not_bol();
@@ -97,7 +98,7 @@ impl<'a> State<'a> {
         self.end(); // end the head-ibox
         if let Some(body) = body {
             self.word_space("=");
-            self.print_expr(body);
+            self.print_expr(body, FixupContext::default());
         }
         self.print_where_clause(&generics.where_clause);
         self.word(";");
@@ -368,7 +369,7 @@ impl<'a> State<'a> {
                 self.nbsp();
                 if !bounds.is_empty() {
                     self.word_nbsp("=");
-                    self.print_type_bounds(&bounds);
+                    self.print_type_bounds(bounds);
                 }
                 self.print_where_clause(&generics.where_clause);
                 self.word(";");
@@ -499,7 +500,7 @@ impl<'a> State<'a> {
                 self.end();
                 self.end(); // Close the outer-box.
             }
-            ast::VariantData::Struct(fields, ..) => {
+            ast::VariantData::Struct { fields, .. } => {
                 self.print_where_clause(&generics.where_clause);
                 self.print_record_struct_body(fields, span);
             }
@@ -514,11 +515,11 @@ impl<'a> State<'a> {
         if let Some(d) = &v.disr_expr {
             self.space();
             self.word_space("=");
-            self.print_expr(&d.value)
+            self.print_expr(&d.value, FixupContext::default())
         }
     }
 
-    pub(crate) fn print_assoc_item(&mut self, item: &ast::AssocItem) {
+    fn print_assoc_item(&mut self, item: &ast::AssocItem) {
         let ast::Item { id, span, ident, ref attrs, ref kind, ref vis, tokens: _ } = *item;
         self.ann.pre(self, AnnNode::SubItem(id));
         self.hardbreak_if_not_bol();
@@ -621,7 +622,7 @@ impl<'a> State<'a> {
         self.print_where_clause_parts(where_clause.has_where_token, &where_clause.predicates);
     }
 
-    pub(crate) fn print_where_clause_parts(
+    fn print_where_clause_parts(
         &mut self,
         has_where_token: bool,
         predicates: &[ast::WherePredicate],
@@ -668,7 +669,7 @@ impl<'a> State<'a> {
         }
     }
 
-    pub fn print_where_bound_predicate(
+    pub(crate) fn print_where_bound_predicate(
         &mut self,
         where_bound_predicate: &ast::WhereBoundPredicate,
     ) {
@@ -712,9 +713,10 @@ impl<'a> State<'a> {
                     self.word("{");
                     self.zerobreak();
                     self.ibox(0);
-                    for use_tree in items.iter().delimited() {
+                    for (pos, use_tree) in items.iter().with_position() {
+                        let is_last = matches!(pos, Position::Last | Position::Only);
                         self.print_use_tree(&use_tree.0);
-                        if !use_tree.is_last {
+                        if !is_last {
                             self.word(",");
                             if let ast::UseTreeKind::Nested(_) = use_tree.0.kind {
                                 self.hardbreak();
diff --git a/compiler/rustc_attr/src/builtin.rs b/compiler/rustc_attr/src/builtin.rs
index ad92d5855..0959e8d30 100644
--- a/compiler/rustc_attr/src/builtin.rs
+++ b/compiler/rustc_attr/src/builtin.rs
@@ -552,7 +552,7 @@ pub fn cfg_matches(
 fn try_gate_cfg(name: Symbol, span: Span, sess: &ParseSess, features: Option<&Features>) {
     let gate = find_gated_cfg(|sym| sym == name);
     if let (Some(feats), Some(gated_cfg)) = (features, gate) {
-        gate_cfg(&gated_cfg, span, sess, feats);
+        gate_cfg(gated_cfg, span, sess, feats);
     }
 }
 
@@ -945,7 +945,7 @@ pub fn parse_repr_attr(sess: &Session, attr: &Attribute) -> Vec<ReprAttr> {
     assert!(attr.has_name(sym::repr), "expected `#[repr(..)]`, found: {attr:?}");
     use ReprAttr::*;
     let mut acc = Vec::new();
-    let diagnostic = &sess.parse_sess.span_diagnostic;
+    let diagnostic = sess.dcx();
 
     if let Some(items) = attr.meta_item_list() {
         for item in items {
@@ -985,7 +985,7 @@ pub fn parse_repr_attr(sess: &Session, attr: &Attribute) -> Vec<ReprAttr> {
                         Ok(literal) => acc.push(ReprPacked(literal)),
                         Err(message) => literal_error = Some(message),
                     };
-                } else if matches!(name, sym::C | sym::simd | sym::transparent)
+                } else if matches!(name, sym::Rust | sym::C | sym::simd | sym::transparent)
                     || int_type_of_word(name).is_some()
                 {
                     recognised = true;
@@ -1018,7 +1018,7 @@ pub fn parse_repr_attr(sess: &Session, attr: &Attribute) -> Vec<ReprAttr> {
                             });
                         } else if matches!(
                             meta_item.name_or_empty(),
-                            sym::C | sym::simd | sym::transparent
+                            sym::Rust | sym::C | sym::simd | sym::transparent
                         ) || int_type_of_word(meta_item.name_or_empty()).is_some()
                         {
                             recognised = true;
@@ -1043,7 +1043,7 @@ pub fn parse_repr_attr(sess: &Session, attr: &Attribute) -> Vec<ReprAttr> {
                             );
                         } else if matches!(
                             meta_item.name_or_empty(),
-                            sym::C | sym::simd | sym::transparent
+                            sym::Rust | sym::C | sym::simd | sym::transparent
                         ) || int_type_of_word(meta_item.name_or_empty()).is_some()
                         {
                             recognised = true;
@@ -1060,9 +1060,9 @@ pub fn parse_repr_attr(sess: &Session, attr: &Attribute) -> Vec<ReprAttr> {
                 // Not a word we recognize. This will be caught and reported by
                 // the `check_mod_attrs` pass, but this pass doesn't always run
                 // (e.g. if we only pretty-print the source), so we have to gate
-                // the `delay_span_bug` call as follows:
+                // the `span_delayed_bug` call as follows:
                 if sess.opts.pretty.map_or(true, |pp| pp.needs_analysis()) {
-                    diagnostic.delay_span_bug(item.span(), "unrecognized representation hint");
+                    diagnostic.span_delayed_bug(item.span(), "unrecognized representation hint");
                 }
             }
         }
diff --git a/compiler/rustc_attr/src/lib.rs b/compiler/rustc_attr/src/lib.rs
index 868c04122..3c5bcecdd 100644
--- a/compiler/rustc_attr/src/lib.rs
+++ b/compiler/rustc_attr/src/lib.rs
@@ -4,9 +4,9 @@
 //! The goal is to move the definition of `MetaItem` and things that don't need to be in `syntax`
 //! to this crate.
 
-#![cfg_attr(not(bootstrap), allow(internal_features))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
+#![allow(internal_features)]
+#![feature(rustdoc_internals)]
+#![doc(rust_logo)]
 #![feature(let_chains)]
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
@@ -14,9 +14,6 @@
 #[macro_use]
 extern crate rustc_macros;
 
-use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
-use rustc_fluent_macro::fluent_messages;
-
 mod builtin;
 mod session_diagnostics;
 
@@ -29,4 +26,4 @@ pub use rustc_ast::attr::*;
 
 pub(crate) use rustc_session::HashStableContext;
 
-fluent_messages! { "../messages.ftl" }
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
diff --git a/compiler/rustc_attr/src/session_diagnostics.rs b/compiler/rustc_attr/src/session_diagnostics.rs
index ca9bbd28b..ce8e04def 100644
--- a/compiler/rustc_attr/src/session_diagnostics.rs
+++ b/compiler/rustc_attr/src/session_diagnostics.rs
@@ -2,7 +2,7 @@ use std::num::IntErrorKind;
 
 use rustc_ast as ast;
 use rustc_errors::{
-    error_code, Applicability, DiagnosticBuilder, ErrorGuaranteed, Handler, IntoDiagnostic,
+    error_code, Applicability, DiagCtxt, DiagnosticBuilder, ErrorGuaranteed, IntoDiagnostic,
 };
 use rustc_macros::Diagnostic;
 use rustc_span::{Span, Symbol};
@@ -51,9 +51,9 @@ pub(crate) struct UnknownMetaItem<'a> {
 
 // Manual implementation to be able to format `expected` items correctly.
 impl<'a> IntoDiagnostic<'a> for UnknownMetaItem<'_> {
-    fn into_diagnostic(self, handler: &'a Handler) -> DiagnosticBuilder<'a, ErrorGuaranteed> {
+    fn into_diagnostic(self, dcx: &'a DiagCtxt) -> DiagnosticBuilder<'a, ErrorGuaranteed> {
         let expected = self.expected.iter().map(|name| format!("`{name}`")).collect::<Vec<_>>();
-        let mut diag = handler.struct_span_err_with_code(
+        let mut diag = dcx.struct_span_err_with_code(
             self.span,
             fluent::attr_unknown_meta_item,
             error_code!(E0541),
@@ -201,8 +201,8 @@ pub(crate) struct UnsupportedLiteral {
 }
 
 impl<'a> IntoDiagnostic<'a> for UnsupportedLiteral {
-    fn into_diagnostic(self, handler: &'a Handler) -> DiagnosticBuilder<'a, ErrorGuaranteed> {
-        let mut diag = handler.struct_span_err_with_code(
+    fn into_diagnostic(self, dcx: &'a DiagCtxt) -> DiagnosticBuilder<'a, ErrorGuaranteed> {
+        let mut diag = dcx.struct_span_err_with_code(
             self.span,
             match self.reason {
                 UnsupportedLiteralReason::Generic => fluent::attr_unsupported_literal_generic,
diff --git a/compiler/rustc_baked_icu_data/src/lib.rs b/compiler/rustc_baked_icu_data/src/lib.rs
index ae8c062d2..ffcb29068 100644
--- a/compiler/rustc_baked_icu_data/src/lib.rs
+++ b/compiler/rustc_baked_icu_data/src/lib.rs
@@ -20,9 +20,9 @@
 //! --cldr-tag latest --icuexport-tag latest -o src/data
 //! ```
 
-#![cfg_attr(not(bootstrap), allow(internal_features))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
+#![allow(internal_features)]
+#![feature(rustdoc_internals)]
+#![doc(rust_logo)]
 #![allow(elided_lifetimes_in_paths)]
 
 mod data {
diff --git a/compiler/rustc_borrowck/Cargo.toml b/compiler/rustc_borrowck/Cargo.toml
index 636817a7c..714f46270 100644
--- a/compiler/rustc_borrowck/Cargo.toml
+++ b/compiler/rustc_borrowck/Cargo.toml
@@ -6,7 +6,7 @@ edition = "2021"
 [dependencies]
 # tidy-alphabetical-start
 either = "1.5.0"
-itertools = "0.10.1"
+itertools = "0.11"
 polonius-engine = "0.13.0"
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
@@ -19,7 +19,6 @@ rustc_lexer = { path = "../rustc_lexer" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_middle = { path = "../rustc_middle" }
 rustc_mir_dataflow = { path = "../rustc_mir_dataflow" }
-rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
diff --git a/compiler/rustc_borrowck/src/borrow_set.rs b/compiler/rustc_borrowck/src/borrow_set.rs
index 5248a649c..948af0395 100644
--- a/compiler/rustc_borrowck/src/borrow_set.rs
+++ b/compiler/rustc_borrowck/src/borrow_set.rs
@@ -107,7 +107,7 @@ impl LocalsStateAtExit {
             LocalsStateAtExit::AllAreInvalidated
         } else {
             let mut has_storage_dead = HasStorageDead(BitSet::new_empty(body.local_decls.len()));
-            has_storage_dead.visit_body(&body);
+            has_storage_dead.visit_body(body);
             let mut has_storage_dead_or_moved = has_storage_dead.0;
             for move_out in &move_data.moves {
                 if let Some(index) = move_data.base_local(move_out.path) {
@@ -128,7 +128,7 @@ impl<'tcx> BorrowSet<'tcx> {
     ) -> Self {
         let mut visitor = GatherBorrows {
             tcx,
-            body: &body,
+            body: body,
             location_map: Default::default(),
             activation_map: Default::default(),
             local_map: Default::default(),
@@ -140,7 +140,7 @@ impl<'tcx> BorrowSet<'tcx> {
             ),
         };
 
-        for (block, block_data) in traversal::preorder(&body) {
+        for (block, block_data) in traversal::preorder(body) {
             visitor.visit_basic_block_data(block, block_data);
         }
 
diff --git a/compiler/rustc_borrowck/src/constraint_generation.rs b/compiler/rustc_borrowck/src/constraint_generation.rs
deleted file mode 100644
index 1f642099f..000000000
--- a/compiler/rustc_borrowck/src/constraint_generation.rs
+++ /dev/null
@@ -1,248 +0,0 @@
-#![deny(rustc::untranslatable_diagnostic)]
-#![deny(rustc::diagnostic_outside_of_impl)]
-use rustc_infer::infer::InferCtxt;
-use rustc_middle::mir::visit::TyContext;
-use rustc_middle::mir::visit::Visitor;
-use rustc_middle::mir::{
-    Body, Local, Location, Place, PlaceRef, ProjectionElem, Rvalue, SourceInfo, Statement,
-    StatementKind, Terminator, TerminatorKind, UserTypeProjection,
-};
-use rustc_middle::ty::visit::TypeVisitable;
-use rustc_middle::ty::GenericArgsRef;
-use rustc_middle::ty::{self, RegionVid, Ty, TyCtxt};
-
-use crate::{
-    borrow_set::BorrowSet, facts::AllFacts, location::LocationTable, places_conflict,
-    region_infer::values::LivenessValues,
-};
-
-pub(super) fn generate_constraints<'tcx>(
-    infcx: &InferCtxt<'tcx>,
-    liveness_constraints: &mut LivenessValues<RegionVid>,
-    all_facts: &mut Option<AllFacts>,
-    location_table: &LocationTable,
-    body: &Body<'tcx>,
-    borrow_set: &BorrowSet<'tcx>,
-) {
-    let mut cg = ConstraintGeneration {
-        borrow_set,
-        infcx,
-        liveness_constraints,
-        location_table,
-        all_facts,
-        body,
-    };
-
-    for (bb, data) in body.basic_blocks.iter_enumerated() {
-        cg.visit_basic_block_data(bb, data);
-    }
-}
-
-/// 'cg = the duration of the constraint generation process itself.
-struct ConstraintGeneration<'cg, 'tcx> {
-    infcx: &'cg InferCtxt<'tcx>,
-    all_facts: &'cg mut Option<AllFacts>,
-    location_table: &'cg LocationTable,
-    liveness_constraints: &'cg mut LivenessValues<RegionVid>,
-    borrow_set: &'cg BorrowSet<'tcx>,
-    body: &'cg Body<'tcx>,
-}
-
-impl<'cg, 'tcx> Visitor<'tcx> for ConstraintGeneration<'cg, 'tcx> {
-    /// We sometimes have `args` within an rvalue, or within a
-    /// call. Make them live at the location where they appear.
-    fn visit_args(&mut self, args: &GenericArgsRef<'tcx>, location: Location) {
-        self.add_regular_live_constraint(*args, location);
-        self.super_args(args);
-    }
-
-    /// We sometimes have `region` within an rvalue, or within a
-    /// call. Make them live at the location where they appear.
-    fn visit_region(&mut self, region: ty::Region<'tcx>, location: Location) {
-        self.add_regular_live_constraint(region, location);
-        self.super_region(region);
-    }
-
-    /// We sometimes have `ty` within an rvalue, or within a
-    /// call. Make them live at the location where they appear.
-    fn visit_ty(&mut self, ty: Ty<'tcx>, ty_context: TyContext) {
-        match ty_context {
-            TyContext::ReturnTy(SourceInfo { span, .. })
-            | TyContext::YieldTy(SourceInfo { span, .. })
-            | TyContext::UserTy(span)
-            | TyContext::LocalDecl { source_info: SourceInfo { span, .. }, .. } => {
-                span_bug!(span, "should not be visiting outside of the CFG: {:?}", ty_context);
-            }
-            TyContext::Location(location) => {
-                self.add_regular_live_constraint(ty, location);
-            }
-        }
-
-        self.super_ty(ty);
-    }
-
-    fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) {
-        if let Some(all_facts) = self.all_facts {
-            let _prof_timer = self.infcx.tcx.prof.generic_activity("polonius_fact_generation");
-            all_facts.cfg_edge.push((
-                self.location_table.start_index(location),
-                self.location_table.mid_index(location),
-            ));
-
-            all_facts.cfg_edge.push((
-                self.location_table.mid_index(location),
-                self.location_table.start_index(location.successor_within_block()),
-            ));
-
-            // If there are borrows on this now dead local, we need to record them as `killed`.
-            if let StatementKind::StorageDead(local) = statement.kind {
-                record_killed_borrows_for_local(
-                    all_facts,
-                    self.borrow_set,
-                    self.location_table,
-                    local,
-                    location,
-                );
-            }
-        }
-
-        self.super_statement(statement, location);
-    }
-
-    fn visit_assign(&mut self, place: &Place<'tcx>, rvalue: &Rvalue<'tcx>, location: Location) {
-        // When we see `X = ...`, then kill borrows of
-        // `(*X).foo` and so forth.
-        self.record_killed_borrows_for_place(*place, location);
-
-        self.super_assign(place, rvalue, location);
-    }
-
-    fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) {
-        if let Some(all_facts) = self.all_facts {
-            let _prof_timer = self.infcx.tcx.prof.generic_activity("polonius_fact_generation");
-            all_facts.cfg_edge.push((
-                self.location_table.start_index(location),
-                self.location_table.mid_index(location),
-            ));
-
-            let successor_blocks = terminator.successors();
-            all_facts.cfg_edge.reserve(successor_blocks.size_hint().0);
-            for successor_block in successor_blocks {
-                all_facts.cfg_edge.push((
-                    self.location_table.mid_index(location),
-                    self.location_table.start_index(successor_block.start_location()),
-                ));
-            }
-        }
-
-        // A `Call` terminator's return value can be a local which has borrows,
-        // so we need to record those as `killed` as well.
-        if let TerminatorKind::Call { destination, .. } = terminator.kind {
-            self.record_killed_borrows_for_place(destination, location);
-        }
-
-        self.super_terminator(terminator, location);
-    }
-
-    fn visit_ascribe_user_ty(
-        &mut self,
-        _place: &Place<'tcx>,
-        _variance: ty::Variance,
-        _user_ty: &UserTypeProjection,
-        _location: Location,
-    ) {
-    }
-}
-
-impl<'cx, 'tcx> ConstraintGeneration<'cx, 'tcx> {
-    /// Some variable with type `live_ty` is "regular live" at
-    /// `location` -- i.e., it may be used later. This means that all
-    /// regions appearing in the type `live_ty` must be live at
-    /// `location`.
-    fn add_regular_live_constraint<T>(&mut self, live_ty: T, location: Location)
-    where
-        T: TypeVisitable<TyCtxt<'tcx>>,
-    {
-        debug!("add_regular_live_constraint(live_ty={:?}, location={:?})", live_ty, location);
-
-        self.infcx.tcx.for_each_free_region(&live_ty, |live_region| {
-            let vid = live_region.as_var();
-            self.liveness_constraints.add_element(vid, location);
-        });
-    }
-
-    /// When recording facts for Polonius, records the borrows on the specified place
-    /// as `killed`. For example, when assigning to a local, or on a call's return destination.
-    fn record_killed_borrows_for_place(&mut self, place: Place<'tcx>, location: Location) {
-        if let Some(all_facts) = self.all_facts {
-            let _prof_timer = self.infcx.tcx.prof.generic_activity("polonius_fact_generation");
-
-            // Depending on the `Place` we're killing:
-            // - if it's a local, or a single deref of a local,
-            //   we kill all the borrows on the local.
-            // - if it's a deeper projection, we have to filter which
-            //   of the borrows are killed: the ones whose `borrowed_place`
-            //   conflicts with the `place`.
-            match place.as_ref() {
-                PlaceRef { local, projection: &[] }
-                | PlaceRef { local, projection: &[ProjectionElem::Deref] } => {
-                    debug!(
-                        "Recording `killed` facts for borrows of local={:?} at location={:?}",
-                        local, location
-                    );
-
-                    record_killed_borrows_for_local(
-                        all_facts,
-                        self.borrow_set,
-                        self.location_table,
-                        local,
-                        location,
-                    );
-                }
-
-                PlaceRef { local, projection: &[.., _] } => {
-                    // Kill conflicting borrows of the innermost local.
-                    debug!(
-                        "Recording `killed` facts for borrows of \
-                            innermost projected local={:?} at location={:?}",
-                        local, location
-                    );
-
-                    if let Some(borrow_indices) = self.borrow_set.local_map.get(&local) {
-                        for &borrow_index in borrow_indices {
-                            let places_conflict = places_conflict::places_conflict(
-                                self.infcx.tcx,
-                                self.body,
-                                self.borrow_set[borrow_index].borrowed_place,
-                                place,
-                                places_conflict::PlaceConflictBias::NoOverlap,
-                            );
-
-                            if places_conflict {
-                                let location_index = self.location_table.mid_index(location);
-                                all_facts.loan_killed_at.push((borrow_index, location_index));
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-}
-
-/// When recording facts for Polonius, records the borrows on the specified local as `killed`.
-fn record_killed_borrows_for_local(
-    all_facts: &mut AllFacts,
-    borrow_set: &BorrowSet<'_>,
-    location_table: &LocationTable,
-    local: Local,
-    location: Location,
-) {
-    if let Some(borrow_indices) = borrow_set.local_map.get(&local) {
-        all_facts.loan_killed_at.reserve(borrow_indices.len());
-        for &borrow_index in borrow_indices {
-            let location_index = location_table.mid_index(location);
-            all_facts.loan_killed_at.push((borrow_index, location_index));
-        }
-    }
-}
diff --git a/compiler/rustc_borrowck/src/constraints/mod.rs b/compiler/rustc_borrowck/src/constraints/mod.rs
index 315886bbe..041ac75ec 100644
--- a/compiler/rustc_borrowck/src/constraints/mod.rs
+++ b/compiler/rustc_borrowck/src/constraints/mod.rs
@@ -122,6 +122,7 @@ rustc_index::newtype_index! {
 }
 
 rustc_index::newtype_index! {
+    #[orderable]
     #[debug_format = "ConstraintSccIndex({})"]
     pub struct ConstraintSccIndex {}
 }
diff --git a/compiler/rustc_borrowck/src/dataflow.rs b/compiler/rustc_borrowck/src/dataflow.rs
index 8676d2ba7..1bd891bdd 100644
--- a/compiler/rustc_borrowck/src/dataflow.rs
+++ b/compiler/rustc_borrowck/src/dataflow.rs
@@ -10,105 +10,93 @@ use rustc_middle::ty::RegionVid;
 use rustc_middle::ty::TyCtxt;
 use rustc_mir_dataflow::impls::{EverInitializedPlaces, MaybeUninitializedPlaces};
 use rustc_mir_dataflow::ResultsVisitable;
-use rustc_mir_dataflow::{self, fmt::DebugWithContext, GenKill};
-use rustc_mir_dataflow::{Analysis, Direction, Results};
+use rustc_mir_dataflow::{fmt::DebugWithContext, GenKill};
+use rustc_mir_dataflow::{Analysis, AnalysisDomain, Results};
 use std::fmt;
 
 use crate::{places_conflict, BorrowSet, PlaceConflictBias, PlaceExt, RegionInferenceContext};
 
-/// A tuple with named fields that can hold either the results or the transient state of the
-/// dataflow analyses used by the borrow checker.
-#[derive(Debug)]
-pub struct BorrowckAnalyses<B, U, E> {
-    pub borrows: B,
-    pub uninits: U,
-    pub ever_inits: E,
-}
-
 /// The results of the dataflow analyses used by the borrow checker.
-pub type BorrowckResults<'mir, 'tcx> = BorrowckAnalyses<
-    Results<'tcx, Borrows<'mir, 'tcx>>,
-    Results<'tcx, MaybeUninitializedPlaces<'mir, 'tcx>>,
-    Results<'tcx, EverInitializedPlaces<'mir, 'tcx>>,
->;
+pub struct BorrowckResults<'mir, 'tcx> {
+    pub(crate) borrows: Results<'tcx, Borrows<'mir, 'tcx>>,
+    pub(crate) uninits: Results<'tcx, MaybeUninitializedPlaces<'mir, 'tcx>>,
+    pub(crate) ever_inits: Results<'tcx, EverInitializedPlaces<'mir, 'tcx>>,
+}
 
 /// The transient state of the dataflow analyses used by the borrow checker.
-pub type BorrowckFlowState<'mir, 'tcx> =
-    <BorrowckResults<'mir, 'tcx> as ResultsVisitable<'tcx>>::FlowState;
-
-macro_rules! impl_visitable {
-    ( $(
-        $T:ident { $( $field:ident : $A:ident ),* $(,)? }
-    )* ) => { $(
-        impl<'tcx, $($A),*, D: Direction> ResultsVisitable<'tcx> for $T<$( Results<'tcx, $A> ),*>
-        where
-            $( $A: Analysis<'tcx, Direction = D>, )*
-        {
-            type Direction = D;
-            type FlowState = $T<$( $A::Domain ),*>;
+#[derive(Debug)]
+pub struct BorrowckFlowState<'mir, 'tcx> {
+    pub(crate) borrows: <Borrows<'mir, 'tcx> as AnalysisDomain<'tcx>>::Domain,
+    pub(crate) uninits: <MaybeUninitializedPlaces<'mir, 'tcx> as AnalysisDomain<'tcx>>::Domain,
+    pub(crate) ever_inits: <EverInitializedPlaces<'mir, 'tcx> as AnalysisDomain<'tcx>>::Domain,
+}
 
-            fn new_flow_state(&self, body: &mir::Body<'tcx>) -> Self::FlowState {
-                $T {
-                    $( $field: self.$field.analysis.bottom_value(body) ),*
-                }
-            }
+impl<'mir, 'tcx> ResultsVisitable<'tcx> for BorrowckResults<'mir, 'tcx> {
+    // All three analyses are forward, but we have to use just one here.
+    type Direction = <Borrows<'mir, 'tcx> as AnalysisDomain<'tcx>>::Direction;
+    type FlowState = BorrowckFlowState<'mir, 'tcx>;
 
-            fn reset_to_block_entry(
-                &self,
-                state: &mut Self::FlowState,
-                block: BasicBlock,
-            ) {
-                $( state.$field.clone_from(&self.$field.entry_set_for_block(block)); )*
-            }
+    fn new_flow_state(&self, body: &mir::Body<'tcx>) -> Self::FlowState {
+        BorrowckFlowState {
+            borrows: self.borrows.analysis.bottom_value(body),
+            uninits: self.uninits.analysis.bottom_value(body),
+            ever_inits: self.ever_inits.analysis.bottom_value(body),
+        }
+    }
 
-            fn reconstruct_before_statement_effect(
-                &mut self,
-                state: &mut Self::FlowState,
-                stmt: &mir::Statement<'tcx>,
-                loc: Location,
-            ) {
-                $( self.$field.analysis
-                    .apply_before_statement_effect(&mut state.$field, stmt, loc); )*
-            }
+    fn reset_to_block_entry(&self, state: &mut Self::FlowState, block: BasicBlock) {
+        state.borrows.clone_from(&self.borrows.entry_set_for_block(block));
+        state.uninits.clone_from(&self.uninits.entry_set_for_block(block));
+        state.ever_inits.clone_from(&self.ever_inits.entry_set_for_block(block));
+    }
 
-            fn reconstruct_statement_effect(
-                &mut self,
-                state: &mut Self::FlowState,
-                stmt: &mir::Statement<'tcx>,
-                loc: Location,
-            ) {
-                $( self.$field.analysis
-                    .apply_statement_effect(&mut state.$field, stmt, loc); )*
-            }
+    fn reconstruct_before_statement_effect(
+        &mut self,
+        state: &mut Self::FlowState,
+        stmt: &mir::Statement<'tcx>,
+        loc: Location,
+    ) {
+        self.borrows.analysis.apply_before_statement_effect(&mut state.borrows, stmt, loc);
+        self.uninits.analysis.apply_before_statement_effect(&mut state.uninits, stmt, loc);
+        self.ever_inits.analysis.apply_before_statement_effect(&mut state.ever_inits, stmt, loc);
+    }
 
-            fn reconstruct_before_terminator_effect(
-                &mut self,
-                state: &mut Self::FlowState,
-                term: &mir::Terminator<'tcx>,
-                loc: Location,
-            ) {
-                $( self.$field.analysis
-                    .apply_before_terminator_effect(&mut state.$field, term, loc); )*
-            }
+    fn reconstruct_statement_effect(
+        &mut self,
+        state: &mut Self::FlowState,
+        stmt: &mir::Statement<'tcx>,
+        loc: Location,
+    ) {
+        self.borrows.analysis.apply_statement_effect(&mut state.borrows, stmt, loc);
+        self.uninits.analysis.apply_statement_effect(&mut state.uninits, stmt, loc);
+        self.ever_inits.analysis.apply_statement_effect(&mut state.ever_inits, stmt, loc);
+    }
 
-            fn reconstruct_terminator_effect(
-                &mut self,
-                state: &mut Self::FlowState,
-                term: &mir::Terminator<'tcx>,
-                loc: Location,
-            ) {
-                $( self.$field.analysis
-                    .apply_terminator_effect(&mut state.$field, term, loc); )*
-            }
-        }
-    )* }
-}
+    fn reconstruct_before_terminator_effect(
+        &mut self,
+        state: &mut Self::FlowState,
+        term: &mir::Terminator<'tcx>,
+        loc: Location,
+    ) {
+        self.borrows.analysis.apply_before_terminator_effect(&mut state.borrows, term, loc);
+        self.uninits.analysis.apply_before_terminator_effect(&mut state.uninits, term, loc);
+        self.ever_inits.analysis.apply_before_terminator_effect(&mut state.ever_inits, term, loc);
+    }
 
-impl_visitable! {
-    BorrowckAnalyses { borrows: B, uninits: U, ever_inits: E }
+    fn reconstruct_terminator_effect(
+        &mut self,
+        state: &mut Self::FlowState,
+        term: &mir::Terminator<'tcx>,
+        loc: Location,
+    ) {
+        self.borrows.analysis.apply_terminator_effect(&mut state.borrows, term, loc);
+        self.uninits.analysis.apply_terminator_effect(&mut state.uninits, term, loc);
+        self.ever_inits.analysis.apply_terminator_effect(&mut state.ever_inits, term, loc);
+    }
 }
 
 rustc_index::newtype_index! {
+    #[orderable]
     #[debug_format = "bw{}"]
     pub struct BorrowIndex {}
 }
@@ -120,24 +108,24 @@ rustc_index::newtype_index! {
 /// `BorrowIndex`, and maps each such index to a `BorrowData`
 /// describing the borrow. These indexes are used for representing the
 /// borrows in compact bitvectors.
-pub struct Borrows<'a, 'tcx> {
+pub struct Borrows<'mir, 'tcx> {
     tcx: TyCtxt<'tcx>,
-    body: &'a Body<'tcx>,
+    body: &'mir Body<'tcx>,
 
-    borrow_set: &'a BorrowSet<'tcx>,
+    borrow_set: &'mir BorrowSet<'tcx>,
     borrows_out_of_scope_at_location: FxIndexMap<Location, Vec<BorrowIndex>>,
 }
 
-struct OutOfScopePrecomputer<'a, 'tcx> {
+struct OutOfScopePrecomputer<'mir, 'tcx> {
     visited: BitSet<mir::BasicBlock>,
     visit_stack: Vec<mir::BasicBlock>,
-    body: &'a Body<'tcx>,
-    regioncx: &'a RegionInferenceContext<'tcx>,
+    body: &'mir Body<'tcx>,
+    regioncx: &'mir RegionInferenceContext<'tcx>,
     borrows_out_of_scope_at_location: FxIndexMap<Location, Vec<BorrowIndex>>,
 }
 
-impl<'a, 'tcx> OutOfScopePrecomputer<'a, 'tcx> {
-    fn new(body: &'a Body<'tcx>, regioncx: &'a RegionInferenceContext<'tcx>) -> Self {
+impl<'mir, 'tcx> OutOfScopePrecomputer<'mir, 'tcx> {
+    fn new(body: &'mir Body<'tcx>, regioncx: &'mir RegionInferenceContext<'tcx>) -> Self {
         OutOfScopePrecomputer {
             visited: BitSet::new_empty(body.basic_blocks.len()),
             visit_stack: vec![],
@@ -240,17 +228,17 @@ pub fn calculate_borrows_out_of_scope_at_location<'tcx>(
     prec.borrows_out_of_scope_at_location
 }
 
-struct PoloniusOutOfScopePrecomputer<'a, 'tcx> {
+struct PoloniusOutOfScopePrecomputer<'mir, 'tcx> {
     visited: BitSet<mir::BasicBlock>,
     visit_stack: Vec<mir::BasicBlock>,
-    body: &'a Body<'tcx>,
-    regioncx: &'a RegionInferenceContext<'tcx>,
+    body: &'mir Body<'tcx>,
+    regioncx: &'mir RegionInferenceContext<'tcx>,
 
     loans_out_of_scope_at_location: FxIndexMap<Location, Vec<BorrowIndex>>,
 }
 
-impl<'a, 'tcx> PoloniusOutOfScopePrecomputer<'a, 'tcx> {
-    fn new(body: &'a Body<'tcx>, regioncx: &'a RegionInferenceContext<'tcx>) -> Self {
+impl<'mir, 'tcx> PoloniusOutOfScopePrecomputer<'mir, 'tcx> {
+    fn new(body: &'mir Body<'tcx>, regioncx: &'mir RegionInferenceContext<'tcx>) -> Self {
         Self {
             visited: BitSet::new_empty(body.basic_blocks.len()),
             visit_stack: vec![],
@@ -403,12 +391,12 @@ impl<'tcx> PoloniusOutOfScopePrecomputer<'_, 'tcx> {
     }
 }
 
-impl<'a, 'tcx> Borrows<'a, 'tcx> {
+impl<'mir, 'tcx> Borrows<'mir, 'tcx> {
     pub fn new(
         tcx: TyCtxt<'tcx>,
-        body: &'a Body<'tcx>,
-        regioncx: &'a RegionInferenceContext<'tcx>,
-        borrow_set: &'a BorrowSet<'tcx>,
+        body: &'mir Body<'tcx>,
+        regioncx: &'mir RegionInferenceContext<'tcx>,
+        borrow_set: &'mir BorrowSet<'tcx>,
     ) -> Self {
         let mut borrows_out_of_scope_at_location =
             calculate_borrows_out_of_scope_at_location(body, regioncx, borrow_set);
@@ -431,7 +419,8 @@ impl<'a, 'tcx> Borrows<'a, 'tcx> {
 
             assert_eq!(
                 borrows_out_of_scope_at_location, polonius_prec.loans_out_of_scope_at_location,
-                "the loans out of scope must be the same as the borrows out of scope"
+                "polonius loan scopes differ from NLL borrow scopes, for body {:?}",
+                body.span,
             );
 
             borrows_out_of_scope_at_location = polonius_prec.loans_out_of_scope_at_location;
@@ -596,7 +585,7 @@ impl<'tcx> rustc_mir_dataflow::GenKillAnalysis<'tcx> for Borrows<'_, 'tcx> {
 
     fn before_terminator_effect(
         &mut self,
-        trans: &mut impl GenKill<Self::Idx>,
+        trans: &mut Self::Domain,
         _terminator: &mir::Terminator<'tcx>,
         location: Location,
     ) {
@@ -623,7 +612,7 @@ impl<'tcx> rustc_mir_dataflow::GenKillAnalysis<'tcx> for Borrows<'_, 'tcx> {
 
     fn call_return_effect(
         &mut self,
-        _trans: &mut impl GenKill<Self::Idx>,
+        _trans: &mut Self::Domain,
         _block: mir::BasicBlock,
         _return_places: CallReturnPlaces<'_, 'tcx>,
     ) {
diff --git a/compiler/rustc_borrowck/src/diagnostics/bound_region_errors.rs b/compiler/rustc_borrowck/src/diagnostics/bound_region_errors.rs
index cfcf31fce..924e68fa9 100644
--- a/compiler/rustc_borrowck/src/diagnostics/bound_region_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/bound_region_errors.rs
@@ -10,6 +10,8 @@ use rustc_infer::infer::RegionVariableOrigin;
 use rustc_infer::infer::{InferCtxt, RegionResolutionError, SubregionOrigin, TyCtxtInferExt as _};
 use rustc_infer::traits::ObligationCause;
 use rustc_middle::ty::error::TypeError;
+use rustc_middle::ty::RePlaceholder;
+use rustc_middle::ty::Region;
 use rustc_middle::ty::RegionVid;
 use rustc_middle::ty::UniverseIndex;
 use rustc_middle::ty::{self, Ty, TyCtxt, TypeFoldable};
@@ -205,6 +207,8 @@ trait TypeOpInfo<'tcx> {
         let span = cause.span;
         let nice_error = self.nice_error(mbcx, cause, placeholder_region, error_region);
 
+        debug!(?nice_error);
+
         if let Some(nice_error) = nice_error {
             mbcx.buffer_error(nice_error);
         } else {
@@ -381,7 +385,7 @@ fn try_extract_error_from_fulfill_cx<'tcx>(
     error_region: Option<ty::Region<'tcx>>,
 ) -> Option<DiagnosticBuilder<'tcx, ErrorGuaranteed>> {
     // We generally shouldn't have errors here because the query was
-    // already run, but there's no point using `delay_span_bug`
+    // already run, but there's no point using `span_delayed_bug`
     // when we're going to emit an error here anyway.
     let _errors = ocx.select_all_or_error();
     let region_constraints = ocx.infcx.with_region_constraints(|r| r.clone());
@@ -404,19 +408,41 @@ fn try_extract_error_from_region_constraints<'tcx>(
     mut region_var_origin: impl FnMut(RegionVid) -> RegionVariableOrigin,
     mut universe_of_region: impl FnMut(RegionVid) -> UniverseIndex,
 ) -> Option<DiagnosticBuilder<'tcx, ErrorGuaranteed>> {
-    let (sub_region, cause) =
-        region_constraints.constraints.iter().find_map(|(constraint, cause)| {
-            match *constraint {
-                Constraint::RegSubReg(sub, sup) if sup == placeholder_region && sup != sub => {
-                    Some((sub, cause.clone()))
-                }
-                // FIXME: Should this check the universe of the var?
-                Constraint::VarSubReg(vid, sup) if sup == placeholder_region => {
-                    Some((ty::Region::new_var(infcx.tcx, vid), cause.clone()))
-                }
-                _ => None,
+    let matches =
+        |a_region: Region<'tcx>, b_region: Region<'tcx>| match (a_region.kind(), b_region.kind()) {
+            (RePlaceholder(a_p), RePlaceholder(b_p)) => a_p.bound == b_p.bound,
+            _ => a_region == b_region,
+        };
+    let check = |constraint: &Constraint<'tcx>, cause: &SubregionOrigin<'tcx>, exact| {
+        match *constraint {
+            Constraint::RegSubReg(sub, sup)
+                if ((exact && sup == placeholder_region)
+                    || (!exact && matches(sup, placeholder_region)))
+                    && sup != sub =>
+            {
+                Some((sub, cause.clone()))
+            }
+            // FIXME: Should this check the universe of the var?
+            Constraint::VarSubReg(vid, sup)
+                if ((exact && sup == placeholder_region)
+                    || (!exact && matches(sup, placeholder_region))) =>
+            {
+                Some((ty::Region::new_var(infcx.tcx, vid), cause.clone()))
             }
-        })?;
+            _ => None,
+        }
+    };
+    let mut info = region_constraints
+        .constraints
+        .iter()
+        .find_map(|(constraint, cause)| check(constraint, cause, true));
+    if info.is_none() {
+        info = region_constraints
+            .constraints
+            .iter()
+            .find_map(|(constraint, cause)| check(constraint, cause, false));
+    }
+    let (sub_region, cause) = info?;
 
     debug!(?sub_region, "cause = {:#?}", cause);
     let error = match (error_region, *sub_region) {
diff --git a/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs b/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
index 9a8f1c97e..db0f4559a 100644
--- a/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
@@ -18,7 +18,7 @@ use rustc_middle::mir::{
     PlaceRef, ProjectionElem, Rvalue, Statement, StatementKind, Terminator, TerminatorKind,
     VarBindingForm,
 };
-use rustc_middle::ty::{self, suggest_constraining_type_params, PredicateKind, Ty};
+use rustc_middle::ty::{self, suggest_constraining_type_params, PredicateKind, Ty, TyCtxt};
 use rustc_middle::util::CallKind;
 use rustc_mir_dataflow::move_paths::{InitKind, MoveOutIndex, MovePathIndex};
 use rustc_span::def_id::LocalDefId;
@@ -398,7 +398,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                 }
                 let typeck = self.infcx.tcx.typeck(self.mir_def_id());
                 let hir_id = hir.parent_id(expr.hir_id);
-                if let Some(parent) = hir.find(hir_id) {
+                if let Some(parent) = self.infcx.tcx.opt_hir_node(hir_id) {
                     let (def_id, args, offset) = if let hir::Node::Expr(parent_expr) = parent
                         && let hir::ExprKind::MethodCall(_, _, args, _) = parent_expr.kind
                         && let Some(def_id) = typeck.type_dependent_def_id(parent_expr.hir_id)
@@ -413,7 +413,10 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                         (None, &[][..], 0)
                     };
                     if let Some(def_id) = def_id
-                        && let Some(node) = hir.find(hir.local_def_id_to_hir_id(def_id))
+                        && let Some(node) = self
+                            .infcx
+                            .tcx
+                            .opt_hir_node(self.infcx.tcx.local_def_id_to_hir_id(def_id))
                         && let Some(fn_sig) = node.fn_sig()
                         && let Some(ident) = node.ident()
                         && let Some(pos) = args.iter().position(|arg| arg.hir_id == expr.hir_id)
@@ -445,7 +448,6 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                         && let hir::ExprKind::Path(hir::QPath::LangItem(
                             LangItem::IntoIterIntoIter,
                             _,
-                            _,
                         )) = call_expr.kind
                     {
                         // Do not suggest `.clone()` in a `for` loop, we already suggest borrowing.
@@ -490,7 +492,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         let mut spans = vec![];
         for init_idx in inits {
             let init = &self.move_data.inits[*init_idx];
-            let span = init.span(&self.body);
+            let span = init.span(self.body);
             if !span.is_dummy() {
                 spans.push(span);
             }
@@ -518,7 +520,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         let body = map.body(body_id);
 
         let mut visitor = ConditionVisitor { spans: &spans, name: &name, errors: vec![] };
-        visitor.visit_body(&body);
+        visitor.visit_body(body);
 
         let mut show_assign_sugg = false;
         let isnt_initialized = if let InitializationRequiringAction::PartialAssignment
@@ -614,7 +616,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
             }
 
             let mut visitor = LetVisitor { decl_span, sugg_span: None };
-            visitor.visit_body(&body);
+            visitor.visit_body(body);
             if let Some(span) = visitor.sugg_span {
                 self.suggest_assign_value(&mut err, moved_place, span);
             }
@@ -779,7 +781,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
             return;
         };
         // Try to find predicates on *generic params* that would allow copying `ty`
-        let ocx = ObligationCtxt::new(&self.infcx);
+        let ocx = ObligationCtxt::new(self.infcx);
         let copy_did = tcx.require_lang_item(LangItem::Copy, Some(span));
         let cause = ObligationCause::misc(span, self.mir_def_id());
 
@@ -856,7 +858,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         self.explain_why_borrow_contains_point(location, borrow, None)
             .add_explanation_to_diagnostic(
                 self.infcx.tcx,
-                &self.body,
+                self.body,
                 &self.local_names,
                 &mut err,
                 "",
@@ -903,7 +905,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         self.explain_why_borrow_contains_point(location, borrow, None)
             .add_explanation_to_diagnostic(
                 self.infcx.tcx,
-                &self.body,
+                self.body,
                 &self.local_names,
                 &mut err,
                 "",
@@ -1136,7 +1138,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
             });
         } else {
             issued_spans.var_subdiag(
-                Some(&self.infcx.tcx.sess.parse_sess.span_diagnostic),
+                Some(self.infcx.tcx.sess.dcx()),
                 &mut err,
                 Some(issued_borrow.kind),
                 |kind, var_span| {
@@ -1153,7 +1155,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
             );
 
             borrow_spans.var_subdiag(
-                Some(&self.infcx.tcx.sess.parse_sess.span_diagnostic),
+                Some(self.infcx.tcx.sess.dcx()),
                 &mut err,
                 Some(gen_borrow_kind),
                 |kind, var_span| {
@@ -1174,7 +1176,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
 
         explanation.add_explanation_to_diagnostic(
             self.infcx.tcx,
-            &self.body,
+            self.body,
             &self.local_names,
             &mut err,
             first_borrow_desc,
@@ -1318,7 +1320,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         let tcx = self.infcx.tcx;
         let hir = tcx.hir();
 
-        let Some(body_id) = hir.get(self.mir_hir_id()).body_id() else { return };
+        let Some(body_id) = tcx.hir_node(self.mir_hir_id()).body_id() else { return };
         let typeck_results = tcx.typeck(self.mir_def_id());
 
         struct ExprFinder<'hir> {
@@ -1346,11 +1348,8 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                 // };
                 // corresponding to the desugaring of a for loop `for <pat> in <head> { <body> }`.
                 if let hir::ExprKind::Call(path, [arg]) = ex.kind
-                    && let hir::ExprKind::Path(hir::QPath::LangItem(
-                        LangItem::IntoIterIntoIter,
-                        _,
-                        _,
-                    )) = path.kind
+                    && let hir::ExprKind::Path(hir::QPath::LangItem(LangItem::IntoIterIntoIter, _)) =
+                        path.kind
                     && arg.span.contains(self.issue_span)
                 {
                     // Find `IntoIterator::into_iter(<head>)`
@@ -1368,10 +1367,10 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                         ..
                     }) = stmt.kind
                     && let hir::ExprKind::Call(path, _args) = call.kind
-                    && let hir::ExprKind::Path(hir::QPath::LangItem(LangItem::IteratorNext, _, _)) =
+                    && let hir::ExprKind::Path(hir::QPath::LangItem(LangItem::IteratorNext, _)) =
                         path.kind
                     && let hir::PatKind::Struct(path, [field, ..], _) = bind.pat.kind
-                    && let hir::QPath::LangItem(LangItem::OptionSome, pat_span, _) = path
+                    && let hir::QPath::LangItem(LangItem::OptionSome, pat_span) = path
                     && call.span.contains(self.issue_span)
                 {
                     // Find `<pat>` and the span for the whole `for` loop.
@@ -1513,7 +1512,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         let local_ty = self.body.local_decls[local].ty;
 
         // Get the body the error happens in
-        let Some(body_id) = hir.get(self.mir_hir_id()).body_id() else { return };
+        let Some(body_id) = tcx.hir_node(self.mir_hir_id()).body_id() else { return };
 
         let body_expr = hir.body(body_id).value;
 
@@ -1562,7 +1561,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         // Check that the parent of the closure is a method call,
         // with receiver matching with local's type (modulo refs)
         let parent = hir.parent_id(closure_expr.hir_id);
-        if let hir::Node::Expr(parent) = hir.get(parent) {
+        if let hir::Node::Expr(parent) = tcx.hir_node(parent) {
             if let hir::ExprKind::MethodCall(_, recv, ..) = parent.kind {
                 let recv_ty = typeck_results.expr_ty(recv);
 
@@ -1578,8 +1577,9 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
             return;
         };
         let sig = args.as_closure().sig();
-        let tupled_params =
-            tcx.erase_late_bound_regions(sig.inputs().iter().next().unwrap().map_bound(|&b| b));
+        let tupled_params = tcx.instantiate_bound_regions_with_erased(
+            sig.inputs().iter().next().unwrap().map_bound(|&b| b),
+        );
         let ty::Tuple(params) = tupled_params.kind() else { return };
 
         // Find the first argument with a matching type, get its name
@@ -1638,15 +1638,14 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         issued_spans: &UseSpans<'tcx>,
     ) {
         let UseSpans::ClosureUse { capture_kind_span, .. } = issued_spans else { return };
-        let hir = self.infcx.tcx.hir();
 
-        struct ExpressionFinder<'hir> {
+        struct ExpressionFinder<'tcx> {
             capture_span: Span,
             closure_change_spans: Vec<Span>,
             closure_arg_span: Option<Span>,
             in_closure: bool,
             suggest_arg: String,
-            hir: rustc_middle::hir::map::Map<'hir>,
+            tcx: TyCtxt<'tcx>,
             closure_local_id: Option<hir::HirId>,
             closure_call_changes: Vec<(Span, String)>,
         }
@@ -1660,7 +1659,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                         fn_decl: hir::FnDecl { inputs, .. },
                         ..
                     }) = e.kind
-                        && let Some(hir::Node::Expr(body)) = self.hir.find(body.hir_id)
+                        && let Some(hir::Node::Expr(body)) = self.tcx.opt_hir_node(body.hir_id)
                     {
                         self.suggest_arg = "this: &Self".to_string();
                         if inputs.len() > 0 {
@@ -1725,8 +1724,8 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         if let Some(hir::Node::ImplItem(hir::ImplItem {
             kind: hir::ImplItemKind::Fn(_fn_sig, body_id),
             ..
-        })) = hir.find(self.mir_hir_id())
-            && let Some(hir::Node::Expr(expr)) = hir.find(body_id.hir_id)
+        })) = self.infcx.tcx.opt_hir_node(self.mir_hir_id())
+            && let Some(hir::Node::Expr(expr)) = self.infcx.tcx.opt_hir_node(body_id.hir_id)
         {
             let mut finder = ExpressionFinder {
                 capture_span: *capture_kind_span,
@@ -1736,7 +1735,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                 suggest_arg: String::new(),
                 closure_local_id: None,
                 closure_call_changes: vec![],
-                hir,
+                tcx: self.infcx.tcx,
             };
             finder.visit_expr(expr);
 
@@ -1931,7 +1930,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         let place_desc = self.describe_place(borrow.borrowed_place.as_ref());
 
         let kind_place = kind.filter(|_| place_desc.is_some()).map(|k| (k, place_span.0));
-        let explanation = self.explain_why_borrow_contains_point(location, &borrow, kind_place);
+        let explanation = self.explain_why_borrow_contains_point(location, borrow, kind_place);
 
         debug!(?place_desc, ?explanation);
 
@@ -2000,14 +1999,14 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
             (Some(name), explanation) => self.report_local_value_does_not_live_long_enough(
                 location,
                 &name,
-                &borrow,
+                borrow,
                 drop_span,
                 borrow_spans,
                 explanation,
             ),
             (None, explanation) => self.report_temporary_value_does_not_live_long_enough(
                 location,
-                &borrow,
+                borrow,
                 drop_span,
                 borrow_spans,
                 proper_span,
@@ -2075,8 +2074,15 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                         .map(|name| format!("function `{name}`"))
                         .unwrap_or_else(|| {
                             match &self.infcx.tcx.def_kind(self.mir_def_id()) {
+                                DefKind::Closure
+                                    if self
+                                        .infcx
+                                        .tcx
+                                        .is_coroutine(self.mir_def_id().to_def_id()) =>
+                                {
+                                    "enclosing coroutine"
+                                }
                                 DefKind::Closure => "enclosing closure",
-                                DefKind::Coroutine => "enclosing coroutine",
                                 kind => bug!("expected closure or coroutine, found {:?}", kind),
                             }
                             .to_string()
@@ -2097,7 +2103,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
             } else {
                 explanation.add_explanation_to_diagnostic(
                     self.infcx.tcx,
-                    &self.body,
+                    self.body,
                     &self.local_names,
                     &mut err,
                     "",
@@ -2118,7 +2124,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
 
             explanation.add_explanation_to_diagnostic(
                 self.infcx.tcx,
-                &self.body,
+                self.body,
                 &self.local_names,
                 &mut err,
                 "",
@@ -2179,7 +2185,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
 
         explanation.add_explanation_to_diagnostic(
             self.infcx.tcx,
-            &self.body,
+            self.body,
             &self.local_names,
             &mut err,
             "",
@@ -2290,7 +2296,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                 let proper_span = proper_span.source_callsite();
                 if let Some(scope) = self.body.source_scopes.get(source_info.scope)
                     && let ClearCrossCrate::Set(scope_data) = &scope.local_data
-                    && let Some(node) = self.infcx.tcx.hir().find(scope_data.lint_root)
+                    && let Some(node) = self.infcx.tcx.opt_hir_node(scope_data.lint_root)
                     && let Some(id) = node.body_id()
                     && let hir::ExprKind::Block(block, _) = self.infcx.tcx.hir().body(id).value.kind
                 {
@@ -2348,7 +2354,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                                     Applicability::MaybeIncorrect,
                                 );
                             } else {
-                                err.note("the result of `format_args!` can only be assigned directly if no placeholders in it's arguments are used");
+                                err.note("the result of `format_args!` can only be assigned directly if no placeholders in its arguments are used");
                                 err.note("to learn more, visit <https://doc.rust-lang.org/std/macro.format_args.html>");
                             }
                             suggested = true;
@@ -2364,7 +2370,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         }
         explanation.add_explanation_to_diagnostic(
             self.infcx.tcx,
-            &self.body,
+            self.body,
             &self.local_names,
             &mut err,
             "",
@@ -2513,12 +2519,23 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                 CoroutineKind::Gen(kind) => match kind {
                     CoroutineSource::Block => "gen block",
                     CoroutineSource::Closure => "gen closure",
-                    _ => bug!("gen block/closure expected, but gen function found."),
+                    CoroutineSource::Fn => {
+                        bug!("gen block/closure expected, but gen function found.")
+                    }
+                },
+                CoroutineKind::AsyncGen(kind) => match kind {
+                    CoroutineSource::Block => "async gen block",
+                    CoroutineSource::Closure => "async gen closure",
+                    CoroutineSource::Fn => {
+                        bug!("gen block/closure expected, but gen function found.")
+                    }
                 },
                 CoroutineKind::Async(async_kind) => match async_kind {
                     CoroutineSource::Block => "async block",
                     CoroutineSource::Closure => "async closure",
-                    _ => bug!("async block/closure expected, but async function found."),
+                    CoroutineSource::Fn => {
+                        bug!("async block/closure expected, but async function found.")
+                    }
                 },
                 CoroutineKind::Coroutine => "coroutine",
             },
@@ -2841,7 +2858,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
 
         self.explain_why_borrow_contains_point(location, loan, None).add_explanation_to_diagnostic(
             self.infcx.tcx,
-            &self.body,
+            self.body,
             &self.local_names,
             &mut err,
             "",
@@ -3019,7 +3036,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
             }
         }
         let mut visitor = FakeReadCauseFinder { place, cause: None };
-        visitor.visit_body(&self.body);
+        visitor.visit_body(self.body);
         match visitor.cause {
             Some(FakeReadCause::ForMatchGuard) => Some("match guard"),
             Some(FakeReadCause::ForIndex) => Some("indexing expression"),
@@ -3246,7 +3263,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
     ) -> Option<AnnotatedBorrowFnSignature<'tcx>> {
         debug!("annotate_fn_sig: did={:?} sig={:?}", did, sig);
         let is_closure = self.infcx.tcx.is_closure(did.to_def_id());
-        let fn_hir_id = self.infcx.tcx.hir().local_def_id_to_hir_id(did);
+        let fn_hir_id = self.infcx.tcx.local_def_id_to_hir_id(did);
         let fn_decl = self.infcx.tcx.hir().fn_decl_by_hir_id(fn_hir_id)?;
 
         // We need to work out which arguments to highlight. We do this by looking
diff --git a/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs b/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs
index 8a930ca59..f1e712d81 100644
--- a/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs
@@ -5,12 +5,13 @@ use rustc_hir as hir;
 use rustc_hir::intravisit::Visitor;
 use rustc_index::IndexSlice;
 use rustc_infer::infer::NllRegionVariableOrigin;
+use rustc_middle::middle::resolve_bound_vars::ObjectLifetimeDefault;
 use rustc_middle::mir::{
     Body, CallSource, CastKind, ConstraintCategory, FakeReadCause, Local, LocalInfo, Location,
     Operand, Place, Rvalue, Statement, StatementKind, TerminatorKind,
 };
 use rustc_middle::ty::adjustment::PointerCoercion;
-use rustc_middle::ty::{self, RegionVid, TyCtxt};
+use rustc_middle::ty::{self, RegionVid, Ty, TyCtxt};
 use rustc_span::symbol::{kw, Symbol};
 use rustc_span::{sym, DesugaringKind, Span};
 use rustc_trait_selection::traits::error_reporting::FindExprBySpan;
@@ -86,7 +87,7 @@ impl<'tcx> BorrowExplanation<'tcx> {
                     if let hir::ExprKind::Path(hir::QPath::Resolved(None, p)) = expr.kind
                         && let [hir::PathSegment { ident, args: None, .. }] = p.segments
                         && let hir::def::Res::Local(hir_id) = p.res
-                        && let Some(hir::Node::Pat(pat)) = tcx.hir().find(hir_id)
+                        && let Some(hir::Node::Pat(pat)) = tcx.opt_hir_node(hir_id)
                     {
                         err.span_label(pat.span, format!("binding `{ident}` declared here"));
                     }
@@ -290,12 +291,69 @@ impl<'tcx> BorrowExplanation<'tcx> {
                     }
                 }
 
+                if let ConstraintCategory::Cast { unsize_to: Some(unsize_ty) } = category {
+                    self.add_object_lifetime_default_note(tcx, err, unsize_ty);
+                }
                 self.add_lifetime_bound_suggestion_to_diagnostic(err, &category, span, region_name);
             }
             _ => {}
         }
     }
 
+    fn add_object_lifetime_default_note(
+        &self,
+        tcx: TyCtxt<'tcx>,
+        err: &mut Diagnostic,
+        unsize_ty: Ty<'tcx>,
+    ) {
+        if let ty::Adt(def, args) = unsize_ty.kind() {
+            // We try to elaborate the object lifetime defaults and present those to the user. This should
+            // make it clear where the region constraint is coming from.
+            let generics = tcx.generics_of(def.did());
+
+            let mut has_dyn = false;
+            let mut failed = false;
+
+            let elaborated_args = std::iter::zip(*args, &generics.params).map(|(arg, param)| {
+                if let Some(ty::Dynamic(obj, _, ty::DynKind::Dyn)) = arg.as_type().map(Ty::kind) {
+                    let default = tcx.object_lifetime_default(param.def_id);
+
+                    let re_static = tcx.lifetimes.re_static;
+
+                    let implied_region = match default {
+                        // This is not entirely precise.
+                        ObjectLifetimeDefault::Empty => re_static,
+                        ObjectLifetimeDefault::Ambiguous => {
+                            failed = true;
+                            re_static
+                        }
+                        ObjectLifetimeDefault::Param(param_def_id) => {
+                            let index = generics.param_def_id_to_index[&param_def_id] as usize;
+                            args.get(index).and_then(|arg| arg.as_region()).unwrap_or_else(|| {
+                                failed = true;
+                                re_static
+                            })
+                        }
+                        ObjectLifetimeDefault::Static => re_static,
+                    };
+
+                    has_dyn = true;
+
+                    Ty::new_dynamic(tcx, obj, implied_region, ty::DynKind::Dyn).into()
+                } else {
+                    arg
+                }
+            });
+            let elaborated_ty = Ty::new_adt(tcx, *def, tcx.mk_args_from_iter(elaborated_args));
+
+            if has_dyn && !failed {
+                err.note(format!(
+                    "due to object lifetime defaults, `{unsize_ty}` actually means `{elaborated_ty}`"
+                ));
+            }
+        }
+    }
+
     fn add_lifetime_bound_suggestion_to_diagnostic(
         &self,
         err: &mut Diagnostic,
@@ -364,7 +422,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         kind_place: Option<(WriteKind, Place<'tcx>)>,
     ) -> BorrowExplanation<'tcx> {
         let regioncx = &self.regioncx;
-        let body: &Body<'_> = &self.body;
+        let body: &Body<'_> = self.body;
         let tcx = self.infcx.tcx;
 
         let borrow_region_vid = borrow.region;
diff --git a/compiler/rustc_borrowck/src/diagnostics/mod.rs b/compiler/rustc_borrowck/src/diagnostics/mod.rs
index c4323fef9..ee3213654 100644
--- a/compiler/rustc_borrowck/src/diagnostics/mod.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/mod.rs
@@ -10,7 +10,8 @@ use rustc_hir as hir;
 use rustc_hir::def::{CtorKind, Namespace};
 use rustc_hir::CoroutineKind;
 use rustc_index::IndexSlice;
-use rustc_infer::infer::LateBoundRegionConversionTime;
+use rustc_infer::infer::BoundRegionConversionTime;
+use rustc_infer::traits::{FulfillmentErrorCode, SelectionError};
 use rustc_middle::mir::tcx::PlaceTy;
 use rustc_middle::mir::{
     AggregateKind, CallSource, ConstOperand, FakeReadCause, Local, LocalInfo, LocalKind, Location,
@@ -24,10 +25,9 @@ use rustc_mir_dataflow::move_paths::{InitLocation, LookupResult};
 use rustc_span::def_id::LocalDefId;
 use rustc_span::{symbol::sym, Span, Symbol, DUMMY_SP};
 use rustc_target::abi::{FieldIdx, VariantIdx};
-use rustc_trait_selection::traits::query::evaluate_obligation::InferCtxtExt;
-use rustc_trait_selection::traits::{
-    type_known_to_meet_bound_modulo_regions, Obligation, ObligationCause,
-};
+use rustc_trait_selection::infer::InferCtxtExt;
+use rustc_trait_selection::traits::error_reporting::suggestions::TypeErrCtxtExt as _;
+use rustc_trait_selection::traits::type_known_to_meet_bound_modulo_regions;
 
 use super::borrow_set::BorrowData;
 use super::MirBorrowckCtxt;
@@ -124,7 +124,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                         let did = did.expect_local();
                         if let Some((span, hir_place)) = self.infcx.tcx.closure_kind_origin(did) {
                             diag.eager_subdiagnostic(
-                                &self.infcx.tcx.sess.parse_sess.span_diagnostic,
+                                self.infcx.tcx.sess.dcx(),
                                 OnClosureNote::InvokedTwice {
                                     place_name: &ty::place_to_string_for_capture(
                                         self.infcx.tcx,
@@ -146,7 +146,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                 let did = did.expect_local();
                 if let Some((span, hir_place)) = self.infcx.tcx.closure_kind_origin(did) {
                     diag.eager_subdiagnostic(
-                        &self.infcx.tcx.sess.parse_sess.span_diagnostic,
+                        self.infcx.tcx.sess.dcx(),
                         OnClosureNote::MovedTwice {
                             place_name: &ty::place_to_string_for_capture(self.infcx.tcx, hir_place),
                             span: *span,
@@ -217,9 +217,9 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                         projection: place.projection.split_at(index + 1).0,
                     }) {
                         let var_index = field.index();
-                        buf = self.upvars[var_index].place.to_string(self.infcx.tcx);
+                        buf = self.upvars[var_index].to_string(self.infcx.tcx);
                         ok = Ok(());
-                        if !self.upvars[var_index].by_ref {
+                        if !self.upvars[var_index].is_by_ref() {
                             buf.insert(0, '*');
                         }
                     } else {
@@ -250,7 +250,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                         local,
                         projection: place.projection.split_at(index + 1).0,
                     }) {
-                        buf = self.upvars[field.index()].place.to_string(self.infcx.tcx);
+                        buf = self.upvars[field.index()].to_string(self.infcx.tcx);
                         ok = Ok(());
                     } else {
                         let field_name = self.describe_field(
@@ -354,7 +354,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                 ty::Adt(def, _) => {
                     let variant = if let Some(idx) = variant_index {
                         assert!(def.is_enum());
-                        &def.variant(idx)
+                        def.variant(idx)
                     } else {
                         def.non_enum_variant()
                     };
@@ -462,7 +462,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         // lifetimes without names with the value `'0`.
         if let ty::Ref(region, ..) = ty.kind() {
             match **region {
-                ty::ReLateBound(_, ty::BoundRegion { kind: br, .. })
+                ty::ReBound(_, ty::BoundRegion { kind: br, .. })
                 | ty::RePlaceholder(ty::PlaceholderRegion {
                     bound: ty::BoundRegion { kind: br, .. },
                     ..
@@ -482,7 +482,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
 
         let region = if let ty::Ref(region, ..) = ty.kind() {
             match **region {
-                ty::ReLateBound(_, ty::BoundRegion { kind: br, .. })
+                ty::ReBound(_, ty::BoundRegion { kind: br, .. })
                 | ty::RePlaceholder(ty::PlaceholderRegion {
                     bound: ty::BoundRegion { kind: br, .. },
                     ..
@@ -624,7 +624,7 @@ impl UseSpans<'_> {
     /// Add a subdiagnostic to the use of the captured variable, if it exists.
     pub(super) fn var_subdiag(
         self,
-        handler: Option<&rustc_errors::Handler>,
+        dcx: Option<&rustc_errors::DiagCtxt>,
         err: &mut Diagnostic,
         kind: Option<rustc_middle::mir::BorrowKind>,
         f: impl FnOnce(Option<CoroutineKind>, Span) -> CaptureVarCause,
@@ -646,7 +646,7 @@ impl UseSpans<'_> {
                 });
             };
             let diag = f(coroutine_kind, path_span);
-            match handler {
+            match dcx {
                 Some(hd) => err.eager_subdiagnostic(hd, diag),
                 None => err.subdiagnostic(diag),
             };
@@ -851,7 +851,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         {
             let Some((method_did, method_args)) = rustc_middle::util::find_self_call(
                 self.infcx.tcx,
-                &self.body,
+                self.body,
                 target_temp,
                 location.block,
             ) else {
@@ -958,7 +958,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
             "closure_span: def_id={:?} target_place={:?} places={:?}",
             def_id, target_place, places
         );
-        let hir_id = self.infcx.tcx.hir().local_def_id_to_hir_id(def_id);
+        let hir_id = self.infcx.tcx.local_def_id_to_hir_id(def_id);
         let expr = &self.infcx.tcx.hir().expect_expr(hir_id).kind;
         debug!("closure_span: hir_id={:?} expr={:?}", hir_id, expr);
         if let hir::ExprKind::Closure(&hir::Closure { body, fn_decl_span, .. }) = expr {
@@ -1043,12 +1043,43 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                 }
                 CallKind::Normal { self_arg, desugaring, method_did, method_args } => {
                     let self_arg = self_arg.unwrap();
+                    let mut has_sugg = false;
                     let tcx = self.infcx.tcx;
+                    // Avoid pointing to the same function in multiple different
+                    // error messages.
+                    if span != DUMMY_SP && self.fn_self_span_reported.insert(self_arg.span) {
+                        self.explain_iterator_advancement_in_for_loop_if_applicable(
+                            err,
+                            span,
+                            &move_spans,
+                        );
+
+                        let func = tcx.def_path_str(method_did);
+                        err.subdiagnostic(CaptureReasonNote::FuncTakeSelf {
+                            func,
+                            place_name: place_name.clone(),
+                            span: self_arg.span,
+                        });
+                    }
+                    let parent_did = tcx.parent(method_did);
+                    let parent_self_ty =
+                        matches!(tcx.def_kind(parent_did), rustc_hir::def::DefKind::Impl { .. })
+                            .then_some(parent_did)
+                            .and_then(|did| match tcx.type_of(did).instantiate_identity().kind() {
+                                ty::Adt(def, ..) => Some(def.did()),
+                                _ => None,
+                            });
+                    let is_option_or_result = parent_self_ty.is_some_and(|def_id| {
+                        matches!(tcx.get_diagnostic_name(def_id), Some(sym::Option | sym::Result))
+                    });
+                    if is_option_or_result && maybe_reinitialized_locations_is_empty {
+                        err.subdiagnostic(CaptureReasonLabel::BorrowContent { var_span });
+                    }
                     if let Some((CallDesugaringKind::ForLoopIntoIter, _)) = desugaring {
                         let ty = moved_place.ty(self.body, tcx).ty;
                         let suggest = match tcx.get_diagnostic_item(sym::IntoIterator) {
                             Some(def_id) => type_known_to_meet_bound_modulo_regions(
-                                &self.infcx,
+                                self.infcx,
                                 self.param_env,
                                 Ty::new_imm_ref(tcx, tcx.lifetimes.re_erased, ty),
                                 def_id,
@@ -1108,72 +1139,92 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                         // Erase and shadow everything that could be passed to the new infcx.
                         let ty = moved_place.ty(self.body, tcx).ty;
 
-                        if let ty::Adt(def, args) = ty.kind()
+                        if let ty::Adt(def, args) = ty.peel_refs().kind()
                             && Some(def.did()) == tcx.lang_items().pin_type()
                             && let ty::Ref(_, _, hir::Mutability::Mut) = args.type_at(0).kind()
                             && let self_ty = self.infcx.instantiate_binder_with_fresh_vars(
                                 fn_call_span,
-                                LateBoundRegionConversionTime::FnCall,
+                                BoundRegionConversionTime::FnCall,
                                 tcx.fn_sig(method_did).instantiate(tcx, method_args).input(0),
                             )
                             && self.infcx.can_eq(self.param_env, ty, self_ty)
                         {
                             err.eager_subdiagnostic(
-                                &self.infcx.tcx.sess.parse_sess.span_diagnostic,
+                                self.infcx.tcx.sess.dcx(),
                                 CaptureReasonSuggest::FreshReborrow {
                                     span: move_span.shrink_to_hi(),
                                 },
                             );
+                            has_sugg = true;
                         }
-                        if let Some(clone_trait) = tcx.lang_items().clone_trait()
-                            && let trait_ref = ty::TraitRef::new(tcx, clone_trait, [ty])
-                            && let o = Obligation::new(
-                                tcx,
-                                ObligationCause::dummy(),
-                                self.param_env,
-                                ty::Binder::dummy(trait_ref),
-                            )
-                            && self.infcx.predicate_must_hold_modulo_regions(&o)
-                        {
-                            err.span_suggestion_verbose(
-                                move_span.shrink_to_hi(),
-                                "you can `clone` the value and consume it, but this might not be \
-                                 your desired behavior",
-                                ".clone()".to_string(),
-                                Applicability::MaybeIncorrect,
-                            );
+                        if let Some(clone_trait) = tcx.lang_items().clone_trait() {
+                            let sugg = if moved_place
+                                .iter_projections()
+                                .any(|(_, elem)| matches!(elem, ProjectionElem::Deref))
+                            {
+                                vec![
+                                    // We use the fully-qualified path because `.clone()` can
+                                    // sometimes choose `<&T as Clone>` instead of `<T as Clone>`
+                                    // when going through auto-deref, so this ensures that doesn't
+                                    // happen, causing suggestions for `.clone().clone()`.
+                                    (move_span.shrink_to_lo(), format!("<{ty} as Clone>::clone(&")),
+                                    (move_span.shrink_to_hi(), ")".to_string()),
+                                ]
+                            } else {
+                                vec![(move_span.shrink_to_hi(), ".clone()".to_string())]
+                            };
+                            if let Some(errors) =
+                                self.infcx.could_impl_trait(clone_trait, ty, self.param_env)
+                                && !has_sugg
+                            {
+                                let msg = match &errors[..] {
+                                    [] => "you can `clone` the value and consume it, but this \
+                                            might not be your desired behavior"
+                                        .to_string(),
+                                    [error] => {
+                                        format!(
+                                            "you could `clone` the value and consume it, if \
+                                                the `{}` trait bound could be satisfied",
+                                            error.obligation.predicate,
+                                        )
+                                    }
+                                    [errors @ .., last] => {
+                                        format!(
+                                            "you could `clone` the value and consume it, if \
+                                                the following trait bounds could be satisfied: {} \
+                                                and `{}`",
+                                            errors
+                                                .iter()
+                                                .map(|e| format!("`{}`", e.obligation.predicate))
+                                                .collect::<Vec<_>>()
+                                                .join(", "),
+                                            last.obligation.predicate,
+                                        )
+                                    }
+                                };
+                                err.multipart_suggestion_verbose(
+                                    msg,
+                                    sugg,
+                                    Applicability::MaybeIncorrect,
+                                );
+                                for error in errors {
+                                    if let FulfillmentErrorCode::CodeSelectionError(
+                                        SelectionError::Unimplemented,
+                                    ) = error.code
+                                        && let ty::PredicateKind::Clause(ty::ClauseKind::Trait(
+                                            pred,
+                                        )) = error.obligation.predicate.kind().skip_binder()
+                                    {
+                                        self.infcx.err_ctxt().suggest_derive(
+                                            &error.obligation,
+                                            err,
+                                            error.obligation.predicate.kind().rebind(pred),
+                                        );
+                                    }
+                                }
+                            }
                         }
                     }
-                    // Avoid pointing to the same function in multiple different
-                    // error messages.
-                    if span != DUMMY_SP && self.fn_self_span_reported.insert(self_arg.span) {
-                        self.explain_iterator_advancement_in_for_loop_if_applicable(
-                            err,
-                            span,
-                            &move_spans,
-                        );
-
-                        let func = tcx.def_path_str(method_did);
-                        err.subdiagnostic(CaptureReasonNote::FuncTakeSelf {
-                            func,
-                            place_name,
-                            span: self_arg.span,
-                        });
-                    }
-                    let parent_did = tcx.parent(method_did);
-                    let parent_self_ty =
-                        matches!(tcx.def_kind(parent_did), rustc_hir::def::DefKind::Impl { .. })
-                            .then_some(parent_did)
-                            .and_then(|did| match tcx.type_of(did).instantiate_identity().kind() {
-                                ty::Adt(def, ..) => Some(def.did()),
-                                _ => None,
-                            });
-                    let is_option_or_result = parent_self_ty.is_some_and(|def_id| {
-                        matches!(tcx.get_diagnostic_name(def_id), Some(sym::Option | sym::Result))
-                    });
-                    if is_option_or_result && maybe_reinitialized_locations_is_empty {
-                        err.subdiagnostic(CaptureReasonLabel::BorrowContent { var_span });
-                    }
                 }
                 // Other desugarings takes &self, which cannot cause a move
                 _ => {}
diff --git a/compiler/rustc_borrowck/src/diagnostics/move_errors.rs b/compiler/rustc_borrowck/src/diagnostics/move_errors.rs
index 41d6b98d7..43487b85a 100644
--- a/compiler/rustc_borrowck/src/diagnostics/move_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/move_errors.rs
@@ -321,7 +321,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
 
         let deref_base = match deref_target_place.projection.as_ref() {
             [proj_base @ .., ProjectionElem::Deref] => {
-                PlaceRef { local: deref_target_place.local, projection: &proj_base }
+                PlaceRef { local: deref_target_place.local, projection: proj_base }
             }
             _ => bug!("deref_target_place is not a deref projection"),
         };
@@ -363,8 +363,8 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
                     format!("captured variable in an `{closure_kind}` closure");
 
                 let upvar = &self.upvars[upvar_field.unwrap().index()];
-                let upvar_hir_id = upvar.place.get_root_variable();
-                let upvar_name = upvar.place.to_string(self.infcx.tcx);
+                let upvar_hir_id = upvar.get_root_variable();
+                let upvar_name = upvar.to_string(self.infcx.tcx);
                 let upvar_span = self.infcx.tcx.hir().span(upvar_hir_id);
 
                 let place_name = self.describe_any_place(move_place.as_ref());
@@ -583,7 +583,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
                 err.subdiagnostic(crate::session_diagnostics::TypeNoCopy::Label {
                     is_partial_move: false,
                     ty: bind_to.ty,
-                    place: &place_desc,
+                    place: place_desc,
                     span: binding_span,
                 });
             }
@@ -607,7 +607,8 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
 
         if let Some(adt) = local_ty.ty_adt_def()
             && adt.repr().packed()
-            && let ExpnKind::Macro(MacroKind::Derive, name) = self.body.span.ctxt().outer_expn_data().kind
+            && let ExpnKind::Macro(MacroKind::Derive, name) =
+                self.body.span.ctxt().outer_expn_data().kind
         {
             err.note(format!("`#[derive({name})]` triggers a move because taking references to the fields of a packed struct is undefined behaviour"));
         }
diff --git a/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs b/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs
index dde46eef6..c3c1f1293 100644
--- a/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs
@@ -3,8 +3,9 @@ use rustc_errors::{Applicability, Diagnostic, DiagnosticBuilder, ErrorGuaranteed
 use rustc_hir as hir;
 use rustc_hir::intravisit::Visitor;
 use rustc_hir::Node;
+use rustc_infer::traits;
 use rustc_middle::mir::{Mutability, Place, PlaceRef, ProjectionElem};
-use rustc_middle::ty::{self, InstanceDef, Ty, TyCtxt};
+use rustc_middle::ty::{self, InstanceDef, ToPredicate, Ty, TyCtxt};
 use rustc_middle::{
     hir::place::PlaceBase,
     mir::{self, BindingForm, Local, LocalDecl, LocalInfo, LocalKind, Location},
@@ -12,6 +13,8 @@ use rustc_middle::{
 use rustc_span::symbol::{kw, Symbol};
 use rustc_span::{sym, BytePos, DesugaringKind, Span};
 use rustc_target::abi::FieldIdx;
+use rustc_trait_selection::infer::InferCtxtExt;
+use rustc_trait_selection::traits::error_reporting::suggestions::TypeErrCtxtExt;
 
 use crate::diagnostics::BorrowedContentSource;
 use crate::util::FindAssignments;
@@ -67,7 +70,6 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
 
                 let imm_borrow_derefed = self.upvars[upvar_index.index()]
                     .place
-                    .place
                     .deref_tys()
                     .any(|ty| matches!(ty.kind(), ty::Ref(.., hir::Mutability::Not)));
 
@@ -85,7 +87,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
                     if self.is_upvar_field_projection(access_place.as_ref()).is_some() {
                         reason = ", as it is not declared as mutable".to_string();
                     } else {
-                        let name = self.upvars[upvar_index.index()].place.to_string(self.infcx.tcx);
+                        let name = self.upvars[upvar_index.index()].to_string(self.infcx.tcx);
                         reason = format!(", as `{name}` is not declared as mutable");
                     }
                 }
@@ -388,13 +390,13 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
                     Place::ty_from(local, proj_base, self.body, self.infcx.tcx).ty
                 ));
 
-                let captured_place = &self.upvars[upvar_index.index()].place;
+                let captured_place = self.upvars[upvar_index.index()];
 
                 err.span_label(span, format!("cannot {act}"));
 
                 let upvar_hir_id = captured_place.get_root_variable();
 
-                if let Some(Node::Pat(pat)) = self.infcx.tcx.hir().find(upvar_hir_id)
+                if let Some(Node::Pat(pat)) = self.infcx.tcx.opt_hir_node(upvar_hir_id)
                     && let hir::PatKind::Binding(hir::BindingAnnotation::NONE, _, upvar_ident, _) =
                         pat.kind
                 {
@@ -659,9 +661,8 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
         if self.body.local_kind(local) != LocalKind::Arg {
             return (false, None);
         }
-        let hir_map = self.infcx.tcx.hir();
         let my_def = self.body.source.def_id();
-        let my_hir = hir_map.local_def_id_to_hir_id(my_def.as_local().unwrap());
+        let my_hir = self.infcx.tcx.local_def_id_to_hir_id(my_def.as_local().unwrap());
         let Some(td) =
             self.infcx.tcx.impl_of_method(my_def).and_then(|x| self.infcx.tcx.trait_id_of_impl(x))
         else {
@@ -669,7 +670,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
         };
         (
             true,
-            td.as_local().and_then(|tld| match hir_map.find_by_def_id(tld) {
+            td.as_local().and_then(|tld| match self.infcx.tcx.opt_hir_node_by_def_id(tld) {
                 Some(Node::Item(hir::Item {
                     kind: hir::ItemKind::Trait(_, _, _, _, items),
                     ..
@@ -680,25 +681,27 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
                         if !matches!(k, hir::AssocItemKind::Fn { .. }) {
                             continue;
                         }
-                        if hir_map.name(hi) != hir_map.name(my_hir) {
+                        if self.infcx.tcx.hir().name(hi) != self.infcx.tcx.hir().name(my_hir) {
                             continue;
                         }
                         f_in_trait_opt = Some(hi);
                         break;
                     }
-                    f_in_trait_opt.and_then(|f_in_trait| match hir_map.find(f_in_trait) {
-                        Some(Node::TraitItem(hir::TraitItem {
-                            kind:
-                                hir::TraitItemKind::Fn(
-                                    hir::FnSig { decl: hir::FnDecl { inputs, .. }, .. },
-                                    _,
-                                ),
-                            ..
-                        })) => {
-                            let hir::Ty { span, .. } = inputs[local.index() - 1];
-                            Some(span)
+                    f_in_trait_opt.and_then(|f_in_trait| {
+                        match self.infcx.tcx.opt_hir_node(f_in_trait) {
+                            Some(Node::TraitItem(hir::TraitItem {
+                                kind:
+                                    hir::TraitItemKind::Fn(
+                                        hir::FnSig { decl: hir::FnDecl { inputs, .. }, .. },
+                                        _,
+                                    ),
+                                ..
+                            })) => {
+                                let hir::Ty { span, .. } = inputs[local.index() - 1];
+                                Some(span)
+                            }
+                            _ => None,
                         }
-                        _ => None,
                     })
                 }
                 _ => None,
@@ -739,12 +742,11 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
             }
         }
 
-        let hir_map = self.infcx.tcx.hir();
         let def_id = self.body.source.def_id();
         let hir_id = if let Some(local_def_id) = def_id.as_local()
-            && let Some(body_id) = hir_map.maybe_body_owned_by(local_def_id)
+            && let Some(body_id) = self.infcx.tcx.hir().maybe_body_owned_by(local_def_id)
         {
-            let body = hir_map.body(body_id);
+            let body = self.infcx.tcx.hir().body(body_id);
             let mut v = BindingFinder { span: pat_span, hir_id: None };
             v.visit_body(body);
             v.hir_id
@@ -760,7 +762,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
             && let Some(hir::Node::Local(hir::Local {
                 pat: hir::Pat { kind: hir::PatKind::Ref(_, _), .. },
                 ..
-            })) = hir_map.find(hir_id)
+            })) = self.infcx.tcx.opt_hir_node(hir_id)
             && let Ok(name) =
                 self.infcx.tcx.sess.source_map().span_to_snippet(local_decl.source_info.span)
         {
@@ -940,7 +942,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
         let closure_id = self.mir_hir_id();
         let closure_span = self.infcx.tcx.def_span(self.mir_def_id());
         let fn_call_id = hir.parent_id(closure_id);
-        let node = hir.get(fn_call_id);
+        let node = self.infcx.tcx.hir_node(fn_call_id);
         let def_id = hir.enclosing_body_owner(fn_call_id);
         let mut look_at_return = true;
         // If we can detect the expression to be an `fn` call where the closure was an argument,
@@ -999,7 +1001,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
         if look_at_return && hir.get_return_block(closure_id).is_some() {
             // ...otherwise we are probably in the tail expression of the function, point at the
             // return type.
-            match hir.get_by_def_id(hir.get_parent_item(fn_call_id).def_id) {
+            match self.infcx.tcx.hir_node_by_def_id(hir.get_parent_item(fn_call_id).def_id) {
                 hir::Node::Item(hir::Item { ident, kind: hir::ItemKind::Fn(sig, ..), .. })
                 | hir::Node::TraitItem(hir::TraitItem {
                     ident,
@@ -1197,12 +1199,11 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
                         hir::intravisit::walk_stmt(self, s);
                     }
                 }
-                let hir_map = self.infcx.tcx.hir();
                 let def_id = self.body.source.def_id();
                 let hir_id = if let Some(local_def_id) = def_id.as_local()
-                    && let Some(body_id) = hir_map.maybe_body_owned_by(local_def_id)
+                    && let Some(body_id) = self.infcx.tcx.hir().maybe_body_owned_by(local_def_id)
                 {
-                    let body = hir_map.body(body_id);
+                    let body = self.infcx.tcx.hir().body(body_id);
                     let mut v = BindingFinder { span: err_label_span, hir_id: None };
                     v.visit_body(body);
                     v.hir_id
@@ -1211,8 +1212,105 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
                 };
 
                 if let Some(hir_id) = hir_id
-                    && let Some(hir::Node::Local(local)) = hir_map.find(hir_id)
+                    && let Some(hir::Node::Local(local)) = self.infcx.tcx.opt_hir_node(hir_id)
                 {
+                    let tables = self.infcx.tcx.typeck(def_id.as_local().unwrap());
+                    if let Some(clone_trait) = self.infcx.tcx.lang_items().clone_trait()
+                        && let Some(expr) = local.init
+                        && let ty = tables.node_type_opt(expr.hir_id)
+                        && let Some(ty) = ty
+                        && let ty::Ref(..) = ty.kind()
+                    {
+                        match self
+                            .infcx
+                            .could_impl_trait(clone_trait, ty.peel_refs(), self.param_env)
+                            .as_deref()
+                        {
+                            Some([]) => {
+                                // The type implements Clone.
+                                err.span_help(
+                                    expr.span,
+                                    format!(
+                                        "you can `clone` the `{}` value and consume it, but this \
+                                         might not be your desired behavior",
+                                        ty.peel_refs(),
+                                    ),
+                                );
+                            }
+                            None => {
+                                if let hir::ExprKind::MethodCall(segment, _rcvr, [], span) =
+                                    expr.kind
+                                    && segment.ident.name == sym::clone
+                                {
+                                    err.span_help(
+                                        span,
+                                        format!(
+                                            "`{}` doesn't implement `Clone`, so this call clones \
+                                             the reference `{ty}`",
+                                            ty.peel_refs(),
+                                        ),
+                                    );
+                                }
+                                // The type doesn't implement Clone.
+                                let trait_ref = ty::Binder::dummy(ty::TraitRef::new(
+                                    self.infcx.tcx,
+                                    clone_trait,
+                                    [ty.peel_refs()],
+                                ));
+                                let obligation = traits::Obligation::new(
+                                    self.infcx.tcx,
+                                    traits::ObligationCause::dummy(),
+                                    self.param_env,
+                                    trait_ref,
+                                );
+                                self.infcx.err_ctxt().suggest_derive(
+                                    &obligation,
+                                    err,
+                                    trait_ref.to_predicate(self.infcx.tcx),
+                                );
+                            }
+                            Some(errors) => {
+                                if let hir::ExprKind::MethodCall(segment, _rcvr, [], span) =
+                                    expr.kind
+                                    && segment.ident.name == sym::clone
+                                {
+                                    err.span_help(
+                                        span,
+                                        format!(
+                                            "`{}` doesn't implement `Clone` because its \
+                                             implementations trait bounds could not be met, so \
+                                             this call clones the reference `{ty}`",
+                                            ty.peel_refs(),
+                                        ),
+                                    );
+                                    err.note(format!(
+                                        "the following trait bounds weren't met: {}",
+                                        errors
+                                            .iter()
+                                            .map(|e| e.obligation.predicate.to_string())
+                                            .collect::<Vec<_>>()
+                                            .join("\n"),
+                                    ));
+                                }
+                                // The type doesn't implement Clone because of unmet obligations.
+                                for error in errors {
+                                    if let traits::FulfillmentErrorCode::CodeSelectionError(
+                                        traits::SelectionError::Unimplemented,
+                                    ) = error.code
+                                        && let ty::PredicateKind::Clause(ty::ClauseKind::Trait(
+                                            pred,
+                                        )) = error.obligation.predicate.kind().skip_binder()
+                                    {
+                                        self.infcx.err_ctxt().suggest_derive(
+                                            &error.obligation,
+                                            err,
+                                            error.obligation.predicate.kind().rebind(pred),
+                                        );
+                                    }
+                                }
+                            }
+                        }
+                    }
                     let (changing, span, sugg) = match local.ty {
                         Some(ty) => ("changing", ty.span, message),
                         None => {
@@ -1397,7 +1495,7 @@ fn get_mut_span_in_struct_field<'tcx>(
         && let ty::Adt(def, _) = ty.kind()
         && let field = def.all_fields().nth(field.index())?
         // Use the HIR types to construct the diagnostic message.
-        && let node = tcx.hir().find_by_def_id(field.did.as_local()?)?
+        && let node = tcx.opt_hir_node_by_def_id(field.did.as_local()?)?
         // Now we're dealing with the actual struct that we're going to suggest a change to,
         // we can expect a field that is an immutable reference to a type.
         && let hir::Node::Field(field) = node
diff --git a/compiler/rustc_borrowck/src/diagnostics/outlives_suggestion.rs b/compiler/rustc_borrowck/src/diagnostics/outlives_suggestion.rs
index b6eb9ae98..66275888c 100644
--- a/compiler/rustc_borrowck/src/diagnostics/outlives_suggestion.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/outlives_suggestion.rs
@@ -50,8 +50,8 @@ impl OutlivesSuggestionBuilder {
     // naming the `'self` lifetime in methods, etc.
     fn region_name_is_suggestable(name: &RegionName) -> bool {
         match name.source {
-            RegionNameSource::NamedEarlyBoundRegion(..)
-            | RegionNameSource::NamedFreeRegion(..)
+            RegionNameSource::NamedEarlyParamRegion(..)
+            | RegionNameSource::NamedLateParamRegion(..)
             | RegionNameSource::Static => true,
 
             // Don't give suggestions for upvars, closure return types, or other unnameable
@@ -206,7 +206,7 @@ impl OutlivesSuggestionBuilder {
         // If there is exactly one suggestable constraints, then just suggest it. Otherwise, emit a
         // list of diagnostics.
         let mut diag = if suggested.len() == 1 {
-            mbcx.infcx.tcx.sess.diagnostic().struct_help(match suggested.last().unwrap() {
+            mbcx.infcx.tcx.sess.dcx().struct_help(match suggested.last().unwrap() {
                 SuggestedConstraint::Outlives(a, bs) => {
                     let bs: SmallVec<[String; 2]> = bs.iter().map(|r| r.to_string()).collect();
                     format!("add bound `{a}: {}`", bs.join(" + "))
@@ -223,7 +223,7 @@ impl OutlivesSuggestionBuilder {
                 .infcx
                 .tcx
                 .sess
-                .diagnostic()
+                .dcx()
                 .struct_help("the following changes may resolve your lifetime errors");
 
             // Add suggestions.
diff --git a/compiler/rustc_borrowck/src/diagnostics/region_errors.rs b/compiler/rustc_borrowck/src/diagnostics/region_errors.rs
index a0a809123..759f5e910 100644
--- a/compiler/rustc_borrowck/src/diagnostics/region_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/region_errors.rs
@@ -35,7 +35,7 @@ use crate::session_diagnostics::{
     LifetimeReturnCategoryErr, RequireStaticErr, VarHereDenote,
 };
 
-use super::{OutlivesSuggestionBuilder, RegionName};
+use super::{OutlivesSuggestionBuilder, RegionName, RegionNameSource};
 use crate::region_infer::{BlameConstraint, ExtraConstraintInfo};
 use crate::{
     nll::ConstraintDescription,
@@ -53,7 +53,7 @@ impl<'tcx> ConstraintDescription for ConstraintCategory<'tcx> {
             ConstraintCategory::Yield => "yielding this value ",
             ConstraintCategory::UseAsConst => "using this value as a constant ",
             ConstraintCategory::UseAsStatic => "using this value as a static ",
-            ConstraintCategory::Cast => "cast ",
+            ConstraintCategory::Cast { .. } => "cast ",
             ConstraintCategory::CallArgument(_) => "argument ",
             ConstraintCategory::TypeAnnotation => "type annotation ",
             ConstraintCategory::ClosureBounds => "closure body ",
@@ -84,7 +84,7 @@ impl<'tcx> RegionErrors<'tcx> {
     #[track_caller]
     pub fn push(&mut self, val: impl Into<RegionErrorKind<'tcx>>) {
         let val = val.into();
-        self.1.sess.delay_span_bug(DUMMY_SP, format!("{val:?}"));
+        self.1.sess.span_delayed_bug(DUMMY_SP, format!("{val:?}"));
         self.0.push(val);
     }
     pub fn is_empty(&self) -> bool {
@@ -95,6 +95,12 @@ impl<'tcx> RegionErrors<'tcx> {
     }
 }
 
+impl std::fmt::Debug for RegionErrors<'_> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_tuple("RegionErrors").field(&self.0).finish()
+    }
+}
+
 #[derive(Clone, Debug)]
 pub(crate) enum RegionErrorKind<'tcx> {
     /// A generic bound failure for a type test (`T: 'a`).
@@ -181,8 +187,8 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
 
     /// Returns `true` if a closure is inferred to be an `FnMut` closure.
     fn is_closure_fn_mut(&self, fr: RegionVid) -> bool {
-        if let Some(ty::ReFree(free_region)) = self.to_error_region(fr).as_deref()
-            && let ty::BoundRegionKind::BrEnv = free_region.bound_region
+        if let Some(ty::ReLateParam(late_param)) = self.to_error_region(fr).as_deref()
+            && let ty::BoundRegionKind::BrEnv = late_param.bound_region
             && let DefiningTy::Closure(_, args) = self.regioncx.universal_regions().defining_ty
         {
             return args.as_closure().kind() == ty::ClosureKind::FnMut;
@@ -209,7 +215,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
             .map(|placeholder| {
                 if let Some(id) = placeholder.bound.kind.get_id()
                     && let Some(placeholder_id) = id.as_local()
-                    && let gat_hir_id = hir.local_def_id_to_hir_id(placeholder_id)
+                    && let gat_hir_id = self.infcx.tcx.local_def_id_to_hir_id(placeholder_id)
                     && let Some(generics_impl) = hir.get_parent(gat_hir_id).generics()
                 {
                     Some((gat_hir_id, generics_impl))
@@ -230,7 +236,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
                 };
                 if bound_generic_params
                     .iter()
-                    .rfind(|bgp| hir.local_def_id_to_hir_id(bgp.def_id) == *gat_hir_id)
+                    .rfind(|bgp| self.infcx.tcx.local_def_id_to_hir_id(bgp.def_id) == *gat_hir_id)
                     .is_some()
                 {
                     for bound in *bounds {
@@ -599,7 +605,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
             };
 
             let captured_place = &self.upvars[upvar_field.index()].place;
-            let defined_hir = match captured_place.place.base {
+            let defined_hir = match captured_place.base {
                 PlaceBase::Local(hirid) => Some(hirid),
                 PlaceBase::Upvar(upvar) => Some(upvar.var_path.hir_id),
                 _ => None,
@@ -644,14 +650,14 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
 
         let fr_name_and_span = self.regioncx.get_var_name_and_span_for_region(
             self.infcx.tcx,
-            &self.body,
+            self.body,
             &self.local_names,
             &self.upvars,
             errci.fr,
         );
         let outlived_fr_name_and_span = self.regioncx.get_var_name_and_span_for_region(
             self.infcx.tcx,
-            &self.body,
+            self.body,
             &self.local_names,
             &self.upvars,
             errci.outlived_fr,
@@ -757,7 +763,14 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
         let err = LifetimeOutliveErr { span: *span };
         let mut diag = self.infcx.tcx.sess.create_err(err);
 
-        let fr_name = self.give_region_a_name(*fr).unwrap();
+        // In certain scenarios, such as the one described in issue #118021,
+        // we might encounter a lifetime that cannot be named.
+        // These situations are bound to result in errors.
+        // To prevent an immediate ICE, we opt to create a dummy name instead.
+        let fr_name = self.give_region_a_name(*fr).unwrap_or(RegionName {
+            name: kw::UnderscoreLifetime,
+            source: RegionNameSource::Static,
+        });
         fr_name.highlight_region_name(&mut diag);
         let outlived_fr_name = self.give_region_a_name(*outlived_fr).unwrap();
         outlived_fr_name.highlight_region_name(&mut diag);
@@ -958,7 +971,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
         for found_did in found_dids {
             let mut traits = vec![];
             let mut hir_v = HirTraitObjectVisitor(&mut traits, *found_did);
-            hir_v.visit_ty(&self_ty);
+            hir_v.visit_ty(self_ty);
             debug!("trait spans found: {:?}", traits);
             for span in &traits {
                 let mut multi_span: MultiSpan = vec![*span].into();
@@ -995,7 +1008,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
             .infcx
             .tcx
             .is_suitable_region(sub)
-            .and_then(|anon_reg| find_anon_type(self.infcx.tcx, sub, &anon_reg.boundregion))
+            .and_then(|anon_reg| find_anon_type(self.infcx.tcx, sub, &anon_reg.bound_region))
         else {
             return;
         };
@@ -1004,7 +1017,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
             .infcx
             .tcx
             .is_suitable_region(sup)
-            .and_then(|anon_reg| find_anon_type(self.infcx.tcx, sup, &anon_reg.boundregion))
+            .and_then(|anon_reg| find_anon_type(self.infcx.tcx, sup, &anon_reg.bound_region))
         else {
             return;
         };
diff --git a/compiler/rustc_borrowck/src/diagnostics/region_name.rs b/compiler/rustc_borrowck/src/diagnostics/region_name.rs
index d38cfbc54..8441dfaa7 100644
--- a/compiler/rustc_borrowck/src/diagnostics/region_name.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/region_name.rs
@@ -23,14 +23,14 @@ pub(crate) struct RegionName {
 }
 
 /// Denotes the source of a region that is named by a `RegionName`. For example, a free region that
-/// was named by the user would get `NamedFreeRegion` and `'static` lifetime would get `Static`.
+/// was named by the user would get `NamedLateParamRegion` and `'static` lifetime would get `Static`.
 /// This helps to print the right kinds of diagnostics.
 #[derive(Debug, Clone)]
 pub(crate) enum RegionNameSource {
     /// A bound (not free) region that was instantiated at the def site (not an HRTB).
-    NamedEarlyBoundRegion(Span),
+    NamedEarlyParamRegion(Span),
     /// A free region that the user has a name (`'a`) for.
-    NamedFreeRegion(Span),
+    NamedLateParamRegion(Span),
     /// The `'static` region.
     Static,
     /// The free region corresponding to the environment of a closure.
@@ -69,8 +69,8 @@ pub(crate) enum RegionNameHighlight {
 impl RegionName {
     pub(crate) fn was_named(&self) -> bool {
         match self.source {
-            RegionNameSource::NamedEarlyBoundRegion(..)
-            | RegionNameSource::NamedFreeRegion(..)
+            RegionNameSource::NamedEarlyParamRegion(..)
+            | RegionNameSource::NamedLateParamRegion(..)
             | RegionNameSource::Static => true,
             RegionNameSource::SynthesizedFreeEnvRegion(..)
             | RegionNameSource::AnonRegionFromArgument(..)
@@ -85,8 +85,8 @@ impl RegionName {
     pub(crate) fn span(&self) -> Option<Span> {
         match self.source {
             RegionNameSource::Static => None,
-            RegionNameSource::NamedEarlyBoundRegion(span)
-            | RegionNameSource::NamedFreeRegion(span)
+            RegionNameSource::NamedEarlyParamRegion(span)
+            | RegionNameSource::NamedLateParamRegion(span)
             | RegionNameSource::SynthesizedFreeEnvRegion(span, _)
             | RegionNameSource::AnonRegionFromUpvar(span, _)
             | RegionNameSource::AnonRegionFromYieldTy(span, _)
@@ -104,8 +104,8 @@ impl RegionName {
 
     pub(crate) fn highlight_region_name(&self, diag: &mut Diagnostic) {
         match &self.source {
-            RegionNameSource::NamedFreeRegion(span)
-            | RegionNameSource::NamedEarlyBoundRegion(span) => {
+            RegionNameSource::NamedLateParamRegion(span)
+            | RegionNameSource::NamedEarlyParamRegion(span) => {
                 diag.span_label(*span, format!("lifetime `{self}` defined here"));
             }
             RegionNameSource::SynthesizedFreeEnvRegion(span, note) => {
@@ -199,7 +199,7 @@ impl<'tcx> MirBorrowckCtxt<'_, 'tcx> {
     }
 
     pub(crate) fn mir_hir_id(&self) -> hir::HirId {
-        self.infcx.tcx.hir().local_def_id_to_hir_id(self.mir_def_id())
+        self.infcx.tcx.local_def_id_to_hir_id(self.mir_def_id())
     }
 
     /// Generate a synthetic region named `'N`, where `N` is the next value of the counter. Then,
@@ -280,28 +280,31 @@ impl<'tcx> MirBorrowckCtxt<'_, 'tcx> {
 
         debug!("give_region_a_name: error_region = {:?}", error_region);
         match *error_region {
-            ty::ReEarlyBound(ebr) => ebr.has_name().then(|| {
+            ty::ReEarlyParam(ebr) => ebr.has_name().then(|| {
                 let span = tcx.hir().span_if_local(ebr.def_id).unwrap_or(DUMMY_SP);
-                RegionName { name: ebr.name, source: RegionNameSource::NamedEarlyBoundRegion(span) }
+                RegionName { name: ebr.name, source: RegionNameSource::NamedEarlyParamRegion(span) }
             }),
 
             ty::ReStatic => {
                 Some(RegionName { name: kw::StaticLifetime, source: RegionNameSource::Static })
             }
 
-            ty::ReFree(free_region) => match free_region.bound_region {
+            ty::ReLateParam(late_param) => match late_param.bound_region {
                 ty::BoundRegionKind::BrNamed(region_def_id, name) => {
                     // Get the span to point to, even if we don't use the name.
                     let span = tcx.hir().span_if_local(region_def_id).unwrap_or(DUMMY_SP);
                     debug!(
                         "bound region named: {:?}, is_named: {:?}",
                         name,
-                        free_region.bound_region.is_named()
+                        late_param.bound_region.is_named()
                     );
 
-                    if free_region.bound_region.is_named() {
+                    if late_param.bound_region.is_named() {
                         // A named region that is actually named.
-                        Some(RegionName { name, source: RegionNameSource::NamedFreeRegion(span) })
+                        Some(RegionName {
+                            name,
+                            source: RegionNameSource::NamedLateParamRegion(span),
+                        })
                     } else if tcx.asyncness(self.mir_hir_id().owner).is_async() {
                         // If we spuriously thought that the region is named, we should let the
                         // system generate a true name for error messages. Currently this can
@@ -357,7 +360,7 @@ impl<'tcx> MirBorrowckCtxt<'_, 'tcx> {
                 ty::BoundRegionKind::BrAnon => None,
             },
 
-            ty::ReLateBound(..)
+            ty::ReBound(..)
             | ty::ReVar(..)
             | ty::RePlaceholder(..)
             | ty::ReErased
@@ -384,7 +387,7 @@ impl<'tcx> MirBorrowckCtxt<'_, 'tcx> {
         let arg_ty = self.regioncx.universal_regions().unnormalized_input_tys
             [implicit_inputs + argument_index];
         let (_, span) = self.regioncx.get_argument_name_and_span_for_region(
-            &self.body,
+            self.body,
             &self.local_names,
             argument_index,
         );
@@ -616,8 +619,8 @@ impl<'tcx> MirBorrowckCtxt<'_, 'tcx> {
                     _,
                 ) => {
                     // HIR lowering sometimes doesn't catch this in erroneous
-                    // programs, so we need to use delay_span_bug here. See #82126.
-                    self.infcx.tcx.sess.delay_span_bug(
+                    // programs, so we need to use span_delayed_bug here. See #82126.
+                    self.infcx.tcx.sess.span_delayed_bug(
                         hir_arg.span(),
                         format!("unmatched arg and hir arg: found {kind:?} vs {hir_arg:?}"),
                     );
@@ -669,7 +672,7 @@ impl<'tcx> MirBorrowckCtxt<'_, 'tcx> {
 
         let mir_hir_id = self.mir_hir_id();
 
-        let (return_span, mir_description, hir_ty) = match hir.get(mir_hir_id) {
+        let (return_span, mir_description, hir_ty) = match tcx.hir_node(mir_hir_id) {
             hir::Node::Expr(hir::Expr {
                 kind: hir::ExprKind::Closure(&hir::Closure { fn_decl, body, fn_decl_span, .. }),
                 ..
@@ -681,12 +684,12 @@ impl<'tcx> MirBorrowckCtxt<'_, 'tcx> {
                     hir::FnRetTy::Return(hir_ty) => (fn_decl.output.span(), Some(hir_ty)),
                 };
                 let mir_description = match hir.body(body).coroutine_kind {
-                    Some(hir::CoroutineKind::Async(gen)) => match gen {
+                    Some(hir::CoroutineKind::Async(src)) => match src {
                         hir::CoroutineSource::Block => " of async block",
                         hir::CoroutineSource::Closure => " of async closure",
                         hir::CoroutineSource::Fn => {
                             let parent_item =
-                                hir.get_by_def_id(hir.get_parent_item(mir_hir_id).def_id);
+                                tcx.hir_node_by_def_id(hir.get_parent_item(mir_hir_id).def_id);
                             let output = &parent_item
                                 .fn_decl()
                                 .expect("coroutine lowered from async fn should be in fn")
@@ -698,12 +701,12 @@ impl<'tcx> MirBorrowckCtxt<'_, 'tcx> {
                             " of async function"
                         }
                     },
-                    Some(hir::CoroutineKind::Gen(gen)) => match gen {
+                    Some(hir::CoroutineKind::Gen(src)) => match src {
                         hir::CoroutineSource::Block => " of gen block",
                         hir::CoroutineSource::Closure => " of gen closure",
                         hir::CoroutineSource::Fn => {
                             let parent_item =
-                                hir.get_by_def_id(hir.get_parent_item(mir_hir_id).def_id);
+                                tcx.hir_node_by_def_id(hir.get_parent_item(mir_hir_id).def_id);
                             let output = &parent_item
                                 .fn_decl()
                                 .expect("coroutine lowered from gen fn should be in fn")
@@ -712,6 +715,21 @@ impl<'tcx> MirBorrowckCtxt<'_, 'tcx> {
                             " of gen function"
                         }
                     },
+
+                    Some(hir::CoroutineKind::AsyncGen(src)) => match src {
+                        hir::CoroutineSource::Block => " of async gen block",
+                        hir::CoroutineSource::Closure => " of async gen closure",
+                        hir::CoroutineSource::Fn => {
+                            let parent_item =
+                                tcx.hir_node_by_def_id(hir.get_parent_item(mir_hir_id).def_id);
+                            let output = &parent_item
+                                .fn_decl()
+                                .expect("coroutine lowered from async gen fn should be in fn")
+                                .output;
+                            span = output.span();
+                            " of async gen function"
+                        }
+                    },
                     Some(hir::CoroutineKind::Coroutine) => " of coroutine",
                     None => " of closure",
                 };
@@ -770,28 +788,18 @@ impl<'tcx> MirBorrowckCtxt<'_, 'tcx> {
         };
         let opaque_ty = hir.item(id);
         if let hir::ItemKind::OpaqueTy(hir::OpaqueTy {
-            bounds:
-                [
-                    hir::GenericBound::LangItemTrait(
-                        hir::LangItem::Future,
-                        _,
-                        _,
-                        hir::GenericArgs {
-                            bindings:
-                                [
-                                    hir::TypeBinding {
-                                        ident: Ident { name: sym::Output, .. },
-                                        kind:
-                                            hir::TypeBindingKind::Equality { term: hir::Term::Ty(ty) },
-                                        ..
-                                    },
-                                ],
-                            ..
-                        },
-                    ),
-                ],
+            bounds: [hir::GenericBound::Trait(trait_ref, _)],
             ..
         }) = opaque_ty.kind
+            && let Some(segment) = trait_ref.trait_ref.path.segments.last()
+            && let Some(args) = segment.args
+            && let [
+                hir::TypeBinding {
+                    ident: Ident { name: sym::Output, .. },
+                    kind: hir::TypeBindingKind::Equality { term: hir::Term::Ty(ty) },
+                    ..
+                },
+            ] = args.bindings
         {
             ty
         } else {
@@ -823,7 +831,7 @@ impl<'tcx> MirBorrowckCtxt<'_, 'tcx> {
         let type_name =
             self.infcx.extract_inference_diagnostics_data(yield_ty.into(), Some(highlight)).name;
 
-        let yield_span = match tcx.hir().get(self.mir_hir_id()) {
+        let yield_span = match tcx.hir_node(self.mir_hir_id()) {
             hir::Node::Expr(hir::Expr {
                 kind: hir::ExprKind::Closure(&hir::Closure { fn_decl_span, .. }),
                 ..
@@ -847,7 +855,7 @@ impl<'tcx> MirBorrowckCtxt<'_, 'tcx> {
         &self,
         fr: RegionVid,
     ) -> Option<RegionName> {
-        let ty::ReEarlyBound(region) = *self.to_error_region(fr)? else {
+        let ty::ReEarlyParam(region) = *self.to_error_region(fr)? else {
             return None;
         };
         if region.has_name() {
@@ -862,7 +870,7 @@ impl<'tcx> MirBorrowckCtxt<'_, 'tcx> {
 
         let found = tcx
             .any_free_region_meets(&tcx.type_of(region_parent).instantiate_identity(), |r| {
-                *r == ty::ReEarlyBound(region)
+                *r == ty::ReEarlyParam(region)
             });
 
         Some(RegionName {
@@ -881,7 +889,7 @@ impl<'tcx> MirBorrowckCtxt<'_, 'tcx> {
         &self,
         fr: RegionVid,
     ) -> Option<RegionName> {
-        let ty::ReEarlyBound(region) = *self.to_error_region(fr)? else {
+        let ty::ReEarlyParam(region) = *self.to_error_region(fr)? else {
             return None;
         };
         if region.has_name() {
@@ -943,7 +951,7 @@ impl<'tcx> MirBorrowckCtxt<'_, 'tcx> {
         &self,
         clauses: &[ty::Clause<'tcx>],
         ty: Ty<'tcx>,
-        region: ty::EarlyBoundRegion,
+        region: ty::EarlyParamRegion,
     ) -> bool {
         let tcx = self.infcx.tcx;
         ty.walk().any(|arg| {
@@ -956,7 +964,7 @@ impl<'tcx> MirBorrowckCtxt<'_, 'tcx> {
                         ty::ClauseKind::Projection(data) if data.projection_ty.self_ty() == ty => {}
                         _ => return false,
                     }
-                    tcx.any_free_region_meets(pred, |r| *r == ty::ReEarlyBound(region))
+                    tcx.any_free_region_meets(pred, |r| *r == ty::ReEarlyParam(region))
                 })
             } else {
                 false
diff --git a/compiler/rustc_borrowck/src/diagnostics/var_name.rs b/compiler/rustc_borrowck/src/diagnostics/var_name.rs
index 3a104c524..28e07f2a8 100644
--- a/compiler/rustc_borrowck/src/diagnostics/var_name.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/var_name.rs
@@ -2,10 +2,9 @@
 #![deny(rustc::diagnostic_outside_of_impl)]
 
 use crate::region_infer::RegionInferenceContext;
-use crate::Upvar;
 use rustc_index::IndexSlice;
 use rustc_middle::mir::{Body, Local};
-use rustc_middle::ty::{RegionVid, TyCtxt};
+use rustc_middle::ty::{self, RegionVid, TyCtxt};
 use rustc_span::symbol::Symbol;
 use rustc_span::Span;
 
@@ -15,7 +14,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         tcx: TyCtxt<'tcx>,
         body: &Body<'tcx>,
         local_names: &IndexSlice<Local, Option<Symbol>>,
-        upvars: &[Upvar<'tcx>],
+        upvars: &[&ty::CapturedPlace<'tcx>],
         fr: RegionVid,
     ) -> Option<(Option<Symbol>, Span)> {
         debug!("get_var_name_and_span_for_region(fr={fr:?})");
@@ -66,10 +65,10 @@ impl<'tcx> RegionInferenceContext<'tcx> {
     pub(crate) fn get_upvar_name_and_span_for_region(
         &self,
         tcx: TyCtxt<'tcx>,
-        upvars: &[Upvar<'tcx>],
+        upvars: &[&ty::CapturedPlace<'tcx>],
         upvar_index: usize,
     ) -> (Symbol, Span) {
-        let upvar_hir_id = upvars[upvar_index].place.get_root_variable();
+        let upvar_hir_id = upvars[upvar_index].get_root_variable();
         debug!("get_upvar_name_and_span_for_region: upvar_hir_id={upvar_hir_id:?}");
 
         let upvar_name = tcx.hir().name(upvar_hir_id);
diff --git a/compiler/rustc_borrowck/src/lib.rs b/compiler/rustc_borrowck/src/lib.rs
index 5787ea13e..43f48f579 100644
--- a/compiler/rustc_borrowck/src/lib.rs
+++ b/compiler/rustc_borrowck/src/lib.rs
@@ -1,8 +1,8 @@
 //! This query borrow-checks the MIR to (further) ensure it is not broken.
 
 #![allow(internal_features)]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
+#![feature(rustdoc_internals)]
+#![doc(rust_logo)]
 #![feature(associated_type_bounds)]
 #![feature(box_patterns)]
 #![feature(let_chains)]
@@ -22,8 +22,7 @@ extern crate tracing;
 
 use rustc_data_structures::fx::{FxIndexMap, FxIndexSet};
 use rustc_data_structures::graph::dominators::Dominators;
-use rustc_errors::{Diagnostic, DiagnosticBuilder, DiagnosticMessage, SubdiagnosticMessage};
-use rustc_fluent_macro::fluent_messages;
+use rustc_errors::{Diagnostic, DiagnosticBuilder};
 use rustc_hir as hir;
 use rustc_hir::def_id::LocalDefId;
 use rustc_index::bit_set::{BitSet, ChunkedBitSet};
@@ -35,7 +34,7 @@ use rustc_middle::mir::tcx::PlaceTy;
 use rustc_middle::mir::*;
 use rustc_middle::query::Providers;
 use rustc_middle::traits::DefiningAnchor;
-use rustc_middle::ty::{self, CapturedPlace, ParamEnv, RegionVid, TyCtxt};
+use rustc_middle::ty::{self, ParamEnv, RegionVid, TyCtxt};
 use rustc_session::lint::builtin::UNUSED_MUT;
 use rustc_span::{Span, Symbol};
 use rustc_target::abi::FieldIdx;
@@ -43,6 +42,7 @@ use rustc_target::abi::FieldIdx;
 use smallvec::SmallVec;
 use std::cell::RefCell;
 use std::collections::BTreeMap;
+use std::marker::PhantomData;
 use std::ops::Deref;
 use std::rc::Rc;
 
@@ -65,19 +65,18 @@ use self::path_utils::*;
 
 pub mod borrow_set;
 mod borrowck_errors;
-mod constraint_generation;
 mod constraints;
 mod dataflow;
 mod def_use;
 mod diagnostics;
 mod facts;
-mod invalidation;
 mod location;
 mod member_constraints;
 mod nll;
 mod path_utils;
 mod place_ext;
 mod places_conflict;
+mod polonius;
 mod prefixes;
 mod region_infer;
 mod renumber;
@@ -98,19 +97,10 @@ use places_conflict::{places_conflict, PlaceConflictBias};
 use region_infer::RegionInferenceContext;
 use renumber::RegionCtxt;
 
-fluent_messages! { "../messages.ftl" }
-
-// FIXME(eddyb) perhaps move this somewhere more centrally.
-#[derive(Debug)]
-struct Upvar<'tcx> {
-    place: CapturedPlace<'tcx>,
-
-    /// If true, the capture is behind a reference.
-    by_ref: bool,
-}
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
 
 /// Associate some local constants with the `'tcx` lifetime
-struct TyCtxtConsts<'tcx>(TyCtxt<'tcx>);
+struct TyCtxtConsts<'tcx>(PhantomData<&'tcx ()>);
 impl<'tcx> TyCtxtConsts<'tcx> {
     const DEREF_PROJECTION: &'tcx [PlaceElem<'tcx>; 1] = &[ProjectionElem::Deref];
 }
@@ -135,7 +125,7 @@ fn mir_borrowck(tcx: TyCtxt<'_>, def: LocalDefId) -> &BorrowCheckResult<'_> {
         return tcx.arena.alloc(result);
     }
 
-    let hir_owner = tcx.hir().local_def_id_to_hir_id(def).owner;
+    let hir_owner = tcx.local_def_id_to_hir_id(def).owner;
 
     let infcx =
         tcx.infer_ctxt().with_opaque_type_inference(DefiningAnchor::Bind(hir_owner.def_id)).build();
@@ -193,18 +183,6 @@ fn do_mir_borrowck<'tcx>(
         infcx.set_tainted_by_errors(e);
         errors.set_tainted_by_errors(e);
     }
-    let upvars: Vec<_> = tcx
-        .closure_captures(def)
-        .iter()
-        .map(|&captured_place| {
-            let capture = captured_place.info.capture_kind;
-            let by_ref = match capture {
-                ty::UpvarCapture::ByValue => false,
-                ty::UpvarCapture::ByRef(..) => true,
-            };
-            Upvar { place: captured_place.clone(), by_ref }
-        })
-        .collect();
 
     // Replace all regions with fresh inference variables. This
     // requires first making our own copy of the MIR. This copy will
@@ -216,21 +194,20 @@ fn do_mir_borrowck<'tcx>(
         nll::replace_regions_in_mir(&infcx, param_env, &mut body_owned, &mut promoted);
     let body = &body_owned; // no further changes
 
-    let location_table_owned = LocationTable::new(body);
-    let location_table = &location_table_owned;
+    let location_table = LocationTable::new(body);
 
-    let move_data = MoveData::gather_moves(&body, tcx, param_env, |_| true);
+    let move_data = MoveData::gather_moves(body, tcx, param_env, |_| true);
     let promoted_move_data = promoted
         .iter_enumerated()
-        .map(|(idx, body)| (idx, MoveData::gather_moves(&body, tcx, param_env, |_| true)));
+        .map(|(idx, body)| (idx, MoveData::gather_moves(body, tcx, param_env, |_| true)));
 
     let mdpe = MoveDataParamEnv { move_data, param_env };
 
-    let mut flow_inits = MaybeInitializedPlaces::new(tcx, &body, &mdpe)
-        .into_engine(tcx, &body)
+    let mut flow_inits = MaybeInitializedPlaces::new(tcx, body, &mdpe)
+        .into_engine(tcx, body)
         .pass_name("borrowck")
         .iterate_to_fixpoint()
-        .into_results_cursor(&body);
+        .into_results_cursor(body);
 
     let locals_are_invalidated_at_exit = tcx.hir().body_owner_kind(def).is_fn_or_closure();
     let borrow_set =
@@ -249,24 +226,24 @@ fn do_mir_borrowck<'tcx>(
         free_regions,
         body,
         &promoted,
-        location_table,
+        &location_table,
         param_env,
         &mut flow_inits,
         &mdpe.move_data,
         &borrow_set,
-        &upvars,
+        tcx.closure_captures(def),
         consumer_options,
     );
 
     // Dump MIR results into a file, if that is enabled. This let us
     // write unit-tests, as well as helping with debugging.
-    nll::dump_mir_results(&infcx, &body, &regioncx, &opt_closure_req);
+    nll::dump_mir_results(&infcx, body, &regioncx, &opt_closure_req);
 
     // We also have a `#[rustc_regions]` annotation that causes us to dump
     // information.
     nll::dump_annotation(
         &infcx,
-        &body,
+        body,
         &regioncx,
         &opt_closure_req,
         &opaque_type_values,
@@ -288,7 +265,7 @@ fn do_mir_borrowck<'tcx>(
         .into_engine(tcx, body)
         .pass_name("borrowck")
         .iterate_to_fixpoint();
-    let flow_ever_inits = EverInitializedPlaces::new(tcx, body, &mdpe)
+    let flow_ever_inits = EverInitializedPlaces::new(body, &mdpe)
         .into_engine(tcx, body)
         .pass_name("borrowck")
         .iterate_to_fixpoint();
@@ -313,7 +290,7 @@ fn do_mir_borrowck<'tcx>(
             param_env,
             body: promoted_body,
             move_data: &move_data,
-            location_table, // no need to create a real one for the promoted, it is not used
+            location_table: &location_table, // no need to create a real one for the promoted, it is not used
             movable_coroutine,
             fn_self_span_reported: Default::default(),
             locals_are_invalidated_at_exit,
@@ -324,7 +301,7 @@ fn do_mir_borrowck<'tcx>(
             used_mut: Default::default(),
             used_mut_upvars: SmallVec::new(),
             borrow_set: Rc::clone(&borrow_set),
-            upvars: Vec::new(),
+            upvars: &[],
             local_names: IndexVec::from_elem(None, &promoted_body.local_decls),
             region_names: RefCell::default(),
             next_region_name: RefCell::new(1),
@@ -354,7 +331,7 @@ fn do_mir_borrowck<'tcx>(
         param_env,
         body,
         move_data: &mdpe.move_data,
-        location_table,
+        location_table: &location_table,
         movable_coroutine,
         locals_are_invalidated_at_exit,
         fn_self_span_reported: Default::default(),
@@ -365,7 +342,7 @@ fn do_mir_borrowck<'tcx>(
         used_mut: Default::default(),
         used_mut_upvars: SmallVec::new(),
         borrow_set: Rc::clone(&borrow_set),
-        upvars,
+        upvars: tcx.closure_captures(def),
         local_names,
         region_names: RefCell::default(),
         next_region_name: RefCell::new(1),
@@ -456,7 +433,7 @@ fn do_mir_borrowck<'tcx>(
             promoted,
             borrow_set,
             region_inference_context: regioncx,
-            location_table: polonius_input.as_ref().map(|_| location_table_owned),
+            location_table: polonius_input.as_ref().map(|_| location_table),
             input_facts: polonius_input,
             output_facts,
         }))
@@ -584,7 +561,7 @@ struct MirBorrowckCtxt<'cx, 'tcx> {
     borrow_set: Rc<BorrowSet<'tcx>>,
 
     /// Information about upvars not necessarily preserved in types or MIR
-    upvars: Vec<Upvar<'tcx>>,
+    upvars: &'tcx [&'tcx ty::CapturedPlace<'tcx>],
 
     /// Names of local (user) variables (extracted from `var_debug_info`).
     local_names: IndexVec<Local, Option<Symbol>>,
@@ -1041,9 +1018,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         flow_state: &Flows<'cx, 'tcx>,
     ) -> bool {
         let mut error_reported = false;
-        let tcx = self.infcx.tcx;
-        let body = self.body;
-        let borrow_set = self.borrow_set.clone();
+        let borrow_set = Rc::clone(&self.borrow_set);
 
         // Use polonius output if it has been enabled.
         let mut polonius_output;
@@ -1060,8 +1035,8 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
 
         each_borrow_involving_path(
             self,
-            tcx,
-            body,
+            self.infcx.tcx,
+            self.body,
             location,
             (sd, place_span.0),
             &borrow_set,
@@ -1538,7 +1513,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
 
         if places_conflict::borrow_conflicts_with_place(
             self.infcx.tcx,
-            &self.body,
+            self.body,
             place,
             borrow.kind,
             root_place,
@@ -2155,11 +2130,11 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                     && !self.has_buffered_errors()
                 {
                     // rust-lang/rust#46908: In pure NLL mode this code path should be
-                    // unreachable, but we use `delay_span_bug` because we can hit this when
+                    // unreachable, but we use `span_delayed_bug` because we can hit this when
                     // dereferencing a non-Copy raw pointer *and* have `-Ztreat-err-as-bug`
                     // enabled. We don't want to ICE for that case, as other errors will have
                     // been emitted (#52262).
-                    self.infcx.tcx.sess.delay_span_bug(
+                    self.infcx.tcx.sess.span_delayed_bug(
                         span,
                         format!(
                             "Accessing `{place:?}` with the kind `{kind:?}` shouldn't be possible",
@@ -2193,7 +2168,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                 // If this is a mutate access to an immutable local variable with no projections
                 // report the error as an illegal reassignment
                 let init = &self.move_data.inits[init_index];
-                let assigned_span = init.span(&self.body);
+                let assigned_span = init.span(self.body);
                 self.report_illegal_reassignment(location, (place, span), assigned_span, place);
             } else {
                 self.report_mutability_error(place, span, the_place_err, error_access, location)
@@ -2294,7 +2269,9 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                                     // unique path to the `&mut`
                                     hir::Mutability::Mut => {
                                         let mode = match self.is_upvar_field_projection(place) {
-                                            Some(field) if self.upvars[field.index()].by_ref => {
+                                            Some(field)
+                                                if self.upvars[field.index()].is_by_ref() =>
+                                            {
                                                 is_local_mutation_allowed
                                             }
                                             _ => LocalMutationIsAllowed::Yes,
@@ -2342,7 +2319,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                                  place={:?}, place_base={:?}",
                                 upvar, is_local_mutation_allowed, place, place_base
                             );
-                            match (upvar.place.mutability, is_local_mutation_allowed) {
+                            match (upvar.mutability, is_local_mutation_allowed) {
                                 (
                                     Mutability::Not,
                                     LocalMutationIsAllowed::No
@@ -2451,7 +2428,7 @@ mod error {
 
         pub fn buffer_error(&mut self, t: DiagnosticBuilder<'_, ErrorGuaranteed>) {
             if let None = self.tainted_by_errors {
-                self.tainted_by_errors = Some(self.tcx.sess.delay_span_bug(
+                self.tainted_by_errors = Some(self.tcx.sess.span_delayed_bug(
                     t.span.clone_ignoring_labels(),
                     "diagnostic buffered but not emitted",
                 ))
@@ -2525,8 +2502,8 @@ mod error {
             if !self.errors.buffered.is_empty() {
                 self.errors.buffered.sort_by_key(|diag| diag.sort_span);
 
-                for mut diag in self.errors.buffered.drain(..) {
-                    self.infcx.tcx.sess.diagnostic().emit_diagnostic(&mut diag);
+                for diag in self.errors.buffered.drain(..) {
+                    self.infcx.tcx.sess.dcx().emit_diagnostic(diag);
                 }
             }
 
diff --git a/compiler/rustc_borrowck/src/location.rs b/compiler/rustc_borrowck/src/location.rs
index 0e669abfd..6f0939316 100644
--- a/compiler/rustc_borrowck/src/location.rs
+++ b/compiler/rustc_borrowck/src/location.rs
@@ -20,6 +20,7 @@ pub struct LocationTable {
 }
 
 rustc_index::newtype_index! {
+    #[orderable]
     #[debug_format = "LocationIndex({})"]
     pub struct LocationIndex {}
 }
diff --git a/compiler/rustc_borrowck/src/nll.rs b/compiler/rustc_borrowck/src/nll.rs
index 0ea4401a8..6781c6a75 100644
--- a/compiler/rustc_borrowck/src/nll.rs
+++ b/compiler/rustc_borrowck/src/nll.rs
@@ -2,16 +2,17 @@
 #![deny(rustc::diagnostic_outside_of_impl)]
 //! The entry point of the NLL borrow checker.
 
+use polonius_engine::{Algorithm, Output};
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_hir::def_id::LocalDefId;
 use rustc_index::IndexSlice;
 use rustc_middle::mir::{create_dump_file, dump_enabled, dump_mir, PassWhere};
-use rustc_middle::mir::{
-    Body, ClosureOutlivesSubject, ClosureRegionRequirements, LocalKind, Location, Promoted,
-    START_BLOCK,
-};
+use rustc_middle::mir::{Body, ClosureOutlivesSubject, ClosureRegionRequirements, Promoted};
 use rustc_middle::ty::print::with_no_trimmed_paths;
 use rustc_middle::ty::{self, OpaqueHiddenType, TyCtxt};
+use rustc_mir_dataflow::impls::MaybeInitializedPlaces;
+use rustc_mir_dataflow::move_paths::MoveData;
+use rustc_mir_dataflow::ResultsCursor;
 use rustc_span::symbol::sym;
 use std::env;
 use std::io;
@@ -19,25 +20,18 @@ use std::path::PathBuf;
 use std::rc::Rc;
 use std::str::FromStr;
 
-use polonius_engine::{Algorithm, Output};
-
-use rustc_mir_dataflow::impls::MaybeInitializedPlaces;
-use rustc_mir_dataflow::move_paths::{InitKind, InitLocation, MoveData};
-use rustc_mir_dataflow::ResultsCursor;
-
 use crate::{
     borrow_set::BorrowSet,
-    constraint_generation,
     consumers::ConsumerOptions,
     diagnostics::RegionErrors,
     facts::{AllFacts, AllFactsExt, RustcFacts},
-    invalidation,
     location::LocationTable,
+    polonius,
     region_infer::{values::RegionValueElements, RegionInferenceContext},
     renumber,
     type_check::{self, MirTypeckRegionConstraints, MirTypeckResults},
     universal_regions::UniversalRegions,
-    BorrowckInferCtxt, Upvar,
+    BorrowckInferCtxt,
 };
 
 pub type PoloniusOutput = Output<RustcFacts>;
@@ -78,81 +72,6 @@ pub(crate) fn replace_regions_in_mir<'tcx>(
     universal_regions
 }
 
-// This function populates an AllFacts instance with base facts related to
-// MovePaths and needed for the move analysis.
-fn populate_polonius_move_facts(
-    all_facts: &mut AllFacts,
-    move_data: &MoveData<'_>,
-    location_table: &LocationTable,
-    body: &Body<'_>,
-) {
-    all_facts
-        .path_is_var
-        .extend(move_data.rev_lookup.iter_locals_enumerated().map(|(l, r)| (r, l)));
-
-    for (child, move_path) in move_data.move_paths.iter_enumerated() {
-        if let Some(parent) = move_path.parent {
-            all_facts.child_path.push((child, parent));
-        }
-    }
-
-    let fn_entry_start =
-        location_table.start_index(Location { block: START_BLOCK, statement_index: 0 });
-
-    // initialized_at
-    for init in move_data.inits.iter() {
-        match init.location {
-            InitLocation::Statement(location) => {
-                let block_data = &body[location.block];
-                let is_terminator = location.statement_index == block_data.statements.len();
-
-                if is_terminator && init.kind == InitKind::NonPanicPathOnly {
-                    // We are at the terminator of an init that has a panic path,
-                    // and where the init should not happen on panic
-
-                    for successor in block_data.terminator().successors() {
-                        if body[successor].is_cleanup {
-                            continue;
-                        }
-
-                        // The initialization happened in (or rather, when arriving at)
-                        // the successors, but not in the unwind block.
-                        let first_statement = Location { block: successor, statement_index: 0 };
-                        all_facts
-                            .path_assigned_at_base
-                            .push((init.path, location_table.start_index(first_statement)));
-                    }
-                } else {
-                    // In all other cases, the initialization just happens at the
-                    // midpoint, like any other effect.
-                    all_facts
-                        .path_assigned_at_base
-                        .push((init.path, location_table.mid_index(location)));
-                }
-            }
-            // Arguments are initialized on function entry
-            InitLocation::Argument(local) => {
-                assert!(body.local_kind(local) == LocalKind::Arg);
-                all_facts.path_assigned_at_base.push((init.path, fn_entry_start));
-            }
-        }
-    }
-
-    for (local, path) in move_data.rev_lookup.iter_locals_enumerated() {
-        if body.local_kind(local) != LocalKind::Arg {
-            // Non-arguments start out deinitialised; we simulate this with an
-            // initial move:
-            all_facts.path_moved_at_base.push((path, fn_entry_start));
-        }
-    }
-
-    // moved_out_at
-    // deinitialisation is assumed to always happen!
-    all_facts
-        .path_moved_at_base
-        .extend(move_data.moves.iter().map(|mo| (mo.path, location_table.mid_index(mo.source))));
-}
-
 /// Computes the (non-lexical) regions from the input MIR.
 ///
 /// This may result in errors being reported.
@@ -166,7 +85,7 @@ pub(crate) fn compute_regions<'cx, 'tcx>(
     flow_inits: &mut ResultsCursor<'cx, 'tcx, MaybeInitializedPlaces<'cx, 'tcx>>,
     move_data: &MoveData<'tcx>,
     borrow_set: &BorrowSet<'tcx>,
-    upvars: &[Upvar<'tcx>],
+    upvars: &[&ty::CapturedPlace<'tcx>],
     consumer_options: Option<ConsumerOptions>,
 ) -> NllOutput<'tcx> {
     let is_polonius_legacy_enabled = infcx.tcx.sess.opts.unstable_opts.polonius.is_legacy_enabled();
@@ -179,70 +98,26 @@ pub(crate) fn compute_regions<'cx, 'tcx>(
 
     let universal_regions = Rc::new(universal_regions);
 
-    let elements = &Rc::new(RegionValueElements::new(&body));
+    let elements = &Rc::new(RegionValueElements::new(body));
 
     // Run the MIR type-checker.
-    let MirTypeckResults {
-        constraints,
-        universal_region_relations,
-        opaque_type_values,
-        live_loans,
-    } = type_check::type_check(
-        infcx,
-        param_env,
-        body,
-        promoted,
-        &universal_regions,
-        location_table,
-        borrow_set,
-        &mut all_facts,
-        flow_inits,
-        move_data,
-        elements,
-        upvars,
-        polonius_input,
-    );
-
-    if let Some(all_facts) = &mut all_facts {
-        let _prof_timer = infcx.tcx.prof.generic_activity("polonius_fact_generation");
-        all_facts.universal_region.extend(universal_regions.universal_regions());
-        populate_polonius_move_facts(all_facts, move_data, location_table, &body);
-
-        // Emit universal regions facts, and their relations, for Polonius.
-        //
-        // 1: universal regions are modeled in Polonius as a pair:
-        // - the universal region vid itself.
-        // - a "placeholder loan" associated to this universal region. Since they don't exist in
-        //   the `borrow_set`, their `BorrowIndex` are synthesized as the universal region index
-        //   added to the existing number of loans, as if they succeeded them in the set.
-        //
-        let borrow_count = borrow_set.len();
-        debug!(
-            "compute_regions: polonius placeholders, num_universals={}, borrow_count={}",
-            universal_regions.len(),
-            borrow_count
+    let MirTypeckResults { constraints, universal_region_relations, opaque_type_values } =
+        type_check::type_check(
+            infcx,
+            param_env,
+            body,
+            promoted,
+            &universal_regions,
+            location_table,
+            borrow_set,
+            &mut all_facts,
+            flow_inits,
+            move_data,
+            elements,
+            upvars,
+            polonius_input,
         );
 
-        for universal_region in universal_regions.universal_regions() {
-            let universal_region_idx = universal_region.index();
-            let placeholder_loan_idx = borrow_count + universal_region_idx;
-            all_facts.placeholder.push((universal_region, placeholder_loan_idx.into()));
-        }
-
-        // 2: the universal region relations `outlives` constraints are emitted as
-        //  `known_placeholder_subset` facts.
-        for (fr1, fr2) in universal_region_relations.known_outlives() {
-            if fr1 != fr2 {
-                debug!(
-                    "compute_regions: emitting polonius `known_placeholder_subset` \
-                     fr1={:?}, fr2={:?}",
-                    fr1, fr2
-                );
-                all_facts.known_placeholder_subset.push((fr1, fr2));
-            }
-        }
-    }
-
     // Create the region inference context, taking ownership of the
     // region inference data that was contained in `infcx`, and the
     // base constraints generated by the type-check.
@@ -250,7 +125,7 @@ pub(crate) fn compute_regions<'cx, 'tcx>(
     let MirTypeckRegionConstraints {
         placeholder_indices,
         placeholder_index_to_region: _,
-        mut liveness_constraints,
+        liveness_constraints,
         outlives_constraints,
         member_constraints,
         universe_causes,
@@ -258,13 +133,16 @@ pub(crate) fn compute_regions<'cx, 'tcx>(
     } = constraints;
     let placeholder_indices = Rc::new(placeholder_indices);
 
-    constraint_generation::generate_constraints(
-        infcx,
-        &mut liveness_constraints,
+    // If requested, emit legacy polonius facts.
+    polonius::emit_facts(
         &mut all_facts,
+        infcx.tcx,
         location_table,
-        &body,
+        body,
         borrow_set,
+        move_data,
+        &universal_regions,
+        &universal_region_relations,
     );
 
     let mut regioncx = RegionInferenceContext::new(
@@ -279,17 +157,12 @@ pub(crate) fn compute_regions<'cx, 'tcx>(
         type_tests,
         liveness_constraints,
         elements,
-        live_loans,
     );
 
-    // Generate various additional constraints.
-    invalidation::generate_invalidates(infcx.tcx, &mut all_facts, location_table, body, borrow_set);
-
-    let def_id = body.source.def_id();
-
-    // Dump facts if requested.
+    // If requested: dump NLL facts, and run legacy polonius analysis.
     let polonius_output = all_facts.as_ref().and_then(|all_facts| {
         if infcx.tcx.sess.opts.unstable_opts.nll_facts {
+            let def_id = body.source.def_id();
             let def_path = infcx.tcx.def_path(def_id);
             let dir_path = PathBuf::from(&infcx.tcx.sess.opts.unstable_opts.nll_facts_dir)
                 .join(def_path.to_filename_friendly_no_crate());
@@ -302,7 +175,7 @@ pub(crate) fn compute_regions<'cx, 'tcx>(
             let algorithm = Algorithm::from_str(&algorithm).unwrap();
             debug!("compute_regions: using polonius algorithm {:?}", algorithm);
             let _prof_timer = infcx.tcx.prof.generic_activity("polonius_analysis");
-            Some(Rc::new(Output::compute(&all_facts, algorithm, false)))
+            Some(Rc::new(Output::compute(all_facts, algorithm, false)))
         } else {
             None
         }
@@ -310,17 +183,17 @@ pub(crate) fn compute_regions<'cx, 'tcx>(
 
     // Solve the region constraints.
     let (closure_region_requirements, nll_errors) =
-        regioncx.solve(infcx, param_env, &body, polonius_output.clone());
+        regioncx.solve(infcx, param_env, body, polonius_output.clone());
 
     if !nll_errors.is_empty() {
         // Suppress unhelpful extra errors in `infer_opaque_types`.
-        infcx.set_tainted_by_errors(infcx.tcx.sess.delay_span_bug(
+        infcx.set_tainted_by_errors(infcx.tcx.sess.span_delayed_bug(
             body.span,
             "`compute_regions` tainted `infcx` with errors but did not emit any errors",
         ));
     }
 
-    let remapped_opaque_tys = regioncx.infer_opaque_types(&infcx, opaque_type_values);
+    let remapped_opaque_tys = regioncx.infer_opaque_types(infcx, opaque_type_values);
 
     NllOutput {
         regioncx,
@@ -407,7 +280,7 @@ pub(super) fn dump_annotation<'tcx>(
 
     let def_span = tcx.def_span(body.source.def_id());
     let mut err = if let Some(closure_region_requirements) = closure_region_requirements {
-        let mut err = tcx.sess.diagnostic().span_note_diag(def_span, "external requirements");
+        let mut err = tcx.sess.dcx().struct_span_note(def_span, "external requirements");
 
         regioncx.annotate(tcx, &mut err);
 
@@ -426,7 +299,7 @@ pub(super) fn dump_annotation<'tcx>(
 
         err
     } else {
-        let mut err = tcx.sess.diagnostic().span_note_diag(def_span, "no external requirements");
+        let mut err = tcx.sess.dcx().struct_span_note(def_span, "no external requirements");
         regioncx.annotate(tcx, &mut err);
 
         err
diff --git a/compiler/rustc_borrowck/src/path_utils.rs b/compiler/rustc_borrowck/src/path_utils.rs
index 51e318f08..2d997dfad 100644
--- a/compiler/rustc_borrowck/src/path_utils.rs
+++ b/compiler/rustc_borrowck/src/path_utils.rs
@@ -4,7 +4,6 @@ use crate::borrow_set::{BorrowData, BorrowSet, TwoPhaseActivation};
 use crate::places_conflict;
 use crate::AccessDepth;
 use crate::BorrowIndex;
-use crate::Upvar;
 use rustc_data_structures::graph::dominators::Dominators;
 use rustc_middle::mir::BorrowKind;
 use rustc_middle::mir::{BasicBlock, Body, Location, Place, PlaceRef, ProjectionElem};
@@ -150,7 +149,7 @@ pub(super) fn borrow_of_local_data(place: Place<'_>) -> bool {
 /// of a closure type.
 pub(crate) fn is_upvar_field_projection<'tcx>(
     tcx: TyCtxt<'tcx>,
-    upvars: &[Upvar<'tcx>],
+    upvars: &[&rustc_middle::ty::CapturedPlace<'tcx>],
     place_ref: PlaceRef<'tcx>,
     body: &Body<'tcx>,
 ) -> Option<FieldIdx> {
@@ -166,7 +165,7 @@ pub(crate) fn is_upvar_field_projection<'tcx>(
         Some((place_base, ProjectionElem::Field(field, _ty))) => {
             let base_ty = place_base.ty(body, tcx).ty;
             if (base_ty.is_closure() || base_ty.is_coroutine())
-                && (!by_ref || upvars[field.index()].by_ref)
+                && (!by_ref || upvars[field.index()].is_by_ref())
             {
                 Some(field)
             } else {
diff --git a/compiler/rustc_borrowck/src/invalidation.rs b/compiler/rustc_borrowck/src/polonius/loan_invalidations.rs
index ec6d7b74e..232bd7418 100644
--- a/compiler/rustc_borrowck/src/invalidation.rs
+++ b/compiler/rustc_borrowck/src/polonius/loan_invalidations.rs
@@ -14,34 +14,21 @@ use crate::{
     ReadOrWrite, Reservation, Shallow, Write, WriteKind,
 };
 
-pub(super) fn generate_invalidates<'tcx>(
+/// Emit `loan_invalidated_at` facts.
+pub(super) fn emit_loan_invalidations<'tcx>(
     tcx: TyCtxt<'tcx>,
-    all_facts: &mut Option<AllFacts>,
+    all_facts: &mut AllFacts,
     location_table: &LocationTable,
     body: &Body<'tcx>,
     borrow_set: &BorrowSet<'tcx>,
 ) {
-    if all_facts.is_none() {
-        // Nothing to do if we don't have any facts
-        return;
-    }
-
-    if let Some(all_facts) = all_facts {
-        let _prof_timer = tcx.prof.generic_activity("polonius_fact_generation");
-        let dominators = body.basic_blocks.dominators();
-        let mut ig = InvalidationGenerator {
-            all_facts,
-            borrow_set,
-            tcx,
-            location_table,
-            body: &body,
-            dominators,
-        };
-        ig.visit_body(body);
-    }
+    let dominators = body.basic_blocks.dominators();
+    let mut visitor =
+        LoanInvalidationsGenerator { all_facts, borrow_set, tcx, location_table, body, dominators };
+    visitor.visit_body(body);
 }
 
-struct InvalidationGenerator<'cx, 'tcx> {
+struct LoanInvalidationsGenerator<'cx, 'tcx> {
     tcx: TyCtxt<'tcx>,
     all_facts: &'cx mut AllFacts,
     location_table: &'cx LocationTable,
@@ -52,7 +39,7 @@ struct InvalidationGenerator<'cx, 'tcx> {
 
 /// Visits the whole MIR and generates `invalidates()` facts.
 /// Most of the code implementing this was stolen from `borrow_check/mod.rs`.
-impl<'cx, 'tcx> Visitor<'tcx> for InvalidationGenerator<'cx, 'tcx> {
+impl<'cx, 'tcx> Visitor<'tcx> for LoanInvalidationsGenerator<'cx, 'tcx> {
     fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) {
         self.check_activations(location);
 
@@ -214,7 +201,7 @@ impl<'cx, 'tcx> Visitor<'tcx> for InvalidationGenerator<'cx, 'tcx> {
     }
 }
 
-impl<'cx, 'tcx> InvalidationGenerator<'cx, 'tcx> {
+impl<'cx, 'tcx> LoanInvalidationsGenerator<'cx, 'tcx> {
     /// Simulates mutation of a place.
     fn mutate_place(&mut self, location: Location, place: Place<'tcx>, kind: AccessDepth) {
         self.access_place(
@@ -348,20 +335,16 @@ impl<'cx, 'tcx> InvalidationGenerator<'cx, 'tcx> {
         rw: ReadOrWrite,
     ) {
         debug!(
-            "invalidation::check_access_for_conflict(location={:?}, place={:?}, sd={:?}, \
-             rw={:?})",
+            "check_access_for_conflict(location={:?}, place={:?}, sd={:?}, rw={:?})",
             location, place, sd, rw,
         );
-        let tcx = self.tcx;
-        let body = self.body;
-        let borrow_set = self.borrow_set;
         each_borrow_involving_path(
             self,
-            tcx,
-            body,
+            self.tcx,
+            self.body,
             location,
             (sd, place),
-            borrow_set,
+            self.borrow_set,
             |_| true,
             |this, borrow_index, borrow| {
                 match (rw, borrow.kind) {
@@ -383,7 +366,7 @@ impl<'cx, 'tcx> InvalidationGenerator<'cx, 'tcx> {
 
                     (Read(_), BorrowKind::Mut { .. }) => {
                         // Reading from mere reservations of mutable-borrows is OK.
-                        if !is_active(&this.dominators, borrow, location) {
+                        if !is_active(this.dominators, borrow, location) {
                             // If the borrow isn't active yet, reads don't invalidate it
                             assert!(allow_two_phase_borrow(borrow.kind));
                             return Control::Continue;
diff --git a/compiler/rustc_borrowck/src/polonius/loan_kills.rs b/compiler/rustc_borrowck/src/polonius/loan_kills.rs
new file mode 100644
index 000000000..5df943837
--- /dev/null
+++ b/compiler/rustc_borrowck/src/polonius/loan_kills.rs
@@ -0,0 +1,147 @@
+#![deny(rustc::untranslatable_diagnostic)]
+#![deny(rustc::diagnostic_outside_of_impl)]
+use rustc_middle::mir::visit::Visitor;
+use rustc_middle::mir::{
+    Body, Local, Location, Place, PlaceRef, ProjectionElem, Rvalue, Statement, StatementKind,
+    Terminator, TerminatorKind,
+};
+use rustc_middle::ty::TyCtxt;
+
+use crate::{borrow_set::BorrowSet, facts::AllFacts, location::LocationTable, places_conflict};
+
+/// Emit `loan_killed_at` and `cfg_edge` facts at the same time.
+pub(super) fn emit_loan_kills<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    all_facts: &mut AllFacts,
+    location_table: &LocationTable,
+    body: &Body<'tcx>,
+    borrow_set: &BorrowSet<'tcx>,
+) {
+    let mut visitor = LoanKillsGenerator { borrow_set, tcx, location_table, all_facts, body };
+    for (bb, data) in body.basic_blocks.iter_enumerated() {
+        visitor.visit_basic_block_data(bb, data);
+    }
+}
+
+struct LoanKillsGenerator<'cx, 'tcx> {
+    tcx: TyCtxt<'tcx>,
+    all_facts: &'cx mut AllFacts,
+    location_table: &'cx LocationTable,
+    borrow_set: &'cx BorrowSet<'tcx>,
+    body: &'cx Body<'tcx>,
+}
+
+impl<'cx, 'tcx> Visitor<'tcx> for LoanKillsGenerator<'cx, 'tcx> {
+    fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) {
+        // Also record CFG facts here.
+        self.all_facts.cfg_edge.push((
+            self.location_table.start_index(location),
+            self.location_table.mid_index(location),
+        ));
+
+        self.all_facts.cfg_edge.push((
+            self.location_table.mid_index(location),
+            self.location_table.start_index(location.successor_within_block()),
+        ));
+
+        // If there are borrows on this now dead local, we need to record them as `killed`.
+        if let StatementKind::StorageDead(local) = statement.kind {
+            self.record_killed_borrows_for_local(local, location);
+        }
+
+        self.super_statement(statement, location);
+    }
+
+    fn visit_assign(&mut self, place: &Place<'tcx>, rvalue: &Rvalue<'tcx>, location: Location) {
+        // When we see `X = ...`, then kill borrows of
+        // `(*X).foo` and so forth.
+        self.record_killed_borrows_for_place(*place, location);
+        self.super_assign(place, rvalue, location);
+    }
+
+    fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) {
+        // Also record CFG facts here.
+        self.all_facts.cfg_edge.push((
+            self.location_table.start_index(location),
+            self.location_table.mid_index(location),
+        ));
+
+        let successor_blocks = terminator.successors();
+        self.all_facts.cfg_edge.reserve(successor_blocks.size_hint().0);
+        for successor_block in successor_blocks {
+            self.all_facts.cfg_edge.push((
+                self.location_table.mid_index(location),
+                self.location_table.start_index(successor_block.start_location()),
+            ));
+        }
+
+        // A `Call` terminator's return value can be a local which has borrows,
+        // so we need to record those as `killed` as well.
+        if let TerminatorKind::Call { destination, .. } = terminator.kind {
+            self.record_killed_borrows_for_place(destination, location);
+        }
+
+        self.super_terminator(terminator, location);
+    }
+}
+
+impl<'tcx> LoanKillsGenerator<'_, 'tcx> {
+    /// Records the borrows on the specified place as `killed`. For example, when assigning to a
+    /// local, or on a call's return destination.
+    fn record_killed_borrows_for_place(&mut self, place: Place<'tcx>, location: Location) {
+        // Depending on the `Place` we're killing:
+        // - if it's a local, or a single deref of a local,
+        //   we kill all the borrows on the local.
+        // - if it's a deeper projection, we have to filter which
+        //   of the borrows are killed: the ones whose `borrowed_place`
+        //   conflicts with the `place`.
+        match place.as_ref() {
+            PlaceRef { local, projection: &[] }
+            | PlaceRef { local, projection: &[ProjectionElem::Deref] } => {
+                debug!(
+                    "Recording `killed` facts for borrows of local={:?} at location={:?}",
+                    local, location
+                );
+
+                self.record_killed_borrows_for_local(local, location);
+            }
+
+            PlaceRef { local, projection: &[.., _] } => {
+                // Kill conflicting borrows of the innermost local.
+                debug!(
+                    "Recording `killed` facts for borrows of \
+                            innermost projected local={:?} at location={:?}",
+                    local, location
+                );
+
+                if let Some(borrow_indices) = self.borrow_set.local_map.get(&local) {
+                    for &borrow_index in borrow_indices {
+                        let places_conflict = places_conflict::places_conflict(
+                            self.tcx,
+                            self.body,
+                            self.borrow_set[borrow_index].borrowed_place,
+                            place,
+                            places_conflict::PlaceConflictBias::NoOverlap,
+                        );
+
+                        if places_conflict {
+                            let location_index = self.location_table.mid_index(location);
+                            self.all_facts.loan_killed_at.push((borrow_index, location_index));
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    /// Records the borrows on the specified local as `killed`.
+    fn record_killed_borrows_for_local(&mut self, local: Local, location: Location) {
+        if let Some(borrow_indices) = self.borrow_set.local_map.get(&local) {
+            let location_index = self.location_table.mid_index(location);
+            self.all_facts.loan_killed_at.reserve(borrow_indices.len());
+            for &borrow_index in borrow_indices {
+                self.all_facts.loan_killed_at.push((borrow_index, location_index));
+            }
+        }
+    }
+}
diff --git a/compiler/rustc_borrowck/src/polonius/mod.rs b/compiler/rustc_borrowck/src/polonius/mod.rs
new file mode 100644
index 000000000..40126d50d
--- /dev/null
+++ b/compiler/rustc_borrowck/src/polonius/mod.rs
@@ -0,0 +1,188 @@
+//! Functions dedicated to fact generation for the `-Zpolonius=legacy` datalog implementation.
+//!
+//! Will be removed in the future, once the in-tree `-Zpolonius=next` implementation reaches feature
+//! parity.
+
+use rustc_middle::mir::{Body, LocalKind, Location, START_BLOCK};
+use rustc_middle::ty::TyCtxt;
+use rustc_mir_dataflow::move_paths::{InitKind, InitLocation, MoveData};
+
+use crate::borrow_set::BorrowSet;
+use crate::facts::AllFacts;
+use crate::location::LocationTable;
+use crate::type_check::free_region_relations::UniversalRegionRelations;
+use crate::universal_regions::UniversalRegions;
+
+mod loan_invalidations;
+mod loan_kills;
+
+/// When requested, emit most of the facts needed by polonius:
+/// - moves and assignments
+/// - universal regions and their relations
+/// - CFG points and edges
+/// - loan kills
+/// - loan invalidations
+///
+/// The rest of the facts are emitted during typeck and liveness.
+pub(crate) fn emit_facts<'tcx>(
+    all_facts: &mut Option<AllFacts>,
+    tcx: TyCtxt<'tcx>,
+    location_table: &LocationTable,
+    body: &Body<'tcx>,
+    borrow_set: &BorrowSet<'tcx>,
+    move_data: &MoveData<'_>,
+    universal_regions: &UniversalRegions<'_>,
+    universal_region_relations: &UniversalRegionRelations<'_>,
+) {
+    let Some(all_facts) = all_facts else {
+        // We don't do anything if there are no facts to fill.
+        return;
+    };
+    let _prof_timer = tcx.prof.generic_activity("polonius_fact_generation");
+    emit_move_facts(all_facts, move_data, location_table, body);
+    emit_universal_region_facts(
+        all_facts,
+        borrow_set,
+        &universal_regions,
+        &universal_region_relations,
+    );
+    emit_cfg_and_loan_kills_facts(all_facts, tcx, location_table, body, borrow_set);
+    emit_loan_invalidations_facts(all_facts, tcx, location_table, body, borrow_set);
+}
+
+/// Emit facts needed for move/init analysis: moves and assignments.
+fn emit_move_facts(
+    all_facts: &mut AllFacts,
+    move_data: &MoveData<'_>,
+    location_table: &LocationTable,
+    body: &Body<'_>,
+) {
+    all_facts
+        .path_is_var
+        .extend(move_data.rev_lookup.iter_locals_enumerated().map(|(l, r)| (r, l)));
+
+    for (child, move_path) in move_data.move_paths.iter_enumerated() {
+        if let Some(parent) = move_path.parent {
+            all_facts.child_path.push((child, parent));
+        }
+    }
+
+    let fn_entry_start =
+        location_table.start_index(Location { block: START_BLOCK, statement_index: 0 });
+
+    // initialized_at
+    for init in move_data.inits.iter() {
+        match init.location {
+            InitLocation::Statement(location) => {
+                let block_data = &body[location.block];
+                let is_terminator = location.statement_index == block_data.statements.len();
+
+                if is_terminator && init.kind == InitKind::NonPanicPathOnly {
+                    // We are at the terminator of an init that has a panic path,
+                    // and where the init should not happen on panic
+
+                    for successor in block_data.terminator().successors() {
+                        if body[successor].is_cleanup {
+                            continue;
+                        }
+
+                        // The initialization happened in (or rather, when arriving at)
+                        // the successors, but not in the unwind block.
+                        let first_statement = Location { block: successor, statement_index: 0 };
+                        all_facts
+                            .path_assigned_at_base
+                            .push((init.path, location_table.start_index(first_statement)));
+                    }
+                } else {
+                    // In all other cases, the initialization just happens at the
+                    // midpoint, like any other effect.
+                    all_facts
+                        .path_assigned_at_base
+                        .push((init.path, location_table.mid_index(location)));
+                }
+            }
+            // Arguments are initialized on function entry
+            InitLocation::Argument(local) => {
+                assert!(body.local_kind(local) == LocalKind::Arg);
+                all_facts.path_assigned_at_base.push((init.path, fn_entry_start));
+            }
+        }
+    }
+
+    for (local, path) in move_data.rev_lookup.iter_locals_enumerated() {
+        if body.local_kind(local) != LocalKind::Arg {
+            // Non-arguments start out deinitialised; we simulate this with an
+            // initial move:
+            all_facts.path_moved_at_base.push((path, fn_entry_start));
+        }
+    }
+
+    // moved_out_at
+    // deinitialisation is assumed to always happen!
+    all_facts
+        .path_moved_at_base
+        .extend(move_data.moves.iter().map(|mo| (mo.path, location_table.mid_index(mo.source))));
+}
+
+/// Emit universal regions facts, and their relations.
+fn emit_universal_region_facts(
+    all_facts: &mut AllFacts,
+    borrow_set: &BorrowSet<'_>,
+    universal_regions: &UniversalRegions<'_>,
+    universal_region_relations: &UniversalRegionRelations<'_>,
+) {
+    // 1: universal regions are modeled in Polonius as a pair:
+    // - the universal region vid itself.
+    // - a "placeholder loan" associated to this universal region. Since they don't exist in
+    //   the `borrow_set`, their `BorrowIndex` are synthesized as the universal region index
+    //   added to the existing number of loans, as if they succeeded them in the set.
+    //
+    all_facts.universal_region.extend(universal_regions.universal_regions());
+    let borrow_count = borrow_set.len();
+    debug!(
+        "emit_universal_region_facts: polonius placeholders, num_universals={}, borrow_count={}",
+        universal_regions.len(),
+        borrow_count
+    );
+
+    for universal_region in universal_regions.universal_regions() {
+        let universal_region_idx = universal_region.index();
+        let placeholder_loan_idx = borrow_count + universal_region_idx;
+        all_facts.placeholder.push((universal_region, placeholder_loan_idx.into()));
+    }
+
+    // 2: the universal region relations `outlives` constraints are emitted as
+    //  `known_placeholder_subset` facts.
+    for (fr1, fr2) in universal_region_relations.known_outlives() {
+        if fr1 != fr2 {
+            debug!(
+                "emit_universal_region_facts: emitting polonius `known_placeholder_subset` \
+                     fr1={:?}, fr2={:?}",
+                fr1, fr2
+            );
+            all_facts.known_placeholder_subset.push((fr1, fr2));
+        }
+    }
+}
+
+/// Emit facts about loan invalidations.
+fn emit_loan_invalidations_facts<'tcx>(
+    all_facts: &mut AllFacts,
+    tcx: TyCtxt<'tcx>,
+    location_table: &LocationTable,
+    body: &Body<'tcx>,
+    borrow_set: &BorrowSet<'tcx>,
+) {
+    loan_invalidations::emit_loan_invalidations(tcx, all_facts, location_table, body, borrow_set);
+}
+
+/// Emit facts about CFG points and edges, as well as locations where loans are killed.
+fn emit_cfg_and_loan_kills_facts<'tcx>(
+    all_facts: &mut AllFacts,
+    tcx: TyCtxt<'tcx>,
+    location_table: &LocationTable,
+    body: &Body<'tcx>,
+    borrow_set: &BorrowSet<'tcx>,
+) {
+    loan_kills::emit_loan_kills(tcx, all_facts, location_table, body, borrow_set);
+}
diff --git a/compiler/rustc_borrowck/src/region_infer/dump_mir.rs b/compiler/rustc_borrowck/src/region_infer/dump_mir.rs
index 4d620ac9d..cfbb2766c 100644
--- a/compiler/rustc_borrowck/src/region_infer/dump_mir.rs
+++ b/compiler/rustc_borrowck/src/region_infer/dump_mir.rs
@@ -67,7 +67,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         with_msg: &mut dyn FnMut(&str) -> io::Result<()>,
     ) -> io::Result<()> {
         for region in self.definitions.indices() {
-            let value = self.liveness_constraints.region_value_str(region);
+            let value = self.liveness_constraints.pretty_print_live_points(region);
             if value != "{}" {
                 with_msg(&format!("{region:?} live at {value}"))?;
             }
diff --git a/compiler/rustc_borrowck/src/region_infer/graphviz.rs b/compiler/rustc_borrowck/src/region_infer/graphviz.rs
index a0cf22e93..408c8390e 100644
--- a/compiler/rustc_borrowck/src/region_infer/graphviz.rs
+++ b/compiler/rustc_borrowck/src/region_infer/graphviz.rs
@@ -8,7 +8,6 @@ use std::borrow::Cow;
 use std::io::{self, Write};
 
 use super::*;
-use crate::constraints::OutlivesConstraint;
 use rustc_graphviz as dot;
 
 impl<'tcx> RegionInferenceContext<'tcx> {
diff --git a/compiler/rustc_borrowck/src/region_infer/mod.rs b/compiler/rustc_borrowck/src/region_infer/mod.rs
index b1f91a056..948221e94 100644
--- a/compiler/rustc_borrowck/src/region_infer/mod.rs
+++ b/compiler/rustc_borrowck/src/region_infer/mod.rs
@@ -7,7 +7,6 @@ use rustc_data_structures::fx::{FxIndexMap, FxIndexSet};
 use rustc_data_structures::graph::scc::Sccs;
 use rustc_errors::Diagnostic;
 use rustc_hir::def_id::CRATE_DEF_ID;
-use rustc_index::bit_set::SparseBitMatrix;
 use rustc_index::{IndexSlice, IndexVec};
 use rustc_infer::infer::outlives::test_type_match;
 use rustc_infer::infer::region_constraints::{GenericKind, VarInfos, VerifyBound, VerifyIfEq};
@@ -31,8 +30,8 @@ use crate::{
     nll::PoloniusOutput,
     region_infer::reverse_sccs::ReverseSccGraph,
     region_infer::values::{
-        LivenessValues, PlaceholderIndices, PointIndex, RegionElement, RegionValueElements,
-        RegionValues, ToElementIndex,
+        LivenessValues, PlaceholderIndices, RegionElement, RegionValueElements, RegionValues,
+        ToElementIndex,
     },
     type_check::{free_region_relations::UniversalRegionRelations, Locations},
     universal_regions::UniversalRegions,
@@ -59,7 +58,7 @@ pub struct RegionInferenceContext<'tcx> {
     /// regions, these start out empty and steadily grow, though for
     /// each universally quantified region R they start out containing
     /// the entire CFG and `end(R)`.
-    liveness_constraints: LivenessValues<RegionVid>,
+    liveness_constraints: LivenessValues,
 
     /// The outlives constraints computed by the type-check.
     constraints: Frozen<OutlivesConstraintSet<'tcx>>,
@@ -120,9 +119,6 @@ pub struct RegionInferenceContext<'tcx> {
     /// Information about how the universally quantified regions in
     /// scope on this function relate to one another.
     universal_region_relations: Frozen<UniversalRegionRelations<'tcx>>,
-
-    /// The set of loans that are live at a given point in the CFG, when using `-Zpolonius=next`.
-    live_loans: SparseBitMatrix<PointIndex, BorrowIndex>,
 }
 
 /// Each time that `apply_member_constraint` is successful, it appends
@@ -151,6 +147,7 @@ pub(crate) struct AppliedMemberConstraint {
     pub(crate) member_constraint_index: NllMemberConstraintIndex,
 }
 
+#[derive(Debug)]
 pub(crate) struct RegionDefinition<'tcx> {
     /// What kind of variable is this -- a free region? existential
     /// variable? etc. (See the `NllRegionVariableOrigin` for more
@@ -332,9 +329,8 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         member_constraints_in: MemberConstraintSet<'tcx, RegionVid>,
         universe_causes: FxIndexMap<ty::UniverseIndex, UniverseInfo<'tcx>>,
         type_tests: Vec<TypeTest<'tcx>>,
-        liveness_constraints: LivenessValues<RegionVid>,
+        liveness_constraints: LivenessValues,
         elements: &Rc<RegionValueElements>,
-        live_loans: SparseBitMatrix<PointIndex, BorrowIndex>,
     ) -> Self {
         debug!("universal_regions: {:#?}", universal_regions);
         debug!("outlives constraints: {:#?}", outlives_constraints);
@@ -359,7 +355,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         let mut scc_values =
             RegionValues::new(elements, universal_regions.len(), &placeholder_indices);
 
-        for region in liveness_constraints.rows() {
+        for region in liveness_constraints.regions() {
             let scc = constraint_sccs.scc(region);
             scc_values.merge_liveness(scc, region, &liveness_constraints);
         }
@@ -388,7 +384,6 @@ impl<'tcx> RegionInferenceContext<'tcx> {
             type_tests,
             universal_regions,
             universal_region_relations,
-            live_loans,
         };
 
         result.init_free_and_bound_regions();
@@ -679,13 +674,10 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         // eagerly.
         let mut outlives_requirements = infcx.tcx.is_typeck_child(mir_def_id).then(Vec::new);
 
-        self.check_type_tests(
-            infcx,
-            param_env,
-            body,
-            outlives_requirements.as_mut(),
-            &mut errors_buffer,
-        );
+        self.check_type_tests(infcx, body, outlives_requirements.as_mut(), &mut errors_buffer);
+
+        debug!(?errors_buffer);
+        debug!(?outlives_requirements);
 
         // In Polonius mode, the errors about missing universal region relations are in the output
         // and need to be emitted or propagated. Otherwise, we need to check whether the
@@ -700,10 +692,14 @@ impl<'tcx> RegionInferenceContext<'tcx> {
             self.check_universal_regions(outlives_requirements.as_mut(), &mut errors_buffer);
         }
 
+        debug!(?errors_buffer);
+
         if errors_buffer.is_empty() {
             self.check_member_constraints(infcx, &mut errors_buffer);
         }
 
+        debug!(?errors_buffer);
+
         let outlives_requirements = outlives_requirements.unwrap_or_default();
 
         if outlives_requirements.is_empty() {
@@ -936,7 +932,6 @@ impl<'tcx> RegionInferenceContext<'tcx> {
     fn check_type_tests(
         &self,
         infcx: &InferCtxt<'tcx>,
-        param_env: ty::ParamEnv<'tcx>,
         body: &Body<'tcx>,
         mut propagated_outlives_requirements: Option<&mut Vec<ClosureOutlivesRequirement<'tcx>>>,
         errors_buffer: &mut RegionErrors<'tcx>,
@@ -954,7 +949,6 @@ impl<'tcx> RegionInferenceContext<'tcx> {
             let generic_ty = type_test.generic_kind.to_ty(tcx);
             if self.eval_verify_bound(
                 infcx,
-                param_env,
                 generic_ty,
                 type_test.lower_bound,
                 &type_test.verify_bound,
@@ -965,7 +959,6 @@ impl<'tcx> RegionInferenceContext<'tcx> {
             if let Some(propagated_outlives_requirements) = &mut propagated_outlives_requirements {
                 if self.try_promote_type_test(
                     infcx,
-                    param_env,
                     body,
                     type_test,
                     propagated_outlives_requirements,
@@ -1023,7 +1016,6 @@ impl<'tcx> RegionInferenceContext<'tcx> {
     fn try_promote_type_test(
         &self,
         infcx: &InferCtxt<'tcx>,
-        param_env: ty::ParamEnv<'tcx>,
         body: &Body<'tcx>,
         type_test: &TypeTest<'tcx>,
         propagated_outlives_requirements: &mut Vec<ClosureOutlivesRequirement<'tcx>>,
@@ -1085,7 +1077,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
             // where `ur` is a local bound -- we are sometimes in a
             // position to prove things that our caller cannot. See
             // #53570 for an example.
-            if self.eval_verify_bound(infcx, param_env, generic_ty, ur, &type_test.verify_bound) {
+            if self.eval_verify_bound(infcx, generic_ty, ur, &type_test.verify_bound) {
                 continue;
             }
 
@@ -1268,7 +1260,6 @@ impl<'tcx> RegionInferenceContext<'tcx> {
     fn eval_verify_bound(
         &self,
         infcx: &InferCtxt<'tcx>,
-        param_env: ty::ParamEnv<'tcx>,
         generic_ty: Ty<'tcx>,
         lower_bound: RegionVid,
         verify_bound: &VerifyBound<'tcx>,
@@ -1277,7 +1268,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
 
         match verify_bound {
             VerifyBound::IfEq(verify_if_eq_b) => {
-                self.eval_if_eq(infcx, param_env, generic_ty, lower_bound, *verify_if_eq_b)
+                self.eval_if_eq(infcx, generic_ty, lower_bound, *verify_if_eq_b)
             }
 
             VerifyBound::IsEmpty => {
@@ -1291,11 +1282,11 @@ impl<'tcx> RegionInferenceContext<'tcx> {
             }
 
             VerifyBound::AnyBound(verify_bounds) => verify_bounds.iter().any(|verify_bound| {
-                self.eval_verify_bound(infcx, param_env, generic_ty, lower_bound, verify_bound)
+                self.eval_verify_bound(infcx, generic_ty, lower_bound, verify_bound)
             }),
 
             VerifyBound::AllBounds(verify_bounds) => verify_bounds.iter().all(|verify_bound| {
-                self.eval_verify_bound(infcx, param_env, generic_ty, lower_bound, verify_bound)
+                self.eval_verify_bound(infcx, generic_ty, lower_bound, verify_bound)
             }),
         }
     }
@@ -1303,19 +1294,13 @@ impl<'tcx> RegionInferenceContext<'tcx> {
     fn eval_if_eq(
         &self,
         infcx: &InferCtxt<'tcx>,
-        param_env: ty::ParamEnv<'tcx>,
         generic_ty: Ty<'tcx>,
         lower_bound: RegionVid,
         verify_if_eq_b: ty::Binder<'tcx, VerifyIfEq<'tcx>>,
     ) -> bool {
         let generic_ty = self.normalize_to_scc_representatives(infcx.tcx, generic_ty);
         let verify_if_eq_b = self.normalize_to_scc_representatives(infcx.tcx, verify_if_eq_b);
-        match test_type_match::extract_verify_if_eq(
-            infcx.tcx,
-            param_env,
-            &verify_if_eq_b,
-            generic_ty,
-        ) {
+        match test_type_match::extract_verify_if_eq(infcx.tcx, &verify_if_eq_b, generic_ty) {
             Some(r) => {
                 let r_vid = self.to_region_vid(r);
                 self.eval_outlives(r_vid, lower_bound)
@@ -1457,6 +1442,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         errors_buffer: &mut RegionErrors<'tcx>,
     ) {
         for (fr, fr_definition) in self.definitions.iter_enumerated() {
+            debug!(?fr, ?fr_definition);
             match fr_definition.origin {
                 NllRegionVariableOrigin::FreeRegion => {
                     // Go through each of the universal regions `fr` and check that
@@ -1963,15 +1949,15 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         None
     }
 
-    /// Finds some region R such that `fr1: R` and `R` is live at `elem`.
+    /// Finds some region R such that `fr1: R` and `R` is live at `location`.
     #[instrument(skip(self), level = "trace", ret)]
-    pub(crate) fn find_sub_region_live_at(&self, fr1: RegionVid, elem: Location) -> RegionVid {
+    pub(crate) fn find_sub_region_live_at(&self, fr1: RegionVid, location: Location) -> RegionVid {
         trace!(scc = ?self.constraint_sccs.scc(fr1));
         trace!(universe = ?self.scc_universes[self.constraint_sccs.scc(fr1)]);
         self.find_constraint_paths_between_regions(fr1, |r| {
-            // First look for some `r` such that `fr1: r` and `r` is live at `elem`
-            trace!(?r, liveness_constraints=?self.liveness_constraints.region_value_str(r));
-            self.liveness_constraints.contains(r, elem)
+            // First look for some `r` such that `fr1: r` and `r` is live at `location`
+            trace!(?r, liveness_constraints=?self.liveness_constraints.pretty_print_live_points(r));
+            self.liveness_constraints.is_live_at(r, location)
         })
         .or_else(|| {
             // If we fail to find that, we may find some `r` such that
@@ -2316,7 +2302,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
     /// Note: for now, the sets of live loans is only available when using `-Zpolonius=next`.
     pub(crate) fn is_loan_live_at(&self, loan_idx: BorrowIndex, location: Location) -> bool {
         let point = self.liveness_constraints.point_from_location(location);
-        self.live_loans.contains(point, loan_idx)
+        self.liveness_constraints.is_loan_live_at(loan_idx, point)
     }
 }
 
diff --git a/compiler/rustc_borrowck/src/region_infer/opaque_types.rs b/compiler/rustc_borrowck/src/region_infer/opaque_types.rs
index fb0e5811c..3764e4c40 100644
--- a/compiler/rustc_borrowck/src/region_infer/opaque_types.rs
+++ b/compiler/rustc_borrowck/src/region_infer/opaque_types.rs
@@ -36,7 +36,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
     /// call `infer_opaque_definition_from_instantiation` to get the inferred
     /// type of `_Return<'_a>`. `infer_opaque_definition_from_instantiation`
     /// compares lifetimes directly, so we need to map the inference variables
-    /// back to concrete lifetimes: `'static`, `ReEarlyBound` or `ReFree`.
+    /// back to concrete lifetimes: `'static`, `ReEarlyParam` or `ReLateParam`.
     ///
     /// First we map all the lifetimes in the concrete type to an equal
     /// universal region that occurs in the concrete type's args, in this case
@@ -315,7 +315,7 @@ fn check_opaque_type_well_formed<'tcx>(
         parent_def_id = tcx.local_parent(parent_def_id);
     }
 
-    // FIXME(-Ztrait-solver=next): We probably should use `DefiningAnchor::Error`
+    // FIXME(-Znext-solver): We probably should use `DefiningAnchor::Error`
     // and prepopulate this `InferCtxt` with known opaque values, rather than
     // using the `Bind` anchor here. For now it's fine.
     let infcx = tcx
@@ -386,7 +386,7 @@ fn check_opaque_type_parameter_valid(
         let arg_is_param = match arg.unpack() {
             GenericArgKind::Type(ty) => matches!(ty.kind(), ty::Param(_)),
             GenericArgKind::Lifetime(lt) if is_ty_alias => {
-                matches!(*lt, ty::ReEarlyBound(_) | ty::ReFree(_))
+                matches!(*lt, ty::ReEarlyParam(_) | ty::ReLateParam(_))
             }
             // FIXME(#113916): we can't currently check for unique lifetime params,
             // see that issue for more. We will also have to ignore unused lifetime
diff --git a/compiler/rustc_borrowck/src/region_infer/values.rs b/compiler/rustc_borrowck/src/region_infer/values.rs
index 38452df32..dc3ee849d 100644
--- a/compiler/rustc_borrowck/src/region_infer/values.rs
+++ b/compiler/rustc_borrowck/src/region_infer/values.rs
@@ -11,6 +11,8 @@ use rustc_middle::ty::{self, RegionVid};
 use std::fmt::Debug;
 use std::rc::Rc;
 
+use crate::dataflow::BorrowIndex;
+
 /// Maps between a `Location` and a `PointIndex` (and vice versa).
 pub(crate) struct RegionValueElements {
     /// For each basic block, how many points are contained within?
@@ -90,6 +92,7 @@ impl RegionValueElements {
 rustc_index::newtype_index! {
     /// A single integer representing a `Location` in the MIR control-flow
     /// graph. Constructed efficiently from `RegionValueElements`.
+    #[orderable]
     #[debug_format = "PointIndex({})"]
     pub struct PointIndex {}
 }
@@ -116,71 +119,132 @@ pub(crate) enum RegionElement {
     PlaceholderRegion(ty::PlaceholderRegion),
 }
 
-/// When we initially compute liveness, we use an interval matrix storing
-/// liveness ranges for each region-vid.
-pub(crate) struct LivenessValues<N: Idx> {
+/// Records the CFG locations where each region is live. When we initially compute liveness, we use
+/// an interval matrix storing liveness ranges for each region-vid.
+pub(crate) struct LivenessValues {
+    /// The map from locations to points.
     elements: Rc<RegionValueElements>,
-    points: SparseIntervalMatrix<N, PointIndex>,
+
+    /// For each region: the points where it is live.
+    points: SparseIntervalMatrix<RegionVid, PointIndex>,
+
+    /// When using `-Zpolonius=next`, for each point: the loans flowing into the live regions at
+    /// that point.
+    pub(crate) loans: Option<LiveLoans>,
 }
 
-impl<N: Idx> LivenessValues<N> {
-    /// Creates a new set of "region values" that tracks causal information.
-    /// Each of the regions in num_region_variables will be initialized with an
-    /// empty set of points and no causal information.
+/// Data used to compute the loans that are live at a given point in the CFG, when using
+/// `-Zpolonius=next`.
+pub(crate) struct LiveLoans {
+    /// The set of loans that flow into a given region. When individual regions are marked as live
+    /// in the CFG, these inflowing loans are recorded as live.
+    pub(crate) inflowing_loans: SparseBitMatrix<RegionVid, BorrowIndex>,
+
+    /// The set of loans that are live at a given point in the CFG.
+    pub(crate) live_loans: SparseBitMatrix<PointIndex, BorrowIndex>,
+}
+
+impl LiveLoans {
+    pub(crate) fn new(num_loans: usize) -> Self {
+        LiveLoans {
+            live_loans: SparseBitMatrix::new(num_loans),
+            inflowing_loans: SparseBitMatrix::new(num_loans),
+        }
+    }
+}
+
+impl LivenessValues {
+    /// Create an empty map of regions to locations where they're live.
     pub(crate) fn new(elements: Rc<RegionValueElements>) -> Self {
-        Self { points: SparseIntervalMatrix::new(elements.num_points), elements }
+        LivenessValues {
+            points: SparseIntervalMatrix::new(elements.num_points),
+            elements,
+            loans: None,
+        }
     }
 
     /// Iterate through each region that has a value in this set.
-    pub(crate) fn rows(&self) -> impl Iterator<Item = N> {
+    pub(crate) fn regions(&self) -> impl Iterator<Item = RegionVid> {
         self.points.rows()
     }
 
-    /// Adds the given element to the value for the given region. Returns whether
-    /// the element is newly added (i.e., was not already present).
-    pub(crate) fn add_element(&mut self, row: N, location: Location) -> bool {
-        debug!("LivenessValues::add(r={:?}, location={:?})", row, location);
-        let index = self.elements.point_from_location(location);
-        self.points.insert(row, index)
+    /// Records `region` as being live at the given `location`.
+    pub(crate) fn add_location(&mut self, region: RegionVid, location: Location) {
+        debug!("LivenessValues::add_location(region={:?}, location={:?})", region, location);
+        let point = self.elements.point_from_location(location);
+        self.points.insert(region, point);
+
+        // When available, record the loans flowing into this region as live at the given point.
+        if let Some(loans) = self.loans.as_mut() {
+            if let Some(inflowing) = loans.inflowing_loans.row(region) {
+                loans.live_loans.union_row(point, inflowing);
+            }
+        }
     }
 
-    /// Adds all the elements in the given bit array into the given
-    /// region. Returns whether any of them are newly added.
-    pub(crate) fn add_elements(&mut self, row: N, locations: &IntervalSet<PointIndex>) -> bool {
-        debug!("LivenessValues::add_elements(row={:?}, locations={:?})", row, locations);
-        self.points.union_row(row, locations)
+    /// Records `region` as being live at all the given `points`.
+    pub(crate) fn add_points(&mut self, region: RegionVid, points: &IntervalSet<PointIndex>) {
+        debug!("LivenessValues::add_points(region={:?}, points={:?})", region, points);
+        self.points.union_row(region, points);
+
+        // When available, record the loans flowing into this region as live at the given points.
+        if let Some(loans) = self.loans.as_mut() {
+            if let Some(inflowing) = loans.inflowing_loans.row(region) {
+                if !inflowing.is_empty() {
+                    for point in points.iter() {
+                        loans.live_loans.union_row(point, inflowing);
+                    }
+                }
+            }
+        }
     }
 
-    /// Adds all the control-flow points to the values for `r`.
-    pub(crate) fn add_all_points(&mut self, row: N) {
-        self.points.insert_all_into_row(row);
+    /// Records `region` as being live at all the control-flow points.
+    pub(crate) fn add_all_points(&mut self, region: RegionVid) {
+        self.points.insert_all_into_row(region);
     }
 
-    /// Returns `true` if the region `r` contains the given element.
-    pub(crate) fn contains(&self, row: N, location: Location) -> bool {
-        let index = self.elements.point_from_location(location);
-        self.points.row(row).is_some_and(|r| r.contains(index))
+    /// Returns whether `region` is marked live at the given `location`.
+    pub(crate) fn is_live_at(&self, region: RegionVid, location: Location) -> bool {
+        let point = self.elements.point_from_location(location);
+        self.points.row(region).is_some_and(|r| r.contains(point))
+    }
+
+    /// Returns whether `region` is marked live at any location.
+    pub(crate) fn is_live_anywhere(&self, region: RegionVid) -> bool {
+        self.live_points(region).next().is_some()
     }
 
-    /// Returns an iterator of all the elements contained by the region `r`
-    pub(crate) fn get_elements(&self, row: N) -> impl Iterator<Item = Location> + '_ {
+    /// Returns an iterator of all the points where `region` is live.
+    fn live_points(&self, region: RegionVid) -> impl Iterator<Item = PointIndex> + '_ {
         self.points
-            .row(row)
+            .row(region)
             .into_iter()
             .flat_map(|set| set.iter())
-            .take_while(move |&p| self.elements.point_in_range(p))
-            .map(move |p| self.elements.to_location(p))
+            .take_while(|&p| self.elements.point_in_range(p))
     }
 
-    /// Returns a "pretty" string value of the region. Meant for debugging.
-    pub(crate) fn region_value_str(&self, r: N) -> String {
-        region_value_str(self.get_elements(r).map(RegionElement::Location))
+    /// For debugging purposes, returns a pretty-printed string of the points where the `region` is
+    /// live.
+    pub(crate) fn pretty_print_live_points(&self, region: RegionVid) -> String {
+        pretty_print_region_elements(
+            self.live_points(region).map(|p| RegionElement::Location(self.elements.to_location(p))),
+        )
     }
 
     #[inline]
     pub(crate) fn point_from_location(&self, location: Location) -> PointIndex {
         self.elements.point_from_location(location)
     }
+
+    /// When using `-Zpolonius=next`, returns whether the `loan_idx` is live at the given `point`.
+    pub(crate) fn is_loan_live_at(&self, loan_idx: BorrowIndex, point: PointIndex) -> bool {
+        self.loans
+            .as_ref()
+            .expect("Accessing live loans requires `-Zpolonius=next`")
+            .live_loans
+            .contains(point, loan_idx)
+    }
 }
 
 /// Maps from `ty::PlaceholderRegion` values that are used in the rest of
@@ -307,7 +371,7 @@ impl<N: Idx> RegionValues<N> {
     /// `self[to] |= values[from]`, essentially: that is, take all the
     /// elements for the region `from` from `values` and add them to
     /// the region `to` in `self`.
-    pub(crate) fn merge_liveness<M: Idx>(&mut self, to: N, from: M, values: &LivenessValues<M>) {
+    pub(crate) fn merge_liveness(&mut self, to: N, from: RegionVid, values: &LivenessValues) {
         if let Some(set) = values.points.row(from) {
             self.points.union_row(to, set);
         }
@@ -376,7 +440,7 @@ impl<N: Idx> RegionValues<N> {
 
     /// Returns a "pretty" string value of the region. Meant for debugging.
     pub(crate) fn region_value_str(&self, r: N) -> String {
-        region_value_str(self.elements_contained_in(r))
+        pretty_print_region_elements(self.elements_contained_in(r))
     }
 }
 
@@ -420,11 +484,12 @@ impl ToElementIndex for ty::PlaceholderRegion {
     }
 }
 
-pub(crate) fn location_set_str(
+/// For debugging purposes, returns a pretty-printed string of the given points.
+pub(crate) fn pretty_print_points(
     elements: &RegionValueElements,
     points: impl IntoIterator<Item = PointIndex>,
 ) -> String {
-    region_value_str(
+    pretty_print_region_elements(
         points
             .into_iter()
             .take_while(|&p| elements.point_in_range(p))
@@ -433,7 +498,8 @@ pub(crate) fn location_set_str(
     )
 }
 
-fn region_value_str(elements: impl IntoIterator<Item = RegionElement>) -> String {
+/// For debugging purposes, returns a pretty-printed string of the given region elements.
+fn pretty_print_region_elements(elements: impl IntoIterator<Item = RegionElement>) -> String {
     let mut result = String::new();
     result.push('{');
 
diff --git a/compiler/rustc_borrowck/src/renumber.rs b/compiler/rustc_borrowck/src/renumber.rs
index ec0131c53..a31d39e14 100644
--- a/compiler/rustc_borrowck/src/renumber.rs
+++ b/compiler/rustc_borrowck/src/renumber.rs
@@ -28,6 +28,9 @@ pub fn renumber_mir<'tcx>(
     renumberer.visit_body(body);
 }
 
+// FIXME(@lcnr): A lot of these variants overlap and it seems like
+// this type is only used to decide which region should be used
+// as representative. This should be cleaned up.
 #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)]
 pub(crate) enum RegionCtxt {
     Location(Location),
diff --git a/compiler/rustc_borrowck/src/type_check/free_region_relations.rs b/compiler/rustc_borrowck/src/type_check/free_region_relations.rs
index c84256f8f..4a76d877a 100644
--- a/compiler/rustc_borrowck/src/type_check/free_region_relations.rs
+++ b/compiler/rustc_borrowck/src/type_check/free_region_relations.rs
@@ -7,7 +7,7 @@ use rustc_infer::infer::region_constraints::GenericKind;
 use rustc_infer::infer::InferCtxt;
 use rustc_middle::mir::ConstraintCategory;
 use rustc_middle::traits::query::OutlivesBound;
-use rustc_middle::ty::{self, RegionVid, Ty};
+use rustc_middle::ty::{self, RegionVid, Ty, TypeVisitableExt};
 use rustc_span::{ErrorGuaranteed, Span, DUMMY_SP};
 use rustc_trait_selection::traits::query::type_op::{self, TypeOp};
 use std::rc::Rc;
@@ -303,7 +303,7 @@ impl<'tcx> UniversalRegionRelationsBuilder<'_, 'tcx> {
             Locations::All(span),
             span,
             ConstraintCategory::Internal,
-            &mut self.constraints,
+            self.constraints,
         )
         .convert_all(data);
     }
@@ -321,6 +321,9 @@ impl<'tcx> UniversalRegionRelationsBuilder<'_, 'tcx> {
             .map_err(|_: ErrorGuaranteed| debug!("failed to compute implied bounds {:?}", ty))
             .ok()?;
         debug!(?bounds, ?constraints);
+        // Because of #109628, we may have unexpected placeholders. Ignore them!
+        // FIXME(#109628): panic in this case once the issue is fixed.
+        let bounds = bounds.into_iter().filter(|bound| !bound.has_placeholders());
         self.add_outlives_bounds(bounds);
         constraints
     }
diff --git a/compiler/rustc_borrowck/src/type_check/input_output.rs b/compiler/rustc_borrowck/src/type_check/input_output.rs
index d053d0a4b..8e141bb38 100644
--- a/compiler/rustc_borrowck/src/type_check/input_output.rs
+++ b/compiler/rustc_borrowck/src/type_check/input_output.rs
@@ -7,7 +7,7 @@
 //! `RETURN_PLACE` the MIR arguments) are always fully normalized (and
 //! contain revealed `impl Trait` values).
 
-use rustc_infer::infer::LateBoundRegionConversionTime;
+use rustc_infer::infer::BoundRegionConversionTime;
 use rustc_middle::mir::*;
 use rustc_middle::ty::{self, Ty};
 use rustc_span::Span;
@@ -35,7 +35,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
             .instantiate_canonical_with_fresh_inference_vars(body.span, &user_provided_poly_sig);
         let user_provided_sig = self.infcx.instantiate_binder_with_fresh_vars(
             body.span,
-            LateBoundRegionConversionTime::FnCall,
+            BoundRegionConversionTime::FnCall,
             user_provided_sig,
         );
 
@@ -77,7 +77,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
             if argument_index + 1 >= body.local_decls.len() {
                 self.tcx()
                     .sess
-                    .delay_span_bug(body.span, "found more normalized_input_ty than local_decls");
+                    .span_delayed_bug(body.span, "found more normalized_input_ty than local_decls");
                 break;
             }
 
@@ -101,10 +101,10 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
         );
 
         // We will not have a universal_regions.yield_ty if we yield (by accident)
-        // outside of a coroutine and return an `impl Trait`, so emit a delay_span_bug
+        // outside of a coroutine and return an `impl Trait`, so emit a span_delayed_bug
         // because we don't want to panic in an assert here if we've already got errors.
         if body.yield_ty().is_some() != universal_regions.yield_ty.is_some() {
-            self.tcx().sess.delay_span_bug(
+            self.tcx().sess.span_delayed_bug(
                 body.span,
                 format!(
                     "Expected body to have yield_ty ({:?}) iff we have a UR yield_ty ({:?})",
diff --git a/compiler/rustc_borrowck/src/type_check/liveness/local_use_map.rs b/compiler/rustc_borrowck/src/type_check/liveness/local_use_map.rs
index a9ca94567..7433c94a0 100644
--- a/compiler/rustc_borrowck/src/type_check/liveness/local_use_map.rs
+++ b/compiler/rustc_borrowck/src/type_check/liveness/local_use_map.rs
@@ -80,7 +80,7 @@ impl LocalUseMap {
         live_locals.iter().for_each(|&local| locals_with_use_data[local] = true);
 
         LocalUseMapBuild { local_use_map: &mut local_use_map, elements, locals_with_use_data }
-            .visit_body(&body);
+            .visit_body(body);
 
         local_use_map
     }
diff --git a/compiler/rustc_borrowck/src/type_check/liveness/mod.rs b/compiler/rustc_borrowck/src/type_check/liveness/mod.rs
index f1ad0ca55..dc4695fd2 100644
--- a/compiler/rustc_borrowck/src/type_check/liveness/mod.rs
+++ b/compiler/rustc_borrowck/src/type_check/liveness/mod.rs
@@ -1,7 +1,9 @@
 use itertools::{Either, Itertools};
 use rustc_data_structures::fx::FxHashSet;
-use rustc_middle::mir::{Body, Local};
-use rustc_middle::ty::{RegionVid, TyCtxt};
+use rustc_middle::mir::visit::{TyContext, Visitor};
+use rustc_middle::mir::{Body, Local, Location, SourceInfo};
+use rustc_middle::ty::visit::TypeVisitable;
+use rustc_middle::ty::{GenericArgsRef, Region, RegionVid, Ty, TyCtxt};
 use rustc_mir_dataflow::impls::MaybeInitializedPlaces;
 use rustc_mir_dataflow::move_paths::MoveData;
 use rustc_mir_dataflow::ResultsCursor;
@@ -11,7 +13,7 @@ use crate::{
     constraints::OutlivesConstraintSet,
     facts::{AllFacts, AllFactsExt},
     location::LocationTable,
-    region_infer::values::RegionValueElements,
+    region_infer::values::{LivenessValues, RegionValueElements},
     universal_regions::UniversalRegions,
 };
 
@@ -42,11 +44,11 @@ pub(super) fn generate<'mir, 'tcx>(
 
     let free_regions = regions_that_outlive_free_regions(
         typeck.infcx.num_region_vars(),
-        &typeck.borrowck_context.universal_regions,
+        typeck.borrowck_context.universal_regions,
         &typeck.borrowck_context.constraints.outlives_constraints,
     );
     let (relevant_live_locals, boring_locals) =
-        compute_relevant_live_locals(typeck.tcx(), &free_regions, &body);
+        compute_relevant_live_locals(typeck.tcx(), &free_regions, body);
     let facts_enabled = use_polonius || AllFacts::enabled(typeck.tcx());
 
     let polonius_drop_used = facts_enabled.then(|| {
@@ -65,6 +67,14 @@ pub(super) fn generate<'mir, 'tcx>(
         boring_locals,
         polonius_drop_used,
     );
+
+    // Mark regions that should be live where they appear within rvalues or within a call: like
+    // args, regions, and types.
+    record_regular_live_regions(
+        typeck.tcx(),
+        &mut typeck.borrowck_context.constraints.liveness_constraints,
+        body,
+    );
 }
 
 // The purpose of `compute_relevant_live_locals` is to define the subset of `Local`
@@ -132,3 +142,71 @@ fn regions_that_outlive_free_regions<'tcx>(
     // Return the final set of things we visited.
     outlives_free_region
 }
+
+/// Some variables are "regular live" at `location` -- i.e., they may be used later. This means that
+/// all regions appearing in their type must be live at `location`.
+fn record_regular_live_regions<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    liveness_constraints: &mut LivenessValues,
+    body: &Body<'tcx>,
+) {
+    let mut visitor = LiveVariablesVisitor { tcx, liveness_constraints };
+    for (bb, data) in body.basic_blocks.iter_enumerated() {
+        visitor.visit_basic_block_data(bb, data);
+    }
+}
+
+/// Visitor looking for regions that should be live within rvalues or calls.
+struct LiveVariablesVisitor<'cx, 'tcx> {
+    tcx: TyCtxt<'tcx>,
+    liveness_constraints: &'cx mut LivenessValues,
+}
+
+impl<'cx, 'tcx> Visitor<'tcx> for LiveVariablesVisitor<'cx, 'tcx> {
+    /// We sometimes have `args` within an rvalue, or within a
+    /// call. Make them live at the location where they appear.
+    fn visit_args(&mut self, args: &GenericArgsRef<'tcx>, location: Location) {
+        self.record_regions_live_at(*args, location);
+        self.super_args(args);
+    }
+
+    /// We sometimes have `region`s within an rvalue, or within a
+    /// call. Make them live at the location where they appear.
+    fn visit_region(&mut self, region: Region<'tcx>, location: Location) {
+        self.record_regions_live_at(region, location);
+        self.super_region(region);
+    }
+
+    /// We sometimes have `ty`s within an rvalue, or within a
+    /// call. Make them live at the location where they appear.
+    fn visit_ty(&mut self, ty: Ty<'tcx>, ty_context: TyContext) {
+        match ty_context {
+            TyContext::ReturnTy(SourceInfo { span, .. })
+            | TyContext::YieldTy(SourceInfo { span, .. })
+            | TyContext::UserTy(span)
+            | TyContext::LocalDecl { source_info: SourceInfo { span, .. }, .. } => {
+                span_bug!(span, "should not be visiting outside of the CFG: {:?}", ty_context);
+            }
+            TyContext::Location(location) => {
+                self.record_regions_live_at(ty, location);
+            }
+        }
+
+        self.super_ty(ty);
+    }
+}
+
+impl<'cx, 'tcx> LiveVariablesVisitor<'cx, 'tcx> {
+    /// Some variable is "regular live" at `location` -- i.e., it may be used later. This means that
+    /// all regions appearing in the type of `value` must be live at `location`.
+    fn record_regions_live_at<T>(&mut self, value: T, location: Location)
+    where
+        T: TypeVisitable<TyCtxt<'tcx>>,
+    {
+        debug!("record_regions_live_at(value={:?}, location={:?})", value, location);
+        self.tcx.for_each_free_region(&value, |live_region| {
+            let live_region_vid = live_region.as_var();
+            self.liveness_constraints.add_location(live_region_vid, location);
+        });
+    }
+}
diff --git a/compiler/rustc_borrowck/src/type_check/liveness/polonius.rs b/compiler/rustc_borrowck/src/type_check/liveness/polonius.rs
index c621df371..45f7b07fd 100644
--- a/compiler/rustc_borrowck/src/type_check/liveness/polonius.rs
+++ b/compiler/rustc_borrowck/src/type_check/liveness/polonius.rs
@@ -100,7 +100,7 @@ pub(super) fn populate_access_facts<'a, 'tcx>(
             location_table,
             move_data,
         };
-        extractor.visit_body(&body);
+        extractor.visit_body(body);
 
         facts.var_dropped_at.extend(
             dropped_at.iter().map(|&(local, location)| (local, location_table.mid_index(location))),
diff --git a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
index e616449cc..c718d57be 100644
--- a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
+++ b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
@@ -1,12 +1,12 @@
 use rustc_data_structures::fx::{FxIndexMap, FxIndexSet};
 use rustc_data_structures::graph::WithSuccessors;
-use rustc_index::bit_set::{HybridBitSet, SparseBitMatrix};
+use rustc_index::bit_set::HybridBitSet;
 use rustc_index::interval::IntervalSet;
 use rustc_infer::infer::canonical::QueryRegionConstraints;
 use rustc_infer::infer::outlives::for_liveness;
 use rustc_middle::mir::{BasicBlock, Body, ConstraintCategory, Local, Location};
 use rustc_middle::traits::query::DropckOutlivesResult;
-use rustc_middle::ty::{RegionVid, Ty, TyCtxt, TypeVisitable, TypeVisitableExt};
+use rustc_middle::ty::{Ty, TyCtxt, TypeVisitable, TypeVisitableExt};
 use rustc_span::DUMMY_SP;
 use rustc_trait_selection::traits::query::type_op::outlives::DropckOutlives;
 use rustc_trait_selection::traits::query::type_op::{TypeOp, TypeOpOutput};
@@ -16,9 +16,8 @@ use rustc_mir_dataflow::impls::MaybeInitializedPlaces;
 use rustc_mir_dataflow::move_paths::{HasMoveData, MoveData, MovePathIndex};
 use rustc_mir_dataflow::ResultsCursor;
 
-use crate::dataflow::BorrowIndex;
 use crate::{
-    region_infer::values::{self, PointIndex, RegionValueElements},
+    region_infer::values::{self, LiveLoans, PointIndex, RegionValueElements},
     type_check::liveness::local_use_map::LocalUseMap,
     type_check::liveness::polonius,
     type_check::NormalizeLocation,
@@ -49,22 +48,17 @@ pub(super) fn trace<'mir, 'tcx>(
     boring_locals: Vec<Local>,
     polonius_drop_used: Option<Vec<(Local, Location)>>,
 ) {
-    debug!("trace()");
-
     let local_use_map = &LocalUseMap::build(&relevant_live_locals, elements, body);
 
     // When using `-Zpolonius=next`, compute the set of loans that can reach a given region.
-    let num_loans = typeck.borrowck_context.borrow_set.len();
-    let mut inflowing_loans = SparseBitMatrix::new(num_loans);
     if typeck.tcx().sess.opts.unstable_opts.polonius.is_next_enabled() {
-        let borrowck_context = &typeck.borrowck_context;
+        let borrowck_context = &mut typeck.borrowck_context;
         let borrow_set = &borrowck_context.borrow_set;
-        let constraint_set = &borrowck_context.constraints.outlives_constraints;
-
-        let num_region_vars = typeck.infcx.num_region_vars();
-        let graph = constraint_set.graph(num_region_vars);
+        let mut live_loans = LiveLoans::new(borrow_set.len());
+        let outlives_constraints = &borrowck_context.constraints.outlives_constraints;
+        let graph = outlives_constraints.graph(typeck.infcx.num_region_vars());
         let region_graph =
-            graph.region_graph(&constraint_set, borrowck_context.universal_regions.fr_static);
+            graph.region_graph(outlives_constraints, borrowck_context.universal_regions.fr_static);
 
         // Traverse each issuing region's constraints, and record the loan as flowing into the
         // outlived region.
@@ -75,9 +69,13 @@ pub(super) fn trace<'mir, 'tcx>(
                     continue;
                 }
 
-                inflowing_loans.insert(succ, loan);
+                live_loans.inflowing_loans.insert(succ, loan);
             }
         }
+
+        // Store the inflowing loans in the liveness constraints: they will be used to compute live
+        // loans when liveness data is recorded there.
+        borrowck_context.constraints.liveness_constraints.loans = Some(live_loans);
     };
 
     let cx = LivenessContext {
@@ -88,7 +86,6 @@ pub(super) fn trace<'mir, 'tcx>(
         local_use_map,
         move_data,
         drop_data: FxIndexMap::default(),
-        inflowing_loans,
     };
 
     let mut results = LivenessResults::new(cx);
@@ -126,9 +123,6 @@ struct LivenessContext<'me, 'typeck, 'flow, 'tcx> {
     /// Index indicating where each variable is assigned, used, or
     /// dropped.
     local_use_map: &'me LocalUseMap,
-
-    /// Set of loans that flow into a given region, when using `-Zpolonius=next`.
-    inflowing_loans: SparseBitMatrix<RegionVid, BorrowIndex>,
 }
 
 struct DropData<'tcx> {
@@ -489,7 +483,7 @@ impl<'tcx> LivenessContext<'_, '_, '_, 'tcx> {
         }
 
         let move_paths = &self.flow_inits.analysis().move_data().move_paths;
-        move_paths[mpi].find_descendant(&move_paths, |mpi| state.contains(mpi)).is_some()
+        move_paths[mpi].find_descendant(move_paths, |mpi| state.contains(mpi)).is_some()
     }
 
     /// Returns `true` if the local variable (or some part of it) is initialized in
@@ -519,14 +513,7 @@ impl<'tcx> LivenessContext<'_, '_, '_, 'tcx> {
         live_at: &IntervalSet<PointIndex>,
     ) {
         debug!("add_use_live_facts_for(value={:?})", value);
-
-        Self::make_all_regions_live(
-            self.elements,
-            &mut self.typeck,
-            value,
-            live_at,
-            &self.inflowing_loans,
-        );
+        Self::make_all_regions_live(self.elements, self.typeck, value, live_at);
     }
 
     /// Some variable with type `live_ty` is "drop live" at `location`
@@ -550,7 +537,7 @@ impl<'tcx> LivenessContext<'_, '_, '_, 'tcx> {
             dropped_local,
             dropped_ty,
             drop_locations,
-            values::location_set_str(self.elements, live_at.iter()),
+            values::pretty_print_points(self.elements, live_at.iter()),
         );
 
         let drop_data = self.drop_data.entry(dropped_ty).or_insert_with({
@@ -577,15 +564,8 @@ impl<'tcx> LivenessContext<'_, '_, '_, 'tcx> {
         // All things in the `outlives` array may be touched by
         // the destructor and must be live at this point.
         for &kind in &drop_data.dropck_result.kinds {
-            Self::make_all_regions_live(
-                self.elements,
-                &mut self.typeck,
-                kind,
-                live_at,
-                &self.inflowing_loans,
-            );
-
-            polonius::add_drop_of_var_derefs_origin(&mut self.typeck, dropped_local, &kind);
+            Self::make_all_regions_live(self.elements, self.typeck, kind, live_at);
+            polonius::add_drop_of_var_derefs_origin(self.typeck, dropped_local, &kind);
         }
     }
 
@@ -594,20 +574,13 @@ impl<'tcx> LivenessContext<'_, '_, '_, 'tcx> {
         typeck: &mut TypeChecker<'_, 'tcx>,
         value: impl TypeVisitable<TyCtxt<'tcx>>,
         live_at: &IntervalSet<PointIndex>,
-        inflowing_loans: &SparseBitMatrix<RegionVid, BorrowIndex>,
     ) {
         debug!("make_all_regions_live(value={:?})", value);
         debug!(
             "make_all_regions_live: live_at={}",
-            values::location_set_str(elements, live_at.iter()),
+            values::pretty_print_points(elements, live_at.iter()),
         );
 
-        // When using `-Zpolonius=next`, we want to record the loans that flow into this value's
-        // regions as being live at the given `live_at` points: this will be used to compute the
-        // location where a loan goes out of scope.
-        let num_loans = typeck.borrowck_context.borrow_set.len();
-        let value_loans = &mut HybridBitSet::new_empty(num_loans);
-
         value.visit_with(&mut for_liveness::FreeRegionsVisitor {
             tcx: typeck.tcx(),
             param_env: typeck.param_env,
@@ -618,22 +591,9 @@ impl<'tcx> LivenessContext<'_, '_, '_, 'tcx> {
                     .borrowck_context
                     .constraints
                     .liveness_constraints
-                    .add_elements(live_region_vid, live_at);
-
-                // There can only be inflowing loans for this region when we are using
-                // `-Zpolonius=next`.
-                if let Some(inflowing) = inflowing_loans.row(live_region_vid) {
-                    value_loans.union(inflowing);
-                }
+                    .add_points(live_region_vid, live_at);
             },
         });
-
-        // Record the loans reaching the value.
-        if !value_loans.is_empty() {
-            for point in live_at.iter() {
-                typeck.borrowck_context.live_loans.union_row(point, value_loans);
-            }
-        }
     }
 
     fn compute_drop_data(
diff --git a/compiler/rustc_borrowck/src/type_check/mod.rs b/compiler/rustc_borrowck/src/type_check/mod.rs
index 608d01039..5247d5f69 100644
--- a/compiler/rustc_borrowck/src/type_check/mod.rs
+++ b/compiler/rustc_borrowck/src/type_check/mod.rs
@@ -14,18 +14,16 @@ use rustc_hir as hir;
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::LocalDefId;
 use rustc_hir::lang_items::LangItem;
-use rustc_index::bit_set::SparseBitMatrix;
 use rustc_index::{IndexSlice, IndexVec};
 use rustc_infer::infer::canonical::QueryRegionConstraints;
 use rustc_infer::infer::outlives::env::RegionBoundPairs;
 use rustc_infer::infer::region_constraints::RegionConstraintData;
 use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
 use rustc_infer::infer::{
-    InferCtxt, LateBoundRegion, LateBoundRegionConversionTime, NllRegionVariableOrigin,
+    BoundRegion, BoundRegionConversionTime, InferCtxt, NllRegionVariableOrigin,
 };
 use rustc_middle::mir::tcx::PlaceTy;
 use rustc_middle::mir::visit::{NonMutatingUseContext, PlaceContext, Visitor};
-use rustc_middle::mir::AssertKind;
 use rustc_middle::mir::*;
 use rustc_middle::traits::query::NoSolution;
 use rustc_middle::traits::ObligationCause;
@@ -51,8 +49,6 @@ use rustc_mir_dataflow::impls::MaybeInitializedPlaces;
 use rustc_mir_dataflow::move_paths::MoveData;
 use rustc_mir_dataflow::ResultsCursor;
 
-use crate::dataflow::BorrowIndex;
-use crate::region_infer::values::PointIndex;
 use crate::session_diagnostics::{MoveUnsized, SimdShuffleLastConst};
 use crate::{
     borrow_set::BorrowSet,
@@ -68,7 +64,7 @@ use crate::{
     region_infer::TypeTest,
     type_check::free_region_relations::{CreateResult, UniversalRegionRelations},
     universal_regions::{DefiningTy, UniversalRegions},
-    BorrowckInferCtxt, Upvar,
+    BorrowckInferCtxt,
 };
 
 macro_rules! span_mirbug {
@@ -138,7 +134,7 @@ pub(crate) fn type_check<'mir, 'tcx>(
     flow_inits: &mut ResultsCursor<'mir, 'tcx, MaybeInitializedPlaces<'mir, 'tcx>>,
     move_data: &MoveData<'tcx>,
     elements: &Rc<RegionValueElements>,
-    upvars: &[Upvar<'tcx>],
+    upvars: &[&ty::CapturedPlace<'tcx>],
     use_polonius: bool,
 ) -> MirTypeckResults<'tcx> {
     let implicit_region_bound = ty::Region::new_var(infcx.tcx, universal_regions.fr_fn_body);
@@ -166,9 +162,6 @@ pub(crate) fn type_check<'mir, 'tcx>(
 
     debug!(?normalized_inputs_and_output);
 
-    // When using `-Zpolonius=next`, liveness will record the set of live loans per point.
-    let mut live_loans = SparseBitMatrix::new(borrow_set.len());
-
     let mut borrowck_context = BorrowCheckContext {
         universal_regions,
         location_table,
@@ -176,7 +169,6 @@ pub(crate) fn type_check<'mir, 'tcx>(
         all_facts,
         constraints: &mut constraints,
         upvars,
-        live_loans: &mut live_loans,
     };
 
     let mut checker = TypeChecker::new(
@@ -191,11 +183,11 @@ pub(crate) fn type_check<'mir, 'tcx>(
     checker.check_user_type_annotations();
 
     let mut verifier = TypeVerifier::new(&mut checker, promoted);
-    verifier.visit_body(&body);
+    verifier.visit_body(body);
 
     checker.typeck_mir(body);
-    checker.equate_inputs_and_outputs(&body, universal_regions, &normalized_inputs_and_output);
-    checker.check_signature_annotation(&body);
+    checker.equate_inputs_and_outputs(body, universal_regions, &normalized_inputs_and_output);
+    checker.check_signature_annotation(body);
 
     liveness::generate(
         &mut checker,
@@ -232,7 +224,7 @@ pub(crate) fn type_check<'mir, 'tcx>(
             let mut hidden_type = infcx.resolve_vars_if_possible(decl.hidden_type);
             trace!("finalized opaque type {:?} to {:#?}", opaque_type_key, hidden_type.ty.kind());
             if hidden_type.has_non_region_infer() {
-                let reported = infcx.tcx.sess.delay_span_bug(
+                let reported = infcx.tcx.sess.span_delayed_bug(
                     decl.hidden_type.span,
                     format!("could not resolve {:#?}", hidden_type.ty.kind()),
                 );
@@ -243,7 +235,7 @@ pub(crate) fn type_check<'mir, 'tcx>(
         })
         .collect();
 
-    MirTypeckResults { constraints, universal_region_relations, opaque_type_values, live_loans }
+    MirTypeckResults { constraints, universal_region_relations, opaque_type_values }
 }
 
 fn translate_outlives_facts(typeck: &mut TypeChecker<'_, '_>) {
@@ -274,9 +266,9 @@ fn translate_outlives_facts(typeck: &mut TypeChecker<'_, '_>) {
 #[track_caller]
 fn mirbug(tcx: TyCtxt<'_>, span: Span, msg: String) {
     // We sometimes see MIR failures (notably predicate failures) due to
-    // the fact that we check rvalue sized predicates here. So use `delay_span_bug`
+    // the fact that we check rvalue sized predicates here. So use `span_delayed_bug`
     // to avoid reporting bugs in those cases.
-    tcx.sess.diagnostic().delay_span_bug(span, msg);
+    tcx.sess.dcx().span_delayed_bug(span, msg);
 }
 
 enum FieldAccessError {
@@ -318,7 +310,7 @@ impl<'a, 'b, 'tcx> Visitor<'tcx> for TypeVerifier<'a, 'b, 'tcx> {
                 .borrowck_context
                 .constraints
                 .liveness_constraints
-                .add_element(live_region_vid, location);
+                .add_location(live_region_vid, location);
         });
 
         // HACK(compiler-errors): Constants that are gathered into Body.required_consts
@@ -389,7 +381,7 @@ impl<'a, 'b, 'tcx> Visitor<'tcx> for TypeVerifier<'a, 'b, 'tcx> {
                     self.cx.ascribe_user_type(
                         constant.const_.ty(),
                         UserType::TypeOf(uv.def, UserArgs { args: uv.args, user_self_ty: None }),
-                        locations.span(&self.cx.body),
+                        locations.span(self.cx.body),
                     );
                 }
             } else if let Some(static_def_id) = constant.check_static_ptr(tcx) {
@@ -553,7 +545,7 @@ impl<'a, 'b, 'tcx> TypeVerifier<'a, 'b, 'tcx> {
         let all_facts = &mut None;
         let mut constraints = Default::default();
         let mut liveness_constraints =
-            LivenessValues::new(Rc::new(RegionValueElements::new(&promoted_body)));
+            LivenessValues::new(Rc::new(RegionValueElements::new(promoted_body)));
         // Don't try to add borrow_region facts for the promoted MIR
 
         let mut swap_constraints = |this: &mut Self| {
@@ -570,7 +562,7 @@ impl<'a, 'b, 'tcx> TypeVerifier<'a, 'b, 'tcx> {
 
         swap_constraints(self);
 
-        self.visit_body(&promoted_body);
+        self.visit_body(promoted_body);
 
         self.cx.typeck_mir(promoted_body);
 
@@ -592,16 +584,16 @@ impl<'a, 'b, 'tcx> TypeVerifier<'a, 'b, 'tcx> {
             }
             self.cx.borrowck_context.constraints.outlives_constraints.push(constraint)
         }
-        for region in liveness_constraints.rows() {
+        for region in liveness_constraints.regions() {
             // If the region is live at at least one location in the promoted MIR,
             // then add a liveness constraint to the main MIR for this region
             // at the location provided as an argument to this method
-            if liveness_constraints.get_elements(region).next().is_some() {
+            if liveness_constraints.is_live_anywhere(region) {
                 self.cx
                     .borrowck_context
                     .constraints
                     .liveness_constraints
-                    .add_element(region, location);
+                    .add_location(region, location);
             }
         }
     }
@@ -857,11 +849,7 @@ struct BorrowCheckContext<'a, 'tcx> {
     all_facts: &'a mut Option<AllFacts>,
     borrow_set: &'a BorrowSet<'tcx>,
     pub(crate) constraints: &'a mut MirTypeckRegionConstraints<'tcx>,
-    upvars: &'a [Upvar<'tcx>],
-
-    /// The set of loans that are live at a given point in the CFG, filled in by `liveness::trace`,
-    /// when using `-Zpolonius=next`.
-    pub(crate) live_loans: &'a mut SparseBitMatrix<PointIndex, BorrowIndex>,
+    upvars: &'a [&'a ty::CapturedPlace<'tcx>],
 }
 
 /// Holder struct for passing results from MIR typeck to the rest of the non-lexical regions
@@ -870,9 +858,6 @@ pub(crate) struct MirTypeckResults<'tcx> {
     pub(crate) constraints: MirTypeckRegionConstraints<'tcx>,
     pub(crate) universal_region_relations: Frozen<UniversalRegionRelations<'tcx>>,
     pub(crate) opaque_type_values: FxIndexMap<OpaqueTypeKey<'tcx>, OpaqueHiddenType<'tcx>>,
-
-    /// The set of loans that are live at a given point in the CFG, when using `-Zpolonius=next`.
-    pub(crate) live_loans: SparseBitMatrix<PointIndex, BorrowIndex>,
 }
 
 /// A collection of region constraints that must be satisfied for the
@@ -899,7 +884,7 @@ pub(crate) struct MirTypeckRegionConstraints<'tcx> {
     /// not otherwise appear in the MIR -- in particular, the
     /// late-bound regions that it instantiates at call-sites -- and
     /// hence it must report on their liveness constraints.
-    pub(crate) liveness_constraints: LivenessValues<RegionVid>,
+    pub(crate) liveness_constraints: LivenessValues,
 
     pub(crate) outlives_constraints: OutlivesConstraintSet<'tcx>,
 
@@ -1018,16 +1003,16 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
             reported_errors: Default::default(),
         };
 
-        // FIXME(-Ztrait-solver=next): A bit dubious that we're only registering
+        // FIXME(-Znext-solver): A bit dubious that we're only registering
         // predefined opaques in the typeck root.
         if infcx.next_trait_solver() && !infcx.tcx.is_typeck_child(body.source.def_id()) {
-            checker.register_predefined_opaques_in_new_solver();
+            checker.register_predefined_opaques_for_next_solver();
         }
 
         checker
     }
 
-    pub(super) fn register_predefined_opaques_in_new_solver(&mut self) {
+    pub(super) fn register_predefined_opaques_for_next_solver(&mut self) {
         // OK to use the identity arguments for each opaque type key, since
         // we remap opaques from HIR typeck back to their definition params.
         let opaques: Vec<_> = self
@@ -1082,7 +1067,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
         );
 
         if result.is_err() {
-            self.infcx.tcx.sess.delay_span_bug(
+            self.infcx.tcx.sess.span_delayed_bug(
                 self.body.span,
                 "failed re-defining predefined opaques in mir typeck",
             );
@@ -1127,7 +1112,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
             locations,
             locations.span(self.body),
             category,
-            &mut self.borrowck_context.constraints,
+            self.borrowck_context.constraints,
         )
         .convert_all(data);
     }
@@ -1202,7 +1187,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
         self.infcx.tcx
     }
 
-    #[instrument(skip(self, body, location), level = "debug")]
+    #[instrument(skip(self, body), level = "debug")]
     fn check_stmt(&mut self, body: &Body<'tcx>, stmt: &Statement<'tcx>, location: Location) {
         let tcx = self.tcx();
         debug!("stmt kind: {:?}", stmt.kind);
@@ -1387,7 +1372,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                         return;
                     }
                 };
-                let (sig, map) = tcx.replace_late_bound_regions(sig, |br| {
+                let (sig, map) = tcx.instantiate_bound_regions(sig, |br| {
                     use crate::renumber::RegionCtxt;
 
                     let region_ctxt_fn = || {
@@ -1401,10 +1386,10 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                     };
 
                     self.infcx.next_region_var(
-                        LateBoundRegion(
+                        BoundRegion(
                             term.source_info.span,
                             br.kind,
-                            LateBoundRegionConversionTime::FnCall,
+                            BoundRegionConversionTime::FnCall,
                         ),
                         region_ctxt_fn,
                     )
@@ -1443,7 +1428,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                     self.borrowck_context
                         .constraints
                         .liveness_constraints
-                        .add_element(region_vid, term_location);
+                        .add_location(region_vid, term_location);
                 }
 
                 self.check_call_inputs(body, term, func, &sig, args, term_location, *call_source);
@@ -1854,7 +1839,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                 for op in ops {
                     self.check_operand(op, location);
                 }
-                self.check_aggregate_rvalue(&body, rvalue, ak, ops, location)
+                self.check_aggregate_rvalue(body, rvalue, ak, ops, location)
             }
 
             Rvalue::Repeat(operand, len) => {
@@ -1934,7 +1919,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                             *ty,
                             ty_fn_ptr_from,
                             location.to_locations(),
-                            ConstraintCategory::Cast,
+                            ConstraintCategory::Cast { unsize_to: None },
                         ) {
                             span_mirbug!(
                                 self,
@@ -1959,7 +1944,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                             *ty,
                             ty_fn_ptr_from,
                             location.to_locations(),
-                            ConstraintCategory::Cast,
+                            ConstraintCategory::Cast { unsize_to: None },
                         ) {
                             span_mirbug!(
                                 self,
@@ -1988,7 +1973,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                             *ty,
                             ty_fn_ptr_from,
                             location.to_locations(),
-                            ConstraintCategory::Cast,
+                            ConstraintCategory::Cast { unsize_to: None },
                         ) {
                             span_mirbug!(
                                 self,
@@ -2013,7 +1998,15 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                         self.prove_trait_ref(
                             trait_ref,
                             location.to_locations(),
-                            ConstraintCategory::Cast,
+                            ConstraintCategory::Cast {
+                                unsize_to: Some(tcx.fold_regions(ty, |r, _| {
+                                    if let ty::ReVar(_) = r.kind() {
+                                        tcx.lifetimes.re_erased
+                                    } else {
+                                        r
+                                    }
+                                })),
+                            },
                         );
                     }
 
@@ -2033,7 +2026,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                                 .iter()
                                 .map(|predicate| predicate.with_self_ty(tcx, self_ty)),
                             location.to_locations(),
-                            ConstraintCategory::Cast,
+                            ConstraintCategory::Cast { unsize_to: None },
                         );
 
                         let outlives_predicate = tcx.mk_predicate(Binder::dummy(
@@ -2044,7 +2037,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                         self.prove_predicate(
                             outlives_predicate,
                             location.to_locations(),
-                            ConstraintCategory::Cast,
+                            ConstraintCategory::Cast { unsize_to: None },
                         );
                     }
 
@@ -2065,7 +2058,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                             *ty_from,
                             *ty_to,
                             location.to_locations(),
-                            ConstraintCategory::Cast,
+                            ConstraintCategory::Cast { unsize_to: None },
                         ) {
                             span_mirbug!(
                                 self,
@@ -2131,7 +2124,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                             *ty_elem,
                             *ty_to,
                             location.to_locations(),
-                            ConstraintCategory::Cast,
+                            ConstraintCategory::Cast { unsize_to: None },
                         ) {
                             span_mirbug!(
                                 self,
@@ -2292,7 +2285,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
             }
 
             Rvalue::Ref(region, _borrow_kind, borrowed_place) => {
-                self.add_reborrow_constraint(&body, location, *region, borrowed_place);
+                self.add_reborrow_constraint(body, location, *region, borrowed_place);
             }
 
             Rvalue::BinaryOp(
@@ -2504,7 +2497,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
         let tcx = self.infcx.tcx;
         let field = path_utils::is_upvar_field_projection(
             tcx,
-            &self.borrowck_context.upvars,
+            self.borrowck_context.upvars,
             borrowed_place.as_ref(),
             body,
         );
@@ -2660,13 +2653,9 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                 location.to_locations(),
                 DUMMY_SP,                   // irrelevant; will be overridden.
                 ConstraintCategory::Boring, // same as above.
-                &mut self.borrowck_context.constraints,
+                self.borrowck_context.constraints,
             )
-            .apply_closure_requirements(
-                &closure_requirements,
-                def_id.to_def_id(),
-                args,
-            );
+            .apply_closure_requirements(closure_requirements, def_id.to_def_id(), args);
         }
 
         // Now equate closure args to regions inherited from `typeck_root_def_id`. Fixes #98589.
@@ -2674,8 +2663,10 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
         let typeck_root_args = ty::GenericArgs::identity_for_item(tcx, typeck_root_def_id);
 
         let parent_args = match tcx.def_kind(def_id) {
+            DefKind::Closure if tcx.is_coroutine(def_id.to_def_id()) => {
+                args.as_coroutine().parent_args()
+            }
             DefKind::Closure => args.as_closure().parent_args(),
-            DefKind::Coroutine => args.as_coroutine().parent_args(),
             DefKind::InlineConst => args.as_inline_const().parent_args(),
             other => bug!("unexpected item {:?}", other),
         };
@@ -2706,7 +2697,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
         debug!(?body.span);
 
         for (local, local_decl) in body.local_decls.iter_enumerated() {
-            self.check_local(&body, local, local_decl);
+            self.check_local(body, local, local_decl);
         }
 
         for (block, block_data) in body.basic_blocks.iter_enumerated() {
@@ -2719,8 +2710,8 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                 location.statement_index += 1;
             }
 
-            self.check_terminator(&body, block_data.terminator(), location);
-            self.check_iscleanup(&body, block_data);
+            self.check_terminator(body, block_data.terminator(), location);
+            self.check_iscleanup(body, block_data);
         }
     }
 }
diff --git a/compiler/rustc_borrowck/src/type_check/relate_tys.rs b/compiler/rustc_borrowck/src/type_check/relate_tys.rs
index c1f82e19c..ee0bd1310 100644
--- a/compiler/rustc_borrowck/src/type_check/relate_tys.rs
+++ b/compiler/rustc_borrowck/src/type_check/relate_tys.rs
@@ -107,12 +107,12 @@ impl<'tcx> TypeRelatingDelegate<'tcx> for NllTypeRelatingDelegate<'_, '_, 'tcx>
     fn next_existential_region_var(
         &mut self,
         from_forall: bool,
-        _name: Option<Symbol>,
+        name: Option<Symbol>,
     ) -> ty::Region<'tcx> {
         let origin = NllRegionVariableOrigin::Existential { from_forall };
 
         let reg_var =
-            self.type_checker.infcx.next_nll_region_var(origin, || RegionCtxt::Existential(_name));
+            self.type_checker.infcx.next_nll_region_var(origin, || RegionCtxt::Existential(name));
 
         reg_var
     }
diff --git a/compiler/rustc_borrowck/src/universal_regions.rs b/compiler/rustc_borrowck/src/universal_regions.rs
index 7897a5a63..2b83c7871 100644
--- a/compiler/rustc_borrowck/src/universal_regions.rs
+++ b/compiler/rustc_borrowck/src/universal_regions.rs
@@ -462,7 +462,6 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> {
 
         // "Liberate" the late-bound regions. These correspond to
         // "local" free regions.
-
         let bound_inputs_and_output = self.compute_inputs_and_output(&indices, defining_ty);
 
         let inputs_and_output = self.infcx.replace_bound_regions_with_nll_infer_vars(
@@ -665,7 +664,7 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> {
                     var: ty::BoundVar::from_usize(bound_vars.len() - 1),
                     kind: ty::BrEnv,
                 };
-                let env_region = ty::Region::new_late_bound(tcx, ty::INNERMOST, br);
+                let env_region = ty::Region::new_bound(tcx, ty::INNERMOST, br);
                 let closure_ty = tcx.closure_env_ty(def_id, args, env_region).unwrap();
 
                 // The "inputs" of the closure in the
@@ -738,18 +737,6 @@ trait InferCtxtExt<'tcx> {
     ) -> T
     where
         T: TypeFoldable<TyCtxt<'tcx>>;
-
-    fn replace_late_bound_regions_with_nll_infer_vars_in_recursive_scope(
-        &self,
-        mir_def_id: LocalDefId,
-        indices: &mut UniversalRegionIndices<'tcx>,
-    );
-
-    fn replace_late_bound_regions_with_nll_infer_vars_in_item(
-        &self,
-        mir_def_id: LocalDefId,
-        indices: &mut UniversalRegionIndices<'tcx>,
-    );
 }
 
 impl<'cx, 'tcx> InferCtxtExt<'tcx> for BorrowckInferCtxt<'cx, 'tcx> {
@@ -781,10 +768,10 @@ impl<'cx, 'tcx> InferCtxtExt<'tcx> for BorrowckInferCtxt<'cx, 'tcx> {
     where
         T: TypeFoldable<TyCtxt<'tcx>>,
     {
-        let (value, _map) = self.tcx.replace_late_bound_regions(value, |br| {
+        let (value, _map) = self.tcx.instantiate_bound_regions(value, |br| {
             debug!(?br);
             let liberated_region =
-                ty::Region::new_free(self.tcx, all_outlive_scope.to_def_id(), br.kind);
+                ty::Region::new_late_param(self.tcx, all_outlive_scope.to_def_id(), br.kind);
             let region_vid = {
                 let name = match br.kind.get_name() {
                     Some(name) => name,
@@ -800,61 +787,13 @@ impl<'cx, 'tcx> InferCtxtExt<'tcx> for BorrowckInferCtxt<'cx, 'tcx> {
         });
         value
     }
-
-    /// Finds late-bound regions that do not appear in the parameter listing and adds them to the
-    /// indices vector. Typically, we identify late-bound regions as we process the inputs and
-    /// outputs of the closure/function. However, sometimes there are late-bound regions which do
-    /// not appear in the fn parameters but which are nonetheless in scope. The simplest case of
-    /// this are unused functions, like fn foo<'a>() { } (see e.g., #51351). Despite not being used,
-    /// users can still reference these regions (e.g., let x: &'a u32 = &22;), so we need to create
-    /// entries for them and store them in the indices map. This code iterates over the complete
-    /// set of late-bound regions and checks for any that we have not yet seen, adding them to the
-    /// inputs vector.
-    #[instrument(skip(self, indices))]
-    fn replace_late_bound_regions_with_nll_infer_vars_in_recursive_scope(
-        &self,
-        mir_def_id: LocalDefId,
-        indices: &mut UniversalRegionIndices<'tcx>,
-    ) {
-        for_each_late_bound_region_in_recursive_scope(self.tcx, mir_def_id, |r| {
-            debug!(?r);
-            if !indices.indices.contains_key(&r) {
-                let region_vid = {
-                    let name = r.get_name_or_anon();
-                    self.next_nll_region_var(FR, || RegionCtxt::LateBound(name))
-                };
-
-                debug!(?region_vid);
-                indices.insert_late_bound_region(r, region_vid.as_var());
-            }
-        });
-    }
-
-    #[instrument(skip(self, indices))]
-    fn replace_late_bound_regions_with_nll_infer_vars_in_item(
-        &self,
-        mir_def_id: LocalDefId,
-        indices: &mut UniversalRegionIndices<'tcx>,
-    ) {
-        for_each_late_bound_region_in_item(self.tcx, mir_def_id, |r| {
-            debug!(?r);
-            if !indices.indices.contains_key(&r) {
-                let region_vid = {
-                    let name = r.get_name_or_anon();
-                    self.next_nll_region_var(FR, || RegionCtxt::LateBound(name))
-                };
-
-                indices.insert_late_bound_region(r, region_vid.as_var());
-            }
-        });
-    }
 }
 
 impl<'tcx> UniversalRegionIndices<'tcx> {
     /// Initially, the `UniversalRegionIndices` map contains only the
     /// early-bound regions in scope. Once that is all setup, we come
     /// in later and instantiate the late-bound regions, and then we
-    /// insert the `ReFree` version of those into the map as
+    /// insert the `ReLateParam` version of those into the map as
     /// well. These are used for error reporting.
     fn insert_late_bound_region(&mut self, r: ty::Region<'tcx>, vid: ty::RegionVid) {
         debug!("insert_late_bound_region({:?}, {:?})", r, vid);
@@ -929,11 +868,12 @@ fn for_each_late_bound_region_in_item<'tcx>(
         return;
     }
 
-    for bound_var in tcx.late_bound_vars(tcx.hir().local_def_id_to_hir_id(mir_def_id)) {
+    for bound_var in tcx.late_bound_vars(tcx.local_def_id_to_hir_id(mir_def_id)) {
         let ty::BoundVariableKind::Region(bound_region) = bound_var else {
             continue;
         };
-        let liberated_region = ty::Region::new_free(tcx, mir_def_id.to_def_id(), bound_region);
+        let liberated_region =
+            ty::Region::new_late_param(tcx, mir_def_id.to_def_id(), bound_region);
         f(liberated_region);
     }
 }
diff --git a/compiler/rustc_borrowck/src/used_muts.rs b/compiler/rustc_borrowck/src/used_muts.rs
index c5991e0bc..6ac8e1ba7 100644
--- a/compiler/rustc_borrowck/src/used_muts.rs
+++ b/compiler/rustc_borrowck/src/used_muts.rs
@@ -35,7 +35,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                 never_initialized_mut_locals: &mut never_initialized_mut_locals,
                 mbcx: self,
             };
-            visitor.visit_body(&visitor.mbcx.body);
+            visitor.visit_body(visitor.mbcx.body);
         }
 
         // Take the union of the existed `used_mut` set with those variables we've found were
diff --git a/compiler/rustc_builtin_macros/src/alloc_error_handler.rs b/compiler/rustc_builtin_macros/src/alloc_error_handler.rs
index 070d50708..dffda8acc 100644
--- a/compiler/rustc_builtin_macros/src/alloc_error_handler.rs
+++ b/compiler/rustc_builtin_macros/src/alloc_error_handler.rs
@@ -31,10 +31,7 @@ pub fn expand(
     {
         (item, true, ecx.with_def_site_ctxt(fn_kind.sig.span))
     } else {
-        ecx.sess
-            .parse_sess
-            .span_diagnostic
-            .emit_err(errors::AllocErrorMustBeFn { span: item.span() });
+        ecx.sess.dcx().emit_err(errors::AllocErrorMustBeFn { span: item.span() });
         return vec![orig_item];
     };
 
diff --git a/compiler/rustc_builtin_macros/src/asm.rs b/compiler/rustc_builtin_macros/src/asm.rs
index 9e66eaf73..6f1acd8e5 100644
--- a/compiler/rustc_builtin_macros/src/asm.rs
+++ b/compiler/rustc_builtin_macros/src/asm.rs
@@ -47,10 +47,10 @@ pub fn parse_asm_args<'a>(
     sp: Span,
     is_global_asm: bool,
 ) -> PResult<'a, AsmArgs> {
-    let diag = &sess.span_diagnostic;
+    let dcx = &sess.dcx;
 
     if p.token == token::Eof {
-        return Err(diag.create_err(errors::AsmRequiresTemplate { span: sp }));
+        return Err(dcx.create_err(errors::AsmRequiresTemplate { span: sp }));
     }
 
     let first_template = p.parse_expr()?;
@@ -69,7 +69,7 @@ pub fn parse_asm_args<'a>(
         if !p.eat(&token::Comma) {
             if allow_templates {
                 // After a template string, we always expect *only* a comma...
-                return Err(diag.create_err(errors::AsmExpectedComma { span: p.token.span }));
+                return Err(dcx.create_err(errors::AsmExpectedComma { span: p.token.span }));
             } else {
                 // ...after that delegate to `expect` to also include the other expected tokens.
                 return Err(p.expect(&token::Comma).err().unwrap());
@@ -110,7 +110,7 @@ pub fn parse_asm_args<'a>(
         let op = if !is_global_asm && p.eat_keyword(kw::In) {
             let reg = parse_reg(p, &mut explicit_reg)?;
             if p.eat_keyword(kw::Underscore) {
-                let err = diag.create_err(errors::AsmUnderscoreInput { span: p.token.span });
+                let err = dcx.create_err(errors::AsmUnderscoreInput { span: p.token.span });
                 return Err(err);
             }
             let expr = p.parse_expr()?;
@@ -126,7 +126,7 @@ pub fn parse_asm_args<'a>(
         } else if !is_global_asm && p.eat_keyword(sym::inout) {
             let reg = parse_reg(p, &mut explicit_reg)?;
             if p.eat_keyword(kw::Underscore) {
-                let err = diag.create_err(errors::AsmUnderscoreInput { span: p.token.span });
+                let err = dcx.create_err(errors::AsmUnderscoreInput { span: p.token.span });
                 return Err(err);
             }
             let expr = p.parse_expr()?;
@@ -140,7 +140,7 @@ pub fn parse_asm_args<'a>(
         } else if !is_global_asm && p.eat_keyword(sym::inlateout) {
             let reg = parse_reg(p, &mut explicit_reg)?;
             if p.eat_keyword(kw::Underscore) {
-                let err = diag.create_err(errors::AsmUnderscoreInput { span: p.token.span });
+                let err = dcx.create_err(errors::AsmUnderscoreInput { span: p.token.span });
                 return Err(err);
             }
             let expr = p.parse_expr()?;
@@ -157,7 +157,7 @@ pub fn parse_asm_args<'a>(
         } else if p.eat_keyword(sym::sym) {
             let expr = p.parse_expr()?;
             let ast::ExprKind::Path(qself, path) = &expr.kind else {
-                let err = diag.create_err(errors::AsmSymNoPath { span: expr.span });
+                let err = dcx.create_err(errors::AsmSymNoPath { span: expr.span });
                 return Err(err);
             };
             let sym = ast::InlineAsmSym {
@@ -178,7 +178,7 @@ pub fn parse_asm_args<'a>(
                     ) => {}
                 ast::ExprKind::MacCall(..) => {}
                 _ => {
-                    let err = diag.create_err(errors::AsmExpectedOther {
+                    let err = dcx.create_err(errors::AsmExpectedOther {
                         span: template.span,
                         is_global_asm,
                     });
@@ -201,12 +201,12 @@ pub fn parse_asm_args<'a>(
         // of the argument available.
         if explicit_reg {
             if name.is_some() {
-                diag.emit_err(errors::AsmExplicitRegisterName { span });
+                dcx.emit_err(errors::AsmExplicitRegisterName { span });
             }
             args.reg_args.insert(slot);
         } else if let Some(name) = name {
             if let Some(&prev) = args.named_args.get(&name) {
-                diag.emit_err(errors::AsmDuplicateArg { span, name, prev: args.operands[prev].1 });
+                dcx.emit_err(errors::AsmDuplicateArg { span, name, prev: args.operands[prev].1 });
                 continue;
             }
             args.named_args.insert(name, slot);
@@ -215,7 +215,7 @@ pub fn parse_asm_args<'a>(
                 let named = args.named_args.values().map(|p| args.operands[*p].1).collect();
                 let explicit = args.reg_args.iter().map(|p| args.operands[p].1).collect();
 
-                diag.emit_err(errors::AsmPositionalAfter { span, named, explicit });
+                dcx.emit_err(errors::AsmPositionalAfter { span, named, explicit });
             }
         }
     }
@@ -224,19 +224,19 @@ pub fn parse_asm_args<'a>(
         && args.options.contains(ast::InlineAsmOptions::READONLY)
     {
         let spans = args.options_spans.clone();
-        diag.emit_err(errors::AsmMutuallyExclusive { spans, opt1: "nomem", opt2: "readonly" });
+        dcx.emit_err(errors::AsmMutuallyExclusive { spans, opt1: "nomem", opt2: "readonly" });
     }
     if args.options.contains(ast::InlineAsmOptions::PURE)
         && args.options.contains(ast::InlineAsmOptions::NORETURN)
     {
         let spans = args.options_spans.clone();
-        diag.emit_err(errors::AsmMutuallyExclusive { spans, opt1: "pure", opt2: "noreturn" });
+        dcx.emit_err(errors::AsmMutuallyExclusive { spans, opt1: "pure", opt2: "noreturn" });
     }
     if args.options.contains(ast::InlineAsmOptions::PURE)
         && !args.options.intersects(ast::InlineAsmOptions::NOMEM | ast::InlineAsmOptions::READONLY)
     {
         let spans = args.options_spans.clone();
-        diag.emit_err(errors::AsmPureCombine { spans });
+        dcx.emit_err(errors::AsmPureCombine { spans });
     }
 
     let mut have_real_output = false;
@@ -263,17 +263,17 @@ pub fn parse_asm_args<'a>(
         }
     }
     if args.options.contains(ast::InlineAsmOptions::PURE) && !have_real_output {
-        diag.emit_err(errors::AsmPureNoOutput { spans: args.options_spans.clone() });
+        dcx.emit_err(errors::AsmPureNoOutput { spans: args.options_spans.clone() });
     }
     if args.options.contains(ast::InlineAsmOptions::NORETURN) && !outputs_sp.is_empty() {
-        let err = diag.create_err(errors::AsmNoReturn { outputs_sp });
+        let err = dcx.create_err(errors::AsmNoReturn { outputs_sp });
         // Bail out now since this is likely to confuse MIR
         return Err(err);
     }
 
     if args.clobber_abis.len() > 0 {
         if is_global_asm {
-            let err = diag.create_err(errors::GlobalAsmClobberAbi {
+            let err = dcx.create_err(errors::GlobalAsmClobberAbi {
                 spans: args.clobber_abis.iter().map(|(_, span)| *span).collect(),
             });
 
@@ -281,7 +281,7 @@ pub fn parse_asm_args<'a>(
             return Err(err);
         }
         if !regclass_outputs.is_empty() {
-            diag.emit_err(errors::AsmClobberNoReg {
+            dcx.emit_err(errors::AsmClobberNoReg {
                 spans: regclass_outputs,
                 clobbers: args.clobber_abis.iter().map(|(_, span)| *span).collect(),
             });
@@ -298,7 +298,7 @@ pub fn parse_asm_args<'a>(
 fn err_duplicate_option(p: &mut Parser<'_>, symbol: Symbol, span: Span) {
     // Tool-only output
     let full_span = if p.token.kind == token::Comma { span.to(p.token.span) } else { span };
-    p.sess.span_diagnostic.emit_err(errors::AsmOptAlreadyprovided { span, symbol, full_span });
+    p.sess.dcx.emit_err(errors::AsmOptAlreadyprovided { span, symbol, full_span });
 }
 
 /// Try to set the provided option in the provided `AsmArgs`.
@@ -370,7 +370,7 @@ fn parse_clobber_abi<'a>(p: &mut Parser<'a>, args: &mut AsmArgs) -> PResult<'a,
     p.expect(&token::OpenDelim(Delimiter::Parenthesis))?;
 
     if p.eat(&token::CloseDelim(Delimiter::Parenthesis)) {
-        return Err(p.sess.span_diagnostic.create_err(errors::NonABI { span: p.token.span }));
+        return Err(p.sess.dcx.create_err(errors::NonABI { span: p.token.span }));
     }
 
     let mut new_abis = Vec::new();
@@ -381,8 +381,7 @@ fn parse_clobber_abi<'a>(p: &mut Parser<'a>, args: &mut AsmArgs) -> PResult<'a,
             }
             Err(opt_lit) => {
                 let span = opt_lit.map_or(p.token.span, |lit| lit.span);
-                let mut err =
-                    p.sess.span_diagnostic.struct_span_err(span, "expected string literal");
+                let mut err = p.sess.dcx.struct_span_err(span, "expected string literal");
                 err.span_label(span, "not a string literal");
                 return Err(err);
             }
diff --git a/compiler/rustc_builtin_macros/src/assert/context.rs b/compiler/rustc_builtin_macros/src/assert/context.rs
index 2a4bfe9e2..b54e11918 100644
--- a/compiler/rustc_builtin_macros/src/assert/context.rs
+++ b/compiler/rustc_builtin_macros/src/assert/context.rs
@@ -151,7 +151,7 @@ impl<'cx, 'a> Context<'cx, 'a> {
     fn build_panic(&self, expr_str: &str, panic_path: Path) -> P<Expr> {
         let escaped_expr_str = escape_to_fmt(expr_str);
         let initial = [
-            TokenTree::token_alone(
+            TokenTree::token_joint_hidden(
                 token::Literal(token::Lit {
                     kind: token::LitKind::Str,
                     symbol: Symbol::intern(&if self.fmt_string.is_empty() {
@@ -170,7 +170,7 @@ impl<'cx, 'a> Context<'cx, 'a> {
         ];
         let captures = self.capture_decls.iter().flat_map(|cap| {
             [
-                TokenTree::token_alone(token::Ident(cap.ident.name, false), cap.ident.span),
+                TokenTree::token_joint_hidden(token::Ident(cap.ident.name, false), cap.ident.span),
                 TokenTree::token_alone(token::Comma, self.span),
             ]
         });
diff --git a/compiler/rustc_builtin_macros/src/cfg_accessible.rs b/compiler/rustc_builtin_macros/src/cfg_accessible.rs
index 37ac09ccd..64be8da59 100644
--- a/compiler/rustc_builtin_macros/src/cfg_accessible.rs
+++ b/compiler/rustc_builtin_macros/src/cfg_accessible.rs
@@ -47,7 +47,7 @@ impl MultiItemModifier for Expander {
         let template = AttributeTemplate { list: Some("path"), ..Default::default() };
         validate_attr::check_builtin_meta_item(
             &ecx.sess.parse_sess,
-            &meta_item,
+            meta_item,
             ast::AttrStyle::Outer,
             sym::cfg_accessible,
             template,
diff --git a/compiler/rustc_builtin_macros/src/cfg_eval.rs b/compiler/rustc_builtin_macros/src/cfg_eval.rs
index f826c6e77..ca26b7ed8 100644
--- a/compiler/rustc_builtin_macros/src/cfg_eval.rs
+++ b/compiler/rustc_builtin_macros/src/cfg_eval.rs
@@ -25,7 +25,7 @@ pub(crate) fn expand(
     annotatable: Annotatable,
 ) -> Vec<Annotatable> {
     check_builtin_macro_attribute(ecx, meta_item, sym::cfg_eval);
-    warn_on_duplicate_attribute(&ecx, &annotatable, sym::cfg_eval);
+    warn_on_duplicate_attribute(ecx, &annotatable, sym::cfg_eval);
     vec![cfg_eval(ecx.sess, ecx.ecfg.features, annotatable, ecx.current_expansion.lint_node_id)]
 }
 
@@ -95,19 +95,19 @@ impl CfgFinder {
     fn has_cfg_or_cfg_attr(annotatable: &Annotatable) -> bool {
         let mut finder = CfgFinder { has_cfg_or_cfg_attr: false };
         match annotatable {
-            Annotatable::Item(item) => finder.visit_item(&item),
-            Annotatable::TraitItem(item) => finder.visit_assoc_item(&item, visit::AssocCtxt::Trait),
-            Annotatable::ImplItem(item) => finder.visit_assoc_item(&item, visit::AssocCtxt::Impl),
-            Annotatable::ForeignItem(item) => finder.visit_foreign_item(&item),
-            Annotatable::Stmt(stmt) => finder.visit_stmt(&stmt),
-            Annotatable::Expr(expr) => finder.visit_expr(&expr),
-            Annotatable::Arm(arm) => finder.visit_arm(&arm),
-            Annotatable::ExprField(field) => finder.visit_expr_field(&field),
-            Annotatable::PatField(field) => finder.visit_pat_field(&field),
-            Annotatable::GenericParam(param) => finder.visit_generic_param(&param),
-            Annotatable::Param(param) => finder.visit_param(&param),
-            Annotatable::FieldDef(field) => finder.visit_field_def(&field),
-            Annotatable::Variant(variant) => finder.visit_variant(&variant),
+            Annotatable::Item(item) => finder.visit_item(item),
+            Annotatable::TraitItem(item) => finder.visit_assoc_item(item, visit::AssocCtxt::Trait),
+            Annotatable::ImplItem(item) => finder.visit_assoc_item(item, visit::AssocCtxt::Impl),
+            Annotatable::ForeignItem(item) => finder.visit_foreign_item(item),
+            Annotatable::Stmt(stmt) => finder.visit_stmt(stmt),
+            Annotatable::Expr(expr) => finder.visit_expr(expr),
+            Annotatable::Arm(arm) => finder.visit_arm(arm),
+            Annotatable::ExprField(field) => finder.visit_expr_field(field),
+            Annotatable::PatField(field) => finder.visit_pat_field(field),
+            Annotatable::GenericParam(param) => finder.visit_generic_param(param),
+            Annotatable::Param(param) => finder.visit_param(param),
+            Annotatable::FieldDef(field) => finder.visit_field_def(field),
+            Annotatable::Variant(variant) => finder.visit_variant(variant),
             Annotatable::Crate(krate) => finder.visit_crate(krate),
         };
         finder.has_cfg_or_cfg_attr
diff --git a/compiler/rustc_builtin_macros/src/cmdline_attrs.rs b/compiler/rustc_builtin_macros/src/cmdline_attrs.rs
index 7b75d7d84..2803ddefb 100644
--- a/compiler/rustc_builtin_macros/src/cmdline_attrs.rs
+++ b/compiler/rustc_builtin_macros/src/cmdline_attrs.rs
@@ -11,7 +11,7 @@ pub fn inject(krate: &mut ast::Crate, parse_sess: &ParseSess, attrs: &[String])
     for raw_attr in attrs {
         let mut parser = rustc_parse::new_parser_from_source_str(
             parse_sess,
-            FileName::cli_crate_attr_source_code(&raw_attr),
+            FileName::cli_crate_attr_source_code(raw_attr),
             raw_attr.clone(),
         );
 
@@ -25,9 +25,7 @@ pub fn inject(krate: &mut ast::Crate, parse_sess: &ParseSess, attrs: &[String])
         };
         let end_span = parser.token.span;
         if parser.token != token::Eof {
-            parse_sess
-                .span_diagnostic
-                .emit_err(errors::InvalidCrateAttr { span: start_span.to(end_span) });
+            parse_sess.dcx.emit_err(errors::InvalidCrateAttr { span: start_span.to(end_span) });
             continue;
         }
 
diff --git a/compiler/rustc_builtin_macros/src/concat_bytes.rs b/compiler/rustc_builtin_macros/src/concat_bytes.rs
index c4f5af384..5499852e1 100644
--- a/compiler/rustc_builtin_macros/src/concat_bytes.rs
+++ b/compiler/rustc_builtin_macros/src/concat_bytes.rs
@@ -159,7 +159,7 @@ pub fn expand_concat_bytes(
                     accumulator.push(val);
                 }
                 Ok(ast::LitKind::ByteStr(ref bytes, _)) => {
-                    accumulator.extend_from_slice(&bytes);
+                    accumulator.extend_from_slice(bytes);
                 }
                 _ => {
                     if !has_errors {
diff --git a/compiler/rustc_builtin_macros/src/derive.rs b/compiler/rustc_builtin_macros/src/derive.rs
index 140853db6..5a77c3276 100644
--- a/compiler/rustc_builtin_macros/src/derive.rs
+++ b/compiler/rustc_builtin_macros/src/derive.rs
@@ -35,7 +35,7 @@ impl MultiItemModifier for Expander {
                     AttributeTemplate { list: Some("Trait1, Trait2, ..."), ..Default::default() };
                 validate_attr::check_builtin_meta_item(
                     &sess.parse_sess,
-                    &meta_item,
+                    meta_item,
                     ast::AttrStyle::Outer,
                     sym::derive,
                     template,
@@ -48,14 +48,14 @@ impl MultiItemModifier for Expander {
                                 NestedMetaItem::MetaItem(meta) => Some(meta),
                                 NestedMetaItem::Lit(lit) => {
                                     // Reject `#[derive("Debug")]`.
-                                    report_unexpected_meta_item_lit(sess, &lit);
+                                    report_unexpected_meta_item_lit(sess, lit);
                                     None
                                 }
                             })
                             .map(|meta| {
                                 // Reject `#[derive(Debug = "value", Debug(abc))]`, but recover the
                                 // paths.
-                                report_path_args(sess, &meta);
+                                report_path_args(sess, meta);
                                 meta.path.clone()
                             })
                             .map(|path| (path, dummy_annotatable(), None, self.0))
diff --git a/compiler/rustc_builtin_macros/src/deriving/clone.rs b/compiler/rustc_builtin_macros/src/deriving/clone.rs
index 1649cc76c..467fa5a2b 100644
--- a/compiler/rustc_builtin_macros/src/deriving/clone.rs
+++ b/compiler/rustc_builtin_macros/src/deriving/clone.rs
@@ -188,7 +188,7 @@ fn cs_clone(
     }
 
     let expr = match *vdata {
-        VariantData::Struct(..) => {
+        VariantData::Struct { .. } => {
             let fields = all_fields
                 .iter()
                 .map(|field| {
diff --git a/compiler/rustc_builtin_macros/src/deriving/cmp/partial_ord.rs b/compiler/rustc_builtin_macros/src/deriving/cmp/partial_ord.rs
index f3164bd2c..7f5589210 100644
--- a/compiler/rustc_builtin_macros/src/deriving/cmp/partial_ord.rs
+++ b/compiler/rustc_builtin_macros/src/deriving/cmp/partial_ord.rs
@@ -136,7 +136,7 @@ fn cs_partial_cmp(
                     && let Some(last) = arms.last_mut()
                     && let PatKind::Wild = last.pat.kind
                 {
-                    last.body = expr2;
+                    last.body = Some(expr2);
                     expr1
                 } else {
                     let eq_arm = cx.arm(
diff --git a/compiler/rustc_builtin_macros/src/deriving/debug.rs b/compiler/rustc_builtin_macros/src/deriving/debug.rs
index 30c9b35bb..50ea86288 100644
--- a/compiler/rustc_builtin_macros/src/deriving/debug.rs
+++ b/compiler/rustc_builtin_macros/src/deriving/debug.rs
@@ -71,7 +71,7 @@ fn show_substructure(cx: &mut ExtCtxt<'_>, span: Span, substr: &Substructure<'_>
             (false, 0)
         }
         ast::VariantData::Tuple(..) => (false, 1),
-        ast::VariantData::Struct(..) => (true, 2),
+        ast::VariantData::Struct { .. } => (true, 2),
     };
 
     // The number of fields that can be handled without an array.
@@ -226,7 +226,7 @@ fn show_fieldless_enum(
                     debug_assert!(fields.is_empty());
                     cx.pat_tuple_struct(span, variant_path, ThinVec::new())
                 }
-                ast::VariantData::Struct(fields, _) => {
+                ast::VariantData::Struct { fields, .. } => {
                     debug_assert!(fields.is_empty());
                     cx.pat_struct(span, variant_path, ThinVec::new())
                 }
diff --git a/compiler/rustc_builtin_macros/src/deriving/default.rs b/compiler/rustc_builtin_macros/src/deriving/default.rs
index 07b172bc7..43874a242 100644
--- a/compiler/rustc_builtin_macros/src/deriving/default.rs
+++ b/compiler/rustc_builtin_macros/src/deriving/default.rs
@@ -127,18 +127,17 @@ fn extract_default_variant<'a>(
         [first, rest @ ..] => {
             let suggs = default_variants
                 .iter()
-                .map(|variant| {
-                    let spans = default_variants
+                .filter_map(|variant| {
+                    let keep = attr::find_by_name(&variant.attrs, kw::Default)?.span;
+                    let spans: Vec<Span> = default_variants
                         .iter()
-                        .filter_map(|v| {
-                            if v.span == variant.span {
-                                None
-                            } else {
-                                Some(attr::find_by_name(&v.attrs, kw::Default)?.span)
-                            }
+                        .flat_map(|v| {
+                            attr::filter_by_name(&v.attrs, kw::Default)
+                                .filter_map(|attr| (attr.span != keep).then_some(attr.span))
                         })
                         .collect();
-                    errors::MultipleDefaultsSugg { spans, ident: variant.ident }
+                    (!spans.is_empty())
+                        .then_some(errors::MultipleDefaultsSugg { spans, ident: variant.ident })
                 })
                 .collect();
             cx.emit_err(errors::MultipleDefaults {
diff --git a/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs b/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs
index aa1ce1b92..841cac781 100644
--- a/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs
+++ b/compiler/rustc_builtin_macros/src/deriving/generic/mod.rs
@@ -467,7 +467,7 @@ impl<'a> TraitDef<'a> {
         match item {
             Annotatable::Item(item) => {
                 let is_packed = item.attrs.iter().any(|attr| {
-                    for r in attr::find_repr_attrs(&cx.sess, attr) {
+                    for r in attr::find_repr_attrs(cx.sess, attr) {
                         if let attr::ReprPacked(_) = r {
                             return true;
                         }
@@ -478,7 +478,7 @@ impl<'a> TraitDef<'a> {
                 let newitem = match &item.kind {
                     ast::ItemKind::Struct(struct_def, generics) => self.expand_struct_def(
                         cx,
-                        &struct_def,
+                        struct_def,
                         item.ident,
                         generics,
                         from_scratch,
@@ -496,7 +496,7 @@ impl<'a> TraitDef<'a> {
                         if self.supports_unions {
                             self.expand_struct_def(
                                 cx,
-                                &struct_def,
+                                struct_def,
                                 item.ident,
                                 generics,
                                 from_scratch,
@@ -1485,7 +1485,7 @@ impl<'a> TraitDef<'a> {
 
                 let struct_path = struct_path.clone();
                 match *struct_def {
-                    VariantData::Struct(..) => {
+                    VariantData::Struct { .. } => {
                         let field_pats = pieces_iter
                             .map(|(sp, ident, pat)| {
                                 if ident.is_none() {
diff --git a/compiler/rustc_builtin_macros/src/deriving/generic/ty.rs b/compiler/rustc_builtin_macros/src/deriving/generic/ty.rs
index 2d5043112..1a45c3279 100644
--- a/compiler/rustc_builtin_macros/src/deriving/generic/ty.rs
+++ b/compiler/rustc_builtin_macros/src/deriving/generic/ty.rs
@@ -182,7 +182,7 @@ impl Bounds {
         let params = self
             .bounds
             .iter()
-            .map(|&(name, ref bounds)| mk_ty_param(cx, span, name, &bounds, self_ty, self_generics))
+            .map(|&(name, ref bounds)| mk_ty_param(cx, span, name, bounds, self_ty, self_generics))
             .collect();
 
         Generics {
diff --git a/compiler/rustc_builtin_macros/src/env.rs b/compiler/rustc_builtin_macros/src/env.rs
index 8c2fa6ee9..d772642b8 100644
--- a/compiler/rustc_builtin_macros/src/env.rs
+++ b/compiler/rustc_builtin_macros/src/env.rs
@@ -13,6 +13,16 @@ use thin_vec::thin_vec;
 
 use crate::errors;
 
+fn lookup_env<'cx>(cx: &'cx ExtCtxt<'_>, var: Symbol) -> Option<Symbol> {
+    let var = var.as_str();
+    if let Some(value) = cx.sess.opts.logical_env.get(var) {
+        return Some(Symbol::intern(value));
+    }
+    // If the environment variable was not defined with the `--env` option, we try to retrieve it
+    // from rustc's environment.
+    env::var(var).ok().as_deref().map(Symbol::intern)
+}
+
 pub fn expand_option_env<'cx>(
     cx: &'cx mut ExtCtxt<'_>,
     sp: Span,
@@ -23,7 +33,7 @@ pub fn expand_option_env<'cx>(
     };
 
     let sp = cx.with_def_site_ctxt(sp);
-    let value = env::var(var.as_str()).ok().as_deref().map(Symbol::intern);
+    let value = lookup_env(cx, var);
     cx.sess.parse_sess.env_depinfo.borrow_mut().insert((var, value));
     let e = match value {
         None => {
@@ -77,7 +87,7 @@ pub fn expand_env<'cx>(
     };
 
     let span = cx.with_def_site_ctxt(sp);
-    let value = env::var(var.as_str()).ok().as_deref().map(Symbol::intern);
+    let value = lookup_env(cx, var);
     cx.sess.parse_sess.env_depinfo.borrow_mut().insert((var, value));
     let e = match value {
         None => {
diff --git a/compiler/rustc_builtin_macros/src/errors.rs b/compiler/rustc_builtin_macros/src/errors.rs
index fde427033..6ffeb4014 100644
--- a/compiler/rustc_builtin_macros/src/errors.rs
+++ b/compiler/rustc_builtin_macros/src/errors.rs
@@ -1,5 +1,5 @@
 use rustc_errors::{
-    AddToDiagnostic, DiagnosticBuilder, EmissionGuarantee, Handler, IntoDiagnostic, MultiSpan,
+    AddToDiagnostic, DiagCtxt, DiagnosticBuilder, ErrorGuaranteed, IntoDiagnostic, MultiSpan,
     SingleLabelManySpans,
 };
 use rustc_macros::{Diagnostic, Subdiagnostic};
@@ -446,14 +446,14 @@ pub(crate) struct EnvNotDefinedWithUserMessage {
 }
 
 // Hand-written implementation to support custom user messages.
-impl<'a, G: EmissionGuarantee> IntoDiagnostic<'a, G> for EnvNotDefinedWithUserMessage {
+impl<'a> IntoDiagnostic<'a> for EnvNotDefinedWithUserMessage {
     #[track_caller]
-    fn into_diagnostic(self, handler: &'a Handler) -> DiagnosticBuilder<'a, G> {
+    fn into_diagnostic(self, dcx: &'a DiagCtxt) -> DiagnosticBuilder<'a, ErrorGuaranteed> {
         #[expect(
             rustc::untranslatable_diagnostic,
             reason = "cannot translate user-provided messages"
         )]
-        let mut diag = handler.struct_diagnostic(self.msg_from_user.to_string());
+        let mut diag = dcx.struct_err(self.msg_from_user.to_string());
         diag.set_span(self.span);
         diag
     }
@@ -801,18 +801,17 @@ pub(crate) struct AsmClobberNoReg {
     pub(crate) clobbers: Vec<Span>,
 }
 
-impl<'a, G: EmissionGuarantee> IntoDiagnostic<'a, G> for AsmClobberNoReg {
-    fn into_diagnostic(self, handler: &'a Handler) -> DiagnosticBuilder<'a, G> {
-        let mut diag =
-            handler.struct_diagnostic(crate::fluent_generated::builtin_macros_asm_clobber_no_reg);
+impl<'a> IntoDiagnostic<'a> for AsmClobberNoReg {
+    fn into_diagnostic(self, dcx: &'a DiagCtxt) -> DiagnosticBuilder<'a, ErrorGuaranteed> {
+        let mut diag = dcx.struct_err(crate::fluent_generated::builtin_macros_asm_clobber_no_reg);
         diag.set_span(self.spans.clone());
         // eager translation as `span_labels` takes `AsRef<str>`
-        let lbl1 = handler.eagerly_translate_to_string(
+        let lbl1 = dcx.eagerly_translate_to_string(
             crate::fluent_generated::builtin_macros_asm_clobber_abi,
             [].into_iter(),
         );
         diag.span_labels(self.clobbers, &lbl1);
-        let lbl2 = handler.eagerly_translate_to_string(
+        let lbl2 = dcx.eagerly_translate_to_string(
             crate::fluent_generated::builtin_macros_asm_clobber_outputs,
             [].into_iter(),
         );
diff --git a/compiler/rustc_builtin_macros/src/format.rs b/compiler/rustc_builtin_macros/src/format.rs
index 214fed8e2..c5fd535b0 100644
--- a/compiler/rustc_builtin_macros/src/format.rs
+++ b/compiler/rustc_builtin_macros/src/format.rs
@@ -547,7 +547,7 @@ fn make_format_args(
                 span: arg_name.span.into(),
                 msg: format!("named argument `{}` is not used by name", arg_name.name).into(),
                 node_id: rustc_ast::CRATE_NODE_ID,
-                lint_id: LintId::of(&NAMED_ARGUMENTS_USED_POSITIONALLY),
+                lint_id: LintId::of(NAMED_ARGUMENTS_USED_POSITIONALLY),
                 diagnostic: BuiltinLintDiagnostics::NamedArgumentUsedPositionally {
                     position_sp_to_replace,
                     position_sp_for_msg,
@@ -617,16 +617,22 @@ fn report_missing_placeholders(
     let placeholders = pieces
         .iter()
         .filter_map(|piece| {
-            if let parse::Piece::NextArgument(argument) = piece && let ArgumentNamed(binding) = argument.position {
-                let span = fmt_span.from_inner(InnerSpan::new(argument.position_span.start, argument.position_span.end));
+            if let parse::Piece::NextArgument(argument) = piece
+                && let ArgumentNamed(binding) = argument.position
+            {
+                let span = fmt_span.from_inner(InnerSpan::new(
+                    argument.position_span.start,
+                    argument.position_span.end,
+                ));
                 Some((span, binding))
-            } else { None }
+            } else {
+                None
+            }
         })
         .collect::<Vec<_>>();
 
     if !placeholders.is_empty() {
-        if let Some(mut new_diag) =
-            report_redundant_format_arguments(ecx, &args, used, placeholders)
+        if let Some(mut new_diag) = report_redundant_format_arguments(ecx, args, used, placeholders)
         {
             diag.cancel();
             new_diag.emit();
@@ -666,30 +672,22 @@ fn report_missing_placeholders(
                     if explained.contains(&sub) {
                         continue;
                     }
-                    explained.insert(sub.clone());
+                    explained.insert(sub);
 
                     if !found_foreign {
                         found_foreign = true;
                         show_doc_note = true;
                     }
 
-                    if let Some(inner_sp) = pos {
-                        let sp = fmt_span.from_inner(inner_sp);
+                    let sp = fmt_span.from_inner(pos);
 
-                        if success {
-                            suggestions.push((sp, trn));
-                        } else {
-                            diag.span_note(
-                                sp,
-                                format!("format specifiers use curly braces, and {}", trn),
-                            );
-                        }
+                    if success {
+                        suggestions.push((sp, trn));
                     } else {
-                        if success {
-                            diag.help(format!("`{}` should be written as `{}`", sub, trn));
-                        } else {
-                            diag.note(format!("`{}` should use curly braces, and {}", sub, trn));
-                        }
+                        diag.span_note(
+                            sp,
+                            format!("format specifiers use curly braces, and {}", trn),
+                        );
                     }
                 }
 
diff --git a/compiler/rustc_builtin_macros/src/format_foreign.rs b/compiler/rustc_builtin_macros/src/format_foreign.rs
index 2fc8a0763..307e582d6 100644
--- a/compiler/rustc_builtin_macros/src/format_foreign.rs
+++ b/compiler/rustc_builtin_macros/src/format_foreign.rs
@@ -19,10 +19,10 @@ pub(crate) mod printf {
             }
         }
 
-        pub fn position(&self) -> Option<InnerSpan> {
+        pub fn position(&self) -> InnerSpan {
             match self {
-                Substitution::Format(fmt) => Some(fmt.position),
-                &Substitution::Escape((start, end)) => Some(InnerSpan::new(start, end)),
+                Substitution::Format(fmt) => fmt.position,
+                &Substitution::Escape((start, end)) => InnerSpan::new(start, end),
             }
         }
 
@@ -302,10 +302,9 @@ pub(crate) mod printf {
         fn next(&mut self) -> Option<Self::Item> {
             let (mut sub, tail) = parse_next_substitution(self.s)?;
             self.s = tail;
-            if let Some(InnerSpan { start, end }) = sub.position() {
-                sub.set_position(start + self.pos, end + self.pos);
-                self.pos += end;
-            }
+            let InnerSpan { start, end } = sub.position();
+            sub.set_position(start + self.pos, end + self.pos);
+            self.pos += end;
             Some(sub)
         }
 
@@ -629,9 +628,9 @@ pub mod shell {
             }
         }
 
-        pub fn position(&self) -> Option<InnerSpan> {
+        pub fn position(&self) -> InnerSpan {
             let (Self::Ordinal(_, pos) | Self::Name(_, pos) | Self::Escape(pos)) = self;
-            Some(InnerSpan::new(pos.0, pos.1))
+            InnerSpan::new(pos.0, pos.1)
         }
 
         pub fn set_position(&mut self, start: usize, end: usize) {
@@ -664,10 +663,9 @@ pub mod shell {
         fn next(&mut self) -> Option<Self::Item> {
             let (mut sub, tail) = parse_next_substitution(self.s)?;
             self.s = tail;
-            if let Some(InnerSpan { start, end }) = sub.position() {
-                sub.set_position(start + self.pos, end + self.pos);
-                self.pos += end;
-            }
+            let InnerSpan { start, end } = sub.position();
+            sub.set_position(start + self.pos, end + self.pos);
+            self.pos += end;
             Some(sub)
         }
 
diff --git a/compiler/rustc_builtin_macros/src/global_allocator.rs b/compiler/rustc_builtin_macros/src/global_allocator.rs
index 33392edf0..00c7907cd 100644
--- a/compiler/rustc_builtin_macros/src/global_allocator.rs
+++ b/compiler/rustc_builtin_macros/src/global_allocator.rs
@@ -34,10 +34,7 @@ pub fn expand(
     {
         (item, true, ecx.with_def_site_ctxt(ty.span))
     } else {
-        ecx.sess
-            .parse_sess
-            .span_diagnostic
-            .emit_err(errors::AllocMustStatics { span: item.span() });
+        ecx.sess.dcx().emit_err(errors::AllocMustStatics { span: item.span() });
         return vec![orig_item];
     };
 
diff --git a/compiler/rustc_builtin_macros/src/lib.rs b/compiler/rustc_builtin_macros/src/lib.rs
index d84742c9b..f60b73fbe 100644
--- a/compiler/rustc_builtin_macros/src/lib.rs
+++ b/compiler/rustc_builtin_macros/src/lib.rs
@@ -1,9 +1,9 @@
 //! This crate contains implementations of built-in macros and other code generating facilities
 //! injecting code into the crate before it is lowered to HIR.
 
-#![cfg_attr(not(bootstrap), allow(internal_features))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
+#![allow(internal_features)]
+#![feature(rustdoc_internals)]
+#![doc(rust_logo)]
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![feature(array_windows)]
 #![feature(box_patterns)]
@@ -23,10 +23,8 @@ extern crate tracing;
 
 use crate::deriving::*;
 
-use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
 use rustc_expand::base::{MacroExpanderFn, ResolverExpand, SyntaxExtensionKind};
 use rustc_expand::proc_macro::BangProcMacro;
-use rustc_fluent_macro::fluent_messages;
 use rustc_span::symbol::sym;
 
 mod alloc_error_handler;
@@ -59,7 +57,7 @@ pub mod proc_macro_harness;
 pub mod standard_library_imports;
 pub mod test_harness;
 
-fluent_messages! { "../messages.ftl" }
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
 
 pub fn register_builtin_macros(resolver: &mut dyn ResolverExpand) {
     let mut register = |name, kind| resolver.register_builtin_macro(name, kind);
diff --git a/compiler/rustc_builtin_macros/src/proc_macro_harness.rs b/compiler/rustc_builtin_macros/src/proc_macro_harness.rs
index dae1bc5bf..4fddaa8ab 100644
--- a/compiler/rustc_builtin_macros/src/proc_macro_harness.rs
+++ b/compiler/rustc_builtin_macros/src/proc_macro_harness.rs
@@ -38,7 +38,7 @@ enum ProcMacro {
 struct CollectProcMacros<'a> {
     macros: Vec<ProcMacro>,
     in_root: bool,
-    handler: &'a rustc_errors::Handler,
+    dcx: &'a rustc_errors::DiagCtxt,
     source_map: &'a SourceMap,
     is_proc_macro_crate: bool,
     is_test_crate: bool,
@@ -52,7 +52,7 @@ pub fn inject(
     is_proc_macro_crate: bool,
     has_proc_macro_decls: bool,
     is_test_crate: bool,
-    handler: &rustc_errors::Handler,
+    dcx: &rustc_errors::DiagCtxt,
 ) {
     let ecfg = ExpansionConfig::default("proc_macro".to_string(), features);
     let mut cx = ExtCtxt::new(sess, ecfg, resolver, None);
@@ -60,7 +60,7 @@ pub fn inject(
     let mut collect = CollectProcMacros {
         macros: Vec::new(),
         in_root: true,
-        handler,
+        dcx,
         source_map: sess.source_map(),
         is_proc_macro_crate,
         is_test_crate,
@@ -86,13 +86,13 @@ pub fn inject(
 impl<'a> CollectProcMacros<'a> {
     fn check_not_pub_in_root(&self, vis: &ast::Visibility, sp: Span) {
         if self.is_proc_macro_crate && self.in_root && vis.kind.is_pub() {
-            self.handler.emit_err(errors::ProcMacro { span: sp });
+            self.dcx.emit_err(errors::ProcMacro { span: sp });
         }
     }
 
     fn collect_custom_derive(&mut self, item: &'a ast::Item, attr: &'a ast::Attribute) {
         let Some((trait_name, proc_attrs)) =
-            parse_macro_name_and_helper_attrs(self.handler, attr, "derive")
+            parse_macro_name_and_helper_attrs(self.dcx, attr, "derive")
         else {
             return;
         };
@@ -112,7 +112,7 @@ impl<'a> CollectProcMacros<'a> {
             } else {
                 "functions tagged with `#[proc_macro_derive]` must be `pub`"
             };
-            self.handler.span_err(self.source_map.guess_head_span(item.span), msg);
+            self.dcx.span_err(self.source_map.guess_head_span(item.span), msg);
         }
     }
 
@@ -130,7 +130,7 @@ impl<'a> CollectProcMacros<'a> {
             } else {
                 "functions tagged with `#[proc_macro_attribute]` must be `pub`"
             };
-            self.handler.span_err(self.source_map.guess_head_span(item.span), msg);
+            self.dcx.span_err(self.source_map.guess_head_span(item.span), msg);
         }
     }
 
@@ -148,7 +148,7 @@ impl<'a> CollectProcMacros<'a> {
             } else {
                 "functions tagged with `#[proc_macro]` must be `pub`"
             };
-            self.handler.span_err(self.source_map.guess_head_span(item.span), msg);
+            self.dcx.span_err(self.source_map.guess_head_span(item.span), msg);
         }
     }
 }
@@ -157,7 +157,7 @@ impl<'a> Visitor<'a> for CollectProcMacros<'a> {
     fn visit_item(&mut self, item: &'a ast::Item) {
         if let ast::ItemKind::MacroDef(..) = item.kind {
             if self.is_proc_macro_crate && attr::contains_name(&item.attrs, sym::macro_export) {
-                self.handler.emit_err(errors::ExportMacroRules {
+                self.dcx.emit_err(errors::ExportMacroRules {
                     span: self.source_map.guess_head_span(item.span),
                 });
             }
@@ -192,7 +192,7 @@ impl<'a> Visitor<'a> for CollectProcMacros<'a> {
                         )
                     };
 
-                    self.handler
+                    self.dcx
                         .struct_span_err(attr.span, msg)
                         .span_label(prev_attr.span, "previous attribute here")
                         .emit();
@@ -218,7 +218,7 @@ impl<'a> Visitor<'a> for CollectProcMacros<'a> {
                 pprust::path_to_string(&attr.get_normal_item().path),
             );
 
-            self.handler.span_err(attr.span, msg);
+            self.dcx.span_err(attr.span, msg);
             return;
         }
 
@@ -232,7 +232,7 @@ impl<'a> Visitor<'a> for CollectProcMacros<'a> {
                 pprust::path_to_string(&attr.get_normal_item().path),
             );
 
-            self.handler.span_err(attr.span, msg);
+            self.dcx.span_err(attr.span, msg);
             return;
         }
 
diff --git a/compiler/rustc_builtin_macros/src/source_util.rs b/compiler/rustc_builtin_macros/src/source_util.rs
index f7bafa285..37808854a 100644
--- a/compiler/rustc_builtin_macros/src/source_util.rs
+++ b/compiler/rustc_builtin_macros/src/source_util.rs
@@ -5,11 +5,11 @@ use rustc_ast::tokenstream::TokenStream;
 use rustc_ast_pretty::pprust;
 use rustc_expand::base::{self, *};
 use rustc_expand::module::DirOwnership;
+use rustc_parse::new_parser_from_file;
 use rustc_parse::parser::{ForceCollect, Parser};
-use rustc_parse::{self, new_parser_from_file};
 use rustc_session::lint::builtin::INCOMPLETE_INCLUDE;
 use rustc_span::symbol::Symbol;
-use rustc_span::{self, Pos, Span};
+use rustc_span::{Pos, Span};
 
 use smallvec::SmallVec;
 use std::rc::Rc;
@@ -133,7 +133,7 @@ pub fn expand_include<'cx>(
             let r = base::parse_expr(&mut self.p)?;
             if self.p.token != token::Eof {
                 self.p.sess.buffer_lint(
-                    &INCOMPLETE_INCLUDE,
+                    INCOMPLETE_INCLUDE,
                     self.p.token.span,
                     self.node_id,
                     "include macro expected single expression in source",
@@ -189,7 +189,7 @@ pub fn expand_include_str(
     match cx.source_map().load_binary_file(&file) {
         Ok(bytes) => match std::str::from_utf8(&bytes) {
             Ok(src) => {
-                let interned_src = Symbol::intern(&src);
+                let interned_src = Symbol::intern(src);
                 base::MacEager::expr(cx.expr_str(sp, interned_src))
             }
             Err(_) => {
diff --git a/compiler/rustc_builtin_macros/src/test.rs b/compiler/rustc_builtin_macros/src/test.rs
index 6d55603c7..ec843a3a0 100644
--- a/compiler/rustc_builtin_macros/src/test.rs
+++ b/compiler/rustc_builtin_macros/src/test.rs
@@ -26,7 +26,7 @@ pub fn expand_test_case(
     anno_item: Annotatable,
 ) -> Vec<Annotatable> {
     check_builtin_macro_attribute(ecx, meta_item, sym::test_case);
-    warn_on_duplicate_attribute(&ecx, &anno_item, sym::test_case);
+    warn_on_duplicate_attribute(ecx, &anno_item, sym::test_case);
 
     if !ecx.ecfg.should_test {
         return vec![];
@@ -79,7 +79,7 @@ pub fn expand_test(
     item: Annotatable,
 ) -> Vec<Annotatable> {
     check_builtin_macro_attribute(cx, meta_item, sym::test);
-    warn_on_duplicate_attribute(&cx, &item, sym::test);
+    warn_on_duplicate_attribute(cx, &item, sym::test);
     expand_test_or_bench(cx, attr_sp, item, false)
 }
 
@@ -90,7 +90,7 @@ pub fn expand_bench(
     item: Annotatable,
 ) -> Vec<Annotatable> {
     check_builtin_macro_attribute(cx, meta_item, sym::bench);
-    warn_on_duplicate_attribute(&cx, &item, sym::bench);
+    warn_on_duplicate_attribute(cx, &item, sym::bench);
     expand_test_or_bench(cx, attr_sp, item, true)
 }
 
@@ -134,9 +134,9 @@ pub fn expand_test_or_bench(
     // will fail. We shouldn't try to expand in this case because the errors
     // would be spurious.
     let check_result = if is_bench {
-        check_bench_signature(cx, &item, &fn_)
+        check_bench_signature(cx, &item, fn_)
     } else {
-        check_test_signature(cx, &item, &fn_)
+        check_test_signature(cx, &item, fn_)
     };
     if check_result.is_err() {
         return if is_stmt {
@@ -389,16 +389,16 @@ pub fn expand_test_or_bench(
 }
 
 fn not_testable_error(cx: &ExtCtxt<'_>, attr_sp: Span, item: Option<&ast::Item>) {
-    let diag = &cx.sess.parse_sess.span_diagnostic;
+    let dcx = cx.sess.dcx();
     let msg = "the `#[test]` attribute may only be used on a non-associated function";
     let mut err = match item.map(|i| &i.kind) {
         // These were a warning before #92959 and need to continue being that to avoid breaking
         // stable user code (#94508).
-        Some(ast::ItemKind::MacCall(_)) => diag.struct_span_warn(attr_sp, msg),
+        Some(ast::ItemKind::MacCall(_)) => dcx.struct_span_warn(attr_sp, msg),
         // `.forget_guarantee()` needed to get these two arms to match types. Because of how
         // locally close the `.emit()` call is I'm comfortable with it, but if it can be
         // reworked in the future to not need it, it'd be nice.
-        _ => diag.struct_span_err(attr_sp, msg).forget_guarantee(),
+        _ => dcx.struct_span_err(attr_sp, msg).forget_guarantee(),
     };
     if let Some(item) = item {
         err.span_label(
@@ -466,7 +466,7 @@ fn should_ignore_message(i: &ast::Item) -> Option<Symbol> {
 fn should_panic(cx: &ExtCtxt<'_>, i: &ast::Item) -> ShouldPanic {
     match attr::find_by_name(&i.attrs, sym::should_panic) {
         Some(attr) => {
-            let sd = &cx.sess.parse_sess.span_diagnostic;
+            let dcx = cx.sess.dcx();
 
             match attr.meta_item_list() {
                 // Handle #[should_panic(expected = "foo")]
@@ -477,7 +477,7 @@ fn should_panic(cx: &ExtCtxt<'_>, i: &ast::Item) -> ShouldPanic {
                         .and_then(|mi| mi.meta_item())
                         .and_then(|mi| mi.value_str());
                     if list.len() != 1 || msg.is_none() {
-                        sd.struct_span_warn(
+                        dcx.struct_span_warn(
                             attr.span,
                             "argument must be of the form: \
                              `expected = \"error message\"`",
@@ -535,14 +535,36 @@ fn check_test_signature(
     f: &ast::Fn,
 ) -> Result<(), ErrorGuaranteed> {
     let has_should_panic_attr = attr::contains_name(&i.attrs, sym::should_panic);
-    let sd = &cx.sess.parse_sess.span_diagnostic;
+    let dcx = cx.sess.dcx();
 
     if let ast::Unsafe::Yes(span) = f.sig.header.unsafety {
-        return Err(sd.emit_err(errors::TestBadFn { span: i.span, cause: span, kind: "unsafe" }));
+        return Err(dcx.emit_err(errors::TestBadFn { span: i.span, cause: span, kind: "unsafe" }));
     }
 
-    if let ast::Async::Yes { span, .. } = f.sig.header.asyncness {
-        return Err(sd.emit_err(errors::TestBadFn { span: i.span, cause: span, kind: "async" }));
+    if let Some(coroutine_kind) = f.sig.header.coroutine_kind {
+        match coroutine_kind {
+            ast::CoroutineKind::Async { span, .. } => {
+                return Err(dcx.emit_err(errors::TestBadFn {
+                    span: i.span,
+                    cause: span,
+                    kind: "async",
+                }));
+            }
+            ast::CoroutineKind::Gen { span, .. } => {
+                return Err(dcx.emit_err(errors::TestBadFn {
+                    span: i.span,
+                    cause: span,
+                    kind: "gen",
+                }));
+            }
+            ast::CoroutineKind::AsyncGen { span, .. } => {
+                return Err(dcx.emit_err(errors::TestBadFn {
+                    span: i.span,
+                    cause: span,
+                    kind: "async gen",
+                }));
+            }
+        }
     }
 
     // If the termination trait is active, the compiler will check that the output
@@ -554,15 +576,15 @@ fn check_test_signature(
     };
 
     if !f.sig.decl.inputs.is_empty() {
-        return Err(sd.span_err(i.span, "functions used as tests can not have any arguments"));
+        return Err(dcx.span_err(i.span, "functions used as tests can not have any arguments"));
     }
 
     if has_should_panic_attr && has_output {
-        return Err(sd.span_err(i.span, "functions using `#[should_panic]` must return `()`"));
+        return Err(dcx.span_err(i.span, "functions using `#[should_panic]` must return `()`"));
     }
 
     if f.generics.params.iter().any(|param| !matches!(param.kind, GenericParamKind::Lifetime)) {
-        return Err(sd.span_err(
+        return Err(dcx.span_err(
             i.span,
             "functions used as tests can not have any non-lifetime generic parameters",
         ));
@@ -579,7 +601,7 @@ fn check_bench_signature(
     // N.B., inadequate check, but we're running
     // well before resolve, can't get too deep.
     if f.sig.decl.inputs.len() != 1 {
-        return Err(cx.sess.parse_sess.span_diagnostic.emit_err(errors::BenchSig { span: i.span }));
+        return Err(cx.sess.dcx().emit_err(errors::BenchSig { span: i.span }));
     }
     Ok(())
 }
diff --git a/compiler/rustc_builtin_macros/src/test_harness.rs b/compiler/rustc_builtin_macros/src/test_harness.rs
index c7999a226..dc28cd2ea 100644
--- a/compiler/rustc_builtin_macros/src/test_harness.rs
+++ b/compiler/rustc_builtin_macros/src/test_harness.rs
@@ -2,7 +2,7 @@
 
 use rustc_ast as ast;
 use rustc_ast::entry::EntryPointType;
-use rustc_ast::mut_visit::{ExpectOne, *};
+use rustc_ast::mut_visit::*;
 use rustc_ast::ptr::P;
 use rustc_ast::visit::{walk_item, Visitor};
 use rustc_ast::{attr, ModKind};
@@ -47,7 +47,7 @@ pub fn inject(
     features: &Features,
     resolver: &mut dyn ResolverExpand,
 ) {
-    let span_diagnostic = sess.diagnostic();
+    let dcx = sess.dcx();
     let panic_strategy = sess.panic_strategy();
     let platform_panic_strategy = sess.target.panic_strategy;
 
@@ -60,7 +60,7 @@ pub fn inject(
 
     // Do this here so that the test_runner crate attribute gets marked as used
     // even in non-test builds
-    let test_runner = get_test_runner(span_diagnostic, &krate);
+    let test_runner = get_test_runner(dcx, krate);
 
     if sess.is_test_crate() {
         let panic_strategy = match (panic_strategy, sess.opts.unstable_opts.panic_abort_tests) {
@@ -70,7 +70,7 @@ pub fn inject(
                     // Silently allow compiling with panic=abort on these platforms,
                     // but with old behavior (abort if a test fails).
                 } else {
-                    span_diagnostic.emit_err(errors::TestsNotSupport {});
+                    dcx.emit_err(errors::TestsNotSupport {});
                 }
                 PanicStrategy::Unwind
             }
@@ -372,7 +372,7 @@ fn mk_tests_slice(cx: &TestCtxt<'_>, sp: Span) -> P<ast::Expr> {
     let ecx = &cx.ext_cx;
 
     let mut tests = cx.test_cases.clone();
-    tests.sort_by(|a, b| a.name.as_str().cmp(&b.name.as_str()));
+    tests.sort_by(|a, b| a.name.as_str().cmp(b.name.as_str()));
 
     ecx.expr_array_ref(
         sp,
@@ -389,7 +389,7 @@ fn get_test_name(i: &ast::Item) -> Option<Symbol> {
     attr::first_attr_value_str_by_name(&i.attrs, sym::rustc_test_marker)
 }
 
-fn get_test_runner(sd: &rustc_errors::Handler, krate: &ast::Crate) -> Option<ast::Path> {
+fn get_test_runner(dcx: &rustc_errors::DiagCtxt, krate: &ast::Crate) -> Option<ast::Path> {
     let test_attr = attr::find_by_name(&krate.attrs, sym::test_runner)?;
     let meta_list = test_attr.meta_item_list()?;
     let span = test_attr.span;
@@ -397,11 +397,11 @@ fn get_test_runner(sd: &rustc_errors::Handler, krate: &ast::Crate) -> Option<ast
         [single] => match single.meta_item() {
             Some(meta_item) if meta_item.is_word() => return Some(meta_item.path.clone()),
             _ => {
-                sd.emit_err(errors::TestRunnerInvalid { span });
+                dcx.emit_err(errors::TestRunnerInvalid { span });
             }
         },
         _ => {
-            sd.emit_err(errors::TestRunnerNargs { span });
+            dcx.emit_err(errors::TestRunnerNargs { span });
         }
     }
     None
diff --git a/compiler/rustc_builtin_macros/src/util.rs b/compiler/rustc_builtin_macros/src/util.rs
index 9463a1418..eeaf00004 100644
--- a/compiler/rustc_builtin_macros/src/util.rs
+++ b/compiler/rustc_builtin_macros/src/util.rs
@@ -10,7 +10,7 @@ pub fn check_builtin_macro_attribute(ecx: &ExtCtxt<'_>, meta_item: &MetaItem, na
     let template = AttributeTemplate { word: true, ..Default::default() };
     validate_attr::check_builtin_meta_item(
         &ecx.sess.parse_sess,
-        &meta_item,
+        meta_item,
         AttrStyle::Outer,
         name,
         template,
diff --git a/compiler/rustc_codegen_cranelift/.github/workflows/abi-cafe.yml b/compiler/rustc_codegen_cranelift/.github/workflows/abi-cafe.yml
index 12aa69d3c..bd3b05118 100644
--- a/compiler/rustc_codegen_cranelift/.github/workflows/abi-cafe.yml
+++ b/compiler/rustc_codegen_cranelift/.github/workflows/abi-cafe.yml
@@ -35,6 +35,10 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
+    - name: CPU features
+      if: matrix.os == 'ubuntu-latest'
+      run: cat /proc/cpuinfo
+
     - name: Cache cargo target dir
       uses: actions/cache@v3
       with:
diff --git a/compiler/rustc_codegen_cranelift/.github/workflows/main.yml b/compiler/rustc_codegen_cranelift/.github/workflows/main.yml
index 47d9a3b93..05dc28d07 100644
--- a/compiler/rustc_codegen_cranelift/.github/workflows/main.yml
+++ b/compiler/rustc_codegen_cranelift/.github/workflows/main.yml
@@ -66,6 +66,10 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
+    - name: CPU features
+      if: matrix.os == 'ubuntu-latest'
+      run: cat /proc/cpuinfo
+
     - name: Cache cargo target dir
       uses: actions/cache@v3
       with:
@@ -136,6 +140,9 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
+    - name: CPU features
+      run: cat /proc/cpuinfo
+
     - name: Prepare dependencies
       run: ./y.sh prepare
 
@@ -159,6 +166,9 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
+    - name: CPU features
+      run: cat /proc/cpuinfo
+
     - name: Cache cargo target dir
       uses: actions/cache@v3
       with:
diff --git a/compiler/rustc_codegen_cranelift/.github/workflows/rustc.yml b/compiler/rustc_codegen_cranelift/.github/workflows/rustc.yml
index b49dc3aff..cb5dd51fe 100644
--- a/compiler/rustc_codegen_cranelift/.github/workflows/rustc.yml
+++ b/compiler/rustc_codegen_cranelift/.github/workflows/rustc.yml
@@ -11,6 +11,9 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
+    - name: CPU features
+      run: cat /proc/cpuinfo
+
     - name: Cache cargo target dir
       uses: actions/cache@v3
       with:
@@ -31,6 +34,9 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
+    - name: CPU features
+      run: cat /proc/cpuinfo
+
     - name: Cache cargo target dir
       uses: actions/cache@v3
       with:
diff --git a/compiler/rustc_codegen_cranelift/Cargo.lock b/compiler/rustc_codegen_cranelift/Cargo.lock
index dcb6cc575..901d1dbea 100644
--- a/compiler/rustc_codegen_cranelift/Cargo.lock
+++ b/compiler/rustc_codegen_cranelift/Cargo.lock
@@ -21,9 +21,9 @@ checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"
 
 [[package]]
 name = "arbitrary"
-version = "1.3.0"
+version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2d098ff73c1ca148721f37baad5ea6a465a13f9573aba8641fbbbae8164a54e"
+checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110"
 
 [[package]]
 name = "bitflags"
@@ -45,18 +45,18 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cranelift-bforest"
-version = "0.101.2"
+version = "0.102.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f773437307980ac0f424bf9b9a5d0cd21a0f17248c6860c9a65bec8b5975f3fe"
+checksum = "76eb38f2af690b5a4411d9a8782b6d77dabff3ca939e0518453ab9f9a4392d41"
 dependencies = [
  "cranelift-entity",
 ]
 
 [[package]]
 name = "cranelift-codegen"
-version = "0.101.2"
+version = "0.102.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "443c2ac50e97fb7de1a0f862753fce3f27215558811a6fcee508eb0c3747fa79"
+checksum = "39526c036b92912417e8931f52c1e235796688068d3efdbbd8b164f299d19156"
 dependencies = [
  "bumpalo",
  "cranelift-bforest",
@@ -75,39 +75,39 @@ dependencies = [
 
 [[package]]
 name = "cranelift-codegen-meta"
-version = "0.101.2"
+version = "0.102.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c5b174c411480c79ce0793c55042fa51bec27e486381d103a53cab3b480cb2db"
+checksum = "fdb0deedc9fccf2db53a5a3c9c9d0163e44143b0d004dca9bf6ab6a0024cd79a"
 dependencies = [
  "cranelift-codegen-shared",
 ]
 
 [[package]]
 name = "cranelift-codegen-shared"
-version = "0.101.2"
+version = "0.102.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "73fa0151a528066a369de6debeea4d4b23a32aba68b5add8c46d3dc8091ff434"
+checksum = "cea2d1b274e45aa8e61e9103efa1ba82d4b5a19d12bd1fd10744c3b7380ba3ff"
 
 [[package]]
 name = "cranelift-control"
-version = "0.101.2"
+version = "0.102.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b8adf1e6398493c9bea1190e37d28a0eb0eca5fddbc80e01e506cda34db92b1f"
+checksum = "6ea5977559a71e63db79a263f0e81a89b996e8a38212c4281e37dd1dbaa8b65c"
 dependencies = [
  "arbitrary",
 ]
 
 [[package]]
 name = "cranelift-entity"
-version = "0.101.2"
+version = "0.102.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4917e2ed3bb5fe87d0ed88395ca6d644018d119a034faedd1f3e1f2c33cd52b2"
+checksum = "2f871ada808b58158d84dfc43a6a2e2d2756baaf4ed1c51fd969ca8330e6ca5c"
 
 [[package]]
 name = "cranelift-frontend"
-version = "0.101.2"
+version = "0.102.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9aaadf1e7cf28886bbf046eaf7ef538997bc8a7e020e578ea4957b39da87d5a1"
+checksum = "e8e6890f587ef59824b3debe577e68fdf9b307b3808c54b8d93a18fd0b70941b"
 dependencies = [
  "cranelift-codegen",
  "log",
@@ -117,15 +117,15 @@ dependencies = [
 
 [[package]]
 name = "cranelift-isle"
-version = "0.101.2"
+version = "0.102.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a67fda31b9d69eaa1c49a2081939454c45857596a9d45af6744680541c628b4c"
+checksum = "a8d5fc6d5d3b52d1917002b17a8ecce448c2621b5bf394bb4e77e2f676893537"
 
 [[package]]
 name = "cranelift-jit"
-version = "0.101.2"
+version = "0.102.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d6bf32710628e7ff298739f1ed80a0bfdafc0c6a3e284c4540b23f18e8889d4b"
+checksum = "e8a2d7744f743f59d9646d7589ad22ea17ed0d71e04906eb77c31e99bc13bd8b"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -143,9 +143,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-module"
-version = "0.101.2"
+version = "0.102.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d693e93a0fbf56b4bc93cffe6b107c2e52f070e1111950505fc8c83ac440b9d"
+checksum = "b96cb196334698e612c197d7d0ae59af5e07667306ec20d7be414717db400873"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -154,9 +154,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-native"
-version = "0.101.2"
+version = "0.102.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "76fb52ba71be98312f35e798d9e98e45ab2586f27584231bf7c644fa9501e8af"
+checksum = "3e10c2e7faa65d4ae7de9a83b44f2c31aca7dc638e17d0a79572fdf8103d720b"
 dependencies = [
  "cranelift-codegen",
  "libc",
@@ -165,9 +165,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-object"
-version = "0.101.2"
+version = "0.102.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2551b2e185022b89e9efa5e04c0f17f679b86ef73d9f7feabc48b608ff23120d"
+checksum = "83ce94e18756058af8a66e3c0ba1123ae15517c72162d8060d0cb0974642adf2"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -295,9 +295,9 @@ checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
 
 [[package]]
 name = "regalloc2"
-version = "0.9.2"
+version = "0.9.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b4dcbd3a2ae7fb94b5813fa0e957c6ab51bf5d0a8ee1b69e0c2d0f1e6eb8485"
+checksum = "ad156d539c879b7a24a363a2016d77961786e71f48f2e2fc8302a92abd2429a6"
 dependencies = [
  "hashbrown 0.13.2",
  "log",
@@ -374,9 +374,9 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 
 [[package]]
 name = "wasmtime-jit-icache-coherence"
-version = "14.0.2"
+version = "15.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0980a96b16abbdaf829858d2389697b1d6cfc6a903873fd74b7e47a6b1045584"
+checksum = "b73ad1395eda136baec5ece7e079e0536a82ef73488e345456cc9b89858ad0ec"
 dependencies = [
  "cfg-if",
  "libc",
diff --git a/compiler/rustc_codegen_cranelift/Cargo.toml b/compiler/rustc_codegen_cranelift/Cargo.toml
index 30db10f74..20fcd2227 100644
--- a/compiler/rustc_codegen_cranelift/Cargo.toml
+++ b/compiler/rustc_codegen_cranelift/Cargo.toml
@@ -8,12 +8,12 @@ crate-type = ["dylib"]
 
 [dependencies]
 # These have to be in sync with each other
-cranelift-codegen = { version = "0.101.2", default-features = false, features = ["std", "unwind", "all-arch"] }
-cranelift-frontend = { version = "0.101.2" }
-cranelift-module = { version = "0.101.2" }
-cranelift-native = { version = "0.101.2" }
-cranelift-jit = { version = "0.101.2", optional = true }
-cranelift-object = { version = "0.101.2" }
+cranelift-codegen = { version = "0.102", default-features = false, features = ["std", "unwind", "all-arch"] }
+cranelift-frontend = { version = "0.102" }
+cranelift-module = { version = "0.102" }
+cranelift-native = { version = "0.102" }
+cranelift-jit = { version = "0.102", optional = true }
+cranelift-object = { version = "0.102" }
 target-lexicon = "0.12.0"
 gimli = { version = "0.28", default-features = false, features = ["write"]}
 object = { version = "0.32", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }
diff --git a/compiler/rustc_codegen_cranelift/Readme.md b/compiler/rustc_codegen_cranelift/Readme.md
index 1a2b2bbc5..ca6ecdf1d 100644
--- a/compiler/rustc_codegen_cranelift/Readme.md
+++ b/compiler/rustc_codegen_cranelift/Readme.md
@@ -5,8 +5,48 @@ This has the potential to improve compilation times in debug mode.
 If your project doesn't use any of the things listed under "Not yet supported", it should work fine.
 If not please open an issue.
 
+## Download using Rustup
+
+The Cranelift codegen backend is distributed in nightly builds on Linux and x86_64 macOS. If you want to
+install it using Rustup, you can do that by running:
+
+```bash
+$ rustup component add rustc-codegen-cranelift-preview --toolchain nightly
+```
+
+Once it is installed, you can enable it with one of the following approaches:
+- `CARGO_PROFILE_DEV_CODEGEN_BACKEND=cranelift cargo +nightly build -Zcodegen-backend`
+- `RUSTFLAGS="-Zcodegen-backend=cranelift" cargo +nightly build`
+- Add the following to `.cargo/config.toml`:
+    ```toml
+    [unstable]
+    codegen-backend = true
+
+    [profile.dev]
+    codegen-backend = "cranelift"
+    ```
+- Add the following to `Cargo.toml`:
+    ```toml
+    # This line needs to come before anything else in Cargo.toml
+    cargo-features = ["codegen-backend"]
+
+    [profile.dev]
+    codegen-backend = "cranelift"
+    ```
+
+## Precompiled builds
+
+You can also download a pre-built version from the [releases] page.
+Extract the `dist` directory in the archive anywhere you want.
+If you want to use `cargo clif build` instead of having to specify the full path to the `cargo-clif` executable, you can add the `bin` subdirectory of the extracted `dist` directory to your `PATH`.
+(tutorial [for Windows](https://stackoverflow.com/a/44272417), and [for Linux/MacOS](https://unix.stackexchange.com/questions/26047/how-to-correctly-add-a-path-to-path/26059#26059)).
+
+[releases]: https://github.com/rust-lang/rustc_codegen_cranelift/releases/tag/dev
+
 ## Building and testing
 
+If you want to build the backend manually, you can download it from GitHub and build it yourself:
+
 ```bash
 $ git clone https://github.com/rust-lang/rustc_codegen_cranelift
 $ cd rustc_codegen_cranelift
@@ -22,15 +62,6 @@ $ ./test.sh
 
 For more docs on how to build and test see [build_system/usage.txt](build_system/usage.txt) or the help message of `./y.sh`.
 
-## Precompiled builds
-
-Alternatively you can download a pre built version from the [releases] page.
-Extract the `dist` directory in the archive anywhere you want.
-If you want to use `cargo clif build` instead of having to specify the full path to the `cargo-clif` executable, you can add the `bin` subdirectory of the extracted `dist` directory to your `PATH`.
-(tutorial [for Windows](https://stackoverflow.com/a/44272417), and [for Linux/MacOS](https://unix.stackexchange.com/questions/26047/how-to-correctly-add-a-path-to-path/26059#26059)).
-
-[releases]: https://github.com/rust-lang/rustc_codegen_cranelift/releases/tag/dev
-
 ## Usage
 
 rustc_codegen_cranelift can be used as a near-drop-in replacement for `cargo build` or `cargo run` for existing projects.
diff --git a/compiler/rustc_codegen_cranelift/build_system/tests.rs b/compiler/rustc_codegen_cranelift/build_system/tests.rs
index 10736ff9a..cb7b2454c 100644
--- a/compiler/rustc_codegen_cranelift/build_system/tests.rs
+++ b/compiler/rustc_codegen_cranelift/build_system/tests.rs
@@ -75,11 +75,6 @@ const BASE_SYSROOT_SUITE: &[TestCase] = &[
         "example/arbitrary_self_types_pointers_and_wrappers.rs",
         &[],
     ),
-    TestCase::build_bin_and_run(
-        "aot.issue_91827_extern_types",
-        "example/issue-91827-extern-types.rs",
-        &[],
-    ),
     TestCase::build_lib("build.alloc_system", "example/alloc_system.rs", "lib"),
     TestCase::build_bin_and_run("aot.alloc_example", "example/alloc_example.rs", &[]),
     TestCase::jit_bin("jit.std_example", "example/std_example.rs", ""),
@@ -99,7 +94,20 @@ const BASE_SYSROOT_SUITE: &[TestCase] = &[
     TestCase::build_bin_and_run("aot.mod_bench", "example/mod_bench.rs", &[]),
     TestCase::build_bin_and_run("aot.issue-72793", "example/issue-72793.rs", &[]),
     TestCase::build_bin("aot.issue-59326", "example/issue-59326.rs"),
+    TestCase::custom("aot.polymorphize_coroutine", &|runner| {
+        runner.run_rustc(&["example/polymorphize_coroutine.rs", "-Zpolymorphize"]);
+        runner.run_out_command("polymorphize_coroutine", &[]);
+    }),
     TestCase::build_bin_and_run("aot.neon", "example/neon.rs", &[]),
+    TestCase::custom("aot.gen_block_iterate", &|runner| {
+        runner.run_rustc([
+            "example/gen_block_iterate.rs",
+            "--edition",
+            "2024",
+            "-Zunstable-options",
+        ]);
+        runner.run_out_command("gen_block_iterate", &[]);
+    }),
 ];
 
 pub(crate) static RAND_REPO: GitRepo = GitRepo::github(
@@ -224,6 +232,13 @@ const EXTENDED_SYSROOT_SUITE: &[TestCase] = &[
         if runner.is_native {
             let mut test_cmd = PORTABLE_SIMD.test(&runner.target_compiler, &runner.dirs);
             test_cmd.arg("-q");
+            // FIXME remove after portable-simd update
+            test_cmd
+                .arg("--")
+                .arg("--skip")
+                .arg("core_simd::swizzle::simd_swizzle")
+                .arg("--skip")
+                .arg("core_simd::vector::Simd<T,N>::lanes");
             spawn_and_wait(test_cmd);
         }
     }),
@@ -457,6 +472,9 @@ impl<'a> TestRunner<'a> {
         cmd.arg("--target");
         cmd.arg(&self.target_compiler.triple);
         cmd.arg("-Cpanic=abort");
+        cmd.arg("-Zunstable-options");
+        cmd.arg("--check-cfg=cfg(no_unstable_features)");
+        cmd.arg("--check-cfg=cfg(jit)");
         cmd.args(args);
         cmd
     }
diff --git a/compiler/rustc_codegen_cranelift/config.txt b/compiler/rustc_codegen_cranelift/config.txt
index 2ccdc7d78..0b7cac188 100644
--- a/compiler/rustc_codegen_cranelift/config.txt
+++ b/compiler/rustc_codegen_cranelift/config.txt
@@ -42,7 +42,9 @@ aot.float-minmax-pass
 aot.mod_bench
 aot.issue-72793
 aot.issue-59326
+aot.polymorphize_coroutine
 aot.neon
+aot.gen_block_iterate
 
 testsuite.extended_sysroot
 test.rust-random/rand
diff --git a/compiler/rustc_codegen_cranelift/example/gen_block_iterate.rs b/compiler/rustc_codegen_cranelift/example/gen_block_iterate.rs
new file mode 100644
index 000000000..25bfe542d
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/example/gen_block_iterate.rs
@@ -0,0 +1,36 @@
+// Copied from https://github.com/rust-lang/rust/blob/46455dc65069387f2dc46612f13fd45452ab301a/tests/ui/coroutine/gen_block_iterate.rs
+// revisions: next old
+//compile-flags: --edition 2024 -Zunstable-options
+//[next] compile-flags: -Znext-solver
+// run-pass
+#![feature(gen_blocks)]
+
+fn foo() -> impl Iterator<Item = u32> {
+    gen { yield 42; for x in 3..6 { yield x } }
+}
+
+fn moved() -> impl Iterator<Item = u32> {
+    let mut x = "foo".to_string();
+    gen move {
+        yield 42;
+        if x == "foo" { return }
+        x.clear();
+        for x in 3..6 { yield x }
+    }
+}
+
+fn main() {
+    let mut iter = foo();
+    assert_eq!(iter.next(), Some(42));
+    assert_eq!(iter.next(), Some(3));
+    assert_eq!(iter.next(), Some(4));
+    assert_eq!(iter.next(), Some(5));
+    assert_eq!(iter.next(), None);
+    // `gen` blocks are fused
+    assert_eq!(iter.next(), None);
+
+    let mut iter = moved();
+    assert_eq!(iter.next(), Some(42));
+    assert_eq!(iter.next(), None);
+
+}
diff --git a/compiler/rustc_codegen_cranelift/example/issue-91827-extern-types.rs b/compiler/rustc_codegen_cranelift/example/issue-91827-extern-types.rs
deleted file mode 100644
index 6f39c5edc..000000000
--- a/compiler/rustc_codegen_cranelift/example/issue-91827-extern-types.rs
+++ /dev/null
@@ -1,55 +0,0 @@
-// Copied from rustc ui test suite
-
-// run-pass
-//
-// Test that we can handle unsized types with an extern type tail part.
-// Regression test for issue #91827.
-
-#![feature(extern_types)]
-
-use std::ptr::addr_of;
-
-extern "C" {
-    type Opaque;
-}
-
-unsafe impl Sync for Opaque {}
-
-#[repr(C)]
-pub struct List<T> {
-    len: usize,
-    data: [T; 0],
-    tail: Opaque,
-}
-
-#[repr(C)]
-pub struct ListImpl<T, const N: usize> {
-    len: usize,
-    data: [T; N],
-}
-
-impl<T> List<T> {
-    const fn as_slice(&self) -> &[T] {
-        unsafe { std::slice::from_raw_parts(self.data.as_ptr(), self.len) }
-    }
-}
-
-impl<T, const N: usize> ListImpl<T, N> {
-    const fn as_list(&self) -> &List<T> {
-        unsafe { std::mem::transmute(self) }
-    }
-}
-
-pub static A: ListImpl<u128, 3> = ListImpl { len: 3, data: [5, 6, 7] };
-pub static A_REF: &'static List<u128> = A.as_list();
-pub static A_TAIL_OFFSET: isize = tail_offset(A.as_list());
-
-const fn tail_offset<T>(list: &List<T>) -> isize {
-    unsafe { (addr_of!(list.tail) as *const u8).offset_from(list as *const List<T> as *const u8) }
-}
-
-fn main() {
-    assert_eq!(A_REF.as_slice(), &[5, 6, 7]);
-    // Check that interpreter and code generation agree about the position of the tail field.
-    assert_eq!(A_TAIL_OFFSET, tail_offset(A_REF));
-}
diff --git a/compiler/rustc_codegen_cranelift/example/mini_core.rs b/compiler/rustc_codegen_cranelift/example/mini_core.rs
index 934e4b178..3607b7cd9 100644
--- a/compiler/rustc_codegen_cranelift/example/mini_core.rs
+++ b/compiler/rustc_codegen_cranelift/example/mini_core.rs
@@ -11,7 +11,7 @@
     thread_local
 )]
 #![no_core]
-#![allow(dead_code, internal_features)]
+#![allow(dead_code, internal_features, ambiguous_wide_pointer_comparisons)]
 
 #[lang = "sized"]
 pub trait Sized {}
diff --git a/compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs b/compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs
index afc51a47f..1d51b499c 100644
--- a/compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs
+++ b/compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs
@@ -337,17 +337,6 @@ fn main() {
     static REF2: &u8 = REF1;
     assert_eq!(*REF1, *REF2);
 
-    extern "C" {
-        type A;
-    }
-
-    fn main() {
-        let x: &A = unsafe { &*(1usize as *const A) };
-
-        assert_eq!(unsafe { intrinsics::size_of_val(x) }, 0);
-        assert_eq!(unsafe { intrinsics::min_align_of_val(x) }, 1);
-    }
-
     #[repr(simd)]
     struct V([f64; 2]);
 
diff --git a/compiler/rustc_codegen_cranelift/example/polymorphize_coroutine.rs b/compiler/rustc_codegen_cranelift/example/polymorphize_coroutine.rs
new file mode 100644
index 000000000..c965b34e1
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/example/polymorphize_coroutine.rs
@@ -0,0 +1,16 @@
+#![feature(coroutines, coroutine_trait)]
+
+use std::ops::Coroutine;
+use std::pin::Pin;
+
+fn main() {
+    run_coroutine::<i32>();
+}
+
+fn run_coroutine<T>() {
+    let mut coroutine = || {
+        yield;
+        return;
+    };
+    Pin::new(&mut coroutine).resume(());
+}
diff --git a/compiler/rustc_codegen_cranelift/patches/0001-portable-simd-Enable-the-exposed_provenance-feature.patch b/compiler/rustc_codegen_cranelift/patches/0001-portable-simd-Enable-the-exposed_provenance-feature.patch
new file mode 100644
index 000000000..b8c0783f5
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/patches/0001-portable-simd-Enable-the-exposed_provenance-feature.patch
@@ -0,0 +1,22 @@
+From a101a43b795431ce617e7782afb451f4853afc00 Mon Sep 17 00:00:00 2001
+From: bjorn3 <17426603+bjorn3@users.noreply.github.com>
+Date: Thu, 7 Dec 2023 14:51:35 +0000
+Subject: [PATCH] Enable the exposed_provenance feature
+
+---
+ crates/core_simd/tests/pointers.rs | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/crates/core_simd/tests/pointers.rs b/crates/core_simd/tests/pointers.rs
+index 0ae8f83..06620d6 100644
+--- a/crates/core_simd/tests/pointers.rs
++++ b/crates/core_simd/tests/pointers.rs
+@@ -1,4 +1,4 @@
+-#![feature(portable_simd, strict_provenance)]
++#![feature(exposed_provenance, portable_simd, strict_provenance)]
+ 
+ use core_simd::simd::{Simd, SimdConstPtr, SimdMutPtr};
+ 
+-- 
+2.34.1
+
diff --git a/compiler/rustc_codegen_cranelift/patches/stdlib-lock.toml b/compiler/rustc_codegen_cranelift/patches/stdlib-lock.toml
index 8a690bada..8e213f71c 100644
--- a/compiler/rustc_codegen_cranelift/patches/stdlib-lock.toml
+++ b/compiler/rustc_codegen_cranelift/patches/stdlib-lock.toml
@@ -36,15 +36,18 @@ dependencies = [
 
 [[package]]
 name = "allocator-api2"
-version = "0.2.15"
+version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "56fc6cf8dc8c4158eed8649f9b8b0ea1518eb62b544fe9490d66fa0b349eafe9"
+checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5"
 
 [[package]]
 name = "cc"
-version = "1.0.79"
+version = "1.0.83"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f"
+checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0"
+dependencies = [
+ "libc",
+]
 
 [[package]]
 name = "cfg-if"
@@ -58,9 +61,9 @@ dependencies = [
 
 [[package]]
 name = "compiler_builtins"
-version = "0.1.103"
+version = "0.1.104"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a3b73c3443a5fd2438d7ba4853c64e4c8efc2404a9e28a9234cc2d5eebc6c242"
+checksum = "99c3f9035afc33f4358773239573f7d121099856753e1bbd2a6a5207098fc741"
 dependencies = [
  "cc",
  "rustc-std-workspace-core",
@@ -124,9 +127,9 @@ dependencies = [
 
 [[package]]
 name = "gimli"
-version = "0.28.0"
+version = "0.28.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0"
+checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253"
 dependencies = [
  "compiler_builtins",
  "rustc-std-workspace-alloc",
@@ -135,9 +138,9 @@ dependencies = [
 
 [[package]]
 name = "hashbrown"
-version = "0.14.0"
+version = "0.14.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a"
+checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604"
 dependencies = [
  "allocator-api2",
  "compiler_builtins",
@@ -147,9 +150,9 @@ dependencies = [
 
 [[package]]
 name = "hermit-abi"
-version = "0.3.2"
+version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b"
+checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7"
 dependencies = [
  "compiler_builtins",
  "rustc-std-workspace-alloc",
@@ -167,9 +170,9 @@ dependencies = [
 
 [[package]]
 name = "memchr"
-version = "2.5.0"
+version = "2.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
+checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167"
 dependencies = [
  "compiler_builtins",
  "rustc-std-workspace-core",
@@ -189,9 +192,9 @@ dependencies = [
 
 [[package]]
 name = "object"
-version = "0.32.0"
+version = "0.32.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77ac5bbd07aea88c60a577a1ce218075ffd59208b2d7ca97adf9bfc5aeb21ebe"
+checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0"
 dependencies = [
  "compiler_builtins",
  "memchr",
@@ -241,9 +244,9 @@ dependencies = [
 
 [[package]]
 name = "r-efi"
-version = "4.2.0"
+version = "4.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "575fc2d9b3da54adbdfaddf6eca48fec256d977c8630a1750b8991347d1ac911"
+checksum = "0e244f96e03a3067f9e521d3167bd42657594cb8588c8d3a2db01545dc1af2e0"
 dependencies = [
  "compiler_builtins",
  "rustc-std-workspace-core",
@@ -402,9 +405,9 @@ dependencies = [
 
 [[package]]
 name = "unicode-width"
-version = "0.1.10"
+version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b"
+checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85"
 dependencies = [
  "compiler_builtins",
  "rustc-std-workspace-core",
@@ -419,6 +422,18 @@ dependencies = [
  "compiler_builtins",
  "core",
  "libc",
+ "unwinding",
+]
+
+[[package]]
+name = "unwinding"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "37a19a21a537f635c16c7576f22d0f2f7d63353c1337ad4ce0d8001c7952a25b"
+dependencies = [
+ "compiler_builtins",
+ "gimli",
+ "rustc-std-workspace-core",
 ]
 
 [[package]]
diff --git a/compiler/rustc_codegen_cranelift/rust-toolchain b/compiler/rustc_codegen_cranelift/rust-toolchain
index b832b06e0..4ba08f1af 100644
--- a/compiler/rustc_codegen_cranelift/rust-toolchain
+++ b/compiler/rustc_codegen_cranelift/rust-toolchain
@@ -1,3 +1,3 @@
 [toolchain]
-channel = "nightly-2023-11-10"
+channel = "nightly-2023-12-19"
 components = ["rust-src", "rustc-dev", "llvm-tools"]
diff --git a/compiler/rustc_codegen_cranelift/rustfmt.toml b/compiler/rustc_codegen_cranelift/rustfmt.toml
index ebeca8662..0f884187a 100644
--- a/compiler/rustc_codegen_cranelift/rustfmt.toml
+++ b/compiler/rustc_codegen_cranelift/rustfmt.toml
@@ -1,4 +1,7 @@
-ignore = ["y.rs"]
+ignore = [
+    "y.rs",
+    "example/gen_block_iterate.rs", # uses edition 2024
+]
 
 # Matches rustfmt.toml of rustc
 version = "Two"
diff --git a/compiler/rustc_codegen_cranelift/scripts/rustc-clif.rs b/compiler/rustc_codegen_cranelift/scripts/rustc-clif.rs
index 33d51bddd..550f20515 100644
--- a/compiler/rustc_codegen_cranelift/scripts/rustc-clif.rs
+++ b/compiler/rustc_codegen_cranelift/scripts/rustc-clif.rs
@@ -27,7 +27,7 @@ fn main() {
         args.push(codegen_backend_arg);
     }
     if !passed_args.iter().any(|arg| {
-        arg == "--sysroot" || arg.to_str().map(|s| s.starts_with("--sysroot=")) == Some(true)
+        arg == "--sysroot" || arg.to_str().is_some_and(|s| s.starts_with("--sysroot="))
     }) {
         args.push(OsString::from("--sysroot"));
         args.push(OsString::from(sysroot.to_str().unwrap()));
diff --git a/compiler/rustc_codegen_cranelift/scripts/rustdoc-clif.rs b/compiler/rustc_codegen_cranelift/scripts/rustdoc-clif.rs
index 10582cc7b..f7d1bdbc4 100644
--- a/compiler/rustc_codegen_cranelift/scripts/rustdoc-clif.rs
+++ b/compiler/rustc_codegen_cranelift/scripts/rustdoc-clif.rs
@@ -27,7 +27,7 @@ fn main() {
         args.push(codegen_backend_arg);
     }
     if !passed_args.iter().any(|arg| {
-        arg == "--sysroot" || arg.to_str().map(|s| s.starts_with("--sysroot=")) == Some(true)
+        arg == "--sysroot" || arg.to_str().is_some_and(|s| s.starts_with("--sysroot="))
     }) {
         args.push(OsString::from("--sysroot"));
         args.push(OsString::from(sysroot.to_str().unwrap()));
diff --git a/compiler/rustc_codegen_cranelift/scripts/rustup.sh b/compiler/rustc_codegen_cranelift/scripts/rustup.sh
index e62788f2e..355282911 100755
--- a/compiler/rustc_codegen_cranelift/scripts/rustup.sh
+++ b/compiler/rustc_codegen_cranelift/scripts/rustup.sh
@@ -46,7 +46,7 @@ case $1 in
         git pull origin master
         branch=sync_cg_clif-$(date +%Y-%m-%d)
         git checkout -b "$branch"
-        "$cg_clif/git-fixed-subtree.sh" pull --prefix=compiler/rustc_codegen_cranelift/ https://github.com/bjorn3/rustc_codegen_cranelift.git master
+        "$cg_clif/git-fixed-subtree.sh" pull --prefix=compiler/rustc_codegen_cranelift/ https://github.com/rust-lang/rustc_codegen_cranelift.git master
         git push -u my "$branch"
 
         # immediately merge the merge commit into cg_clif to prevent merge conflicts when syncing
diff --git a/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh b/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh
index bbb8a010d..731828caa 100644
--- a/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh
+++ b/compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh
@@ -1,15 +1,17 @@
 #!/usr/bin/env bash
 set -e
 
+# CG_CLIF_FORCE_GNU_AS will force usage of as instead of the LLVM backend of rustc as we
+# the LLVM backend isn't compiled in here.
+export CG_CLIF_FORCE_GNU_AS=1
+
 # Compiletest expects all standard library paths to start with /rustc/FAKE_PREFIX.
 # CG_CLIF_STDLIB_REMAP_PATH_PREFIX will cause cg_clif's build system to pass
 # --remap-path-prefix to handle this.
-# CG_CLIF_FORCE_GNU_AS will force usage of as instead of the LLVM backend of rustc as we
-# the LLVM backend isn't compiled in here.
-CG_CLIF_FORCE_GNU_AS=1 CG_CLIF_STDLIB_REMAP_PATH_PREFIX=/rustc/FAKE_PREFIX ./y.sh build
+CG_CLIF_STDLIB_REMAP_PATH_PREFIX=/rustc/FAKE_PREFIX ./y.sh build
 
 echo "[SETUP] Rust fork"
-git clone https://github.com/rust-lang/rust.git || true
+git clone https://github.com/rust-lang/rust.git --filter=tree:0 || true
 pushd rust
 git fetch
 git checkout -- .
diff --git a/compiler/rustc_codegen_cranelift/scripts/test_bootstrap.sh b/compiler/rustc_codegen_cranelift/scripts/test_bootstrap.sh
index a8f6d7a20..791d45799 100755
--- a/compiler/rustc_codegen_cranelift/scripts/test_bootstrap.sh
+++ b/compiler/rustc_codegen_cranelift/scripts/test_bootstrap.sh
@@ -11,7 +11,5 @@ rm -r compiler/rustc_codegen_cranelift/{Cargo.*,src}
 cp ../Cargo.* compiler/rustc_codegen_cranelift/
 cp -r ../src compiler/rustc_codegen_cranelift/src
 
-# CG_CLIF_FORCE_GNU_AS will force usage of as instead of the LLVM backend of rustc as we
-# the LLVM backend isn't compiled in here.
-CG_CLIF_FORCE_GNU_AS=1 ./x.py build --stage 1 library/std
+./x.py build --stage 1 library/std
 popd
diff --git a/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh b/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh
index cdc78adcf..7d7ffdadc 100755
--- a/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh
+++ b/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh
@@ -44,6 +44,7 @@ rm tests/ui/proc-macro/no-mangle-in-proc-macro-issue-111888.rs
 # vendor intrinsics
 rm tests/ui/sse2.rs # CodegenBackend::target_features not yet implemented
 rm tests/ui/simd/array-type.rs # "Index argument for `simd_insert` is not a constant"
+rm tests/ui/simd/masked-load-store.rs
 
 # exotic linkages
 rm tests/ui/issues/issue-33992.rs # unsupported linkages
@@ -146,11 +147,6 @@ rm tests/ui/process/nofile-limit.rs # TODO some AArch64 linking issue
 
 rm tests/ui/stdio-is-blocking.rs # really slow with unoptimized libstd
 
-# rustc bugs
-# ==========
-# https://github.com/rust-lang/rust/pull/116447#issuecomment-1790451463
-rm tests/ui/coroutine/gen_block_*.rs
-
 cp ../dist/bin/rustdoc-clif ../dist/bin/rustdoc # some tests expect bin/rustdoc to exist
 
 # prevent $(RUSTDOC) from picking up the sysroot built by x.py. It conflicts with the one used by
diff --git a/compiler/rustc_codegen_cranelift/src/abi/comments.rs b/compiler/rustc_codegen_cranelift/src/abi/comments.rs
index ade6968de..a318cae17 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/comments.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/comments.rs
@@ -3,7 +3,6 @@
 
 use std::borrow::Cow;
 
-use rustc_middle::mir;
 use rustc_target::abi::call::PassMode;
 
 use crate::prelude::*;
diff --git a/compiler/rustc_codegen_cranelift/src/abi/mod.rs b/compiler/rustc_codegen_cranelift/src/abi/mod.rs
index c4572e035..2c194f6d6 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/mod.rs
@@ -6,7 +6,7 @@ mod returning;
 
 use std::borrow::Cow;
 
-use cranelift_codegen::ir::{AbiParam, SigRef};
+use cranelift_codegen::ir::SigRef;
 use cranelift_module::ModuleError;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::ty::layout::FnAbiOf;
@@ -383,6 +383,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
                 args,
                 ret_place,
                 target,
+                source_info.span,
             );
             return;
         }
diff --git a/compiler/rustc_codegen_cranelift/src/analyze.rs b/compiler/rustc_codegen_cranelift/src/analyze.rs
index 321612238..c5762638a 100644
--- a/compiler/rustc_codegen_cranelift/src/analyze.rs
+++ b/compiler/rustc_codegen_cranelift/src/analyze.rs
@@ -2,7 +2,6 @@
 
 use rustc_index::IndexVec;
 use rustc_middle::mir::StatementKind::*;
-use rustc_middle::ty::Ty;
 
 use crate::prelude::*;
 
diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs
index 91b1547cb..df40a5eb4 100644
--- a/compiler/rustc_codegen_cranelift/src/base.rs
+++ b/compiler/rustc_codegen_cranelift/src/base.rs
@@ -176,10 +176,10 @@ pub(crate) fn compile_fn(
         match module.define_function(codegened_func.func_id, context) {
             Ok(()) => {}
             Err(ModuleError::Compilation(CodegenError::ImplLimitExceeded)) => {
-                let handler = rustc_session::EarlyErrorHandler::new(
+                let early_dcx = rustc_session::EarlyDiagCtxt::new(
                     rustc_session::config::ErrorOutputType::default(),
                 );
-                handler.early_error(format!(
+                early_dcx.early_error(format!(
                     "backend implementation limit exceeded while compiling {name}",
                     name = codegened_func.symbol_name
                 ));
@@ -353,7 +353,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                             fx,
                             rustc_hir::LangItem::PanicBoundsCheck,
                             &[index, len, location],
-                            source_info.span,
+                            Some(source_info.span),
                         );
                     }
                     AssertKind::MisalignedPointerDereference { ref required, ref found } => {
@@ -365,7 +365,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                             fx,
                             rustc_hir::LangItem::PanicMisalignedPointerDereference,
                             &[required, found, location],
-                            source_info.span,
+                            Some(source_info.span),
                         );
                     }
                     _ => {
@@ -456,7 +456,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                     );
                 }
 
-                crate::inline_asm::codegen_inline_asm(
+                crate::inline_asm::codegen_inline_asm_terminator(
                     fx,
                     source_info.span,
                     template,
@@ -945,19 +945,19 @@ pub(crate) fn codegen_panic<'tcx>(
     let msg_len = fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(msg_str.len()).unwrap());
     let args = [msg_ptr, msg_len, location];
 
-    codegen_panic_inner(fx, rustc_hir::LangItem::Panic, &args, source_info.span);
+    codegen_panic_inner(fx, rustc_hir::LangItem::Panic, &args, Some(source_info.span));
 }
 
 pub(crate) fn codegen_panic_nounwind<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     msg_str: &str,
-    source_info: mir::SourceInfo,
+    span: Option<Span>,
 ) {
     let msg_ptr = fx.anonymous_str(msg_str);
     let msg_len = fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(msg_str.len()).unwrap());
     let args = [msg_ptr, msg_len];
 
-    codegen_panic_inner(fx, rustc_hir::LangItem::PanicNounwind, &args, source_info.span);
+    codegen_panic_inner(fx, rustc_hir::LangItem::PanicNounwind, &args, span);
 }
 
 pub(crate) fn codegen_unwind_terminate<'tcx>(
@@ -967,16 +967,16 @@ pub(crate) fn codegen_unwind_terminate<'tcx>(
 ) {
     let args = [];
 
-    codegen_panic_inner(fx, reason.lang_item(), &args, source_info.span);
+    codegen_panic_inner(fx, reason.lang_item(), &args, Some(source_info.span));
 }
 
 fn codegen_panic_inner<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     lang_item: rustc_hir::LangItem,
     args: &[Value],
-    span: Span,
+    span: Option<Span>,
 ) {
-    let def_id = fx.tcx.require_lang_item(lang_item, Some(span));
+    let def_id = fx.tcx.require_lang_item(lang_item, span);
 
     let instance = Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx);
     let symbol_name = fx.tcx.symbol_name(instance).name;
diff --git a/compiler/rustc_codegen_cranelift/src/common.rs b/compiler/rustc_codegen_cranelift/src/common.rs
index 63562d335..bd19a7ed0 100644
--- a/compiler/rustc_codegen_cranelift/src/common.rs
+++ b/compiler/rustc_codegen_cranelift/src/common.rs
@@ -98,11 +98,15 @@ fn clif_pair_type_from_ty<'tcx>(
 
 /// Is a pointer to this type a fat ptr?
 pub(crate) fn has_ptr_meta<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> bool {
-    let ptr_ty = Ty::new_ptr(tcx, TypeAndMut { ty, mutbl: rustc_hir::Mutability::Not });
-    match &tcx.layout_of(ParamEnv::reveal_all().and(ptr_ty)).unwrap().abi {
-        Abi::Scalar(_) => false,
-        Abi::ScalarPair(_, _) => true,
-        abi => unreachable!("Abi of ptr to {:?} is {:?}???", ty, abi),
+    if ty.is_sized(tcx, ParamEnv::reveal_all()) {
+        return false;
+    }
+
+    let tail = tcx.struct_tail_erasing_lifetimes(ty, ParamEnv::reveal_all());
+    match tail.kind() {
+        ty::Foreign(..) => false,
+        ty::Str | ty::Slice(..) | ty::Dynamic(..) => true,
+        _ => bug!("unexpected unsized tail: {:?}", tail),
     }
 }
 
diff --git a/compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs b/compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs
index 20f2ee4c7..967896913 100644
--- a/compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs
+++ b/compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs
@@ -46,7 +46,7 @@ impl ConcurrencyLimiter {
         }
     }
 
-    pub(super) fn acquire(&mut self, handler: &rustc_errors::Handler) -> ConcurrencyLimiterToken {
+    pub(super) fn acquire(&mut self, dcx: &rustc_errors::DiagCtxt) -> ConcurrencyLimiterToken {
         let mut state = self.state.lock().unwrap();
         loop {
             state.assert_invariants();
@@ -64,7 +64,7 @@ impl ConcurrencyLimiter {
                     // Make sure to drop the mutex guard first to prevent poisoning the mutex.
                     drop(state);
                     if let Some(err) = err {
-                        handler.fatal(err).raise();
+                        dcx.fatal(err);
                     } else {
                         // The error was already emitted, but compilation continued. Raise a silent
                         // fatal error.
diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs
index b0853d30e..9ffa006e5 100644
--- a/compiler/rustc_codegen_cranelift/src/constant.rs
+++ b/compiler/rustc_codegen_cranelift/src/constant.rs
@@ -1,10 +1,12 @@
 //! Handling of `static`s, `const`s and promoted allocations
 
+use std::cmp::Ordering;
+
 use cranelift_module::*;
-use rustc_data_structures::fx::{FxHashMap, FxHashSet};
+use rustc_data_structures::fx::FxHashSet;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::mir::interpret::{read_target_uint, AllocId, GlobalAlloc, Scalar};
-use rustc_middle::mir::ConstValue;
+use rustc_middle::ty::ScalarInt;
 
 use crate::prelude::*;
 
@@ -123,7 +125,8 @@ pub(crate) fn codegen_const_value<'tcx>(
                 }
             }
             Scalar::Ptr(ptr, _size) => {
-                let (alloc_id, offset) = ptr.into_parts(); // we know the `offset` is relative
+                let (prov, offset) = ptr.into_parts(); // we know the `offset` is relative
+                let alloc_id = prov.alloc_id();
                 let base_addr = match fx.tcx.global_alloc(alloc_id) {
                     GlobalAlloc::Memory(alloc) => {
                         let data_id = data_id_for_alloc_id(
@@ -371,7 +374,8 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant
         let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()).to_vec();
         data.define(bytes.into_boxed_slice());
 
-        for &(offset, alloc_id) in alloc.provenance().ptrs().iter() {
+        for &(offset, prov) in alloc.provenance().ptrs().iter() {
+            let alloc_id = prov.alloc_id();
             let addend = {
                 let endianness = tcx.data_layout.endian;
                 let offset = offset.bytes() as usize;
@@ -430,9 +434,9 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant
 pub(crate) fn mir_operand_get_const_val<'tcx>(
     fx: &FunctionCx<'_, '_, 'tcx>,
     operand: &Operand<'tcx>,
-) -> Option<ConstValue<'tcx>> {
+) -> Option<ScalarInt> {
     match operand {
-        Operand::Constant(const_) => Some(eval_mir_constant(fx, const_).0),
+        Operand::Constant(const_) => eval_mir_constant(fx, const_).0.try_to_scalar_int(),
         // FIXME(rust-lang/rust#85105): Casts like `IMM8 as u32` result in the const being stored
         // inside a temporary before being passed to the intrinsic requiring the const argument.
         // This code tries to find a single constant defining definition of the referenced local.
@@ -440,7 +444,7 @@ pub(crate) fn mir_operand_get_const_val<'tcx>(
             if !place.projection.is_empty() {
                 return None;
             }
-            let mut computed_const_val = None;
+            let mut computed_scalar_int = None;
             for bb_data in fx.mir.basic_blocks.iter() {
                 for stmt in &bb_data.statements {
                     match &stmt.kind {
@@ -456,22 +460,38 @@ pub(crate) fn mir_operand_get_const_val<'tcx>(
                                     operand,
                                     ty,
                                 ) => {
-                                    if computed_const_val.is_some() {
+                                    if computed_scalar_int.is_some() {
                                         return None; // local assigned twice
                                     }
                                     if !matches!(ty.kind(), ty::Uint(_) | ty::Int(_)) {
                                         return None;
                                     }
-                                    let const_val = mir_operand_get_const_val(fx, operand)?;
-                                    if fx.layout_of(*ty).size
-                                        != const_val.try_to_scalar_int()?.size()
+                                    let scalar_int = mir_operand_get_const_val(fx, operand)?;
+                                    let scalar_int = match fx
+                                        .layout_of(*ty)
+                                        .size
+                                        .cmp(&scalar_int.size())
                                     {
-                                        return None;
-                                    }
-                                    computed_const_val = Some(const_val);
+                                        Ordering::Equal => scalar_int,
+                                        Ordering::Less => match ty.kind() {
+                                            ty::Uint(_) => ScalarInt::try_from_uint(
+                                                scalar_int.try_to_uint(scalar_int.size()).unwrap(),
+                                                fx.layout_of(*ty).size,
+                                            )
+                                            .unwrap(),
+                                            ty::Int(_) => ScalarInt::try_from_int(
+                                                scalar_int.try_to_int(scalar_int.size()).unwrap(),
+                                                fx.layout_of(*ty).size,
+                                            )
+                                            .unwrap(),
+                                            _ => unreachable!(),
+                                        },
+                                        Ordering::Greater => return None,
+                                    };
+                                    computed_scalar_int = Some(scalar_int);
                                 }
                                 Rvalue::Use(operand) => {
-                                    computed_const_val = mir_operand_get_const_val(fx, operand)
+                                    computed_scalar_int = mir_operand_get_const_val(fx, operand)
                                 }
                                 _ => return None,
                             }
@@ -522,7 +542,7 @@ pub(crate) fn mir_operand_get_const_val<'tcx>(
                     TerminatorKind::Call { .. } => {}
                 }
             }
-            computed_const_val
+            computed_scalar_int
         }
     }
 }
diff --git a/compiler/rustc_codegen_cranelift/src/driver/aot.rs b/compiler/rustc_codegen_cranelift/src/driver/aot.rs
index 11229dd42..b3ab533df 100644
--- a/compiler/rustc_codegen_cranelift/src/driver/aot.rs
+++ b/compiler/rustc_codegen_cranelift/src/driver/aot.rs
@@ -422,7 +422,7 @@ pub(crate) fn run_aot(
                                     backend_config.clone(),
                                     global_asm_config.clone(),
                                     cgu.name(),
-                                    concurrency_limiter.acquire(tcx.sess.diagnostic()),
+                                    concurrency_limiter.acquire(tcx.sess.dcx()),
                                 ),
                                 module_codegen,
                                 Some(rustc_middle::dep_graph::hash_result),
diff --git a/compiler/rustc_codegen_cranelift/src/inline_asm.rs b/compiler/rustc_codegen_cranelift/src/inline_asm.rs
index ce0eecca8..73f4bc7c1 100644
--- a/compiler/rustc_codegen_cranelift/src/inline_asm.rs
+++ b/compiler/rustc_codegen_cranelift/src/inline_asm.rs
@@ -3,14 +3,13 @@
 use std::fmt::Write;
 
 use rustc_ast::ast::{InlineAsmOptions, InlineAsmTemplatePiece};
-use rustc_middle::mir::InlineAsmOperand;
 use rustc_span::sym;
 use rustc_target::asm::*;
 use target_lexicon::BinaryFormat;
 
 use crate::prelude::*;
 
-enum CInlineAsmOperand<'tcx> {
+pub(crate) enum CInlineAsmOperand<'tcx> {
     In {
         reg: InlineAsmRegOrRegClass,
         value: Value,
@@ -34,7 +33,7 @@ enum CInlineAsmOperand<'tcx> {
     },
 }
 
-pub(crate) fn codegen_inline_asm<'tcx>(
+pub(crate) fn codegen_inline_asm_terminator<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     span: Span,
     template: &[InlineAsmTemplatePiece],
@@ -42,8 +41,6 @@ pub(crate) fn codegen_inline_asm<'tcx>(
     options: InlineAsmOptions,
     destination: Option<mir::BasicBlock>,
 ) {
-    // FIXME add .eh_frame unwind info directives
-
     // Used by panic_abort on Windows, but uses a syntax which only happens to work with
     // asm!() by accident and breaks with the GNU assembler as well as global_asm!() for
     // the LLVM backend.
@@ -135,15 +132,33 @@ pub(crate) fn codegen_inline_asm<'tcx>(
         })
         .collect::<Vec<_>>();
 
-    let mut inputs = Vec::new();
-    let mut outputs = Vec::new();
+    codegen_inline_asm_inner(fx, template, &operands, options);
+
+    match destination {
+        Some(destination) => {
+            let destination_block = fx.get_block(destination);
+            fx.bcx.ins().jump(destination_block, &[]);
+        }
+        None => {
+            fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
+        }
+    }
+}
+
+pub(crate) fn codegen_inline_asm_inner<'tcx>(
+    fx: &mut FunctionCx<'_, '_, 'tcx>,
+    template: &[InlineAsmTemplatePiece],
+    operands: &[CInlineAsmOperand<'tcx>],
+    options: InlineAsmOptions,
+) {
+    // FIXME add .eh_frame unwind info directives
 
     let mut asm_gen = InlineAssemblyGenerator {
         tcx: fx.tcx,
         arch: fx.tcx.sess.asm_arch.unwrap(),
         enclosing_def_id: fx.instance.def_id(),
         template,
-        operands: &operands,
+        operands,
         options,
         registers: Vec::new(),
         stack_slots_clobber: Vec::new(),
@@ -165,6 +180,8 @@ pub(crate) fn codegen_inline_asm<'tcx>(
     let generated_asm = asm_gen.generate_asm_wrapper(&asm_name);
     fx.cx.global_asm.push_str(&generated_asm);
 
+    let mut inputs = Vec::new();
+    let mut outputs = Vec::new();
     for (i, operand) in operands.iter().enumerate() {
         match operand {
             CInlineAsmOperand::In { reg: _, value } => {
@@ -186,16 +203,6 @@ pub(crate) fn codegen_inline_asm<'tcx>(
     }
 
     call_inline_asm(fx, &asm_name, asm_gen.stack_slot_size, inputs, outputs);
-
-    match destination {
-        Some(destination) => {
-            let destination_block = fx.get_block(destination);
-            fx.bcx.ins().jump(destination_block, &[]);
-        }
-        None => {
-            fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
-        }
-    }
 }
 
 struct InlineAssemblyGenerator<'a, 'tcx> {
@@ -637,8 +644,21 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
     ) {
         match arch {
             InlineAsmArch::X86_64 => {
-                write!(generated_asm, "    mov [rbx+0x{:x}], ", offset.bytes()).unwrap();
-                reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap();
+                match reg {
+                    InlineAsmReg::X86(reg)
+                        if reg as u32 >= X86InlineAsmReg::xmm0 as u32
+                            && reg as u32 <= X86InlineAsmReg::xmm15 as u32 =>
+                    {
+                        // rustc emits x0 rather than xmm0
+                        write!(generated_asm, "    movups [rbx+0x{:x}], ", offset.bytes()).unwrap();
+                        write!(generated_asm, "xmm{}", reg as u32 - X86InlineAsmReg::xmm0 as u32)
+                            .unwrap();
+                    }
+                    _ => {
+                        write!(generated_asm, "    mov [rbx+0x{:x}], ", offset.bytes()).unwrap();
+                        reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap();
+                    }
+                }
                 generated_asm.push('\n');
             }
             InlineAsmArch::AArch64 => {
@@ -663,8 +683,24 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
     ) {
         match arch {
             InlineAsmArch::X86_64 => {
-                generated_asm.push_str("    mov ");
-                reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap();
+                match reg {
+                    InlineAsmReg::X86(reg)
+                        if reg as u32 >= X86InlineAsmReg::xmm0 as u32
+                            && reg as u32 <= X86InlineAsmReg::xmm15 as u32 =>
+                    {
+                        // rustc emits x0 rather than xmm0
+                        write!(
+                            generated_asm,
+                            "    movups xmm{}",
+                            reg as u32 - X86InlineAsmReg::xmm0 as u32
+                        )
+                        .unwrap();
+                    }
+                    _ => {
+                        generated_asm.push_str("    mov ");
+                        reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap()
+                    }
+                }
                 writeln!(generated_asm, ", [rbx+0x{:x}]", offset.bytes()).unwrap();
             }
             InlineAsmArch::AArch64 => {
@@ -720,7 +756,12 @@ fn call_inline_asm<'tcx>(
     fx.bcx.ins().call(inline_asm_func, &[stack_slot_addr]);
 
     for (offset, place) in outputs {
-        let ty = fx.clif_type(place.layout().ty).unwrap();
+        let ty = if place.layout().ty.is_simd() {
+            let (lane_count, lane_type) = place.layout().ty.simd_size_and_type(fx.tcx);
+            fx.clif_type(lane_type).unwrap().by(lane_count.try_into().unwrap()).unwrap()
+        } else {
+            fx.clif_type(place.layout().ty).unwrap()
+        };
         let value = stack_slot.offset(fx, i32::try_from(offset.bytes()).unwrap().into()).load(
             fx,
             ty,
@@ -729,83 +770,3 @@ fn call_inline_asm<'tcx>(
         place.write_cvalue(fx, CValue::by_val(value, place.layout()));
     }
 }
-
-pub(crate) fn codegen_xgetbv<'tcx>(
-    fx: &mut FunctionCx<'_, '_, 'tcx>,
-    xcr_no: Value,
-    ret: CPlace<'tcx>,
-) {
-    // FIXME add .eh_frame unwind info directives
-
-    let operands = vec![
-        CInlineAsmOperand::In {
-            reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::cx)),
-            value: xcr_no,
-        },
-        CInlineAsmOperand::Out {
-            reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)),
-            late: true,
-            place: Some(ret),
-        },
-        CInlineAsmOperand::Out {
-            reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)),
-            late: true,
-            place: None,
-        },
-    ];
-    let options = InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM;
-
-    let mut inputs = Vec::new();
-    let mut outputs = Vec::new();
-
-    let mut asm_gen = InlineAssemblyGenerator {
-        tcx: fx.tcx,
-        arch: fx.tcx.sess.asm_arch.unwrap(),
-        enclosing_def_id: fx.instance.def_id(),
-        template: &[InlineAsmTemplatePiece::String(
-            "
-            xgetbv
-            // out = rdx << 32 | rax
-            shl rdx, 32
-            or rax, rdx
-            "
-            .to_string(),
-        )],
-        operands: &operands,
-        options,
-        registers: Vec::new(),
-        stack_slots_clobber: Vec::new(),
-        stack_slots_input: Vec::new(),
-        stack_slots_output: Vec::new(),
-        stack_slot_size: Size::from_bytes(0),
-    };
-    asm_gen.allocate_registers();
-    asm_gen.allocate_stack_slots();
-
-    let inline_asm_index = fx.cx.inline_asm_index.get();
-    fx.cx.inline_asm_index.set(inline_asm_index + 1);
-    let asm_name = format!(
-        "__inline_asm_{}_n{}",
-        fx.cx.cgu_name.as_str().replace('.', "__").replace('-', "_"),
-        inline_asm_index
-    );
-
-    let generated_asm = asm_gen.generate_asm_wrapper(&asm_name);
-    fx.cx.global_asm.push_str(&generated_asm);
-
-    for (i, operand) in operands.iter().enumerate() {
-        match operand {
-            CInlineAsmOperand::In { reg: _, value } => {
-                inputs.push((asm_gen.stack_slots_input[i].unwrap(), *value));
-            }
-            CInlineAsmOperand::Out { reg: _, late: _, place } => {
-                if let Some(place) = place {
-                    outputs.push((asm_gen.stack_slots_output[i].unwrap(), *place));
-                }
-            }
-            _ => unreachable!(),
-        }
-    }
-
-    call_inline_asm(fx, &asm_name, asm_gen.stack_slot_size, inputs, outputs);
-}
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs
index e9b7daf14..dbd5db875 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs
@@ -1,7 +1,5 @@
 //! Emulate LLVM intrinsics
 
-use rustc_middle::ty::GenericArgsRef;
-
 use crate::intrinsics::*;
 use crate::prelude::*;
 
@@ -12,6 +10,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
     args: &[mir::Operand<'tcx>],
     ret: CPlace<'tcx>,
     target: Option<BasicBlock>,
+    span: Span,
 ) {
     if intrinsic.starts_with("llvm.aarch64") {
         return llvm_aarch64::codegen_aarch64_llvm_intrinsic_call(
@@ -31,6 +30,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
             args,
             ret,
             target,
+            span,
         );
     }
 
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_aarch64.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_aarch64.rs
index ee098be1f..e1e514dca 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_aarch64.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_aarch64.rs
@@ -1,7 +1,5 @@
 //! Emulate AArch64 LLVM intrinsics
 
-use rustc_middle::ty::GenericArgsRef;
-
 use crate::intrinsics::*;
 use crate::prelude::*;
 
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs
index 4c5360486..99bb5c4ea 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs
@@ -1,7 +1,9 @@
 //! Emulate x86 LLVM intrinsics
 
-use rustc_middle::ty::GenericArgsRef;
+use rustc_ast::ast::{InlineAsmOptions, InlineAsmTemplatePiece};
+use rustc_target::asm::*;
 
+use crate::inline_asm::{codegen_inline_asm_inner, CInlineAsmOperand};
 use crate::intrinsics::*;
 use crate::prelude::*;
 
@@ -12,19 +14,53 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
     args: &[mir::Operand<'tcx>],
     ret: CPlace<'tcx>,
     target: Option<BasicBlock>,
+    span: Span,
 ) {
     match intrinsic {
         "llvm.x86.sse2.pause" | "llvm.aarch64.isb" => {
             // Spin loop hint
         }
 
+        "llvm.x86.avx.vzeroupper" => {
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_zeroupper&ig_expand=7218
+            // Do nothing. It is a perf hint anyway.
+        }
+
         // Used by is_x86_feature_detected!();
         "llvm.x86.xgetbv" => {
             intrinsic_args!(fx, args => (xcr_no); intrinsic);
 
             let xcr_no = xcr_no.load_scalar(fx);
 
-            crate::inline_asm::codegen_xgetbv(fx, xcr_no, ret);
+            codegen_inline_asm_inner(
+                fx,
+                &[InlineAsmTemplatePiece::String(
+                    "
+                    xgetbv
+                    // out = rdx << 32 | rax
+                    shl rdx, 32
+                    or rax, rdx
+                    "
+                    .to_string(),
+                )],
+                &[
+                    CInlineAsmOperand::In {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::cx)),
+                        value: xcr_no,
+                    },
+                    CInlineAsmOperand::Out {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)),
+                        late: true,
+                        place: Some(ret),
+                    },
+                    CInlineAsmOperand::Out {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)),
+                        late: true,
+                        place: None,
+                    },
+                ],
+                InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM,
+            );
         }
 
         "llvm.x86.sse3.ldu.dq" | "llvm.x86.avx.ldu.dq.256" => {
@@ -37,6 +73,103 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
             ret.write_cvalue(fx, val);
         }
 
+        "llvm.x86.avx2.gather.d.d"
+        | "llvm.x86.avx2.gather.d.q"
+        | "llvm.x86.avx2.gather.d.ps"
+        | "llvm.x86.avx2.gather.d.pd"
+        | "llvm.x86.avx2.gather.d.d.256"
+        | "llvm.x86.avx2.gather.d.q.256"
+        | "llvm.x86.avx2.gather.d.ps.256"
+        | "llvm.x86.avx2.gather.d.pd.256"
+        | "llvm.x86.avx2.gather.q.d"
+        | "llvm.x86.avx2.gather.q.q"
+        | "llvm.x86.avx2.gather.q.ps"
+        | "llvm.x86.avx2.gather.q.pd"
+        | "llvm.x86.avx2.gather.q.d.256"
+        | "llvm.x86.avx2.gather.q.q.256"
+        | "llvm.x86.avx2.gather.q.ps.256"
+        | "llvm.x86.avx2.gather.q.pd.256" => {
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_i64gather_pd&ig_expand=3818
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_mask_i64gather_pd&ig_expand=3819
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_i64gather_pd&ig_expand=3821
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_mask_i64gather_pd&ig_expand=3822
+            // ...
+
+            intrinsic_args!(fx, args => (src, ptr, index, mask, scale); intrinsic);
+
+            let (src_lane_count, src_lane_ty) = src.layout().ty.simd_size_and_type(fx.tcx);
+            let (index_lane_count, index_lane_ty) = index.layout().ty.simd_size_and_type(fx.tcx);
+            let (mask_lane_count, mask_lane_ty) = mask.layout().ty.simd_size_and_type(fx.tcx);
+            let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
+            assert_eq!(src_lane_ty, ret_lane_ty);
+            assert!(index_lane_ty.is_integral());
+            assert_eq!(src_lane_count, mask_lane_count);
+            assert_eq!(src_lane_count, ret_lane_count);
+
+            let lane_clif_ty = fx.clif_type(ret_lane_ty).unwrap();
+            let index_lane_clif_ty = fx.clif_type(index_lane_ty).unwrap();
+            let mask_lane_clif_ty = fx.clif_type(mask_lane_ty).unwrap();
+            let ret_lane_layout = fx.layout_of(ret_lane_ty);
+
+            let ptr = ptr.load_scalar(fx);
+            let scale = scale.load_scalar(fx);
+            let scale = fx.bcx.ins().uextend(types::I64, scale);
+            for lane_idx in 0..std::cmp::min(src_lane_count, index_lane_count) {
+                let src_lane = src.value_lane(fx, lane_idx).load_scalar(fx);
+                let index_lane = index.value_lane(fx, lane_idx).load_scalar(fx);
+                let mask_lane = mask.value_lane(fx, lane_idx).load_scalar(fx);
+                let mask_lane =
+                    fx.bcx.ins().bitcast(mask_lane_clif_ty.as_int(), MemFlags::new(), mask_lane);
+
+                let if_enabled = fx.bcx.create_block();
+                let if_disabled = fx.bcx.create_block();
+                let next = fx.bcx.create_block();
+                let res_lane = fx.bcx.append_block_param(next, lane_clif_ty);
+
+                let mask_lane = match mask_lane_clif_ty {
+                    types::I32 | types::F32 => {
+                        fx.bcx.ins().band_imm(mask_lane, 0x8000_0000u64 as i64)
+                    }
+                    types::I64 | types::F64 => {
+                        fx.bcx.ins().band_imm(mask_lane, 0x8000_0000_0000_0000u64 as i64)
+                    }
+                    _ => unreachable!(),
+                };
+                fx.bcx.ins().brif(mask_lane, if_enabled, &[], if_disabled, &[]);
+                fx.bcx.seal_block(if_enabled);
+                fx.bcx.seal_block(if_disabled);
+
+                fx.bcx.switch_to_block(if_enabled);
+                let index_lane = if index_lane_clif_ty != types::I64 {
+                    fx.bcx.ins().sextend(types::I64, index_lane)
+                } else {
+                    index_lane
+                };
+                let offset = fx.bcx.ins().imul(index_lane, scale);
+                let lane_ptr = fx.bcx.ins().iadd(ptr, offset);
+                let res = fx.bcx.ins().load(lane_clif_ty, MemFlags::trusted(), lane_ptr, 0);
+                fx.bcx.ins().jump(next, &[res]);
+
+                fx.bcx.switch_to_block(if_disabled);
+                fx.bcx.ins().jump(next, &[src_lane]);
+
+                fx.bcx.seal_block(next);
+                fx.bcx.switch_to_block(next);
+
+                fx.bcx.ins().nop();
+
+                ret.place_lane(fx, lane_idx)
+                    .write_cvalue(fx, CValue::by_val(res_lane, ret_lane_layout));
+            }
+
+            for lane_idx in std::cmp::min(src_lane_count, index_lane_count)..ret_lane_count {
+                let zero_lane = fx.bcx.ins().iconst(mask_lane_clif_ty.as_int(), 0);
+                let zero_lane = fx.bcx.ins().bitcast(mask_lane_clif_ty, MemFlags::new(), zero_lane);
+                ret.place_lane(fx, lane_idx)
+                    .write_cvalue(fx, CValue::by_val(zero_lane, ret_lane_layout));
+            }
+        }
+
         "llvm.x86.sse.cmp.ps" | "llvm.x86.sse2.cmp.pd" => {
             let (x, y, kind) = match args {
                 [x, y, kind] => (x, y, kind),
@@ -241,16 +374,31 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
             );
         }
         "llvm.x86.ssse3.pabs.b.128" | "llvm.x86.ssse3.pabs.w.128" | "llvm.x86.ssse3.pabs.d.128" => {
-            let a = match args {
-                [a] => a,
-                _ => bug!("wrong number of args for intrinsic {intrinsic}"),
-            };
-            let a = codegen_operand(fx, a);
+            intrinsic_args!(fx, args => (a); intrinsic);
 
             simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| {
                 fx.bcx.ins().iabs(lane)
             });
         }
+        "llvm.x86.sse2.cvttps2dq" => {
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvttps_epi32&ig_expand=2429
+            intrinsic_args!(fx, args => (a); intrinsic);
+            let a = a.load_scalar(fx);
+
+            // Using inline asm instead of fcvt_to_sint_sat as unrepresentable values are turned
+            // into 0x80000000 for which Cranelift doesn't have a native instruction.
+            codegen_inline_asm_inner(
+                fx,
+                &[InlineAsmTemplatePiece::String(format!("cvttps2dq xmm0, xmm0"))],
+                &[CInlineAsmOperand::InOut {
+                    reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)),
+                    _late: true,
+                    in_value: a,
+                    out_place: Some(ret),
+                }],
+                InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM,
+            );
+        }
         "llvm.x86.addcarry.32" | "llvm.x86.addcarry.64" => {
             intrinsic_args!(fx, args => (c_in, a, b); intrinsic);
             let c_in = c_in.load_scalar(fx);
@@ -332,9 +480,11 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
             for out_lane_idx in 0..lane_count / 8 {
                 let mut lane_diff_acc = fx.bcx.ins().iconst(types::I64, 0);
 
-                for lane_idx in out_lane_idx * 8..out_lane_idx * 8 + 1 {
+                for lane_idx in out_lane_idx * 8..out_lane_idx * 8 + 8 {
                     let a_lane = a.value_lane(fx, lane_idx).load_scalar(fx);
+                    let a_lane = fx.bcx.ins().uextend(types::I16, a_lane);
                     let b_lane = b.value_lane(fx, lane_idx).load_scalar(fx);
+                    let b_lane = fx.bcx.ins().uextend(types::I16, b_lane);
 
                     let lane_diff = fx.bcx.ins().isub(a_lane, b_lane);
                     let abs_lane_diff = fx.bcx.ins().iabs(lane_diff);
@@ -405,12 +555,12 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
             let ret_lane_layout = fx.layout_of(fx.tcx.types.i32);
             for out_lane_idx in 0..lane_count / 2 {
                 let a_lane0 = a.value_lane(fx, out_lane_idx * 2).load_scalar(fx);
-                let a_lane0 = fx.bcx.ins().uextend(types::I32, a_lane0);
+                let a_lane0 = fx.bcx.ins().sextend(types::I32, a_lane0);
                 let b_lane0 = b.value_lane(fx, out_lane_idx * 2).load_scalar(fx);
                 let b_lane0 = fx.bcx.ins().sextend(types::I32, b_lane0);
 
                 let a_lane1 = a.value_lane(fx, out_lane_idx * 2 + 1).load_scalar(fx);
-                let a_lane1 = fx.bcx.ins().uextend(types::I32, a_lane1);
+                let a_lane1 = fx.bcx.ins().sextend(types::I32, a_lane1);
                 let b_lane1 = b.value_lane(fx, out_lane_idx * 2 + 1).load_scalar(fx);
                 let b_lane1 = fx.bcx.ins().sextend(types::I32, b_lane1);
 
@@ -565,14 +715,14 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
             assert_eq!(ret_lane_ty, fx.tcx.types.i16);
             assert_eq!(lane_count * 2, ret_lane_count);
 
-            let min_i16 = fx.bcx.ins().iconst(types::I32, i64::from(i16::MIN as u16));
-            let max_i16 = fx.bcx.ins().iconst(types::I32, i64::from(i16::MAX as u16));
+            let min_i16 = fx.bcx.ins().iconst(types::I32, i32::from(i16::MIN) as u32 as i64);
+            let max_i16 = fx.bcx.ins().iconst(types::I32, i32::from(i16::MAX) as u32 as i64);
             let ret_lane_layout = fx.layout_of(fx.tcx.types.i16);
 
             for idx in 0..lane_count {
                 let lane = a.value_lane(fx, idx).load_scalar(fx);
                 let sat = fx.bcx.ins().smax(lane, min_i16);
-                let sat = fx.bcx.ins().umin(sat, max_i16);
+                let sat = fx.bcx.ins().smin(sat, max_i16);
                 let res = fx.bcx.ins().ireduce(types::I16, sat);
 
                 let res_lane = CValue::by_val(res, ret_lane_layout);
@@ -582,7 +732,7 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
             for idx in 0..lane_count {
                 let lane = b.value_lane(fx, idx).load_scalar(fx);
                 let sat = fx.bcx.ins().smax(lane, min_i16);
-                let sat = fx.bcx.ins().umin(sat, max_i16);
+                let sat = fx.bcx.ins().smin(sat, max_i16);
                 let res = fx.bcx.ins().ireduce(types::I16, sat);
 
                 let res_lane = CValue::by_val(res, ret_lane_layout);
@@ -609,8 +759,8 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
 
             for idx in 0..lane_count {
                 let lane = a.value_lane(fx, idx).load_scalar(fx);
-                let sat = fx.bcx.ins().umax(lane, min_u16);
-                let sat = fx.bcx.ins().umin(sat, max_u16);
+                let sat = fx.bcx.ins().smax(lane, min_u16);
+                let sat = fx.bcx.ins().smin(sat, max_u16);
                 let res = fx.bcx.ins().ireduce(types::I16, sat);
 
                 let res_lane = CValue::by_val(res, ret_lane_layout);
@@ -619,8 +769,8 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
 
             for idx in 0..lane_count {
                 let lane = b.value_lane(fx, idx).load_scalar(fx);
-                let sat = fx.bcx.ins().umax(lane, min_u16);
-                let sat = fx.bcx.ins().umin(sat, max_u16);
+                let sat = fx.bcx.ins().smax(lane, min_u16);
+                let sat = fx.bcx.ins().smin(sat, max_u16);
                 let res = fx.bcx.ins().ireduce(types::I16, sat);
 
                 let res_lane = CValue::by_val(res, ret_lane_layout);
@@ -641,14 +791,14 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
             assert_eq!(ret_lane_ty, fx.tcx.types.i16);
             assert_eq!(lane_count * 2, ret_lane_count);
 
-            let min_i16 = fx.bcx.ins().iconst(types::I32, i64::from(i16::MIN as u16));
-            let max_i16 = fx.bcx.ins().iconst(types::I32, i64::from(i16::MAX as u16));
+            let min_i16 = fx.bcx.ins().iconst(types::I32, i32::from(i16::MIN) as u32 as i64);
+            let max_i16 = fx.bcx.ins().iconst(types::I32, i32::from(i16::MAX) as u32 as i64);
             let ret_lane_layout = fx.layout_of(fx.tcx.types.i16);
 
             for idx in 0..lane_count / 2 {
                 let lane = a.value_lane(fx, idx).load_scalar(fx);
                 let sat = fx.bcx.ins().smax(lane, min_i16);
-                let sat = fx.bcx.ins().umin(sat, max_i16);
+                let sat = fx.bcx.ins().smin(sat, max_i16);
                 let res = fx.bcx.ins().ireduce(types::I16, sat);
 
                 let res_lane = CValue::by_val(res, ret_lane_layout);
@@ -658,7 +808,7 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
             for idx in 0..lane_count / 2 {
                 let lane = b.value_lane(fx, idx).load_scalar(fx);
                 let sat = fx.bcx.ins().smax(lane, min_i16);
-                let sat = fx.bcx.ins().umin(sat, max_i16);
+                let sat = fx.bcx.ins().smin(sat, max_i16);
                 let res = fx.bcx.ins().ireduce(types::I16, sat);
 
                 let res_lane = CValue::by_val(res, ret_lane_layout);
@@ -668,7 +818,7 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
             for idx in 0..lane_count / 2 {
                 let lane = a.value_lane(fx, idx).load_scalar(fx);
                 let sat = fx.bcx.ins().smax(lane, min_i16);
-                let sat = fx.bcx.ins().umin(sat, max_i16);
+                let sat = fx.bcx.ins().smin(sat, max_i16);
                 let res = fx.bcx.ins().ireduce(types::I16, sat);
 
                 let res_lane = CValue::by_val(res, ret_lane_layout);
@@ -678,7 +828,7 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
             for idx in 0..lane_count / 2 {
                 let lane = b.value_lane(fx, idx).load_scalar(fx);
                 let sat = fx.bcx.ins().smax(lane, min_i16);
-                let sat = fx.bcx.ins().umin(sat, max_i16);
+                let sat = fx.bcx.ins().smin(sat, max_i16);
                 let res = fx.bcx.ins().ireduce(types::I16, sat);
 
                 let res_lane = CValue::by_val(res, ret_lane_layout);
@@ -686,66 +836,489 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
             }
         }
 
-        "llvm.x86.pclmulqdq" => {
-            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_clmulepi64_si128&ig_expand=772
-            intrinsic_args!(fx, args => (a, b, imm8); intrinsic);
+        "llvm.x86.fma.vfmaddsub.ps"
+        | "llvm.x86.fma.vfmaddsub.pd"
+        | "llvm.x86.fma.vfmaddsub.ps.256"
+        | "llvm.x86.fma.vfmaddsub.pd.256" => {
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_fmaddsub_ps&ig_expand=3205
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_fmaddsub_pd&ig_expand=3181
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_fmaddsub_ps&ig_expand=3209
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_fmaddsub_pd&ig_expand=3185
+            intrinsic_args!(fx, args => (a, b, c); intrinsic);
 
             assert_eq!(a.layout(), b.layout());
+            assert_eq!(a.layout(), c.layout());
             let layout = a.layout();
 
             let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
             let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
-            assert_eq!(lane_ty, fx.tcx.types.i64);
-            assert_eq!(ret_lane_ty, fx.tcx.types.i64);
-            assert_eq!(lane_count, 2);
-            assert_eq!(ret_lane_count, 2);
+            assert!(lane_ty.is_floating_point());
+            assert!(ret_lane_ty.is_floating_point());
+            assert_eq!(lane_count, ret_lane_count);
+            let ret_lane_layout = fx.layout_of(ret_lane_ty);
 
-            let imm8 = imm8.load_scalar(fx);
+            for idx in 0..lane_count {
+                let a_lane = a.value_lane(fx, idx).load_scalar(fx);
+                let b_lane = b.value_lane(fx, idx).load_scalar(fx);
+                let c_lane = c.value_lane(fx, idx).load_scalar(fx);
+
+                let mul = fx.bcx.ins().fmul(a_lane, b_lane);
+                let res = if idx & 1 == 0 {
+                    fx.bcx.ins().fsub(mul, c_lane)
+                } else {
+                    fx.bcx.ins().fadd(mul, c_lane)
+                };
 
-            let control0 = fx.bcx.ins().band_imm(imm8, 0b0000_0001);
-            let a_lane0 = a.value_lane(fx, 0).load_scalar(fx);
-            let a_lane1 = a.value_lane(fx, 1).load_scalar(fx);
-            let temp1 = fx.bcx.ins().select(control0, a_lane1, a_lane0);
+                let res_lane = CValue::by_val(res, ret_lane_layout);
+                ret.place_lane(fx, idx).write_cvalue(fx, res_lane);
+            }
+        }
 
-            let control4 = fx.bcx.ins().band_imm(imm8, 0b0001_0000);
-            let b_lane0 = b.value_lane(fx, 0).load_scalar(fx);
-            let b_lane1 = b.value_lane(fx, 1).load_scalar(fx);
-            let temp2 = fx.bcx.ins().select(control4, b_lane1, b_lane0);
+        "llvm.x86.fma.vfmsubadd.ps"
+        | "llvm.x86.fma.vfmsubadd.pd"
+        | "llvm.x86.fma.vfmsubadd.ps.256"
+        | "llvm.x86.fma.vfmsubadd.pd.256" => {
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_fmsubadd_ps&ig_expand=3325
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_fmsubadd_pd&ig_expand=3301
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_fmsubadd_ps&ig_expand=3329
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_fmsubadd_pd&ig_expand=3305
+            intrinsic_args!(fx, args => (a, b, c); intrinsic);
 
-            fn extract_bit(fx: &mut FunctionCx<'_, '_, '_>, val: Value, bit: i64) -> Value {
-                let tmp = fx.bcx.ins().ushr_imm(val, bit);
-                fx.bcx.ins().band_imm(tmp, 1)
-            }
+            assert_eq!(a.layout(), b.layout());
+            assert_eq!(a.layout(), c.layout());
+            let layout = a.layout();
 
-            let mut res1 = fx.bcx.ins().iconst(types::I64, 0);
-            for i in 0..=63 {
-                let x = extract_bit(fx, temp1, 0);
-                let y = extract_bit(fx, temp2, i);
-                let mut temp = fx.bcx.ins().band(x, y);
-                for j in 1..=i {
-                    let x = extract_bit(fx, temp1, j);
-                    let y = extract_bit(fx, temp2, i - j);
-                    let z = fx.bcx.ins().band(x, y);
-                    temp = fx.bcx.ins().bxor(temp, z);
-                }
-                let temp = fx.bcx.ins().ishl_imm(temp, i);
-                res1 = fx.bcx.ins().bor(res1, temp);
+            let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
+            let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
+            assert!(lane_ty.is_floating_point());
+            assert!(ret_lane_ty.is_floating_point());
+            assert_eq!(lane_count, ret_lane_count);
+            let ret_lane_layout = fx.layout_of(ret_lane_ty);
+
+            for idx in 0..lane_count {
+                let a_lane = a.value_lane(fx, idx).load_scalar(fx);
+                let b_lane = b.value_lane(fx, idx).load_scalar(fx);
+                let c_lane = c.value_lane(fx, idx).load_scalar(fx);
+
+                let mul = fx.bcx.ins().fmul(a_lane, b_lane);
+                let res = if idx & 1 == 0 {
+                    fx.bcx.ins().fadd(mul, c_lane)
+                } else {
+                    fx.bcx.ins().fsub(mul, c_lane)
+                };
+
+                let res_lane = CValue::by_val(res, ret_lane_layout);
+                ret.place_lane(fx, idx).write_cvalue(fx, res_lane);
             }
-            ret.place_lane(fx, 0).to_ptr().store(fx, res1, MemFlags::trusted());
-
-            let mut res2 = fx.bcx.ins().iconst(types::I64, 0);
-            for i in 64..=127 {
-                let mut temp = fx.bcx.ins().iconst(types::I64, 0);
-                for j in i - 63..=63 {
-                    let x = extract_bit(fx, temp1, j);
-                    let y = extract_bit(fx, temp2, i - j);
-                    let z = fx.bcx.ins().band(x, y);
-                    temp = fx.bcx.ins().bxor(temp, z);
-                }
-                let temp = fx.bcx.ins().ishl_imm(temp, i);
-                res2 = fx.bcx.ins().bor(res2, temp);
+        }
+
+        "llvm.x86.fma.vfnmadd.ps"
+        | "llvm.x86.fma.vfnmadd.pd"
+        | "llvm.x86.fma.vfnmadd.ps.256"
+        | "llvm.x86.fma.vfnmadd.pd.256" => {
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_fnmadd_ps&ig_expand=3391
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_fnmadd_pd&ig_expand=3367
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_fnmadd_ps&ig_expand=3395
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm256_fnmadd_pd&ig_expand=3371
+            intrinsic_args!(fx, args => (a, b, c); intrinsic);
+
+            assert_eq!(a.layout(), b.layout());
+            assert_eq!(a.layout(), c.layout());
+            let layout = a.layout();
+
+            let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
+            let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
+            assert!(lane_ty.is_floating_point());
+            assert!(ret_lane_ty.is_floating_point());
+            assert_eq!(lane_count, ret_lane_count);
+            let ret_lane_layout = fx.layout_of(ret_lane_ty);
+
+            for idx in 0..lane_count {
+                let a_lane = a.value_lane(fx, idx).load_scalar(fx);
+                let b_lane = b.value_lane(fx, idx).load_scalar(fx);
+                let c_lane = c.value_lane(fx, idx).load_scalar(fx);
+
+                let mul = fx.bcx.ins().fmul(a_lane, b_lane);
+                let neg_mul = fx.bcx.ins().fneg(mul);
+                let res = fx.bcx.ins().fadd(neg_mul, c_lane);
+
+                let res_lane = CValue::by_val(res, ret_lane_layout);
+                ret.place_lane(fx, idx).write_cvalue(fx, res_lane);
             }
-            ret.place_lane(fx, 1).to_ptr().store(fx, res2, MemFlags::trusted());
+        }
+
+        "llvm.x86.sse42.pcmpestri128" => {
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpestri&ig_expand=939
+            intrinsic_args!(fx, args => (a, la, b, lb, _imm8); intrinsic);
+
+            let a = a.load_scalar(fx);
+            let la = la.load_scalar(fx);
+            let b = b.load_scalar(fx);
+            let lb = lb.load_scalar(fx);
+
+            let imm8 = if let Some(imm8) = crate::constant::mir_operand_get_const_val(fx, &args[4])
+            {
+                imm8
+            } else {
+                fx.tcx.sess.span_fatal(span, "Index argument for `_mm_cmpestri` is not a constant");
+            };
+
+            let imm8 = imm8.try_to_u8().unwrap_or_else(|_| panic!("kind not scalar: {:?}", imm8));
+
+            codegen_inline_asm_inner(
+                fx,
+                &[InlineAsmTemplatePiece::String(format!("pcmpestri xmm0, xmm1, {imm8}"))],
+                &[
+                    CInlineAsmOperand::In {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)),
+                        value: a,
+                    },
+                    CInlineAsmOperand::In {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)),
+                        value: b,
+                    },
+                    // Implicit argument to the pcmpestri intrinsic
+                    CInlineAsmOperand::In {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)),
+                        value: la,
+                    },
+                    // Implicit argument to the pcmpestri intrinsic
+                    CInlineAsmOperand::In {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)),
+                        value: lb,
+                    },
+                    // Implicit result of the pcmpestri intrinsic
+                    CInlineAsmOperand::Out {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::cx)),
+                        late: true,
+                        place: Some(ret),
+                    },
+                ],
+                InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM,
+            );
+        }
+
+        "llvm.x86.sse42.pcmpestrm128" => {
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cmpestrm&ig_expand=940
+            intrinsic_args!(fx, args => (a, la, b, lb, _imm8); intrinsic);
+
+            let a = a.load_scalar(fx);
+            let la = la.load_scalar(fx);
+            let b = b.load_scalar(fx);
+            let lb = lb.load_scalar(fx);
+
+            let imm8 = if let Some(imm8) = crate::constant::mir_operand_get_const_val(fx, &args[4])
+            {
+                imm8
+            } else {
+                fx.tcx.sess.span_fatal(span, "Index argument for `_mm_cmpestrm` is not a constant");
+            };
+
+            let imm8 = imm8.try_to_u8().unwrap_or_else(|_| panic!("kind not scalar: {:?}", imm8));
+
+            codegen_inline_asm_inner(
+                fx,
+                &[InlineAsmTemplatePiece::String(format!("pcmpestrm xmm0, xmm1, {imm8}"))],
+                &[
+                    CInlineAsmOperand::InOut {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)),
+                        _late: true,
+                        in_value: a,
+                        out_place: Some(ret),
+                    },
+                    CInlineAsmOperand::In {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)),
+                        value: b,
+                    },
+                    // Implicit argument to the pcmpestri intrinsic
+                    CInlineAsmOperand::In {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)),
+                        value: la,
+                    },
+                    // Implicit argument to the pcmpestri intrinsic
+                    CInlineAsmOperand::In {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)),
+                        value: lb,
+                    },
+                ],
+                InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM,
+            );
+        }
+
+        "llvm.x86.pclmulqdq" => {
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_clmulepi64_si128&ig_expand=772
+            intrinsic_args!(fx, args => (a, b, _imm8); intrinsic);
+
+            let a = a.load_scalar(fx);
+            let b = b.load_scalar(fx);
+
+            let imm8 = if let Some(imm8) = crate::constant::mir_operand_get_const_val(fx, &args[2])
+            {
+                imm8
+            } else {
+                fx.tcx.sess.span_fatal(
+                    span,
+                    "Index argument for `_mm_clmulepi64_si128` is not a constant",
+                );
+            };
+
+            let imm8 = imm8.try_to_u8().unwrap_or_else(|_| panic!("kind not scalar: {:?}", imm8));
+
+            codegen_inline_asm_inner(
+                fx,
+                &[InlineAsmTemplatePiece::String(format!("pclmulqdq xmm0, xmm1, {imm8}"))],
+                &[
+                    CInlineAsmOperand::InOut {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)),
+                        _late: true,
+                        in_value: a,
+                        out_place: Some(ret),
+                    },
+                    CInlineAsmOperand::In {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)),
+                        value: b,
+                    },
+                ],
+                InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM,
+            );
+        }
+
+        "llvm.x86.aesni.aeskeygenassist" => {
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aeskeygenassist_si128&ig_expand=261
+            intrinsic_args!(fx, args => (a, _imm8); intrinsic);
+
+            let a = a.load_scalar(fx);
+
+            let imm8 = if let Some(imm8) = crate::constant::mir_operand_get_const_val(fx, &args[1])
+            {
+                imm8
+            } else {
+                fx.tcx.sess.span_fatal(
+                    span,
+                    "Index argument for `_mm_aeskeygenassist_si128` is not a constant",
+                );
+            };
+
+            let imm8 = imm8.try_to_u8().unwrap_or_else(|_| panic!("kind not scalar: {:?}", imm8));
+
+            codegen_inline_asm_inner(
+                fx,
+                &[InlineAsmTemplatePiece::String(format!("aeskeygenassist xmm0, xmm0, {imm8}"))],
+                &[CInlineAsmOperand::InOut {
+                    reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)),
+                    _late: true,
+                    in_value: a,
+                    out_place: Some(ret),
+                }],
+                InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM,
+            );
+        }
+
+        "llvm.x86.aesni.aesimc" => {
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesimc_si128&ig_expand=260
+            intrinsic_args!(fx, args => (a); intrinsic);
+
+            let a = a.load_scalar(fx);
+
+            codegen_inline_asm_inner(
+                fx,
+                &[InlineAsmTemplatePiece::String("aesimc xmm0, xmm0".to_string())],
+                &[CInlineAsmOperand::InOut {
+                    reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)),
+                    _late: true,
+                    in_value: a,
+                    out_place: Some(ret),
+                }],
+                InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM,
+            );
+        }
+
+        "llvm.x86.aesni.aesenc" => {
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesenc_si128&ig_expand=252
+            intrinsic_args!(fx, args => (a, round_key); intrinsic);
+
+            let a = a.load_scalar(fx);
+            let round_key = round_key.load_scalar(fx);
+
+            codegen_inline_asm_inner(
+                fx,
+                &[InlineAsmTemplatePiece::String("aesenc xmm0, xmm1".to_string())],
+                &[
+                    CInlineAsmOperand::InOut {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)),
+                        _late: true,
+                        in_value: a,
+                        out_place: Some(ret),
+                    },
+                    CInlineAsmOperand::In {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)),
+                        value: round_key,
+                    },
+                ],
+                InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM,
+            );
+        }
+
+        "llvm.x86.aesni.aesenclast" => {
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesenclast_si128&ig_expand=257
+            intrinsic_args!(fx, args => (a, round_key); intrinsic);
+
+            let a = a.load_scalar(fx);
+            let round_key = round_key.load_scalar(fx);
+
+            codegen_inline_asm_inner(
+                fx,
+                &[InlineAsmTemplatePiece::String("aesenclast xmm0, xmm1".to_string())],
+                &[
+                    CInlineAsmOperand::InOut {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)),
+                        _late: true,
+                        in_value: a,
+                        out_place: Some(ret),
+                    },
+                    CInlineAsmOperand::In {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)),
+                        value: round_key,
+                    },
+                ],
+                InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM,
+            );
+        }
+
+        "llvm.x86.aesni.aesdec" => {
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesdec_si128&ig_expand=242
+            intrinsic_args!(fx, args => (a, round_key); intrinsic);
+
+            let a = a.load_scalar(fx);
+            let round_key = round_key.load_scalar(fx);
+
+            codegen_inline_asm_inner(
+                fx,
+                &[InlineAsmTemplatePiece::String("aesdec xmm0, xmm1".to_string())],
+                &[
+                    CInlineAsmOperand::InOut {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)),
+                        _late: true,
+                        in_value: a,
+                        out_place: Some(ret),
+                    },
+                    CInlineAsmOperand::In {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)),
+                        value: round_key,
+                    },
+                ],
+                InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM,
+            );
+        }
+
+        "llvm.x86.aesni.aesdeclast" => {
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_aesdeclast_si128&ig_expand=247
+            intrinsic_args!(fx, args => (a, round_key); intrinsic);
+
+            let a = a.load_scalar(fx);
+            let round_key = round_key.load_scalar(fx);
+
+            codegen_inline_asm_inner(
+                fx,
+                &[InlineAsmTemplatePiece::String("aesdeclast xmm0, xmm1".to_string())],
+                &[
+                    CInlineAsmOperand::InOut {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)),
+                        _late: true,
+                        in_value: a,
+                        out_place: Some(ret),
+                    },
+                    CInlineAsmOperand::In {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)),
+                        value: round_key,
+                    },
+                ],
+                InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM,
+            );
+        }
+
+        "llvm.x86.sha256rnds2" => {
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sha256rnds2_epu32&ig_expand=5977
+            intrinsic_args!(fx, args => (a, b, k); intrinsic);
+
+            let a = a.load_scalar(fx);
+            let b = b.load_scalar(fx);
+            let k = k.load_scalar(fx);
+
+            codegen_inline_asm_inner(
+                fx,
+                &[InlineAsmTemplatePiece::String("sha256rnds2 xmm1, xmm2".to_string())],
+                &[
+                    CInlineAsmOperand::InOut {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)),
+                        _late: true,
+                        in_value: a,
+                        out_place: Some(ret),
+                    },
+                    CInlineAsmOperand::In {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm2)),
+                        value: b,
+                    },
+                    // Implicit argument to the sha256rnds2 instruction
+                    CInlineAsmOperand::In {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)),
+                        value: k,
+                    },
+                ],
+                InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM,
+            );
+        }
+
+        "llvm.x86.sha256msg1" => {
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sha256msg1_epu32&ig_expand=5975
+            intrinsic_args!(fx, args => (a, b); intrinsic);
+
+            let a = a.load_scalar(fx);
+            let b = b.load_scalar(fx);
+
+            codegen_inline_asm_inner(
+                fx,
+                &[InlineAsmTemplatePiece::String("sha256msg1 xmm1, xmm2".to_string())],
+                &[
+                    CInlineAsmOperand::InOut {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)),
+                        _late: true,
+                        in_value: a,
+                        out_place: Some(ret),
+                    },
+                    CInlineAsmOperand::In {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm2)),
+                        value: b,
+                    },
+                ],
+                InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM,
+            );
+        }
+
+        "llvm.x86.sha256msg2" => {
+            // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_sha256msg2_epu32&ig_expand=5976
+            intrinsic_args!(fx, args => (a, b); intrinsic);
+
+            let a = a.load_scalar(fx);
+            let b = b.load_scalar(fx);
+
+            codegen_inline_asm_inner(
+                fx,
+                &[InlineAsmTemplatePiece::String("sha256msg2 xmm1, xmm2".to_string())],
+                &[
+                    CInlineAsmOperand::InOut {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm1)),
+                        _late: true,
+                        in_value: a,
+                        out_place: Some(ret),
+                    },
+                    CInlineAsmOperand::In {
+                        reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm2)),
+                        value: b,
+                    },
+                ],
+                InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM,
+            );
         }
 
         "llvm.x86.avx.ptestz.256" => {
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
index bfeeb117f..68126f124 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
@@ -487,13 +487,12 @@ fn codegen_regular_intrinsic_call<'tcx>(
             let layout = fx.layout_of(generic_args.type_at(0));
             // Note: Can't use is_unsized here as truly unsized types need to take the fixed size
             // branch
-            let size = if let Abi::ScalarPair(_, _) = ptr.layout().abi {
-                let (_ptr, info) = ptr.load_scalar_pair(fx);
-                let (size, _align) = crate::unsize::size_and_align_of_dst(fx, layout, info);
-                size
+            let meta = if let Abi::ScalarPair(_, _) = ptr.layout().abi {
+                Some(ptr.load_scalar_pair(fx).1)
             } else {
-                fx.bcx.ins().iconst(fx.pointer_type, layout.size.bytes() as i64)
+                None
             };
+            let (size, _align) = crate::unsize::size_and_align_of(fx, layout, meta);
             ret.write_cvalue(fx, CValue::by_val(size, usize_layout));
         }
         sym::min_align_of_val => {
@@ -502,13 +501,12 @@ fn codegen_regular_intrinsic_call<'tcx>(
             let layout = fx.layout_of(generic_args.type_at(0));
             // Note: Can't use is_unsized here as truly unsized types need to take the fixed size
             // branch
-            let align = if let Abi::ScalarPair(_, _) = ptr.layout().abi {
-                let (_ptr, info) = ptr.load_scalar_pair(fx);
-                let (_size, align) = crate::unsize::size_and_align_of_dst(fx, layout, info);
-                align
+            let meta = if let Abi::ScalarPair(_, _) = ptr.layout().abi {
+                Some(ptr.load_scalar_pair(fx).1)
             } else {
-                fx.bcx.ins().iconst(fx.pointer_type, layout.align.abi.bytes() as i64)
+                None
             };
+            let (_size, align) = crate::unsize::size_and_align_of(fx, layout, meta);
             ret.write_cvalue(fx, CValue::by_val(align, usize_layout));
         }
 
@@ -688,7 +686,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
                             }
                         })
                     });
-                    crate::base::codegen_panic_nounwind(fx, &msg_str, source_info);
+                    crate::base::codegen_panic_nounwind(fx, &msg_str, Some(source_info.span));
                     return;
                 }
             }
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
index ea137c4ca..fe4f073f7 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
@@ -1,7 +1,6 @@
 //! Codegen `extern "platform-intrinsic"` intrinsics.
 
-use rustc_middle::ty::GenericArgsRef;
-use rustc_span::Symbol;
+use cranelift_codegen::ir::immediates::Offset32;
 use rustc_target::abi::Endian;
 
 use super::*;
@@ -282,11 +281,11 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                 fx.tcx.sess.span_fatal(span, "Index argument for `simd_insert` is not a constant");
             };
 
-            let idx = idx_const
-                .try_to_bits(Size::from_bytes(4 /* u32*/))
-                .unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
+            let idx: u32 = idx_const
+                .try_to_u32()
+                .unwrap_or_else(|_| panic!("kind not scalar: {:?}", idx_const));
             let (lane_count, _lane_ty) = base.layout().ty.simd_size_and_type(fx.tcx);
-            if idx >= lane_count.into() {
+            if u64::from(idx) >= lane_count {
                 fx.tcx.sess.span_fatal(
                     fx.mir.span,
                     format!("[simd_insert] idx {} >= lane_count {}", idx, lane_count),
@@ -331,10 +330,10 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             };
 
             let idx = idx_const
-                .try_to_bits(Size::from_bytes(4 /* u32*/))
-                .unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
+                .try_to_u32()
+                .unwrap_or_else(|_| panic!("kind not scalar: {:?}", idx_const));
             let (lane_count, _lane_ty) = v.layout().ty.simd_size_and_type(fx.tcx);
-            if idx >= lane_count.into() {
+            if u64::from(idx) >= lane_count {
                 fx.tcx.sess.span_fatal(
                     fx.mir.span,
                     format!("[simd_extract] idx {} >= lane_count {}", idx, lane_count),
@@ -1008,8 +1007,57 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
             }
         }
 
+        sym::simd_masked_load => {
+            intrinsic_args!(fx, args => (mask, ptr, val); intrinsic);
+
+            let (val_lane_count, val_lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
+            let (mask_lane_count, _mask_lane_ty) = mask.layout().ty.simd_size_and_type(fx.tcx);
+            let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
+            assert_eq!(val_lane_count, mask_lane_count);
+            assert_eq!(val_lane_count, ret_lane_count);
+
+            let lane_clif_ty = fx.clif_type(val_lane_ty).unwrap();
+            let ret_lane_layout = fx.layout_of(ret_lane_ty);
+            let ptr_val = ptr.load_scalar(fx);
+
+            for lane_idx in 0..ret_lane_count {
+                let val_lane = val.value_lane(fx, lane_idx).load_scalar(fx);
+                let mask_lane = mask.value_lane(fx, lane_idx).load_scalar(fx);
+
+                let if_enabled = fx.bcx.create_block();
+                let if_disabled = fx.bcx.create_block();
+                let next = fx.bcx.create_block();
+                let res_lane = fx.bcx.append_block_param(next, lane_clif_ty);
+
+                fx.bcx.ins().brif(mask_lane, if_enabled, &[], if_disabled, &[]);
+                fx.bcx.seal_block(if_enabled);
+                fx.bcx.seal_block(if_disabled);
+
+                fx.bcx.switch_to_block(if_enabled);
+                let offset = lane_idx as i32 * lane_clif_ty.bytes() as i32;
+                let res = fx.bcx.ins().load(
+                    lane_clif_ty,
+                    MemFlags::trusted(),
+                    ptr_val,
+                    Offset32::new(offset),
+                );
+                fx.bcx.ins().jump(next, &[res]);
+
+                fx.bcx.switch_to_block(if_disabled);
+                fx.bcx.ins().jump(next, &[val_lane]);
+
+                fx.bcx.seal_block(next);
+                fx.bcx.switch_to_block(next);
+
+                fx.bcx.ins().nop();
+
+                ret.place_lane(fx, lane_idx)
+                    .write_cvalue(fx, CValue::by_val(res_lane, ret_lane_layout));
+            }
+        }
+
         sym::simd_scatter => {
-            intrinsic_args!(fx, args => (val, ptr, mask); intrinsic);
+            intrinsic_args!(fx, args => (mask, ptr, val); intrinsic);
 
             let (val_lane_count, _val_lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
             let (ptr_lane_count, _ptr_lane_ty) = ptr.layout().ty.simd_size_and_type(fx.tcx);
diff --git a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs
index da84e54a9..196418023 100644
--- a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs
+++ b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs
@@ -58,11 +58,10 @@
 use std::fmt;
 use std::io::Write;
 
-use cranelift_codegen::{
-    entity::SecondaryMap,
-    ir::entities::AnyEntity,
-    write::{FuncWriter, PlainWriter},
-};
+use cranelift_codegen::entity::SecondaryMap;
+use cranelift_codegen::ir::entities::AnyEntity;
+use cranelift_codegen::ir::Fact;
+use cranelift_codegen::write::{FuncWriter, PlainWriter};
 use rustc_middle::ty::layout::FnAbiOf;
 use rustc_middle::ty::print::with_no_trimmed_paths;
 use rustc_session::config::{OutputFilenames, OutputType};
@@ -155,8 +154,13 @@ impl FuncWriter for &'_ CommentWriter {
         _func: &Function,
         entity: AnyEntity,
         value: &dyn fmt::Display,
+        maybe_fact: Option<&Fact>,
     ) -> fmt::Result {
-        write!(w, "    {} = {}", entity, value)?;
+        if let Some(fact) = maybe_fact {
+            write!(w, "    {} ! {} = {}", entity, fact, value)?;
+        } else {
+            write!(w, "    {} = {}", entity, value)?;
+        }
 
         if let Some(comment) = self.entity_comments.get(&entity) {
             writeln!(w, " ; {}", comment.replace('\n', "\n; "))
@@ -227,9 +231,8 @@ pub(crate) fn write_ir_file(
     let res = std::fs::File::create(clif_file_name).and_then(|mut file| write(&mut file));
     if let Err(err) = res {
         // Using early_warn as no Session is available here
-        let handler = rustc_session::EarlyErrorHandler::new(
-            rustc_session::config::ErrorOutputType::default(),
-        );
+        let handler =
+            rustc_session::EarlyDiagCtxt::new(rustc_session::config::ErrorOutputType::default());
         handler.early_warn(format!("error writing ir file: {}", err));
     }
 }
diff --git a/compiler/rustc_codegen_cranelift/src/unsize.rs b/compiler/rustc_codegen_cranelift/src/unsize.rs
index c6133f2b3..f777e1137 100644
--- a/compiler/rustc_codegen_cranelift/src/unsize.rs
+++ b/compiler/rustc_codegen_cranelift/src/unsize.rs
@@ -2,6 +2,9 @@
 //!
 //! [`PointerCoercion::Unsize`]: `rustc_middle::ty::adjustment::PointerCoercion::Unsize`
 
+use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths};
+
+use crate::base::codegen_panic_nounwind;
 use crate::prelude::*;
 
 // Adapted from https://github.com/rust-lang/rust/blob/2a663555ddf36f6b041445894a8c175cd1bc718c/src/librustc_codegen_ssa/base.rs#L159-L307
@@ -187,63 +190,113 @@ pub(crate) fn coerce_dyn_star<'tcx>(
 
 // Adapted from https://github.com/rust-lang/rust/blob/2a663555ddf36f6b041445894a8c175cd1bc718c/src/librustc_codegen_ssa/glue.rs
 
-pub(crate) fn size_and_align_of_dst<'tcx>(
+pub(crate) fn size_and_align_of<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     layout: TyAndLayout<'tcx>,
-    info: Value,
+    info: Option<Value>,
 ) -> (Value, Value) {
-    assert!(layout.is_unsized() || layout.abi == Abi::Uninhabited);
-    match layout.ty.kind() {
+    if layout.is_sized() {
+        return (
+            fx.bcx.ins().iconst(fx.pointer_type, layout.size.bytes() as i64),
+            fx.bcx.ins().iconst(fx.pointer_type, layout.align.abi.bytes() as i64),
+        );
+    }
+
+    let ty = layout.ty;
+    match ty.kind() {
         ty::Dynamic(..) => {
             // load size/align from vtable
-            (crate::vtable::size_of_obj(fx, info), crate::vtable::min_align_of_obj(fx, info))
+            (
+                crate::vtable::size_of_obj(fx, info.unwrap()),
+                crate::vtable::min_align_of_obj(fx, info.unwrap()),
+            )
         }
         ty::Slice(_) | ty::Str => {
             let unit = layout.field(fx, 0);
             // The info in this case is the length of the str, so the size is that
             // times the unit size.
             (
-                fx.bcx.ins().imul_imm(info, unit.size.bytes() as i64),
+                fx.bcx.ins().imul_imm(info.unwrap(), unit.size.bytes() as i64),
                 fx.bcx.ins().iconst(fx.pointer_type, unit.align.abi.bytes() as i64),
             )
         }
-        _ => {
+        ty::Foreign(_) => {
+            let trap_block = fx.bcx.create_block();
+            let true_ = fx.bcx.ins().iconst(types::I8, 1);
+            let next_block = fx.bcx.create_block();
+            fx.bcx.ins().brif(true_, trap_block, &[], next_block, &[]);
+            fx.bcx.seal_block(trap_block);
+            fx.bcx.seal_block(next_block);
+            fx.bcx.switch_to_block(trap_block);
+
+            // `extern` type. We cannot compute the size, so panic.
+            let msg_str = with_no_visible_paths!({
+                with_no_trimmed_paths!({
+                    format!("attempted to compute the size or alignment of extern type `{ty}`")
+                })
+            });
+
+            codegen_panic_nounwind(fx, &msg_str, None);
+
+            fx.bcx.switch_to_block(next_block);
+
+            // This function does not return so we can now return whatever we want.
+            let size = fx.bcx.ins().iconst(fx.pointer_type, 42);
+            let align = fx.bcx.ins().iconst(fx.pointer_type, 42);
+            (size, align)
+        }
+        ty::Adt(..) | ty::Tuple(..) => {
             // First get the size of all statically known fields.
             // Don't use size_of because it also rounds up to alignment, which we
             // want to avoid, as the unsized field's alignment could be smaller.
             assert!(!layout.ty.is_simd());
 
             let i = layout.fields.count() - 1;
-            let sized_size = layout.fields.offset(i).bytes();
+            let unsized_offset_unadjusted = layout.fields.offset(i).bytes();
+            let unsized_offset_unadjusted =
+                fx.bcx.ins().iconst(fx.pointer_type, unsized_offset_unadjusted as i64);
             let sized_align = layout.align.abi.bytes();
             let sized_align = fx.bcx.ins().iconst(fx.pointer_type, sized_align as i64);
 
             // Recurse to get the size of the dynamically sized field (must be
             // the last field).
             let field_layout = layout.field(fx, i);
-            let (unsized_size, mut unsized_align) = size_and_align_of_dst(fx, field_layout, info);
-
-            // FIXME (#26403, #27023): We should be adding padding
-            // to `sized_size` (to accommodate the `unsized_align`
-            // required of the unsized field that follows) before
-            // summing it with `sized_size`. (Note that since #26403
-            // is unfixed, we do not yet add the necessary padding
-            // here. But this is where the add would go.)
-
-            // Return the sum of sizes and max of aligns.
-            let size = fx.bcx.ins().iadd_imm(unsized_size, sized_size as i64);
-
-            // Packed types ignore the alignment of their fields.
-            if let ty::Adt(def, _) = layout.ty.kind() {
-                if def.repr().packed() {
-                    unsized_align = sized_align;
+            let (unsized_size, mut unsized_align) = size_and_align_of(fx, field_layout, info);
+
+            // # First compute the dynamic alignment
+
+            // For packed types, we need to cap the alignment.
+            if let ty::Adt(def, _) = ty.kind() {
+                if let Some(packed) = def.repr().pack {
+                    if packed.bytes() == 1 {
+                        // We know this will be capped to 1.
+                        unsized_align = fx.bcx.ins().iconst(fx.pointer_type, 1);
+                    } else {
+                        // We have to dynamically compute `min(unsized_align, packed)`.
+                        let packed = fx.bcx.ins().iconst(fx.pointer_type, packed.bytes() as i64);
+                        let cmp = fx.bcx.ins().icmp(IntCC::UnsignedLessThan, unsized_align, packed);
+                        unsized_align = fx.bcx.ins().select(cmp, unsized_align, packed);
+                    }
                 }
             }
 
             // Choose max of two known alignments (combined value must
             // be aligned according to more restrictive of the two).
             let cmp = fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, sized_align, unsized_align);
-            let align = fx.bcx.ins().select(cmp, sized_align, unsized_align);
+            let full_align = fx.bcx.ins().select(cmp, sized_align, unsized_align);
+
+            // # Then compute the dynamic size
+
+            // The full formula for the size would be:
+            // let unsized_offset_adjusted = unsized_offset_unadjusted.align_to(unsized_align);
+            // let full_size = (unsized_offset_adjusted + unsized_size).align_to(full_align);
+            // However, `unsized_size` is a multiple of `unsized_align`.
+            // Therefore, we can equivalently do the `align_to(unsized_align)` *after* adding `unsized_size`:
+            // let full_size = (unsized_offset_unadjusted + unsized_size).align_to(unsized_align).align_to(full_align);
+            // Furthermore, `align >= unsized_align`, and therefore we only need to do:
+            // let full_size = (unsized_offset_unadjusted + unsized_size).align_to(full_align);
+
+            let full_size = fx.bcx.ins().iadd(unsized_offset_unadjusted, unsized_size);
 
             // Issue #27023: must add any necessary padding to `size`
             // (to make it a multiple of `align`) before returning it.
@@ -255,12 +308,13 @@ pub(crate) fn size_and_align_of_dst<'tcx>(
             // emulated via the semi-standard fast bit trick:
             //
             //   `(size + (align-1)) & -align`
-            let addend = fx.bcx.ins().iadd_imm(align, -1);
-            let add = fx.bcx.ins().iadd(size, addend);
-            let neg = fx.bcx.ins().ineg(align);
-            let size = fx.bcx.ins().band(add, neg);
+            let addend = fx.bcx.ins().iadd_imm(full_align, -1);
+            let add = fx.bcx.ins().iadd(full_size, addend);
+            let neg = fx.bcx.ins().ineg(full_align);
+            let full_size = fx.bcx.ins().band(add, neg);
 
-            (size, align)
+            (full_size, full_align)
         }
+        _ => bug!("size_and_align_of_dst: {ty} not supported"),
     }
 }
diff --git a/compiler/rustc_codegen_cranelift/src/value_and_place.rs b/compiler/rustc_codegen_cranelift/src/value_and_place.rs
index 21ad2a835..567a5669d 100644
--- a/compiler/rustc_codegen_cranelift/src/value_and_place.rs
+++ b/compiler/rustc_codegen_cranelift/src/value_and_place.rs
@@ -20,34 +20,36 @@ fn codegen_field<'tcx>(
         (base.offset_i64(fx, i64::try_from(field_offset.bytes()).unwrap()), field_layout)
     };
 
-    if let Some(extra) = extra {
-        if field_layout.is_sized() {
-            return simple(fx);
-        }
-        match field_layout.ty.kind() {
-            ty::Slice(..) | ty::Str | ty::Foreign(..) => simple(fx),
-            ty::Adt(def, _) if def.repr().packed() => {
-                assert_eq!(layout.align.abi.bytes(), 1);
-                simple(fx)
-            }
-            _ => {
-                // We have to align the offset for DST's
-                let unaligned_offset = field_offset.bytes();
-                let (_, unsized_align) =
-                    crate::unsize::size_and_align_of_dst(fx, field_layout, extra);
+    if field_layout.is_sized() {
+        return simple(fx);
+    }
+    match field_layout.ty.kind() {
+        ty::Slice(..) | ty::Str => simple(fx),
+        _ => {
+            let unaligned_offset = field_offset.bytes();
 
-                let one = fx.bcx.ins().iconst(fx.pointer_type, 1);
-                let align_sub_1 = fx.bcx.ins().isub(unsized_align, one);
-                let and_lhs = fx.bcx.ins().iadd_imm(align_sub_1, unaligned_offset as i64);
-                let zero = fx.bcx.ins().iconst(fx.pointer_type, 0);
-                let and_rhs = fx.bcx.ins().isub(zero, unsized_align);
-                let offset = fx.bcx.ins().band(and_lhs, and_rhs);
+            // Get the alignment of the field
+            let (_, mut unsized_align) = crate::unsize::size_and_align_of(fx, field_layout, extra);
 
-                (base.offset_value(fx, offset), field_layout)
+            // For packed types, we need to cap alignment.
+            if let ty::Adt(def, _) = layout.ty.kind() {
+                if let Some(packed) = def.repr().pack {
+                    let packed = fx.bcx.ins().iconst(fx.pointer_type, packed.bytes() as i64);
+                    let cmp = fx.bcx.ins().icmp(IntCC::UnsignedLessThan, unsized_align, packed);
+                    unsized_align = fx.bcx.ins().select(cmp, unsized_align, packed);
+                }
             }
+
+            // Bump the unaligned offset up to the appropriate alignment
+            let one = fx.bcx.ins().iconst(fx.pointer_type, 1);
+            let align_sub_1 = fx.bcx.ins().isub(unsized_align, one);
+            let and_lhs = fx.bcx.ins().iadd_imm(align_sub_1, unaligned_offset as i64);
+            let zero = fx.bcx.ins().iconst(fx.pointer_type, 0);
+            let and_rhs = fx.bcx.ins().isub(zero, unsized_align);
+            let offset = fx.bcx.ins().band(and_lhs, and_rhs);
+
+            (base.offset_value(fx, offset), field_layout)
         }
-    } else {
-        simple(fx)
     }
 }
 
@@ -329,7 +331,13 @@ impl<'tcx> CValue<'tcx> {
                 let msb = fx.bcx.ins().iconst(types::I64, (const_val >> 64) as u64 as i64);
                 fx.bcx.ins().iconcat(lsb, msb)
             }
-            ty::Bool | ty::Char | ty::Uint(_) | ty::Int(_) | ty::Ref(..) | ty::RawPtr(..) => {
+            ty::Bool
+            | ty::Char
+            | ty::Uint(_)
+            | ty::Int(_)
+            | ty::Ref(..)
+            | ty::RawPtr(..)
+            | ty::FnPtr(..) => {
                 let raw_val = const_val.size().truncate(const_val.to_bits(layout.size).unwrap());
                 fx.bcx.ins().iconst(clif_ty, raw_val as i64)
             }
@@ -725,13 +733,8 @@ impl<'tcx> CPlace<'tcx> {
         };
 
         let (field_ptr, field_layout) = codegen_field(fx, base, extra, layout, field);
-        if field_layout.is_unsized() {
-            if let ty::Foreign(_) = field_layout.ty.kind() {
-                assert!(extra.is_none());
-                CPlace::for_ptr(field_ptr, field_layout)
-            } else {
-                CPlace::for_ptr_with_extra(field_ptr, extra.unwrap(), field_layout)
-            }
+        if has_ptr_meta(fx.tcx, field_layout.ty) {
+            CPlace::for_ptr_with_extra(field_ptr, extra.unwrap(), field_layout)
         } else {
             CPlace::for_ptr(field_ptr, field_layout)
         }
@@ -971,6 +974,32 @@ pub(crate) fn assert_assignable<'tcx>(
                 }
             }
         }
+        (&ty::Coroutine(def_id_a, args_a, mov_a), &ty::Coroutine(def_id_b, args_b, mov_b))
+            if def_id_a == def_id_b && mov_a == mov_b =>
+        {
+            let mut types_a = args_a.types();
+            let mut types_b = args_b.types();
+            loop {
+                match (types_a.next(), types_b.next()) {
+                    (Some(a), Some(b)) => assert_assignable(fx, a, b, limit - 1),
+                    (None, None) => return,
+                    (Some(_), None) | (None, Some(_)) => panic!("{:#?}/{:#?}", from_ty, to_ty),
+                }
+            }
+        }
+        (&ty::CoroutineWitness(def_id_a, args_a), &ty::CoroutineWitness(def_id_b, args_b))
+            if def_id_a == def_id_b =>
+        {
+            let mut types_a = args_a.types();
+            let mut types_b = args_b.types();
+            loop {
+                match (types_a.next(), types_b.next()) {
+                    (Some(a), Some(b)) => assert_assignable(fx, a, b, limit - 1),
+                    (None, None) => return,
+                    (Some(_), None) | (None, Some(_)) => panic!("{:#?}/{:#?}", from_ty, to_ty),
+                }
+            }
+        }
         (ty::Param(_), _) | (_, ty::Param(_)) if fx.tcx.sess.opts.unstable_opts.polymorphize => {
             // No way to check if it is correct or not with polymorphization enabled
         }
diff --git a/compiler/rustc_codegen_cranelift/y.cmd b/compiler/rustc_codegen_cranelift/y.cmd
new file mode 100644
index 000000000..e9b688645
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/y.cmd
@@ -0,0 +1,9 @@
+@echo off
+echo [BUILD] build system >&2
+mkdir build 2>nul
+rustc build_system/main.rs -o build\y.exe -Cdebuginfo=1 --edition 2021 || goto :error
+build\y.exe %* || goto :error
+goto :EOF
+
+:error
+exit /b
diff --git a/compiler/rustc_codegen_cranelift/y.ps1 b/compiler/rustc_codegen_cranelift/y.ps1
new file mode 100644
index 000000000..02ef0fcbd
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/y.ps1
@@ -0,0 +1,12 @@
+$ErrorActionPreference = "Stop"
+
+$host.ui.WriteErrorLine("[BUILD] build system")
+New-Item -ItemType Directory -Force -Path build | Out-Null
+& rustc build_system/main.rs -o build\y.exe -Cdebuginfo=1 --edition 2021
+if ($LASTEXITCODE -ne 0) {
+    exit $LASTEXITCODE
+}
+& build\y.exe $args
+if ($LASTEXITCODE -ne 0) {
+    exit $LASTEXITCODE
+}
diff --git a/compiler/rustc_codegen_gcc/.github/workflows/ci.yml b/compiler/rustc_codegen_gcc/.github/workflows/ci.yml
index 65e7a697a..308bc55ea 100644
--- a/compiler/rustc_codegen_gcc/.github/workflows/ci.yml
+++ b/compiler/rustc_codegen_gcc/.github/workflows/ci.yml
@@ -99,8 +99,10 @@ jobs:
     - name: Build
       run: |
         ./y.sh prepare --only-libcore
-        ./y.sh build
-        cargo test
+        # TODO: remove --features master when it is back to the default.
+        ./y.sh build --features master
+        # TODO: remove --features master when it is back to the default.
+        cargo test --features master
         ./clean_all.sh
 
     - name: Prepare dependencies
@@ -121,7 +123,8 @@ jobs:
 
     - name: Run tests
       run: |
-        ./test.sh --release --clean --build-sysroot ${{ matrix.commands }}
+        # TODO: remove --features master when it is back to the default.
+        ./test.sh --features master --release --clean --build-sysroot ${{ matrix.commands }}
 
   duplicates:
     runs-on: ubuntu-latest
diff --git a/compiler/rustc_codegen_gcc/.github/workflows/failures.yml b/compiler/rustc_codegen_gcc/.github/workflows/failures.yml
index 27864dcad..ae8de79b7 100644
--- a/compiler/rustc_codegen_gcc/.github/workflows/failures.yml
+++ b/compiler/rustc_codegen_gcc/.github/workflows/failures.yml
@@ -21,11 +21,14 @@ jobs:
         libgccjit_version:
           - gcc: "libgccjit.so"
             artifacts_branch: "master"
+            # TODO: switch back to --no-default-features in the case of libgccjit 12 when the default is to enable
+            # master again.
+            extra: "--features master"
           - gcc: "libgccjit_without_int128.so"
             artifacts_branch: "master-without-128bit-integers"
+            extra: "--features master"
           - gcc: "libgccjit12.so"
             artifacts_branch: "gcc12"
-            extra: "--no-default-features"
             # FIXME(antoyo): we need to set GCC_EXEC_PREFIX so that the linker can find the linker plugin.
             # Not sure why it's not found otherwise.
             env_extra: "TEST_FLAGS='-Cpanic=abort -Zpanic-abort-tests' GCC_EXEC_PREFIX=/usr/lib/gcc/"
diff --git a/compiler/rustc_codegen_gcc/.github/workflows/m68k.yml b/compiler/rustc_codegen_gcc/.github/workflows/m68k.yml
index 55ee0a212..4d9d7e23d 100644
--- a/compiler/rustc_codegen_gcc/.github/workflows/m68k.yml
+++ b/compiler/rustc_codegen_gcc/.github/workflows/m68k.yml
@@ -114,8 +114,10 @@ jobs:
     - name: Build
       run: |
         ./y.sh prepare --only-libcore --cross
-        ./y.sh build --target-triple m68k-unknown-linux-gnu
-        CG_GCC_TEST_TARGET=m68k-unknown-linux-gnu cargo test
+        # TODO: remove --features master when it is back to the default.
+        ./y.sh build --target-triple m68k-unknown-linux-gnu --features master
+        # TODO: remove --features master when it is back to the default.
+        CG_GCC_TEST_TARGET=m68k-unknown-linux-gnu cargo test --features master
         ./clean_all.sh
 
     - name: Prepare dependencies
@@ -136,4 +138,5 @@ jobs:
 
     - name: Run tests
       run: |
-        ./test.sh --release --clean --build-sysroot ${{ matrix.commands }}
+        # TODO: remove --features master when it is back to the default.
+        ./test.sh --release --features master --clean --build-sysroot ${{ matrix.commands }}
diff --git a/compiler/rustc_codegen_gcc/.github/workflows/release.yml b/compiler/rustc_codegen_gcc/.github/workflows/release.yml
index ae1134177..43b90fcec 100644
--- a/compiler/rustc_codegen_gcc/.github/workflows/release.yml
+++ b/compiler/rustc_codegen_gcc/.github/workflows/release.yml
@@ -78,8 +78,10 @@ jobs:
     - name: Build
       run: |
         ./y.sh prepare --only-libcore
-        EMBED_LTO_BITCODE=1 ./y.sh build --release --release-sysroot
-        cargo test
+        # TODO: remove --features master when it is back to the default.
+        EMBED_LTO_BITCODE=1 ./y.sh build --release --release-sysroot --features master
+        # TODO: remove --features master when it is back to the default.
+        cargo test --features master
         ./clean_all.sh
 
     - name: Prepare dependencies
@@ -102,4 +104,5 @@ jobs:
 
     - name: Run tests
       run: |
-        EMBED_LTO_BITCODE=1 ./test.sh --release --clean --release-sysroot --build-sysroot ${{ matrix.commands }}
+        # TODO: remove --features master when it is back to the default.
+        EMBED_LTO_BITCODE=1 ./test.sh --release --clean --release-sysroot --build-sysroot ${{ matrix.commands }} --features master
diff --git a/compiler/rustc_codegen_gcc/.github/workflows/stdarch.yml b/compiler/rustc_codegen_gcc/.github/workflows/stdarch.yml
index 28ac3cb65..42109ba3e 100644
--- a/compiler/rustc_codegen_gcc/.github/workflows/stdarch.yml
+++ b/compiler/rustc_codegen_gcc/.github/workflows/stdarch.yml
@@ -92,8 +92,10 @@ jobs:
     - name: Build
       run: |
         ./y.sh prepare --only-libcore
-        ./y.sh build --release --release-sysroot
-        cargo test
+        # TODO: remove `--features master` when it is back to the default.
+        ./y.sh build --release --release-sysroot --features master
+        # TODO: remove --features master when it is back to the default.
+        cargo test --features master
 
     - name: Clean
       if: ${{ !matrix.cargo_runner }}
@@ -111,12 +113,14 @@ jobs:
       uses: actions-rs/cargo@v1.0.3
       with:
         command: build
-        args: --release
+        # TODO: remove `--features master` when it is back to the default.
+        args: --release --features master
 
     - name: Run tests
       if: ${{ !matrix.cargo_runner }}
       run: |
-        ./test.sh --release --clean --release-sysroot --build-sysroot --mini-tests --std-tests --test-libcore
+        # TODO: remove `--features master` when it is back to the default.
+        ./test.sh --release --clean --release-sysroot --build-sysroot --mini-tests --std-tests --test-libcore --features master
 
     - name: Run stdarch tests
       if: ${{ !matrix.cargo_runner }}
diff --git a/compiler/rustc_codegen_gcc/Cargo.lock b/compiler/rustc_codegen_gcc/Cargo.lock
index b8e2e5d80..ddfce5d59 100644
--- a/compiler/rustc_codegen_gcc/Cargo.lock
+++ b/compiler/rustc_codegen_gcc/Cargo.lock
@@ -74,7 +74,7 @@ dependencies = [
 [[package]]
 name = "gccjit"
 version = "1.0.0"
-source = "git+https://github.com/antoyo/gccjit.rs#c52a218f5529321285b4489e5562a00e5428e033"
+source = "git+https://github.com/antoyo/gccjit.rs#6e290f25b1d1edab5ae9ace486fd2dc8c08d6421"
 dependencies = [
  "gccjit_sys",
 ]
@@ -82,7 +82,7 @@ dependencies = [
 [[package]]
 name = "gccjit_sys"
 version = "0.0.1"
-source = "git+https://github.com/antoyo/gccjit.rs#c52a218f5529321285b4489e5562a00e5428e033"
+source = "git+https://github.com/antoyo/gccjit.rs#6e290f25b1d1edab5ae9ace486fd2dc8c08d6421"
 dependencies = [
  "libc",
 ]
@@ -120,9 +120,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.147"
+version = "0.2.150"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3"
+checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c"
 
 [[package]]
 name = "linux-raw-sys"
diff --git a/compiler/rustc_codegen_gcc/build_sysroot/build_sysroot.sh b/compiler/rustc_codegen_gcc/build_sysroot/build_sysroot.sh
index 116fd36e7..ebc7dc375 100755
--- a/compiler/rustc_codegen_gcc/build_sysroot/build_sysroot.sh
+++ b/compiler/rustc_codegen_gcc/build_sysroot/build_sysroot.sh
@@ -28,3 +28,7 @@ fi
 # Copy files to sysroot
 mkdir -p sysroot/lib/rustlib/$TARGET_TRIPLE/lib/
 cp -r target/$TARGET_TRIPLE/$sysroot_channel/deps/* sysroot/lib/rustlib/$TARGET_TRIPLE/lib/
+# Copy the source files to the sysroot (Rust for Linux needs this).
+source_dir=sysroot/lib/rustlib/src/rust
+mkdir -p $source_dir
+cp -r sysroot_src/library/ $source_dir
diff --git a/compiler/rustc_codegen_gcc/build_system/src/build.rs b/compiler/rustc_codegen_gcc/build_system/src/build.rs
index eaca7a987..f1c3701a9 100644
--- a/compiler/rustc_codegen_gcc/build_system/src/build.rs
+++ b/compiler/rustc_codegen_gcc/build_system/src/build.rs
@@ -194,6 +194,12 @@ fn build_sysroot(
         copier,
     )?;
 
+    // Copy the source files to the sysroot (Rust for Linux needs this).
+    let sysroot_src_path = "sysroot/lib/rustlib/src/rust";
+    fs::create_dir_all(&sysroot_src_path)
+        .map_err(|error| format!("Failed to create directory `{}`: {:?}", sysroot_src_path, error))?;
+    run_command(&[&"cp", &"-r", &"sysroot_src/library/", &sysroot_src_path], None)?;
+
     Ok(())
 }
 
diff --git a/compiler/rustc_codegen_gcc/example/mini_core.rs b/compiler/rustc_codegen_gcc/example/mini_core.rs
index 343285203..db94bc1c8 100644
--- a/compiler/rustc_codegen_gcc/example/mini_core.rs
+++ b/compiler/rustc_codegen_gcc/example/mini_core.rs
@@ -4,7 +4,7 @@
     thread_local
 )]
 #![no_core]
-#![allow(dead_code, internal_features)]
+#![allow(dead_code, internal_features, ambiguous_wide_pointer_comparisons)]
 
 #[no_mangle]
 unsafe extern "C" fn _Unwind_Resume() {
diff --git a/compiler/rustc_codegen_gcc/example/std_example.rs b/compiler/rustc_codegen_gcc/example/std_example.rs
index 0f6325c89..ad69409eb 100644
--- a/compiler/rustc_codegen_gcc/example/std_example.rs
+++ b/compiler/rustc_codegen_gcc/example/std_example.rs
@@ -1,3 +1,4 @@
+#![allow(internal_features)]
 #![feature(core_intrinsics, coroutines, coroutine_trait, is_sorted)]
 
 #[cfg(feature="master")]
diff --git a/compiler/rustc_codegen_gcc/failing-ui-tests12.txt b/compiler/rustc_codegen_gcc/failing-ui-tests12.txt
index f91aa9253..4af93939b 100644
--- a/compiler/rustc_codegen_gcc/failing-ui-tests12.txt
+++ b/compiler/rustc_codegen_gcc/failing-ui-tests12.txt
@@ -38,3 +38,5 @@ tests/ui/target-feature/missing-plusminus.rs
 tests/ui/sse2.rs
 tests/ui/codegen/issue-79865-llvm-miscompile.rs
 tests/ui/intrinsics/intrinsics-integer.rs
+tests/ui/std-backtrace.rs
+tests/ui/mir/alignment/packed.rs
diff --git a/compiler/rustc_codegen_gcc/patches/libgccjit12/0001-core-Disable-portable-simd-test.patch b/compiler/rustc_codegen_gcc/patches/libgccjit12/0001-core-Disable-portable-simd-test.patch
index 9520a5a39..914ae986b 100644
--- a/compiler/rustc_codegen_gcc/patches/libgccjit12/0001-core-Disable-portable-simd-test.patch
+++ b/compiler/rustc_codegen_gcc/patches/libgccjit12/0001-core-Disable-portable-simd-test.patch
@@ -1,6 +1,6 @@
-From 7bcd24ec6d4a96121874cb1ae5a23ea274aeff34 Mon Sep 17 00:00:00 2001
+From a5663265f797a43c502915c356fe7899c16cee92 Mon Sep 17 00:00:00 2001
 From: None <none@example.com>
-Date: Thu, 19 Oct 2023 13:12:51 -0400
+Date: Sat, 18 Nov 2023 10:50:36 -0500
 Subject: [PATCH] [core] Disable portable-simd test
 
 ---
@@ -8,18 +8,18 @@ Subject: [PATCH] [core] Disable portable-simd test
  1 file changed, 2 deletions(-)
 
 diff --git a/library/core/tests/lib.rs b/library/core/tests/lib.rs
-index 5814ed4..194ad4c 100644
+index d0a119c..76fdece 100644
 --- a/library/core/tests/lib.rs
 +++ b/library/core/tests/lib.rs
-@@ -90,7 +90,6 @@
+@@ -89,7 +89,6 @@
+ #![feature(never_type)]
  #![feature(unwrap_infallible)]
- #![feature(pointer_byte_offsets)]
  #![feature(pointer_is_aligned)]
 -#![feature(portable_simd)]
  #![feature(ptr_metadata)]
  #![feature(lazy_cell)]
  #![feature(unsized_tuple_coercion)]
-@@ -157,7 +156,6 @@ mod pin;
+@@ -155,7 +154,6 @@ mod pin;
  mod pin_macro;
  mod ptr;
  mod result;
@@ -28,5 +28,5 @@ index 5814ed4..194ad4c 100644
  mod str;
  mod str_lossy;
 -- 
-2.42.0
+2.42.1
 
diff --git a/compiler/rustc_codegen_gcc/rust-toolchain b/compiler/rustc_codegen_gcc/rust-toolchain
index 205ec53b4..1962c2172 100644
--- a/compiler/rustc_codegen_gcc/rust-toolchain
+++ b/compiler/rustc_codegen_gcc/rust-toolchain
@@ -1,3 +1,3 @@
 [toolchain]
-channel = "nightly-2023-10-21"
+channel = "nightly-2023-11-17"
 components = ["rust-src", "rustc-dev", "llvm-tools-preview"]
diff --git a/compiler/rustc_codegen_gcc/src/allocator.rs b/compiler/rustc_codegen_gcc/src/allocator.rs
index c8c098e29..7c7044830 100644
--- a/compiler/rustc_codegen_gcc/src/allocator.rs
+++ b/compiler/rustc_codegen_gcc/src/allocator.rs
@@ -90,7 +90,7 @@ fn create_wrapper_function(
         .collect();
     let func = context.new_function(None, FunctionType::Exported, output.unwrap_or(void), &args, from_name, false);
 
-    if tcx.sess.target.options.default_hidden_visibility {
+    if tcx.sess.default_hidden_visibility() {
         #[cfg(feature="master")]
         func.add_attribute(FnAttribute::Visibility(gccjit::Visibility::Hidden));
     }
diff --git a/compiler/rustc_codegen_gcc/src/back/lto.rs b/compiler/rustc_codegen_gcc/src/back/lto.rs
index 529454b11..c21b76868 100644
--- a/compiler/rustc_codegen_gcc/src/back/lto.rs
+++ b/compiler/rustc_codegen_gcc/src/back/lto.rs
@@ -30,7 +30,7 @@ use rustc_codegen_ssa::back::write::{CodegenContext, FatLtoInput};
 use rustc_codegen_ssa::traits::*;
 use rustc_codegen_ssa::{looks_like_rust_object_file, ModuleCodegen, ModuleKind};
 use rustc_data_structures::memmap::Mmap;
-use rustc_errors::{FatalError, Handler};
+use rustc_errors::{FatalError, DiagCtxt};
 use rustc_hir::def_id::LOCAL_CRATE;
 use rustc_middle::dep_graph::WorkProduct;
 use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel};
@@ -61,7 +61,7 @@ struct LtoData {
     tmp_path: TempDir,
 }
 
-fn prepare_lto(cgcx: &CodegenContext<GccCodegenBackend>, diag_handler: &Handler) -> Result<LtoData, FatalError> {
+fn prepare_lto(cgcx: &CodegenContext<GccCodegenBackend>, dcx: &DiagCtxt) -> Result<LtoData, FatalError> {
     let export_threshold = match cgcx.lto {
         // We're just doing LTO for our one crate
         Lto::ThinLocal => SymbolExportLevel::Rust,
@@ -106,18 +106,18 @@ fn prepare_lto(cgcx: &CodegenContext<GccCodegenBackend>, diag_handler: &Handler)
         // Make sure we actually can run LTO
         for crate_type in cgcx.crate_types.iter() {
             if !crate_type_allows_lto(*crate_type) {
-                diag_handler.emit_err(LtoDisallowed);
+                dcx.emit_err(LtoDisallowed);
                 return Err(FatalError);
             } else if *crate_type == CrateType::Dylib {
                 if !cgcx.opts.unstable_opts.dylib_lto {
-                    diag_handler.emit_err(LtoDylib);
+                    dcx.emit_err(LtoDylib);
                     return Err(FatalError);
                 }
             }
         }
 
         if cgcx.opts.cg.prefer_dynamic && !cgcx.opts.unstable_opts.dylib_lto {
-            diag_handler.emit_err(DynamicLinkingWithLTO);
+            dcx.emit_err(DynamicLinkingWithLTO);
             return Err(FatalError);
         }
 
@@ -154,7 +154,7 @@ fn prepare_lto(cgcx: &CodegenContext<GccCodegenBackend>, diag_handler: &Handler)
                         upstream_modules.push((module, CString::new(name).unwrap()));
                     }
                     Err(e) => {
-                        diag_handler.emit_err(e);
+                        dcx.emit_err(e);
                         return Err(FatalError);
                     }
                 }
@@ -183,16 +183,16 @@ pub(crate) fn run_fat(
     modules: Vec<FatLtoInput<GccCodegenBackend>>,
     cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
 ) -> Result<LtoModuleCodegen<GccCodegenBackend>, FatalError> {
-    let diag_handler = cgcx.create_diag_handler();
-    let lto_data = prepare_lto(cgcx, &diag_handler)?;
+    let dcx = cgcx.create_dcx();
+    let lto_data = prepare_lto(cgcx, &dcx)?;
     /*let symbols_below_threshold =
         lto_data.symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::<Vec<_>>();*/
-    fat_lto(cgcx, &diag_handler, modules, cached_modules, lto_data.upstream_modules, lto_data.tmp_path,
+    fat_lto(cgcx, &dcx, modules, cached_modules, lto_data.upstream_modules, lto_data.tmp_path,
         //&symbols_below_threshold,
     )
 }
 
-fn fat_lto(cgcx: &CodegenContext<GccCodegenBackend>, _diag_handler: &Handler, modules: Vec<FatLtoInput<GccCodegenBackend>>, cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>, mut serialized_modules: Vec<(SerializedModule<ModuleBuffer>, CString)>, tmp_path: TempDir,
+fn fat_lto(cgcx: &CodegenContext<GccCodegenBackend>, _dcx: &DiagCtxt, modules: Vec<FatLtoInput<GccCodegenBackend>>, cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>, mut serialized_modules: Vec<(SerializedModule<ModuleBuffer>, CString)>, tmp_path: TempDir,
     //symbols_below_threshold: &[*const libc::c_char],
 ) -> Result<LtoModuleCodegen<GccCodegenBackend>, FatalError> {
     let _timer = cgcx.prof.generic_activity("GCC_fat_lto_build_monolithic_module");
@@ -257,7 +257,7 @@ fn fat_lto(cgcx: &CodegenContext<GccCodegenBackend>, _diag_handler: &Handler, mo
             let (buffer, name) = serialized_modules.remove(0);
             info!("no in-memory regular modules to choose from, parsing {:?}", name);
             ModuleCodegen {
-                module_llvm: GccContext::parse(cgcx, &name, buffer.data(), diag_handler)?,
+                module_llvm: GccContext::parse(cgcx, &name, buffer.data(), dcx)?,
                 name: name.into_string().unwrap(),
                 kind: ModuleKind::Regular,
             }*/
diff --git a/compiler/rustc_codegen_gcc/src/back/write.rs b/compiler/rustc_codegen_gcc/src/back/write.rs
index 04772d770..2f8a54f52 100644
--- a/compiler/rustc_codegen_gcc/src/back/write.rs
+++ b/compiler/rustc_codegen_gcc/src/back/write.rs
@@ -4,7 +4,7 @@ use gccjit::OutputKind;
 use rustc_codegen_ssa::{CompiledModule, ModuleCodegen};
 use rustc_codegen_ssa::back::link::ensure_removed;
 use rustc_codegen_ssa::back::write::{BitcodeSection, CodegenContext, EmitObj, ModuleConfig};
-use rustc_errors::Handler;
+use rustc_errors::DiagCtxt;
 use rustc_fs_util::link_or_copy;
 use rustc_session::config::OutputType;
 use rustc_span::fatal_error::FatalError;
@@ -13,7 +13,7 @@ use rustc_target::spec::SplitDebuginfo;
 use crate::{GccCodegenBackend, GccContext};
 use crate::errors::CopyBitcode;
 
-pub(crate) unsafe fn codegen(cgcx: &CodegenContext<GccCodegenBackend>, diag_handler: &Handler, module: ModuleCodegen<GccContext>, config: &ModuleConfig) -> Result<CompiledModule, FatalError> {
+pub(crate) unsafe fn codegen(cgcx: &CodegenContext<GccCodegenBackend>, dcx: &DiagCtxt, module: ModuleCodegen<GccContext>, config: &ModuleConfig) -> Result<CompiledModule, FatalError> {
     let _timer = cgcx.prof.generic_activity_with_arg("GCC_module_codegen", &*module.name);
     {
         let context = &module.module_llvm.context;
@@ -127,12 +127,12 @@ pub(crate) unsafe fn codegen(cgcx: &CodegenContext<GccCodegenBackend>, diag_hand
             EmitObj::Bitcode => {
                 debug!("copying bitcode {:?} to obj {:?}", bc_out, obj_out);
                 if let Err(err) = link_or_copy(&bc_out, &obj_out) {
-                    diag_handler.emit_err(CopyBitcode { err });
+                    dcx.emit_err(CopyBitcode { err });
                 }
 
                 if !config.emit_bc {
                     debug!("removing_bitcode {:?}", bc_out);
-                    ensure_removed(diag_handler, &bc_out);
+                    ensure_removed(dcx, &bc_out);
                 }
             }
 
@@ -148,7 +148,7 @@ pub(crate) unsafe fn codegen(cgcx: &CodegenContext<GccCodegenBackend>, diag_hand
     ))
 }
 
-pub(crate) fn link(_cgcx: &CodegenContext<GccCodegenBackend>, _diag_handler: &Handler, mut _modules: Vec<ModuleCodegen<GccContext>>) -> Result<ModuleCodegen<GccContext>, FatalError> {
+pub(crate) fn link(_cgcx: &CodegenContext<GccCodegenBackend>, _dcx: &DiagCtxt, mut _modules: Vec<ModuleCodegen<GccContext>>) -> Result<ModuleCodegen<GccContext>, FatalError> {
     unimplemented!();
 }
 
diff --git a/compiler/rustc_codegen_gcc/src/base.rs b/compiler/rustc_codegen_gcc/src/base.rs
index 5073066c1..b0788718d 100644
--- a/compiler/rustc_codegen_gcc/src/base.rs
+++ b/compiler/rustc_codegen_gcc/src/base.rs
@@ -3,7 +3,6 @@ use std::env;
 use std::time::Instant;
 
 use gccjit::{
-    Context,
     FunctionType,
     GlobalKind,
 };
@@ -18,8 +17,9 @@ use rustc_codegen_ssa::mono_item::MonoItemExt;
 use rustc_codegen_ssa::traits::DebugInfoMethods;
 use rustc_session::config::DebugInfo;
 use rustc_span::Symbol;
+use rustc_target::spec::PanicStrategy;
 
-use crate::{LockedTargetInfo, gcc_util};
+use crate::{LockedTargetInfo, gcc_util, new_context};
 use crate::GccContext;
 use crate::builder::Builder;
 use crate::context::CodegenCx;
@@ -88,20 +88,18 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol, target_info: Lock
     fn module_codegen(tcx: TyCtxt<'_>, (cgu_name, target_info): (Symbol, LockedTargetInfo)) -> ModuleCodegen<GccContext> {
         let cgu = tcx.codegen_unit(cgu_name);
         // Instantiate monomorphizations without filling out definitions yet...
-        let context = Context::default();
+        let context = new_context(tcx);
 
-        context.add_command_line_option("-fexceptions");
-        context.add_driver_option("-fexceptions");
+        if tcx.sess.panic_strategy() == PanicStrategy::Unwind {
+            context.add_command_line_option("-fexceptions");
+            context.add_driver_option("-fexceptions");
+        }
 
         let disabled_features: HashSet<_> = tcx.sess.opts.cg.target_feature.split(',')
             .filter(|feature| feature.starts_with('-'))
             .map(|string| &string[1..])
             .collect();
 
-        if tcx.sess.target.arch == "x86" || tcx.sess.target.arch == "x86_64" {
-            context.add_command_line_option("-masm=intel");
-        }
-
         if !disabled_features.contains("avx") && tcx.sess.target.arch == "x86_64" {
             // NOTE: we always enable AVX because the equivalent of llvm.x86.sse2.cmp.pd in GCC for
             // SSE2 is multiple builtins, so we use the AVX __builtin_ia32_cmppd instead.
diff --git a/compiler/rustc_codegen_gcc/src/common.rs b/compiler/rustc_codegen_gcc/src/common.rs
index 93fe27e54..c6edd52d1 100644
--- a/compiler/rustc_codegen_gcc/src/common.rs
+++ b/compiler/rustc_codegen_gcc/src/common.rs
@@ -199,7 +199,8 @@ impl<'gcc, 'tcx> ConstMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
                 }
             }
             Scalar::Ptr(ptr, _size) => {
-                let (alloc_id, offset) = ptr.into_parts();
+                let (prov, offset) = ptr.into_parts(); // we know the `offset` is relative
+                let alloc_id = prov.alloc_id();
                 let base_addr =
                     match self.tcx.global_alloc(alloc_id) {
                         GlobalAlloc::Memory(alloc) => {
@@ -376,9 +377,6 @@ pub trait TypeReflection<'gcc, 'tcx>  {
     fn is_i128(&self, cx: &CodegenCx<'gcc, 'tcx>) -> bool;
     fn is_u128(&self, cx: &CodegenCx<'gcc, 'tcx>) -> bool;
 
-    fn is_f32(&self, cx: &CodegenCx<'gcc, 'tcx>) -> bool;
-    fn is_f64(&self, cx: &CodegenCx<'gcc, 'tcx>) -> bool;
-
     fn is_vector(&self) -> bool;
 }
 
@@ -463,14 +461,6 @@ impl<'gcc, 'tcx> TypeReflection<'gcc, 'tcx> for Type<'gcc> {
         self.unqualified() == cx.u128_type.unqualified()
     }
 
-    fn is_f32(&self, cx: &CodegenCx<'gcc, 'tcx>) -> bool {
-        self.unqualified() == cx.context.new_type::<f32>()
-    }
-
-    fn is_f64(&self, cx: &CodegenCx<'gcc, 'tcx>) -> bool {
-        self.unqualified() == cx.context.new_type::<f64>()
-    }
-
     fn is_vector(&self) -> bool {
         let mut typ = self.clone();
         loop {
diff --git a/compiler/rustc_codegen_gcc/src/consts.rs b/compiler/rustc_codegen_gcc/src/consts.rs
index d8a1fd315..f06416b9b 100644
--- a/compiler/rustc_codegen_gcc/src/consts.rs
+++ b/compiler/rustc_codegen_gcc/src/consts.rs
@@ -285,7 +285,8 @@ pub fn const_alloc_to_gcc<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, alloc: ConstAl
     let pointer_size = dl.pointer_size.bytes() as usize;
 
     let mut next_offset = 0;
-    for &(offset, alloc_id) in alloc.provenance().ptrs().iter() {
+    for &(offset, prov) in alloc.provenance().ptrs().iter() {
+        let alloc_id = prov.alloc_id();
         let offset = offset.bytes();
         assert_eq!(offset as usize as u64, offset);
         let offset = offset as usize;
@@ -313,7 +314,7 @@ pub fn const_alloc_to_gcc<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, alloc: ConstAl
 
         llvals.push(cx.scalar_to_backend(
             InterpScalar::from_pointer(
-                interpret::Pointer::new(alloc_id, Size::from_bytes(ptr_offset)),
+                interpret::Pointer::new(prov, Size::from_bytes(ptr_offset)),
                 &cx.tcx,
             ),
             abi::Scalar::Initialized { value: Primitive::Pointer(address_space), valid_range: WrappingRange::full(dl.pointer_size) },
diff --git a/compiler/rustc_codegen_gcc/src/context.rs b/compiler/rustc_codegen_gcc/src/context.rs
index a043660ea..ab9c703db 100644
--- a/compiler/rustc_codegen_gcc/src/context.rs
+++ b/compiler/rustc_codegen_gcc/src/context.rs
@@ -22,12 +22,6 @@ use rustc_target::spec::{HasTargetSpec, Target, TlsModel};
 use crate::callee::get_fn;
 use crate::common::SignType;
 
-#[derive(Clone)]
-pub struct FuncSig<'gcc> {
-    pub params: Vec<Type<'gcc>>,
-    pub return_type: Type<'gcc>,
-}
-
 pub struct CodegenCx<'gcc, 'tcx> {
     pub check_overflow: bool,
     pub codegen_unit: &'tcx CodegenUnit<'tcx>,
@@ -569,5 +563,6 @@ fn to_gcc_tls_mode(tls_model: TlsModel) -> gccjit::TlsModel {
         TlsModel::LocalDynamic => gccjit::TlsModel::LocalDynamic,
         TlsModel::InitialExec => gccjit::TlsModel::InitialExec,
         TlsModel::LocalExec => gccjit::TlsModel::LocalExec,
+        TlsModel::Emulated => gccjit::TlsModel::GlobalDynamic,
     }
 }
diff --git a/compiler/rustc_codegen_gcc/src/errors.rs b/compiler/rustc_codegen_gcc/src/errors.rs
index 4bf3b71f5..766d90cf7 100644
--- a/compiler/rustc_codegen_gcc/src/errors.rs
+++ b/compiler/rustc_codegen_gcc/src/errors.rs
@@ -1,5 +1,6 @@
 use rustc_errors::{
-    DiagnosticArgValue, DiagnosticBuilder, ErrorGuaranteed, Handler, IntoDiagnostic, IntoDiagnosticArg,
+    DiagCtxt, DiagnosticArgValue, DiagnosticBuilder, ErrorGuaranteed, IntoDiagnostic,
+    IntoDiagnosticArg,
 };
 use rustc_macros::{Diagnostic, Subdiagnostic};
 use rustc_span::Span;
@@ -111,8 +112,8 @@ pub(crate) struct TargetFeatureDisableOrEnable<'a> {
 pub(crate) struct MissingFeatures;
 
 impl IntoDiagnostic<'_, ErrorGuaranteed> for TargetFeatureDisableOrEnable<'_> {
-    fn into_diagnostic(self, sess: &'_ Handler) -> DiagnosticBuilder<'_, ErrorGuaranteed> {
-        let mut diag = sess.struct_err(fluent::codegen_gcc_target_feature_disable_or_enable);
+    fn into_diagnostic(self, dcx: &'_ DiagCtxt) -> DiagnosticBuilder<'_, ErrorGuaranteed> {
+        let mut diag = dcx.struct_err(fluent::codegen_gcc_target_feature_disable_or_enable);
         if let Some(span) = self.span {
             diag.set_span(span);
         };
diff --git a/compiler/rustc_codegen_gcc/src/gcc_util.rs b/compiler/rustc_codegen_gcc/src/gcc_util.rs
index 1248fdcd2..2aa84f267 100644
--- a/compiler/rustc_codegen_gcc/src/gcc_util.rs
+++ b/compiler/rustc_codegen_gcc/src/gcc_util.rs
@@ -2,12 +2,10 @@
 use gccjit::Context;
 use smallvec::{smallvec, SmallVec};
 
-use rustc_codegen_ssa::target_features::{
-    supported_target_features, tied_target_features, RUSTC_SPECIFIC_FEATURES,
-};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_middle::bug;
 use rustc_session::Session;
+use rustc_target::target_features::RUSTC_SPECIFIC_FEATURES;
 
 use crate::errors::{PossibleFeature, TargetFeatureDisableOrEnable, UnknownCTargetFeature, UnknownCTargetFeaturePrefix};
 
@@ -44,7 +42,7 @@ pub(crate) fn global_gcc_features(sess: &Session, diagnostics: bool) -> Vec<Stri
     );
 
     // -Ctarget-features
-    let supported_features = supported_target_features(sess);
+    let supported_features = sess.target.supported_target_features();
     let mut featsmap = FxHashMap::default();
     let feats = sess.opts.cg.target_feature
         .split(',')
@@ -187,7 +185,7 @@ pub fn to_gcc_features<'a>(sess: &Session, s: &'a str) -> SmallVec<[&'a str; 2]>
 // Given a map from target_features to whether they are enabled or disabled,
 // ensure only valid combinations are allowed.
 pub fn check_tied_features(sess: &Session, features: &FxHashMap<&str, bool>) -> Option<&'static [&'static str]> {
-    for tied in tied_target_features(sess) {
+    for tied in sess.target.tied_target_features() {
         // Tied features must be set to the same value, or not set at all
         let mut tied_iter = tied.iter();
         let enabled = features.get(tied_iter.next().unwrap());
diff --git a/compiler/rustc_codegen_gcc/src/int.rs b/compiler/rustc_codegen_gcc/src/int.rs
index ea8550d20..9b9b3ea4f 100644
--- a/compiler/rustc_codegen_gcc/src/int.rs
+++ b/compiler/rustc_codegen_gcc/src/int.rs
@@ -76,6 +76,9 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
                 a >> b
             }
         }
+        else if a_type.is_vector() && a_type.is_vector() {
+            a >> b
+        }
         else if a_native && !b_native {
             self.gcc_lshr(a, self.gcc_int_cast(b, a_type))
         }
@@ -144,7 +147,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
     fn additive_operation(&self, operation: BinaryOp, a: RValue<'gcc>, mut b: RValue<'gcc>) -> RValue<'gcc> {
         let a_type = a.get_type();
         let b_type = b.get_type();
-        if self.is_native_int_type_or_bool(a_type) && self.is_native_int_type_or_bool(b_type) {
+        if (self.is_native_int_type_or_bool(a_type) && self.is_native_int_type_or_bool(b_type)) || (a_type.is_vector() && b_type.is_vector()) {
             if a_type != b_type {
                 if a_type.is_vector() {
                     // Vector types need to be bitcast.
@@ -158,6 +161,8 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
             self.context.new_binary_op(None, operation, a_type, a, b)
         }
         else {
+            debug_assert!(a_type.dyncast_array().is_some());
+            debug_assert!(b_type.dyncast_array().is_some());
             let signed = a_type.is_compatible_with(self.i128_type);
             let func_name =
                 match (operation, signed) {
@@ -189,10 +194,12 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
     fn multiplicative_operation(&self, operation: BinaryOp, operation_name: &str, signed: bool, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
         let a_type = a.get_type();
         let b_type = b.get_type();
-        if self.is_native_int_type_or_bool(a_type) && self.is_native_int_type_or_bool(b_type) {
+        if (self.is_native_int_type_or_bool(a_type) && self.is_native_int_type_or_bool(b_type)) || (a_type.is_vector() && b_type.is_vector()) {
             self.context.new_binary_op(None, operation, a_type, a, b)
         }
         else {
+            debug_assert!(a_type.dyncast_array().is_some());
+            debug_assert!(b_type.dyncast_array().is_some());
             let sign =
                 if signed {
                     ""
@@ -337,6 +344,8 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
     pub fn operation_with_overflow(&self, func_name: &str, lhs: RValue<'gcc>, rhs: RValue<'gcc>) -> (RValue<'gcc>, RValue<'gcc>) {
         let a_type = lhs.get_type();
         let b_type = rhs.get_type();
+        debug_assert!(a_type.dyncast_array().is_some());
+        debug_assert!(b_type.dyncast_array().is_some());
         let param_a = self.context.new_parameter(None, a_type, "a");
         let param_b = self.context.new_parameter(None, b_type, "b");
         let result_field = self.context.new_field(None, a_type, "result");
@@ -496,7 +505,11 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
     pub fn gcc_xor(&self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
         let a_type = a.get_type();
         let b_type = b.get_type();
-        if self.is_native_int_type_or_bool(a_type) && self.is_native_int_type_or_bool(b_type) {
+        if a_type.is_vector() && b_type.is_vector() {
+            let b = self.bitcast_if_needed(b, a_type);
+            a ^ b
+        }
+        else if self.is_native_int_type_or_bool(a_type) && self.is_native_int_type_or_bool(b_type) {
             a ^ b
         }
         else {
@@ -527,6 +540,9 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
                 a << b
             }
         }
+        else if a_type.is_vector() && a_type.is_vector() {
+            a << b
+        }
         else if a_native && !b_native {
             self.gcc_shl(a, self.gcc_int_cast(b, a_type))
         }
@@ -690,6 +706,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
         let a_native = self.is_native_int_type_or_bool(a_type);
         let b_native = self.is_native_int_type_or_bool(b_type);
         if a_type.is_vector() && b_type.is_vector() {
+            let b = self.bitcast_if_needed(b, a_type);
             self.context.new_binary_op(None, operation, a_type, a, b)
         }
         else if a_native && b_native {
@@ -748,6 +765,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
             return self.context.new_cast(None, value, dest_typ);
         }
 
+        debug_assert!(value_type.dyncast_array().is_some());
         let name_suffix =
             match self.type_kind(dest_typ) {
                 TypeKind::Float => "tisf",
@@ -781,6 +799,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
             return self.context.new_cast(None, value, dest_typ);
         }
 
+        debug_assert!(value_type.dyncast_array().is_some());
         let name_suffix =
             match self.type_kind(value_type) {
                 TypeKind::Float => "sfti",
diff --git a/compiler/rustc_codegen_gcc/src/lib.rs b/compiler/rustc_codegen_gcc/src/lib.rs
index a530fc994..1f3f909d8 100644
--- a/compiler/rustc_codegen_gcc/src/lib.rs
+++ b/compiler/rustc_codegen_gcc/src/lib.rs
@@ -12,9 +12,9 @@
  * TODO(antoyo): remove the patches.
  */
 
-#![cfg_attr(not(bootstrap), allow(internal_features))]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
+#![allow(internal_features)]
+#![doc(rust_logo)]
+#![feature(rustdoc_internals)]
 #![feature(
     rustc_private,
     decl_macro,
@@ -39,6 +39,8 @@ extern crate rustc_errors;
 extern crate rustc_fluent_macro;
 extern crate rustc_fs_util;
 extern crate rustc_hir;
+#[cfg(feature="master")]
+extern crate rustc_interface;
 extern crate rustc_macros;
 extern crate rustc_metadata;
 extern crate rustc_middle;
@@ -86,7 +88,7 @@ use std::sync::atomic::Ordering;
 
 use gccjit::{Context, OptimizationLevel};
 #[cfg(feature="master")]
-use gccjit::TargetInfo;
+use gccjit::{TargetInfo, Version};
 #[cfg(not(feature="master"))]
 use gccjit::CType;
 use errors::LTONotSupported;
@@ -95,12 +97,10 @@ use rustc_codegen_ssa::{CodegenResults, CompiledModule, ModuleCodegen};
 use rustc_codegen_ssa::base::codegen_crate;
 use rustc_codegen_ssa::back::write::{CodegenContext, FatLtoInput, ModuleConfig, TargetMachineFactoryFn};
 use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule};
-use rustc_codegen_ssa::target_features::supported_target_features;
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_data_structures::sync::IntoDynSyncSend;
 use rustc_codegen_ssa::traits::{CodegenBackend, ExtraBackendMethods, ThinBufferMethods, WriteBackendMethods};
-use rustc_errors::{DiagnosticMessage, ErrorGuaranteed, Handler, SubdiagnosticMessage};
-use rustc_fluent_macro::fluent_messages;
+use rustc_errors::{ErrorGuaranteed, DiagCtxt};
 use rustc_metadata::EncodedMetadata;
 use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
 use rustc_middle::util::Providers;
@@ -114,7 +114,7 @@ use tempfile::TempDir;
 use crate::back::lto::ModuleBuffer;
 use crate::gcc_util::target_cpu;
 
-fluent_messages! { "../messages.ftl" }
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
 
 pub struct PrintOnPanic<F: Fn() -> String>(pub F);
 
@@ -244,17 +244,33 @@ impl CodegenBackend for GccCodegenBackend {
     }
 }
 
+fn new_context<'gcc, 'tcx>(tcx: TyCtxt<'tcx>) -> Context<'gcc> {
+    let context = Context::default();
+    if tcx.sess.target.arch == "x86" || tcx.sess.target.arch == "x86_64" {
+        context.add_command_line_option("-masm=intel");
+    }
+    #[cfg(feature="master")]
+    {
+        let version = Version::get();
+        let version = format!("{}.{}.{}", version.major, version.minor, version.patch);
+        context.set_output_ident(&format!("rustc version {} with libgccjit {}",
+                rustc_interface::util::rustc_version_str().unwrap_or("unknown version"),
+                version,
+        ));
+    }
+    // TODO(antoyo): check if this should only be added when using -Cforce-unwind-tables=n.
+    context.add_command_line_option("-fno-asynchronous-unwind-tables");
+    context
+}
+
 impl ExtraBackendMethods for GccCodegenBackend {
     fn codegen_allocator<'tcx>(&self, tcx: TyCtxt<'tcx>, module_name: &str, kind: AllocatorKind, alloc_error_handler_kind: AllocatorKind) -> Self::Module {
         let mut mods = GccContext {
-            context: Context::default(),
+            context: new_context(tcx),
             should_combine_object_files: false,
             temp_dir: None,
         };
 
-        if tcx.sess.target.arch == "x86" || tcx.sess.target.arch == "x86_64" {
-            mods.context.add_command_line_option("-masm=intel");
-        }
         unsafe { allocator::codegen(tcx, &mut mods, module_name, kind, alloc_error_handler_kind); }
         mods
     }
@@ -314,7 +330,7 @@ impl WriteBackendMethods for GccCodegenBackend {
         unimplemented!()
     }
 
-    unsafe fn optimize(_cgcx: &CodegenContext<Self>, _diag_handler: &Handler, module: &ModuleCodegen<Self::Module>, config: &ModuleConfig) -> Result<(), FatalError> {
+    unsafe fn optimize(_cgcx: &CodegenContext<Self>, _dcx: &DiagCtxt, module: &ModuleCodegen<Self::Module>, config: &ModuleConfig) -> Result<(), FatalError> {
         module.module_llvm.context.set_optimization_level(to_gcc_opt_level(config.opt_level));
         Ok(())
     }
@@ -328,8 +344,8 @@ impl WriteBackendMethods for GccCodegenBackend {
         unimplemented!();
     }
 
-    unsafe fn codegen(cgcx: &CodegenContext<Self>, diag_handler: &Handler, module: ModuleCodegen<Self::Module>, config: &ModuleConfig) -> Result<CompiledModule, FatalError> {
-        back::write::codegen(cgcx, diag_handler, module, config)
+    unsafe fn codegen(cgcx: &CodegenContext<Self>, dcx: &DiagCtxt, module: ModuleCodegen<Self::Module>, config: &ModuleConfig) -> Result<CompiledModule, FatalError> {
+        back::write::codegen(cgcx, dcx, module, config)
     }
 
     fn prepare_thin(_module: ModuleCodegen<Self::Module>) -> (String, Self::ThinBuffer) {
@@ -340,8 +356,8 @@ impl WriteBackendMethods for GccCodegenBackend {
         unimplemented!();
     }
 
-    fn run_link(cgcx: &CodegenContext<Self>, diag_handler: &Handler, modules: Vec<ModuleCodegen<Self::Module>>) -> Result<ModuleCodegen<Self::Module>, FatalError> {
-        back::write::link(cgcx, diag_handler, modules)
+    fn run_link(cgcx: &CodegenContext<Self>, dcx: &DiagCtxt, modules: Vec<ModuleCodegen<Self::Module>>) -> Result<ModuleCodegen<Self::Module>, FatalError> {
+        back::write::link(cgcx, dcx, modules)
     }
 }
 
@@ -380,11 +396,13 @@ fn to_gcc_opt_level(optlevel: Option<OptLevel>) -> OptimizationLevel {
 }
 
 pub fn target_features(sess: &Session, allow_unstable: bool, target_info: &LockedTargetInfo) -> Vec<Symbol> {
-    supported_target_features(sess)
+    sess
+        .target
+        .supported_target_features()
         .iter()
         .filter_map(
             |&(feature, gate)| {
-                if sess.is_nightly_build() || allow_unstable || gate.is_none() { Some(feature) } else { None }
+                if sess.is_nightly_build() || allow_unstable || gate.is_stable() { Some(feature) } else { None }
             },
         )
         .filter(|_feature| {
diff --git a/compiler/rustc_codegen_llvm/Cargo.toml b/compiler/rustc_codegen_llvm/Cargo.toml
index 1d309eb90..7122c055e 100644
--- a/compiler/rustc_codegen_llvm/Cargo.toml
+++ b/compiler/rustc_codegen_llvm/Cargo.toml
@@ -9,8 +9,7 @@ test = false
 [dependencies]
 # tidy-alphabetical-start
 bitflags = "1.0"
-cstr = "0.2"
-itertools = "0.10.5"
+itertools = "0.11"
 libc = "0.2"
 measureme = "10.0.0"
 object = { version = "0.32.0", default-features = false, features = ["std", "read"] }
diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs
index 6e3a4cae2..97dc40125 100644
--- a/compiler/rustc_codegen_llvm/src/abi.rs
+++ b/compiler/rustc_codegen_llvm/src/abi.rs
@@ -15,7 +15,6 @@ use rustc_middle::ty::layout::LayoutOf;
 pub use rustc_middle::ty::layout::{FAT_PTR_ADDR, FAT_PTR_EXTRA};
 use rustc_middle::ty::Ty;
 use rustc_session::config;
-use rustc_target::abi::call::ArgAbi;
 pub use rustc_target::abi::call::*;
 use rustc_target::abi::{self, HasDataLayout, Int};
 pub use rustc_target::spec::abi::Abi;
@@ -348,50 +347,18 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                 PassMode::Direct(_) => {
                     // ABI-compatible Rust types have the same `layout.abi` (up to validity ranges),
                     // and for Scalar ABIs the LLVM type is fully determined by `layout.abi`,
-                    // guarnateeing that we generate ABI-compatible LLVM IR. Things get tricky for
-                    // aggregates...
-                    if matches!(arg.layout.abi, abi::Abi::Aggregate { .. }) {
-                        assert!(
-                            arg.layout.is_sized(),
-                            "`PassMode::Direct` for unsized type: {}",
-                            arg.layout.ty
-                        );
-                        // This really shouldn't happen, since `immediate_llvm_type` will use
-                        // `layout.fields` to turn this Rust type into an LLVM type. This means all
-                        // sorts of Rust type details leak into the ABI. However wasm sadly *does*
-                        // currently use this mode so we have to allow it -- but we absolutely
-                        // shouldn't let any more targets do that.
-                        // (Also see <https://github.com/rust-lang/rust/issues/115666>.)
-                        //
-                        // The unstable abi `PtxKernel` also uses Direct for now.
-                        // It needs to switch to something else before stabilization can happen.
-                        // (See issue: https://github.com/rust-lang/rust/issues/117271)
-                        assert!(
-                            matches!(&*cx.tcx.sess.target.arch, "wasm32" | "wasm64")
-                                || self.conv == Conv::PtxKernel,
-                            "`PassMode::Direct` for aggregates only allowed on wasm and `extern \"ptx-kernel\"` fns\nProblematic type: {:#?}",
-                            arg.layout,
-                        );
-                    }
+                    // guaranteeing that we generate ABI-compatible LLVM IR.
                     arg.layout.immediate_llvm_type(cx)
                 }
                 PassMode::Pair(..) => {
                     // ABI-compatible Rust types have the same `layout.abi` (up to validity ranges),
                     // so for ScalarPair we can easily be sure that we are generating ABI-compatible
                     // LLVM IR.
-                    assert!(
-                        matches!(arg.layout.abi, abi::Abi::ScalarPair(..)),
-                        "PassMode::Pair for type {}",
-                        arg.layout.ty
-                    );
                     llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(cx, 0, true));
                     llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(cx, 1, true));
                     continue;
                 }
-                PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack } => {
-                    // `Indirect` with metadata is only for unsized types, and doesn't work with
-                    // on-stack passing.
-                    assert!(arg.layout.is_unsized() && !on_stack);
+                PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => {
                     // Construct the type of a (wide) pointer to `ty`, and pass its two fields.
                     // Any two ABI-compatible unsized types have the same metadata type and
                     // moreover the same metadata value leads to the same dynamic size and
@@ -402,13 +369,8 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                     llargument_tys.push(ptr_layout.scalar_pair_element_llvm_type(cx, 1, true));
                     continue;
                 }
-                PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => {
-                    assert!(arg.layout.is_sized());
-                    cx.type_ptr()
-                }
+                PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => cx.type_ptr(),
                 PassMode::Cast { cast, pad_i32 } => {
-                    // `Cast` means "transmute to `CastType`"; that only makes sense for sized types.
-                    assert!(arg.layout.is_sized());
                     // add padding
                     if *pad_i32 {
                         llargument_tys.push(Reg::i32().llvm_type(cx));
@@ -530,7 +492,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
             PassMode::Cast { cast, pad_i32: _ } => {
                 cast.attrs.apply_attrs_to_callsite(
                     llvm::AttributePlace::ReturnValue,
-                    &bx.cx,
+                    bx.cx,
                     callsite,
                 );
             }
diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs
index db5c1388e..58b3aa438 100644
--- a/compiler/rustc_codegen_llvm/src/allocator.rs
+++ b/compiler/rustc_codegen_llvm/src/allocator.rs
@@ -67,7 +67,7 @@ pub(crate) unsafe fn codegen(
         llcx,
         llmod,
         "__rust_alloc_error_handler",
-        &alloc_error_handler_name(alloc_error_handler_kind),
+        alloc_error_handler_name(alloc_error_handler_kind),
         &[usize, usize], // size, align
         None,
         true,
@@ -76,7 +76,7 @@ pub(crate) unsafe fn codegen(
     // __rust_alloc_error_handler_should_panic
     let name = OomStrategy::SYMBOL;
     let ll_g = llvm::LLVMRustGetOrInsertGlobal(llmod, name.as_ptr().cast(), name.len(), i8);
-    if tcx.sess.target.default_hidden_visibility {
+    if tcx.sess.default_hidden_visibility() {
         llvm::LLVMRustSetVisibility(ll_g, llvm::Visibility::Hidden);
     }
     let val = tcx.sess.opts.unstable_opts.oom.should_panic();
@@ -85,7 +85,7 @@ pub(crate) unsafe fn codegen(
 
     let name = NO_ALLOC_SHIM_IS_UNSTABLE;
     let ll_g = llvm::LLVMRustGetOrInsertGlobal(llmod, name.as_ptr().cast(), name.len(), i8);
-    if tcx.sess.target.default_hidden_visibility {
+    if tcx.sess.default_hidden_visibility() {
         llvm::LLVMRustSetVisibility(ll_g, llvm::Visibility::Hidden);
     }
     let llval = llvm::LLVMConstInt(i8, 0, False);
@@ -130,7 +130,7 @@ fn create_wrapper_function(
             None
         };
 
-        if tcx.sess.target.default_hidden_visibility {
+        if tcx.sess.default_hidden_visibility() {
             llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden);
         }
         if tcx.sess.must_emit_unwind_tables() {
@@ -146,7 +146,7 @@ fn create_wrapper_function(
         }
         llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden);
 
-        let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, "entry\0".as_ptr().cast());
+        let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, c"entry".as_ptr().cast());
 
         let llbuilder = llvm::LLVMCreateBuilderInContext(llcx);
         llvm::LLVMPositionBuilderAtEnd(llbuilder, llbb);
diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs
index b6c01545f..481741bb1 100644
--- a/compiler/rustc_codegen_llvm/src/attributes.rs
+++ b/compiler/rustc_codegen_llvm/src/attributes.rs
@@ -4,7 +4,7 @@ use rustc_codegen_ssa::traits::*;
 use rustc_hir::def_id::DefId;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::ty::{self, TyCtxt};
-use rustc_session::config::OptLevel;
+use rustc_session::config::{FunctionReturn, OptLevel};
 use rustc_span::symbol::sym;
 use rustc_target::spec::abi::Abi;
 use rustc_target::spec::{FramePointer, SanitizerSet, StackProbeType, StackProtector};
@@ -118,6 +118,15 @@ pub fn frame_pointer_type_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attr
     Some(llvm::CreateAttrStringValue(cx.llcx, "frame-pointer", attr_value))
 }
 
+fn function_return_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> {
+    let function_return_attr = match cx.sess().opts.unstable_opts.function_return {
+        FunctionReturn::Keep => return None,
+        FunctionReturn::ThunkExtern => AttributeKind::FnRetThunkExtern,
+    };
+
+    Some(function_return_attr.create_attr(cx.llcx))
+}
+
 /// Tell LLVM what instrument function to insert.
 #[inline]
 fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attribute; 4]> {
@@ -136,7 +145,7 @@ fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attr
         attrs.push(llvm::CreateAttrStringValue(
             cx.llcx,
             "instrument-function-entry-inlined",
-            &mcount_name,
+            mcount_name,
         ));
     }
     if let Some(options) = &cx.sess().opts.unstable_opts.instrument_xray {
@@ -331,8 +340,9 @@ pub fn from_fn_attrs<'ll, 'tcx>(
         to_add.push(llvm::CreateAttrString(cx.llcx, "use-sample-profile"));
     }
 
-    // FIXME: none of these three functions interact with source level attributes.
+    // FIXME: none of these functions interact with source level attributes.
     to_add.extend(frame_pointer_type_attr(cx));
+    to_add.extend(function_return_attr(cx));
     to_add.extend(instrument_function_attr(cx));
     to_add.extend(nojumptables_attr(cx));
     to_add.extend(probestack_attr(cx));
@@ -459,7 +469,7 @@ pub fn from_fn_attrs<'ll, 'tcx>(
         // If this function is an import from the environment but the wasm
         // import has a specific module/name, apply them here.
         if let Some(module) = wasm_import_module(cx.tcx, instance.def_id()) {
-            to_add.push(llvm::CreateAttrStringValue(cx.llcx, "wasm-import-module", &module));
+            to_add.push(llvm::CreateAttrStringValue(cx.llcx, "wasm-import-module", module));
 
             let name =
                 codegen_fn_attrs.link_name.unwrap_or_else(|| cx.tcx.item_name(instance.def_id()));
diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs
index f33075a88..cf47c94a8 100644
--- a/compiler/rustc_codegen_llvm/src/back/archive.rs
+++ b/compiler/rustc_codegen_llvm/src/back/archive.rs
@@ -68,7 +68,7 @@ impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> {
     ) -> io::Result<()> {
         let mut archive = archive.to_path_buf();
         if self.sess.target.llvm_target.contains("-apple-macosx") {
-            if let Some(new_archive) = try_extract_macho_fat_archive(&self.sess, &archive)? {
+            if let Some(new_archive) = try_extract_macho_fat_archive(self.sess, &archive)? {
                 archive = new_archive
             }
         }
diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs
index db297425b..42bd86870 100644
--- a/compiler/rustc_codegen_llvm/src/back/lto.rs
+++ b/compiler/rustc_codegen_llvm/src/back/lto.rs
@@ -14,7 +14,7 @@ use rustc_codegen_ssa::traits::*;
 use rustc_codegen_ssa::{looks_like_rust_object_file, ModuleCodegen, ModuleKind};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::memmap::Mmap;
-use rustc_errors::{FatalError, Handler};
+use rustc_errors::{DiagCtxt, FatalError};
 use rustc_hir::def_id::LOCAL_CRATE;
 use rustc_middle::bug;
 use rustc_middle::dep_graph::WorkProduct;
@@ -47,7 +47,7 @@ pub fn crate_type_allows_lto(crate_type: CrateType) -> bool {
 
 fn prepare_lto(
     cgcx: &CodegenContext<LlvmCodegenBackend>,
-    diag_handler: &Handler,
+    dcx: &DiagCtxt,
 ) -> Result<(Vec<CString>, Vec<(SerializedModule<ModuleBuffer>, CString)>), FatalError> {
     let export_threshold = match cgcx.lto {
         // We're just doing LTO for our one crate
@@ -84,23 +84,23 @@ fn prepare_lto(
         // Make sure we actually can run LTO
         for crate_type in cgcx.crate_types.iter() {
             if !crate_type_allows_lto(*crate_type) {
-                diag_handler.emit_err(LtoDisallowed);
+                dcx.emit_err(LtoDisallowed);
                 return Err(FatalError);
             } else if *crate_type == CrateType::Dylib {
                 if !cgcx.opts.unstable_opts.dylib_lto {
-                    diag_handler.emit_err(LtoDylib);
+                    dcx.emit_err(LtoDylib);
                     return Err(FatalError);
                 }
             } else if *crate_type == CrateType::ProcMacro {
                 if !cgcx.opts.unstable_opts.dylib_lto {
-                    diag_handler.emit_err(LtoProcMacro);
+                    dcx.emit_err(LtoProcMacro);
                     return Err(FatalError);
                 }
             }
         }
 
         if cgcx.opts.cg.prefer_dynamic && !cgcx.opts.unstable_opts.dylib_lto {
-            diag_handler.emit_err(DynamicLinkingWithLTO);
+            dcx.emit_err(DynamicLinkingWithLTO);
             return Err(FatalError);
         }
 
@@ -138,7 +138,7 @@ fn prepare_lto(
                         upstream_modules.push((module, CString::new(name).unwrap()));
                     }
                     Err(e) => {
-                        diag_handler.emit_err(e);
+                        dcx.emit_err(e);
                         return Err(FatalError);
                     }
                 }
@@ -200,18 +200,11 @@ pub(crate) fn run_fat(
     modules: Vec<FatLtoInput<LlvmCodegenBackend>>,
     cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
 ) -> Result<LtoModuleCodegen<LlvmCodegenBackend>, FatalError> {
-    let diag_handler = cgcx.create_diag_handler();
-    let (symbols_below_threshold, upstream_modules) = prepare_lto(cgcx, &diag_handler)?;
+    let dcx = cgcx.create_dcx();
+    let (symbols_below_threshold, upstream_modules) = prepare_lto(cgcx, &dcx)?;
     let symbols_below_threshold =
         symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::<Vec<_>>();
-    fat_lto(
-        cgcx,
-        &diag_handler,
-        modules,
-        cached_modules,
-        upstream_modules,
-        &symbols_below_threshold,
-    )
+    fat_lto(cgcx, &dcx, modules, cached_modules, upstream_modules, &symbols_below_threshold)
 }
 
 /// Performs thin LTO by performing necessary global analysis and returning two
@@ -222,8 +215,8 @@ pub(crate) fn run_thin(
     modules: Vec<(String, ThinBuffer)>,
     cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
 ) -> Result<(Vec<LtoModuleCodegen<LlvmCodegenBackend>>, Vec<WorkProduct>), FatalError> {
-    let diag_handler = cgcx.create_diag_handler();
-    let (symbols_below_threshold, upstream_modules) = prepare_lto(cgcx, &diag_handler)?;
+    let dcx = cgcx.create_dcx();
+    let (symbols_below_threshold, upstream_modules) = prepare_lto(cgcx, &dcx)?;
     let symbols_below_threshold =
         symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::<Vec<_>>();
     if cgcx.opts.cg.linker_plugin_lto.enabled() {
@@ -232,14 +225,7 @@ pub(crate) fn run_thin(
                       is deferred to the linker"
         );
     }
-    thin_lto(
-        cgcx,
-        &diag_handler,
-        modules,
-        upstream_modules,
-        cached_modules,
-        &symbols_below_threshold,
-    )
+    thin_lto(cgcx, &dcx, modules, upstream_modules, cached_modules, &symbols_below_threshold)
 }
 
 pub(crate) fn prepare_thin(module: ModuleCodegen<ModuleLlvm>) -> (String, ThinBuffer) {
@@ -250,7 +236,7 @@ pub(crate) fn prepare_thin(module: ModuleCodegen<ModuleLlvm>) -> (String, ThinBu
 
 fn fat_lto(
     cgcx: &CodegenContext<LlvmCodegenBackend>,
-    diag_handler: &Handler,
+    dcx: &DiagCtxt,
     modules: Vec<FatLtoInput<LlvmCodegenBackend>>,
     cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
     mut serialized_modules: Vec<(SerializedModule<ModuleBuffer>, CString)>,
@@ -316,7 +302,7 @@ fn fat_lto(
             let (buffer, name) = serialized_modules.remove(0);
             info!("no in-memory regular modules to choose from, parsing {:?}", name);
             ModuleCodegen {
-                module_llvm: ModuleLlvm::parse(cgcx, &name, buffer.data(), diag_handler)?,
+                module_llvm: ModuleLlvm::parse(cgcx, &name, buffer.data(), dcx)?,
                 name: name.into_string().unwrap(),
                 kind: ModuleKind::Regular,
             }
@@ -333,13 +319,8 @@ fn fat_lto(
         // The linking steps below may produce errors and diagnostics within LLVM
         // which we'd like to handle and print, so set up our diagnostic handlers
         // (which get unregistered when they go out of scope below).
-        let _handler = DiagnosticHandlers::new(
-            cgcx,
-            diag_handler,
-            llcx,
-            &module,
-            CodegenDiagnosticsStage::LTO,
-        );
+        let _handler =
+            DiagnosticHandlers::new(cgcx, dcx, llcx, &module, CodegenDiagnosticsStage::LTO);
 
         // For all other modules we codegened we'll need to link them into our own
         // bitcode. All modules were codegened in their own LLVM context, however,
@@ -367,9 +348,7 @@ fn fat_lto(
                 });
             info!("linking {:?}", name);
             let data = bc_decoded.data();
-            linker
-                .add(data)
-                .map_err(|()| write::llvm_err(diag_handler, LlvmError::LoadBitcode { name }))?;
+            linker.add(data).map_err(|()| write::llvm_err(dcx, LlvmError::LoadBitcode { name }))?;
             serialized_bitcode.push(bc_decoded);
         }
         drop(linker);
@@ -452,7 +431,7 @@ impl Drop for Linker<'_> {
 /// they all go out of scope.
 fn thin_lto(
     cgcx: &CodegenContext<LlvmCodegenBackend>,
-    diag_handler: &Handler,
+    dcx: &DiagCtxt,
     modules: Vec<(String, ThinBuffer)>,
     serialized_modules: Vec<(SerializedModule<ModuleBuffer>, CString)>,
     cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
@@ -527,7 +506,7 @@ fn thin_lto(
             symbols_below_threshold.as_ptr(),
             symbols_below_threshold.len() as u32,
         )
-        .ok_or_else(|| write::llvm_err(diag_handler, LlvmError::PrepareThinLtoContext))?;
+        .ok_or_else(|| write::llvm_err(dcx, LlvmError::PrepareThinLtoContext))?;
 
         let data = ThinData(data);
 
@@ -599,7 +578,7 @@ fn thin_lto(
         // session, overwriting the previous serialized data (if any).
         if let Some(path) = key_map_path {
             if let Err(err) = curr_key_map.save_to_file(&path) {
-                return Err(write::llvm_err(diag_handler, LlvmError::WriteThinLtoKey { err }));
+                return Err(write::llvm_err(dcx, LlvmError::WriteThinLtoKey { err }));
             }
         }
 
@@ -609,7 +588,7 @@ fn thin_lto(
 
 pub(crate) fn run_pass_manager(
     cgcx: &CodegenContext<LlvmCodegenBackend>,
-    diag_handler: &Handler,
+    dcx: &DiagCtxt,
     module: &mut ModuleCodegen<ModuleLlvm>,
     thin: bool,
 ) -> Result<(), FatalError> {
@@ -631,13 +610,13 @@ pub(crate) fn run_pass_manager(
             llvm::LLVMRustAddModuleFlag(
                 module.module_llvm.llmod(),
                 llvm::LLVMModFlagBehavior::Error,
-                "LTOPostLink\0".as_ptr().cast(),
+                c"LTOPostLink".as_ptr().cast(),
                 1,
             );
         }
         let opt_stage = if thin { llvm::OptStage::ThinLTO } else { llvm::OptStage::FatLTO };
         let opt_level = config.opt_level.unwrap_or(config::OptLevel::No);
-        write::llvm_optimize(cgcx, diag_handler, module, config, opt_level, opt_stage)?;
+        write::llvm_optimize(cgcx, dcx, module, config, opt_level, opt_stage)?;
     }
     debug!("lto done");
     Ok(())
@@ -721,11 +700,11 @@ pub unsafe fn optimize_thin_module(
     thin_module: ThinModule<LlvmCodegenBackend>,
     cgcx: &CodegenContext<LlvmCodegenBackend>,
 ) -> Result<ModuleCodegen<ModuleLlvm>, FatalError> {
-    let diag_handler = cgcx.create_diag_handler();
+    let dcx = cgcx.create_dcx();
 
     let module_name = &thin_module.shared.module_names[thin_module.idx];
     let tm_factory_config = TargetMachineFactoryConfig::new(cgcx, module_name.to_str().unwrap());
-    let tm = (cgcx.tm_factory)(tm_factory_config).map_err(|e| write::llvm_err(&diag_handler, e))?;
+    let tm = (cgcx.tm_factory)(tm_factory_config).map_err(|e| write::llvm_err(&dcx, e))?;
 
     // Right now the implementation we've got only works over serialized
     // modules, so we create a fresh new LLVM context and parse the module
@@ -733,7 +712,7 @@ pub unsafe fn optimize_thin_module(
     // crates but for locally codegened modules we may be able to reuse
     // that LLVM Context and Module.
     let llcx = llvm::LLVMRustContextCreate(cgcx.fewer_names);
-    let llmod_raw = parse_module(llcx, module_name, thin_module.data(), &diag_handler)? as *const _;
+    let llmod_raw = parse_module(llcx, module_name, thin_module.data(), &dcx)? as *const _;
     let mut module = ModuleCodegen {
         module_llvm: ModuleLlvm { llmod_raw, llcx, tm: ManuallyDrop::new(tm) },
         name: thin_module.name().to_string(),
@@ -756,7 +735,7 @@ pub unsafe fn optimize_thin_module(
             let _timer =
                 cgcx.prof.generic_activity_with_arg("LLVM_thin_lto_rename", thin_module.name());
             if !llvm::LLVMRustPrepareThinLTORename(thin_module.shared.data.0, llmod, target) {
-                return Err(write::llvm_err(&diag_handler, LlvmError::PrepareThinLtoModule));
+                return Err(write::llvm_err(&dcx, LlvmError::PrepareThinLtoModule));
             }
             save_temp_bitcode(cgcx, &module, "thin-lto-after-rename");
         }
@@ -766,7 +745,7 @@ pub unsafe fn optimize_thin_module(
                 .prof
                 .generic_activity_with_arg("LLVM_thin_lto_resolve_weak", thin_module.name());
             if !llvm::LLVMRustPrepareThinLTOResolveWeak(thin_module.shared.data.0, llmod) {
-                return Err(write::llvm_err(&diag_handler, LlvmError::PrepareThinLtoModule));
+                return Err(write::llvm_err(&dcx, LlvmError::PrepareThinLtoModule));
             }
             save_temp_bitcode(cgcx, &module, "thin-lto-after-resolve");
         }
@@ -776,7 +755,7 @@ pub unsafe fn optimize_thin_module(
                 .prof
                 .generic_activity_with_arg("LLVM_thin_lto_internalize", thin_module.name());
             if !llvm::LLVMRustPrepareThinLTOInternalize(thin_module.shared.data.0, llmod) {
-                return Err(write::llvm_err(&diag_handler, LlvmError::PrepareThinLtoModule));
+                return Err(write::llvm_err(&dcx, LlvmError::PrepareThinLtoModule));
             }
             save_temp_bitcode(cgcx, &module, "thin-lto-after-internalize");
         }
@@ -785,7 +764,7 @@ pub unsafe fn optimize_thin_module(
             let _timer =
                 cgcx.prof.generic_activity_with_arg("LLVM_thin_lto_import", thin_module.name());
             if !llvm::LLVMRustPrepareThinLTOImport(thin_module.shared.data.0, llmod, target) {
-                return Err(write::llvm_err(&diag_handler, LlvmError::PrepareThinLtoModule));
+                return Err(write::llvm_err(&dcx, LlvmError::PrepareThinLtoModule));
             }
             save_temp_bitcode(cgcx, &module, "thin-lto-after-import");
         }
@@ -797,7 +776,7 @@ pub unsafe fn optimize_thin_module(
         // little differently.
         {
             info!("running thin lto passes over {}", module.name);
-            run_pass_manager(cgcx, &diag_handler, &mut module, true)?;
+            run_pass_manager(cgcx, &dcx, &mut module, true)?;
             save_temp_bitcode(cgcx, &module, "thin-lto-after-pm");
         }
     }
@@ -816,6 +795,9 @@ impl ThinLTOKeysMap {
         use std::io::Write;
         let file = File::create(path)?;
         let mut writer = io::BufWriter::new(file);
+        // The entries are loaded back into a hash map in `load_from_file()`, so
+        // the order in which we write them to file here does not matter.
+        #[allow(rustc::potential_query_instability)]
         for (module, key) in &self.keys {
             writeln!(writer, "{module} {key}")?;
         }
@@ -865,10 +847,10 @@ pub fn parse_module<'a>(
     cx: &'a llvm::Context,
     name: &CStr,
     data: &[u8],
-    diag_handler: &Handler,
+    dcx: &DiagCtxt,
 ) -> Result<&'a llvm::Module, FatalError> {
     unsafe {
         llvm::LLVMRustParseBitcodeForLTO(cx, data.as_ptr(), data.len(), name.as_ptr())
-            .ok_or_else(|| write::llvm_err(diag_handler, LlvmError::ParseBitcode))
+            .ok_or_else(|| write::llvm_err(dcx, LlvmError::ParseBitcode))
     }
 }
diff --git a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs
index 36484c3c3..28a88dd2e 100644
--- a/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs
+++ b/compiler/rustc_codegen_llvm/src/back/owned_target_machine.rs
@@ -39,7 +39,7 @@ impl OwnedTargetMachine {
         split_dwarf_file: &CStr,
         output_obj_file: &CStr,
         debug_info_compression: &CStr,
-        force_emulated_tls: bool,
+        use_emulated_tls: bool,
         args_cstr_buff: &[u8],
     ) -> Result<Self, LlvmError<'static>> {
         assert!(args_cstr_buff.len() > 0);
@@ -71,7 +71,7 @@ impl OwnedTargetMachine {
                 split_dwarf_file.as_ptr(),
                 output_obj_file.as_ptr(),
                 debug_info_compression.as_ptr(),
-                force_emulated_tls,
+                use_emulated_tls,
                 args_cstr_buff.as_ptr() as *const c_char,
                 args_cstr_buff.len(),
             )
diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs
index 9d5204034..074188db9 100644
--- a/compiler/rustc_codegen_llvm/src/back/write.rs
+++ b/compiler/rustc_codegen_llvm/src/back/write.rs
@@ -26,14 +26,14 @@ use rustc_codegen_ssa::traits::*;
 use rustc_codegen_ssa::{CompiledModule, ModuleCodegen};
 use rustc_data_structures::profiling::SelfProfilerRef;
 use rustc_data_structures::small_c_str::SmallCStr;
-use rustc_errors::{FatalError, Handler, Level};
+use rustc_errors::{DiagCtxt, FatalError, Level};
 use rustc_fs_util::{link_or_copy, path_to_c_string};
 use rustc_middle::ty::TyCtxt;
 use rustc_session::config::{self, Lto, OutputType, Passes, SplitDwarfKind, SwitchWithOptPath};
 use rustc_session::Session;
 use rustc_span::symbol::sym;
 use rustc_span::InnerSpan;
-use rustc_target::spec::{CodeModel, RelocModel, SanitizerSet, SplitDebuginfo};
+use rustc_target::spec::{CodeModel, RelocModel, SanitizerSet, SplitDebuginfo, TlsModel};
 
 use crate::llvm::diagnostic::OptimizationDiagnosticKind;
 use libc::{c_char, c_int, c_uint, c_void, size_t};
@@ -45,15 +45,15 @@ use std::slice;
 use std::str;
 use std::sync::Arc;
 
-pub fn llvm_err<'a>(handler: &rustc_errors::Handler, err: LlvmError<'a>) -> FatalError {
+pub fn llvm_err<'a>(dcx: &rustc_errors::DiagCtxt, err: LlvmError<'a>) -> FatalError {
     match llvm::last_error() {
-        Some(llvm_err) => handler.emit_almost_fatal(WithLlvmError(err, llvm_err)),
-        None => handler.emit_almost_fatal(err),
+        Some(llvm_err) => dcx.emit_almost_fatal(WithLlvmError(err, llvm_err)),
+        None => dcx.emit_almost_fatal(err),
     }
 }
 
 pub fn write_output_file<'ll>(
-    handler: &rustc_errors::Handler,
+    dcx: &rustc_errors::DiagCtxt,
     target: &'ll llvm::TargetMachine,
     pm: &llvm::PassManager<'ll>,
     m: &'ll llvm::Module,
@@ -93,9 +93,7 @@ pub fn write_output_file<'ll>(
             }
         }
 
-        result
-            .into_result()
-            .map_err(|()| llvm_err(handler, LlvmError::WriteOutput { path: output }))
+        result.into_result().map_err(|()| llvm_err(dcx, LlvmError::WriteOutput { path: output }))
     }
 }
 
@@ -105,7 +103,7 @@ pub fn create_informational_target_machine(sess: &Session) -> OwnedTargetMachine
     // system/tcx is set up.
     let features = llvm_util::global_llvm_features(sess, false);
     target_machine_factory(sess, config::OptLevel::No, &features)(config)
-        .unwrap_or_else(|err| llvm_err(sess.diagnostic(), err).raise())
+        .unwrap_or_else(|err| llvm_err(sess.dcx(), err).raise())
 }
 
 pub fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> OwnedTargetMachine {
@@ -124,11 +122,11 @@ pub fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> OwnedTargetMach
     let config = TargetMachineFactoryConfig { split_dwarf_file, output_obj_file };
 
     target_machine_factory(
-        &tcx.sess,
+        tcx.sess,
         tcx.backend_optimization_level(()),
         tcx.global_backend_features(()),
     )(config)
-    .unwrap_or_else(|err| llvm_err(tcx.sess.diagnostic(), err).raise())
+    .unwrap_or_else(|err| llvm_err(tcx.sess.dcx(), err).raise())
 }
 
 pub fn to_llvm_opt_settings(
@@ -223,7 +221,7 @@ pub fn target_machine_factory(
 
     let path_mapping = sess.source_map().path_mapping().clone();
 
-    let force_emulated_tls = sess.target.force_emulated_tls;
+    let use_emulated_tls = matches!(sess.tls_model(), TlsModel::Emulated);
 
     // copy the exe path, followed by path all into one buffer
     // null terminating them so we can use them as null terminated strings
@@ -297,7 +295,7 @@ pub fn target_machine_factory(
             &split_dwarf_file,
             &output_obj_file,
             &debuginfo_compression,
-            force_emulated_tls,
+            use_emulated_tls,
             &args_cstr_buff,
         )
     })
@@ -332,7 +330,7 @@ pub enum CodegenDiagnosticsStage {
 }
 
 pub struct DiagnosticHandlers<'a> {
-    data: *mut (&'a CodegenContext<LlvmCodegenBackend>, &'a Handler),
+    data: *mut (&'a CodegenContext<LlvmCodegenBackend>, &'a DiagCtxt),
     llcx: &'a llvm::Context,
     old_handler: Option<&'a llvm::DiagnosticHandler>,
 }
@@ -340,7 +338,7 @@ pub struct DiagnosticHandlers<'a> {
 impl<'a> DiagnosticHandlers<'a> {
     pub fn new(
         cgcx: &'a CodegenContext<LlvmCodegenBackend>,
-        handler: &'a Handler,
+        dcx: &'a DiagCtxt,
         llcx: &'a llvm::Context,
         module: &ModuleCodegen<ModuleLlvm>,
         stage: CodegenDiagnosticsStage,
@@ -375,7 +373,7 @@ impl<'a> DiagnosticHandlers<'a> {
             .and_then(|dir| dir.to_str().and_then(|p| CString::new(p).ok()));
 
         let pgo_available = cgcx.opts.cg.profile_use.is_some();
-        let data = Box::into_raw(Box::new((cgcx, handler)));
+        let data = Box::into_raw(Box::new((cgcx, dcx)));
         unsafe {
             let old_handler = llvm::LLVMRustContextGetDiagnosticHandler(llcx);
             llvm::LLVMRustContextConfigureDiagnosticHandler(
@@ -429,7 +427,7 @@ unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void
     if user.is_null() {
         return;
     }
-    let (cgcx, diag_handler) = *(user as *const (&CodegenContext<LlvmCodegenBackend>, &Handler));
+    let (cgcx, dcx) = *(user as *const (&CodegenContext<LlvmCodegenBackend>, &DiagCtxt));
 
     match llvm::diagnostic::Diagnostic::unpack(info) {
         llvm::diagnostic::InlineAsm(inline) => {
@@ -437,7 +435,7 @@ unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void
         }
 
         llvm::diagnostic::Optimization(opt) => {
-            diag_handler.emit_note(FromLlvmOptimizationDiag {
+            dcx.emit_note(FromLlvmOptimizationDiag {
                 filename: &opt.filename,
                 line: opt.line,
                 column: opt.column,
@@ -459,14 +457,14 @@ unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void
                 llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s)
             })
             .expect("non-UTF8 diagnostic");
-            diag_handler.emit_warning(FromLlvmDiag { message });
+            dcx.emit_warning(FromLlvmDiag { message });
         }
         llvm::diagnostic::Unsupported(diagnostic_ref) => {
             let message = llvm::build_string(|s| {
                 llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s)
             })
             .expect("non-UTF8 diagnostic");
-            diag_handler.emit_err(FromLlvmDiag { message });
+            dcx.emit_err(FromLlvmDiag { message });
         }
         llvm::diagnostic::UnknownDiagnostic(..) => {}
     }
@@ -507,7 +505,7 @@ fn get_instr_profile_output_path(config: &ModuleConfig) -> Option<CString> {
 
 pub(crate) unsafe fn llvm_optimize(
     cgcx: &CodegenContext<LlvmCodegenBackend>,
-    diag_handler: &Handler,
+    dcx: &DiagCtxt,
     module: &ModuleCodegen<ModuleLlvm>,
     config: &ModuleConfig,
     opt_level: config::OptLevel,
@@ -589,13 +587,13 @@ pub(crate) unsafe fn llvm_optimize(
         llvm_plugins.as_ptr().cast(),
         llvm_plugins.len(),
     );
-    result.into_result().map_err(|()| llvm_err(diag_handler, LlvmError::RunLlvmPasses))
+    result.into_result().map_err(|()| llvm_err(dcx, LlvmError::RunLlvmPasses))
 }
 
 // Unsafe due to LLVM calls.
 pub(crate) unsafe fn optimize(
     cgcx: &CodegenContext<LlvmCodegenBackend>,
-    diag_handler: &Handler,
+    dcx: &DiagCtxt,
     module: &ModuleCodegen<ModuleLlvm>,
     config: &ModuleConfig,
 ) -> Result<(), FatalError> {
@@ -603,8 +601,7 @@ pub(crate) unsafe fn optimize(
 
     let llmod = module.module_llvm.llmod();
     let llcx = &*module.module_llvm.llcx;
-    let _handlers =
-        DiagnosticHandlers::new(cgcx, diag_handler, llcx, module, CodegenDiagnosticsStage::Opt);
+    let _handlers = DiagnosticHandlers::new(cgcx, dcx, llcx, module, CodegenDiagnosticsStage::Opt);
 
     let module_name = module.name.clone();
     let module_name = Some(&module_name[..]);
@@ -622,14 +619,14 @@ pub(crate) unsafe fn optimize(
             _ if cgcx.opts.cg.linker_plugin_lto.enabled() => llvm::OptStage::PreLinkThinLTO,
             _ => llvm::OptStage::PreLinkNoLTO,
         };
-        return llvm_optimize(cgcx, diag_handler, module, config, opt_level, opt_stage);
+        return llvm_optimize(cgcx, dcx, module, config, opt_level, opt_stage);
     }
     Ok(())
 }
 
 pub(crate) fn link(
     cgcx: &CodegenContext<LlvmCodegenBackend>,
-    diag_handler: &Handler,
+    dcx: &DiagCtxt,
     mut modules: Vec<ModuleCodegen<ModuleLlvm>>,
 ) -> Result<ModuleCodegen<ModuleLlvm>, FatalError> {
     use super::lto::{Linker, ModuleBuffer};
@@ -642,9 +639,9 @@ pub(crate) fn link(
     for module in elements {
         let _timer = cgcx.prof.generic_activity_with_arg("LLVM_link_module", &*module.name);
         let buffer = ModuleBuffer::new(module.module_llvm.llmod());
-        linker.add(buffer.data()).map_err(|()| {
-            llvm_err(diag_handler, LlvmError::SerializeModule { name: &module.name })
-        })?;
+        linker
+            .add(buffer.data())
+            .map_err(|()| llvm_err(dcx, LlvmError::SerializeModule { name: &module.name }))?;
     }
     drop(linker);
     Ok(modules.remove(0))
@@ -652,7 +649,7 @@ pub(crate) fn link(
 
 pub(crate) unsafe fn codegen(
     cgcx: &CodegenContext<LlvmCodegenBackend>,
-    diag_handler: &Handler,
+    dcx: &DiagCtxt,
     module: ModuleCodegen<ModuleLlvm>,
     config: &ModuleConfig,
 ) -> Result<CompiledModule, FatalError> {
@@ -663,13 +660,8 @@ pub(crate) unsafe fn codegen(
         let tm = &*module.module_llvm.tm;
         let module_name = module.name.clone();
         let module_name = Some(&module_name[..]);
-        let _handlers = DiagnosticHandlers::new(
-            cgcx,
-            diag_handler,
-            llcx,
-            &module,
-            CodegenDiagnosticsStage::Codegen,
-        );
+        let _handlers =
+            DiagnosticHandlers::new(cgcx, dcx, llcx, &module, CodegenDiagnosticsStage::Codegen);
 
         if cgcx.msvc_imps_needed {
             create_msvc_imps(cgcx, llcx, llmod);
@@ -728,7 +720,7 @@ pub(crate) unsafe fn codegen(
                     .prof
                     .generic_activity_with_arg("LLVM_module_codegen_emit_bitcode", &*module.name);
                 if let Err(err) = fs::write(&bc_out, data) {
-                    diag_handler.emit_err(WriteBytecode { path: &bc_out, err });
+                    dcx.emit_err(WriteBytecode { path: &bc_out, err });
                 }
             }
 
@@ -778,9 +770,7 @@ pub(crate) unsafe fn codegen(
                 record_artifact_size(&cgcx.prof, "llvm_ir", &out);
             }
 
-            result
-                .into_result()
-                .map_err(|()| llvm_err(diag_handler, LlvmError::WriteIr { path: &out }))?;
+            result.into_result().map_err(|()| llvm_err(dcx, LlvmError::WriteIr { path: &out }))?;
         }
 
         if config.emit_asm {
@@ -799,7 +789,7 @@ pub(crate) unsafe fn codegen(
             };
             with_codegen(tm, llmod, config.no_builtins, |cpm| {
                 write_output_file(
-                    diag_handler,
+                    dcx,
                     tm,
                     cpm,
                     llmod,
@@ -834,7 +824,7 @@ pub(crate) unsafe fn codegen(
 
                 with_codegen(tm, llmod, config.no_builtins, |cpm| {
                     write_output_file(
-                        diag_handler,
+                        dcx,
                         tm,
                         cpm,
                         llmod,
@@ -849,12 +839,12 @@ pub(crate) unsafe fn codegen(
             EmitObj::Bitcode => {
                 debug!("copying bitcode {:?} to obj {:?}", bc_out, obj_out);
                 if let Err(err) = link_or_copy(&bc_out, &obj_out) {
-                    diag_handler.emit_err(CopyBitcode { err });
+                    dcx.emit_err(CopyBitcode { err });
                 }
 
                 if !config.emit_bc {
                     debug!("removing_bitcode {:?}", bc_out);
-                    ensure_removed(diag_handler, &bc_out);
+                    ensure_removed(dcx, &bc_out);
                 }
             }
 
@@ -918,6 +908,7 @@ fn target_is_aix(cgcx: &CodegenContext<LlvmCodegenBackend>) -> bool {
     cgcx.opts.target_triple.triple().contains("-aix")
 }
 
+//FIXME use c string literals here too
 pub(crate) fn bitcode_section_name(cgcx: &CodegenContext<LlvmCodegenBackend>) -> &'static str {
     if target_is_apple(cgcx) {
         "__LLVM,__bitcode\0"
@@ -990,17 +981,13 @@ unsafe fn embed_bitcode(
     // reason (see issue #90326 for historical background).
     let is_aix = target_is_aix(cgcx);
     let is_apple = target_is_apple(cgcx);
-    if is_apple
-        || is_aix
-        || cgcx.opts.target_triple.triple().starts_with("wasm")
-        || cgcx.opts.target_triple.triple().starts_with("asmjs")
-    {
+    if is_apple || is_aix || cgcx.opts.target_triple.triple().starts_with("wasm") {
         // We don't need custom section flags, create LLVM globals.
         let llconst = common::bytes_in_context(llcx, bitcode);
         let llglobal = llvm::LLVMAddGlobal(
             llmod,
             common::val_ty(llconst),
-            "rustc.embedded.module\0".as_ptr().cast(),
+            c"rustc.embedded.module".as_ptr().cast(),
         );
         llvm::LLVMSetInitializer(llglobal, llconst);
 
@@ -1013,15 +1000,15 @@ unsafe fn embed_bitcode(
         let llglobal = llvm::LLVMAddGlobal(
             llmod,
             common::val_ty(llconst),
-            "rustc.embedded.cmdline\0".as_ptr().cast(),
+            c"rustc.embedded.cmdline".as_ptr().cast(),
         );
         llvm::LLVMSetInitializer(llglobal, llconst);
         let section = if is_apple {
-            "__LLVM,__cmdline\0"
+            c"__LLVM,__cmdline"
         } else if is_aix {
-            ".info\0"
+            c".info"
         } else {
-            ".llvmcmd\0"
+            c".llvmcmd"
         };
         llvm::LLVMSetSection(llglobal, section.as_ptr().cast());
         llvm::LLVMRustSetLinkage(llglobal, llvm::Linkage::PrivateLinkage);
@@ -1110,7 +1097,7 @@ fn record_llvm_cgu_instructions_stats(prof: &SelfProfilerRef, llmod: &llvm::Modu
     }
 
     let raw_stats =
-        llvm::build_string(|s| unsafe { llvm::LLVMRustModuleInstructionStats(&llmod, s) })
+        llvm::build_string(|s| unsafe { llvm::LLVMRustModuleInstructionStats(llmod, s) })
             .expect("cannot get module instruction stats");
 
     #[derive(serde::Deserialize)]
diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs
index b659fd02e..5dc271ccd 100644
--- a/compiler/rustc_codegen_llvm/src/base.rs
+++ b/compiler/rustc_codegen_llvm/src/base.rs
@@ -19,8 +19,6 @@ use crate::context::CodegenCx;
 use crate::llvm;
 use crate::value::Value;
 
-use cstr::cstr;
-
 use rustc_codegen_ssa::base::maybe_create_entry_wrapper;
 use rustc_codegen_ssa::mono_item::MonoItemExt;
 use rustc_codegen_ssa::traits::*;
@@ -110,11 +108,11 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol) -> (ModuleCodegen
 
             // Create the llvm.used and llvm.compiler.used variables.
             if !cx.used_statics.borrow().is_empty() {
-                cx.create_used_variable_impl(cstr!("llvm.used"), &*cx.used_statics.borrow());
+                cx.create_used_variable_impl(c"llvm.used", &*cx.used_statics.borrow());
             }
             if !cx.compiler_used_statics.borrow().is_empty() {
                 cx.create_used_variable_impl(
-                    cstr!("llvm.compiler.used"),
+                    c"llvm.compiler.used",
                     &*cx.compiler_used_statics.borrow(),
                 );
             }
diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs
index 7b259055d..8f60175a6 100644
--- a/compiler/rustc_codegen_llvm/src/builder.rs
+++ b/compiler/rustc_codegen_llvm/src/builder.rs
@@ -7,7 +7,6 @@ use crate::llvm_util;
 use crate::type_::Type;
 use crate::type_of::LayoutLlvmExt;
 use crate::value::Value;
-use cstr::cstr;
 use libc::{c_char, c_uint};
 use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, SynchronizationScope, TypeKind};
 use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
@@ -27,7 +26,6 @@ use rustc_target::abi::{self, call::FnAbi, Align, Size, WrappingRange};
 use rustc_target::spec::{HasTargetSpec, SanitizerSet, Target};
 use smallvec::SmallVec;
 use std::borrow::Cow;
-use std::ffi::CStr;
 use std::iter;
 use std::ops::Deref;
 use std::ptr;
@@ -47,13 +45,10 @@ impl Drop for Builder<'_, '_, '_> {
     }
 }
 
-// FIXME(eddyb) use a checked constructor when they become `const fn`.
-const EMPTY_C_STR: &CStr = unsafe { CStr::from_bytes_with_nul_unchecked(b"\0") };
-
 /// Empty string, to be used where LLVM expects an instruction name, indicating
 /// that the instruction is to be left unnamed (i.e. numbered, in textual IR).
 // FIXME(eddyb) pass `&CStr` directly to FFI once it's a thin pointer.
-const UNNAMED: *const c_char = EMPTY_C_STR.as_ptr();
+const UNNAMED: *const c_char = c"".as_ptr();
 
 impl<'ll, 'tcx> BackendTypes for Builder<'_, 'll, 'tcx> {
     type Value = <CodegenCx<'ll, 'tcx> as BackendTypes>::Value;
@@ -358,7 +353,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         let new_kind = match ty.kind() {
             Int(t @ Isize) => Int(t.normalize(self.tcx.sess.target.pointer_width)),
             Uint(t @ Usize) => Uint(t.normalize(self.tcx.sess.target.pointer_width)),
-            t @ (Uint(_) | Int(_)) => t.clone(),
+            t @ (Uint(_) | Int(_)) => *t,
             _ => panic!("tried to get overflow intrinsic for op applied to non-int type"),
         };
 
@@ -489,6 +484,15 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
 
     #[instrument(level = "trace", skip(self))]
     fn load_operand(&mut self, place: PlaceRef<'tcx, &'ll Value>) -> OperandRef<'tcx, &'ll Value> {
+        if place.layout.is_unsized() {
+            let tail = self.tcx.struct_tail_with_normalize(place.layout.ty, |ty| ty, || {});
+            if matches!(tail.kind(), ty::Foreign(..)) {
+                // Unsized locals and, at least conceptually, even unsized arguments must be copied
+                // around, which requires dynamically determining their size. Therefore, we cannot
+                // allow `extern` types here. Consult t-opsem before removing this check.
+                panic!("unsized locals must not be `extern` types");
+            }
+        }
         assert_eq!(place.llextra.is_some(), place.layout.is_unsized());
 
         if place.layout.is_zst() {
@@ -1003,14 +1007,13 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
     }
 
     fn cleanup_pad(&mut self, parent: Option<&'ll Value>, args: &[&'ll Value]) -> Funclet<'ll> {
-        let name = cstr!("cleanuppad");
         let ret = unsafe {
             llvm::LLVMBuildCleanupPad(
                 self.llbuilder,
                 parent,
                 args.as_ptr(),
                 args.len() as c_uint,
-                name.as_ptr(),
+                c"cleanuppad".as_ptr(),
             )
         };
         Funclet::new(ret.expect("LLVM does not have support for cleanuppad"))
@@ -1024,14 +1027,13 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
     }
 
     fn catch_pad(&mut self, parent: &'ll Value, args: &[&'ll Value]) -> Funclet<'ll> {
-        let name = cstr!("catchpad");
         let ret = unsafe {
             llvm::LLVMBuildCatchPad(
                 self.llbuilder,
                 parent,
                 args.as_ptr(),
                 args.len() as c_uint,
-                name.as_ptr(),
+                c"catchpad".as_ptr(),
             )
         };
         Funclet::new(ret.expect("LLVM does not have support for catchpad"))
@@ -1043,14 +1045,13 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         unwind: Option<&'ll BasicBlock>,
         handlers: &[&'ll BasicBlock],
     ) -> &'ll Value {
-        let name = cstr!("catchswitch");
         let ret = unsafe {
             llvm::LLVMBuildCatchSwitch(
                 self.llbuilder,
                 parent,
                 unwind,
                 handlers.len() as c_uint,
-                name.as_ptr(),
+                c"catchswitch".as_ptr(),
             )
         };
         let ret = ret.expect("LLVM does not have support for catchswitch");
diff --git a/compiler/rustc_codegen_llvm/src/callee.rs b/compiler/rustc_codegen_llvm/src/callee.rs
index d5778757c..e675362ac 100644
--- a/compiler/rustc_codegen_llvm/src/callee.rs
+++ b/compiler/rustc_codegen_llvm/src/callee.rs
@@ -59,10 +59,10 @@ pub fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) ->
             // To avoid this, we set the Storage Class to "DllImport" so that
             // LLVM will prefix the name with `__imp_`. Ideally, we'd like the
             // existing logic below to set the Storage Class, but it has an
-            // exemption for MinGW for backwards compatability.
+            // exemption for MinGW for backwards compatibility.
             let llfn = cx.declare_fn(
                 &common::i686_decorated_name(
-                    &dllimport,
+                    dllimport,
                     common::is_mingw_gnu_toolchain(&tcx.sess.target),
                     true,
                 ),
diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs
index 0b0816c27..8173e41af 100644
--- a/compiler/rustc_codegen_llvm/src/common.rs
+++ b/compiler/rustc_codegen_llvm/src/common.rs
@@ -203,6 +203,7 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
                 unsafe {
                     llvm::LLVMSetInitializer(g, sc);
                     llvm::LLVMSetGlobalConstant(g, True);
+                    llvm::LLVMSetUnnamedAddress(g, llvm::UnnamedAddr::Global);
                     llvm::LLVMRustSetLinkage(g, llvm::Linkage::InternalLinkage);
                 }
                 (s.to_owned(), g)
@@ -245,8 +246,8 @@ impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
                 }
             }
             Scalar::Ptr(ptr, _size) => {
-                let (alloc_id, offset) = ptr.into_parts();
-                let (base_addr, base_addr_space) = match self.tcx.global_alloc(alloc_id) {
+                let (prov, offset) = ptr.into_parts();
+                let (base_addr, base_addr_space) = match self.tcx.global_alloc(prov.alloc_id()) {
                     GlobalAlloc::Memory(alloc) => {
                         let init = const_alloc_to_llvm(self, alloc);
                         let alloc = alloc.inner();
diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs
index 307c1264d..77e893c81 100644
--- a/compiler/rustc_codegen_llvm/src/consts.rs
+++ b/compiler/rustc_codegen_llvm/src/consts.rs
@@ -8,7 +8,6 @@ use crate::llvm::{self, True};
 use crate::type_::Type;
 use crate::type_of::LayoutLlvmExt;
 use crate::value::Value;
-use cstr::cstr;
 use rustc_codegen_ssa::traits::*;
 use rustc_hir::def_id::DefId;
 use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs};
@@ -72,7 +71,7 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation<
     }
 
     let mut next_offset = 0;
-    for &(offset, alloc_id) in alloc.provenance().ptrs().iter() {
+    for &(offset, prov) in alloc.provenance().ptrs().iter() {
         let offset = offset.bytes();
         assert_eq!(offset as usize as u64, offset);
         let offset = offset as usize;
@@ -92,13 +91,10 @@ pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation<
         .expect("const_alloc_to_llvm: could not read relocation pointer")
             as u64;
 
-        let address_space = cx.tcx.global_alloc(alloc_id).address_space(cx);
+        let address_space = cx.tcx.global_alloc(prov.alloc_id()).address_space(cx);
 
         llvals.push(cx.scalar_to_backend(
-            InterpScalar::from_pointer(
-                Pointer::new(alloc_id, Size::from_bytes(ptr_offset)),
-                &cx.tcx,
-            ),
+            InterpScalar::from_pointer(Pointer::new(prov, Size::from_bytes(ptr_offset)), &cx.tcx),
             Scalar::Initialized {
                 value: Primitive::Pointer(address_space),
                 valid_range: WrappingRange::full(dl.pointer_size),
@@ -187,7 +183,7 @@ fn check_and_apply_linkage<'ll, 'tcx>(
     {
         cx.declare_global(
             &common::i686_decorated_name(
-                &dllimport,
+                dllimport,
                 common::is_mingw_gnu_toolchain(&cx.tcx.sess.target),
                 true,
             ),
@@ -476,9 +472,9 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> {
                             .all(|&byte| byte == 0);
 
                     let sect_name = if all_bytes_are_zero {
-                        cstr!("__DATA,__thread_bss")
+                        c"__DATA,__thread_bss"
                     } else {
-                        cstr!("__DATA,__thread_data")
+                        c"__DATA,__thread_data"
                     };
                     llvm::LLVMSetSection(g, sect_name.as_ptr());
                 }
@@ -507,7 +503,7 @@ impl<'ll> StaticMethods for CodegenCx<'ll, '_> {
                     let val = llvm::LLVMMetadataAsValue(self.llcx, meta);
                     llvm::LLVMAddNamedMetadataOperand(
                         self.llmod,
-                        "wasm.custom_sections\0".as_ptr().cast(),
+                        c"wasm.custom_sections".as_ptr().cast(),
                         val,
                     );
                 }
diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs
index 4dd6372b5..3053c4e0d 100644
--- a/compiler/rustc_codegen_llvm/src/context.rs
+++ b/compiler/rustc_codegen_llvm/src/context.rs
@@ -8,7 +8,6 @@ use crate::llvm_util;
 use crate::type_::Type;
 use crate::value::Value;
 
-use cstr::cstr;
 use rustc_codegen_ssa::base::{wants_msvc_seh, wants_wasm_eh};
 use rustc_codegen_ssa::errors as ssa_errors;
 use rustc_codegen_ssa::traits::*;
@@ -120,6 +119,7 @@ fn to_llvm_tls_model(tls_model: TlsModel) -> llvm::ThreadLocalMode {
         TlsModel::LocalDynamic => llvm::ThreadLocalMode::LocalDynamic,
         TlsModel::InitialExec => llvm::ThreadLocalMode::InitialExec,
         TlsModel::LocalExec => llvm::ThreadLocalMode::LocalExec,
+        TlsModel::Emulated => llvm::ThreadLocalMode::GeneralDynamic,
     }
 }
 
@@ -134,18 +134,6 @@ pub unsafe fn create_module<'ll>(
 
     let mut target_data_layout = sess.target.data_layout.to_string();
     let llvm_version = llvm_util::get_version();
-    if llvm_version < (16, 0, 0) {
-        if sess.target.arch == "s390x" {
-            // LLVM 16 data layout changed to always set 64-bit vector alignment,
-            // which is conditional in earlier LLVM versions.
-            // https://reviews.llvm.org/D131158 for the discussion.
-            target_data_layout = target_data_layout.replace("-v128:64", "");
-        } else if sess.target.arch == "riscv64" {
-            // LLVM 16 introduced this change so as to produce more efficient code.
-            // See https://reviews.llvm.org/D116735 for the discussion.
-            target_data_layout = target_data_layout.replace("-n32:64-", "-n64-");
-        }
-    }
     if llvm_version < (17, 0, 0) {
         if sess.target.arch.starts_with("powerpc") {
             // LLVM 17 specifies function pointer alignment for ppc:
@@ -226,13 +214,13 @@ pub unsafe fn create_module<'ll>(
     // If skipping the PLT is enabled, we need to add some module metadata
     // to ensure intrinsic calls don't use it.
     if !sess.needs_plt() {
-        let avoid_plt = "RtLibUseGOT\0".as_ptr().cast();
+        let avoid_plt = c"RtLibUseGOT".as_ptr().cast();
         llvm::LLVMRustAddModuleFlag(llmod, llvm::LLVMModFlagBehavior::Warning, avoid_plt, 1);
     }
 
     // Enable canonical jump tables if CFI is enabled. (See https://reviews.llvm.org/D65629.)
     if sess.is_sanitizer_cfi_canonical_jump_tables_enabled() && sess.is_sanitizer_cfi_enabled() {
-        let canonical_jump_tables = "CFI Canonical Jump Tables\0".as_ptr().cast();
+        let canonical_jump_tables = c"CFI Canonical Jump Tables".as_ptr().cast();
         llvm::LLVMRustAddModuleFlag(
             llmod,
             llvm::LLVMModFlagBehavior::Override,
@@ -243,7 +231,7 @@ pub unsafe fn create_module<'ll>(
 
     // Enable LTO unit splitting if specified or if CFI is enabled. (See https://reviews.llvm.org/D53891.)
     if sess.is_split_lto_unit_enabled() || sess.is_sanitizer_cfi_enabled() {
-        let enable_split_lto_unit = "EnableSplitLTOUnit\0".as_ptr().cast();
+        let enable_split_lto_unit = c"EnableSplitLTOUnit".as_ptr().cast();
         llvm::LLVMRustAddModuleFlag(
             llmod,
             llvm::LLVMModFlagBehavior::Override,
@@ -254,7 +242,7 @@ pub unsafe fn create_module<'ll>(
 
     // Add "kcfi" module flag if KCFI is enabled. (See https://reviews.llvm.org/D119296.)
     if sess.is_sanitizer_kcfi_enabled() {
-        let kcfi = "kcfi\0".as_ptr().cast();
+        let kcfi = c"kcfi".as_ptr().cast();
         llvm::LLVMRustAddModuleFlag(llmod, llvm::LLVMModFlagBehavior::Override, kcfi, 1);
     }
 
@@ -267,7 +255,7 @@ pub unsafe fn create_module<'ll>(
                 llvm::LLVMRustAddModuleFlag(
                     llmod,
                     llvm::LLVMModFlagBehavior::Warning,
-                    "cfguard\0".as_ptr() as *const _,
+                    c"cfguard".as_ptr() as *const _,
                     1,
                 )
             }
@@ -276,7 +264,7 @@ pub unsafe fn create_module<'ll>(
                 llvm::LLVMRustAddModuleFlag(
                     llmod,
                     llvm::LLVMModFlagBehavior::Warning,
-                    "cfguard\0".as_ptr() as *const _,
+                    c"cfguard".as_ptr() as *const _,
                     2,
                 )
             }
@@ -294,26 +282,26 @@ pub unsafe fn create_module<'ll>(
             llvm::LLVMRustAddModuleFlag(
                 llmod,
                 behavior,
-                "branch-target-enforcement\0".as_ptr().cast(),
+                c"branch-target-enforcement".as_ptr().cast(),
                 bti.into(),
             );
             llvm::LLVMRustAddModuleFlag(
                 llmod,
                 behavior,
-                "sign-return-address\0".as_ptr().cast(),
+                c"sign-return-address".as_ptr().cast(),
                 pac_ret.is_some().into(),
             );
             let pac_opts = pac_ret.unwrap_or(PacRet { leaf: false, key: PAuthKey::A });
             llvm::LLVMRustAddModuleFlag(
                 llmod,
                 behavior,
-                "sign-return-address-all\0".as_ptr().cast(),
+                c"sign-return-address-all".as_ptr().cast(),
                 pac_opts.leaf.into(),
             );
             llvm::LLVMRustAddModuleFlag(
                 llmod,
                 behavior,
-                "sign-return-address-with-bkey\0".as_ptr().cast(),
+                c"sign-return-address-with-bkey".as_ptr().cast(),
                 u32::from(pac_opts.key == PAuthKey::B),
             );
         } else {
@@ -329,7 +317,7 @@ pub unsafe fn create_module<'ll>(
         llvm::LLVMRustAddModuleFlag(
             llmod,
             llvm::LLVMModFlagBehavior::Override,
-            "cf-protection-branch\0".as_ptr().cast(),
+            c"cf-protection-branch".as_ptr().cast(),
             1,
         )
     }
@@ -337,7 +325,7 @@ pub unsafe fn create_module<'ll>(
         llvm::LLVMRustAddModuleFlag(
             llmod,
             llvm::LLVMModFlagBehavior::Override,
-            "cf-protection-return\0".as_ptr().cast(),
+            c"cf-protection-return".as_ptr().cast(),
             1,
         )
     }
@@ -346,11 +334,21 @@ pub unsafe fn create_module<'ll>(
         llvm::LLVMRustAddModuleFlag(
             llmod,
             llvm::LLVMModFlagBehavior::Error,
-            "Virtual Function Elim\0".as_ptr().cast(),
+            c"Virtual Function Elim".as_ptr().cast(),
             1,
         );
     }
 
+    // Set module flag to enable Windows EHCont Guard (/guard:ehcont).
+    if sess.opts.unstable_opts.ehcont_guard {
+        llvm::LLVMRustAddModuleFlag(
+            llmod,
+            llvm::LLVMModFlagBehavior::Warning,
+            c"ehcontguard".as_ptr() as *const _,
+            1,
+        )
+    }
+
     // Insert `llvm.ident` metadata.
     //
     // On the wasm targets it will get hooked up to the "producer" sections
@@ -364,10 +362,28 @@ pub unsafe fn create_module<'ll>(
     );
     llvm::LLVMAddNamedMetadataOperand(
         llmod,
-        cstr!("llvm.ident").as_ptr(),
+        c"llvm.ident".as_ptr(),
         llvm::LLVMMDNodeInContext(llcx, &name_metadata, 1),
     );
 
+    // Add module flags specified via -Z llvm_module_flag
+    for (key, value, behavior) in &sess.opts.unstable_opts.llvm_module_flag {
+        let key = format!("{key}\0");
+        let behavior = match behavior.as_str() {
+            "error" => llvm::LLVMModFlagBehavior::Error,
+            "warning" => llvm::LLVMModFlagBehavior::Warning,
+            "require" => llvm::LLVMModFlagBehavior::Require,
+            "override" => llvm::LLVMModFlagBehavior::Override,
+            "append" => llvm::LLVMModFlagBehavior::Append,
+            "appendunique" => llvm::LLVMModFlagBehavior::AppendUnique,
+            "max" => llvm::LLVMModFlagBehavior::Max,
+            "min" => llvm::LLVMModFlagBehavior::Min,
+            // We already checked this during option parsing
+            _ => unreachable!(),
+        };
+        llvm::LLVMRustAddModuleFlag(llmod, behavior, key.as_ptr().cast(), *value)
+    }
+
     llmod
 }
 
@@ -494,14 +510,13 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
     }
 
     pub(crate) fn create_used_variable_impl(&self, name: &'static CStr, values: &[&'ll Value]) {
-        let section = cstr!("llvm.metadata");
         let array = self.const_array(self.type_ptr(), values);
 
         unsafe {
             let g = llvm::LLVMAddGlobal(self.llmod, self.val_ty(array), name.as_ptr());
             llvm::LLVMSetInitializer(g, array);
             llvm::LLVMRustSetLinkage(g, llvm::Linkage::AppendingLinkage);
-            llvm::LLVMSetSection(g, section.as_ptr());
+            llvm::LLVMSetSection(g, c"llvm.metadata".as_ptr());
         }
     }
 }
diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs
index 274e0aeaa..33bfde03a 100644
--- a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs
+++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs
@@ -58,6 +58,11 @@ pub fn finalize(cx: &CodegenCx<'_, '_>) {
         return;
     }
 
+    // The entries of the map are only used to get a list of all files with
+    // coverage info. In the end the list of files is passed into
+    // `GlobalFileTable::new()` which internally do `.sort_unstable_by()`, so
+    // the iteration order here does not matter.
+    #[allow(rustc::potential_query_instability)]
     let function_coverage_entries = function_coverage_map
         .into_iter()
         .map(|(instance, function_coverage)| (instance, function_coverage.into_finished()))
@@ -176,7 +181,7 @@ impl GlobalFileTable {
         // compilation directory can be combined with the relative paths
         // to get absolute paths, if needed.
         use rustc_session::RemapFileNameExt;
-        let working_dir: &str = &tcx.sess.opts.working_dir.for_codegen(&tcx.sess).to_string_lossy();
+        let working_dir: &str = &tcx.sess.opts.working_dir.for_codegen(tcx.sess).to_string_lossy();
 
         llvm::build_byte_buffer(|buffer| {
             coverageinfo::write_filenames_section_to_buffer(
@@ -189,8 +194,6 @@ impl GlobalFileTable {
 }
 
 rustc_index::newtype_index! {
-    // Tell the newtype macro to not generate `Encode`/`Decode` impls.
-    #[custom_encodable]
     struct LocalFileId {}
 }
 
@@ -375,10 +378,7 @@ fn add_unused_functions(cx: &CodegenCx<'_, '_>) {
             // just "functions", like consts, statics, etc. Filter those out.
             // If `ignore_unused_generics` was specified, filter out any
             // generic functions from consideration as well.
-            if !matches!(
-                kind,
-                DefKind::Fn | DefKind::AssocFn | DefKind::Closure | DefKind::Coroutine
-            ) {
+            if !matches!(kind, DefKind::Fn | DefKind::AssocFn | DefKind::Closure) {
                 return None;
             }
             if ignore_unused_generics && tcx.generics_of(def_id).requires_monomorphization(tcx) {
diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs
index 7d6975618..0befbb5a3 100644
--- a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs
+++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs
@@ -85,6 +85,14 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> {
 
         let bx = self;
 
+        match coverage.kind {
+            // Marker statements have no effect during codegen,
+            // so return early and don't create `func_coverage`.
+            CoverageKind::SpanMarker => return,
+            // Match exhaustively to ensure that newly-added kinds are classified correctly.
+            CoverageKind::CounterIncrement { .. } | CoverageKind::ExpressionUsed { .. } => {}
+        }
+
         let Some(function_coverage_info) =
             bx.tcx.instance_mir(instance.def).function_coverage_info.as_deref()
         else {
@@ -100,6 +108,9 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> {
 
         let Coverage { kind } = coverage;
         match *kind {
+            CoverageKind::SpanMarker => unreachable!(
+                "unexpected marker statement {kind:?} should have caused an early return"
+            ),
             CoverageKind::CounterIncrement { id } => {
                 func_coverage.mark_counter_id_seen(id);
                 // We need to explicitly drop the `RefMut` before calling into `instrprof_increment`,
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs
index 425e935bc..d82b1e1e7 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs
@@ -30,14 +30,13 @@ pub fn insert_reference_to_gdb_debug_scripts_section_global(bx: &mut Builder<'_,
 /// Allocates the global variable responsible for the .debug_gdb_scripts binary
 /// section.
 pub fn get_or_insert_gdb_debug_scripts_section_global<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll Value {
-    let c_section_var_name = "__rustc_debug_gdb_scripts_section__\0";
-    let section_var_name = &c_section_var_name[..c_section_var_name.len() - 1];
+    let c_section_var_name = c"__rustc_debug_gdb_scripts_section__";
+    let section_var_name = c_section_var_name.to_str().unwrap();
 
     let section_var =
         unsafe { llvm::LLVMGetNamedGlobal(cx.llmod, c_section_var_name.as_ptr().cast()) };
 
     section_var.unwrap_or_else(|| {
-        let section_name = b".debug_gdb_scripts\0";
         let mut section_contents = Vec::new();
 
         // Add the pretty printers for the standard library first.
@@ -70,7 +69,7 @@ pub fn get_or_insert_gdb_debug_scripts_section_global<'ll>(cx: &CodegenCx<'ll, '
             let section_var = cx
                 .define_global(section_var_name, llvm_type)
                 .unwrap_or_else(|| bug!("symbol `{}` is already defined", section_var_name));
-            llvm::LLVMSetSection(section_var, section_name.as_ptr().cast());
+            llvm::LLVMSetSection(section_var, c".debug_gdb_scripts".as_ptr().cast());
             llvm::LLVMSetInitializer(section_var, cx.const_bytes(section_contents));
             llvm::LLVMSetGlobalConstant(section_var, llvm::True);
             llvm::LLVMSetUnnamedAddress(section_var, llvm::UnnamedAddr::Global);
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs
index 865bf01c8..883f82caa 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs
@@ -17,10 +17,10 @@ use crate::debuginfo::utils::FatPtrKind;
 use crate::llvm;
 use crate::llvm::debuginfo::{
     DIDescriptor, DIFile, DIFlags, DILexicalBlock, DIScope, DIType, DebugEmissionKind,
+    DebugNameTableKind,
 };
 use crate::value::Value;
 
-use cstr::cstr;
 use rustc_codegen_ssa::debuginfo::type_names::cpp_like_debuginfo;
 use rustc_codegen_ssa::debuginfo::type_names::VTableNameKind;
 use rustc_codegen_ssa::traits::*;
@@ -35,9 +35,10 @@ use rustc_middle::ty::{
 use rustc_session::config::{self, DebugInfo, Lto};
 use rustc_span::symbol::Symbol;
 use rustc_span::FileName;
-use rustc_span::{self, FileNameDisplayPreference, SourceFile};
+use rustc_span::{FileNameDisplayPreference, SourceFile};
 use rustc_symbol_mangling::typeid_for_trait_ref;
 use rustc_target::abi::{Align, Size};
+use rustc_target::spec::DebuginfoKind;
 use smallvec::smallvec;
 
 use libc::{c_char, c_longlong, c_uint};
@@ -606,7 +607,7 @@ pub fn file_metadata<'ll>(cx: &CodegenCx<'ll, '_>, source_file: &SourceFile) ->
 
                     if let Ok(rel_path) = abs_path.strip_prefix(working_directory) {
                         (
-                            working_directory.to_string_lossy().into(),
+                            working_directory.to_string_lossy(),
                             rel_path.to_string_lossy().into_owned(),
                         )
                     } else {
@@ -853,8 +854,7 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>(
 
     use rustc_session::RemapFileNameExt;
     let name_in_debuginfo = name_in_debuginfo.to_string_lossy();
-    let work_dir = tcx.sess.opts.working_dir.for_codegen(&tcx.sess).to_string_lossy();
-    let flags = "\0";
+    let work_dir = tcx.sess.opts.working_dir.for_codegen(tcx.sess).to_string_lossy();
     let output_filenames = tcx.output_filenames(());
     let split_name = if tcx.sess.target_can_use_split_dwarf() {
         output_filenames
@@ -878,6 +878,17 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>(
     let split_name = split_name.to_str().unwrap();
     let kind = DebugEmissionKind::from_generic(tcx.sess.opts.debuginfo);
 
+    let dwarf_version =
+        tcx.sess.opts.unstable_opts.dwarf_version.unwrap_or(tcx.sess.target.default_dwarf_version);
+    let is_dwarf_kind =
+        matches!(tcx.sess.target.debuginfo_kind, DebuginfoKind::Dwarf | DebuginfoKind::DwarfDsym);
+    // Don't emit `.debug_pubnames` and `.debug_pubtypes` on DWARFv4 or lower.
+    let debug_name_table_kind = if is_dwarf_kind && dwarf_version <= 4 {
+        DebugNameTableKind::None
+    } else {
+        DebugNameTableKind::Default
+    };
+
     unsafe {
         let compile_unit_file = llvm::LLVMRustDIBuilderCreateFile(
             debug_context.builder,
@@ -897,7 +908,7 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>(
             producer.as_ptr().cast(),
             producer.len(),
             tcx.sess.opts.optimize != config::OptLevel::No,
-            flags.as_ptr().cast(),
+            c"".as_ptr().cast(),
             0,
             // NB: this doesn't actually have any perceptible effect, it seems. LLVM will instead
             // put the path supplied to `MCSplitDwarfFile` into the debug info of the final
@@ -907,6 +918,7 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>(
             kind,
             0,
             tcx.sess.opts.unstable_opts.split_dwarf_inlining,
+            debug_name_table_kind,
         );
 
         if tcx.sess.opts.unstable_opts.profile {
@@ -926,8 +938,7 @@ pub fn build_compile_unit_di_node<'ll, 'tcx>(
             );
             let val = llvm::LLVMMetadataAsValue(debug_context.llcontext, gcov_metadata);
 
-            let llvm_gcov_ident = cstr!("llvm.gcov");
-            llvm::LLVMAddNamedMetadataOperand(debug_context.llmod, llvm_gcov_ident.as_ptr(), val);
+            llvm::LLVMAddNamedMetadataOperand(debug_context.llmod, c"llvm.gcov".as_ptr(), val);
         }
 
         return unit_metadata;
@@ -966,6 +977,27 @@ fn build_field_di_node<'ll, 'tcx>(
     }
 }
 
+/// Returns the `DIFlags` corresponding to the visibility of the item identified by `did`.
+///
+/// `DIFlags::Flag{Public,Protected,Private}` correspond to `DW_AT_accessibility`
+/// (public/protected/private) aren't exactly right for Rust, but neither is `DW_AT_visibility`
+/// (local/exported/qualified), and there's no way to set `DW_AT_visibility` in LLVM's API.
+fn visibility_di_flags<'ll, 'tcx>(
+    cx: &CodegenCx<'ll, 'tcx>,
+    did: DefId,
+    type_did: DefId,
+) -> DIFlags {
+    let parent_did = cx.tcx.parent(type_did);
+    let visibility = cx.tcx.visibility(did);
+    match visibility {
+        Visibility::Public => DIFlags::FlagPublic,
+        // Private fields have a restricted visibility of the module containing the type.
+        Visibility::Restricted(did) if did == parent_did => DIFlags::FlagPrivate,
+        // `pub(crate)`/`pub(super)` visibilities are any other restricted visibility.
+        Visibility::Restricted(..) => DIFlags::FlagProtected,
+    }
+}
+
 /// Creates the debuginfo node for a Rust struct type. Maybe be a regular struct or a tuple-struct.
 fn build_struct_type_di_node<'ll, 'tcx>(
     cx: &CodegenCx<'ll, 'tcx>,
@@ -989,7 +1021,7 @@ fn build_struct_type_di_node<'ll, 'tcx>(
             &compute_debuginfo_type_name(cx.tcx, struct_type, false),
             size_and_align_of(struct_type_and_layout),
             Some(containing_scope),
-            DIFlags::FlagZero,
+            visibility_di_flags(cx, adt_def.did(), adt_def.did()),
         ),
         // Fields:
         |cx, owner| {
@@ -1012,7 +1044,7 @@ fn build_struct_type_di_node<'ll, 'tcx>(
                         &field_name[..],
                         (field_layout.size, field_layout.align.abi),
                         struct_type_and_layout.fields.offset(i),
-                        DIFlags::FlagZero,
+                        visibility_di_flags(cx, f.did, adt_def.did()),
                         type_di_node(cx, field_layout.ty),
                     )
                 })
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs
index ca7bfbeac..4a2861af4 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs
@@ -26,8 +26,8 @@ use crate::{
             enums::{tag_base_type, DiscrResult},
             file_metadata, size_and_align_of, type_di_node,
             type_map::{self, Stub, UniqueTypeId},
-            unknown_file_metadata, DINodeCreationResult, SmallVec, NO_GENERICS, NO_SCOPE_METADATA,
-            UNKNOWN_LINE_NUMBER,
+            unknown_file_metadata, visibility_di_flags, DINodeCreationResult, SmallVec,
+            NO_GENERICS, NO_SCOPE_METADATA, UNKNOWN_LINE_NUMBER,
         },
         utils::DIB,
     },
@@ -215,7 +215,7 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>(
             &enum_type_name,
             cx.size_and_align_of(enum_type),
             NO_SCOPE_METADATA,
-            DIFlags::FlagZero,
+            visibility_di_flags(cx, enum_adt_def.did(), enum_adt_def.did()),
         ),
         |cx, enum_type_di_node| {
             match enum_type_and_layout.variants {
@@ -320,6 +320,7 @@ fn build_single_variant_union_fields<'ll, 'tcx>(
     variant_index: VariantIdx,
 ) -> SmallVec<&'ll DIType> {
     let variant_layout = enum_type_and_layout.for_variant(cx, variant_index);
+    let visibility_flags = visibility_di_flags(cx, enum_adt_def.did(), enum_adt_def.did());
     let variant_struct_type_di_node = super::build_enum_variant_struct_type_di_node(
         cx,
         enum_type_and_layout,
@@ -327,6 +328,7 @@ fn build_single_variant_union_fields<'ll, 'tcx>(
         variant_index,
         enum_adt_def.variant(variant_index),
         variant_layout,
+        visibility_flags,
     );
 
     let tag_base_type = cx.tcx.types.u32;
@@ -364,7 +366,7 @@ fn build_single_variant_union_fields<'ll, 'tcx>(
             //       since the later is sometimes smaller (if it has fewer fields).
             size_and_align_of(enum_type_and_layout),
             Size::ZERO,
-            DIFlags::FlagZero,
+            visibility_flags,
             variant_struct_type_wrapper_di_node,
         ),
         unsafe {
@@ -376,7 +378,7 @@ fn build_single_variant_union_fields<'ll, 'tcx>(
                 unknown_file_metadata(cx),
                 UNKNOWN_LINE_NUMBER,
                 variant_names_type_di_node,
-                DIFlags::FlagZero,
+                visibility_flags,
                 Some(cx.const_u64(SINGLE_VARIANT_VIRTUAL_DISR)),
                 tag_base_type_align.bits() as u32,
             )
@@ -403,6 +405,7 @@ fn build_union_fields_for_enum<'ll, 'tcx>(
             (variant_index, variant_name)
         }),
     );
+    let visibility_flags = visibility_di_flags(cx, enum_adt_def.did(), enum_adt_def.did());
 
     let variant_field_infos: SmallVec<VariantFieldInfo<'ll>> = variant_indices
         .map(|variant_index| {
@@ -417,6 +420,7 @@ fn build_union_fields_for_enum<'ll, 'tcx>(
                 variant_index,
                 variant_def,
                 variant_layout,
+                visibility_flags,
             );
 
             VariantFieldInfo {
@@ -437,6 +441,7 @@ fn build_union_fields_for_enum<'ll, 'tcx>(
         tag_base_type,
         tag_field,
         untagged_variant_index,
+        visibility_flags,
     )
 }
 
@@ -715,7 +720,7 @@ fn build_union_fields_for_direct_tag_coroutine<'ll, 'tcx>(
                 coroutine_type_and_layout,
                 coroutine_type_di_node,
                 coroutine_layout,
-                &common_upvar_names,
+                common_upvar_names,
             );
 
             let span = coroutine_layout.variant_source_info[variant_index].span;
@@ -744,6 +749,7 @@ fn build_union_fields_for_direct_tag_coroutine<'ll, 'tcx>(
         tag_base_type,
         tag_field,
         None,
+        DIFlags::FlagZero,
     )
 }
 
@@ -758,6 +764,7 @@ fn build_union_fields_for_direct_tag_enum_or_coroutine<'ll, 'tcx>(
     tag_base_type: Ty<'tcx>,
     tag_field: usize,
     untagged_variant_index: Option<VariantIdx>,
+    di_flags: DIFlags,
 ) -> SmallVec<&'ll DIType> {
     let tag_base_type_di_node = type_di_node(cx, tag_base_type);
     let mut unions_fields = SmallVec::with_capacity(variant_field_infos.len() + 1);
@@ -801,7 +808,7 @@ fn build_union_fields_for_direct_tag_enum_or_coroutine<'ll, 'tcx>(
                 align.bits() as u32,
                 // Union fields are always at offset zero
                 Size::ZERO.bits(),
-                DIFlags::FlagZero,
+                di_flags,
                 variant_struct_type_wrapper,
             )
         }
@@ -835,7 +842,7 @@ fn build_union_fields_for_direct_tag_enum_or_coroutine<'ll, 'tcx>(
             TAG_FIELD_NAME_128_LO,
             size_and_align,
             lo_offset,
-            DIFlags::FlagZero,
+            di_flags,
             type_di_node,
         ));
 
@@ -855,7 +862,7 @@ fn build_union_fields_for_direct_tag_enum_or_coroutine<'ll, 'tcx>(
             TAG_FIELD_NAME,
             cx.size_and_align_of(enum_type_and_layout.field(cx, tag_field).ty),
             enum_type_and_layout.fields.offset(tag_field),
-            DIFlags::FlagZero,
+            di_flags,
             tag_base_type_di_node,
         ));
     }
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs
index df1df6d19..eef8dbb33 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs
@@ -250,6 +250,7 @@ fn build_enum_variant_struct_type_di_node<'ll, 'tcx>(
     variant_index: VariantIdx,
     variant_def: &VariantDef,
     variant_layout: TyAndLayout<'tcx>,
+    di_flags: DIFlags,
 ) -> &'ll DIType {
     debug_assert_eq!(variant_layout.ty, enum_type_and_layout.ty);
 
@@ -267,7 +268,7 @@ fn build_enum_variant_struct_type_di_node<'ll, 'tcx>(
             // NOTE: We use size and align of enum_type, not from variant_layout:
             size_and_align_of(enum_type_and_layout),
             Some(enum_type_di_node),
-            DIFlags::FlagZero,
+            di_flags,
         ),
         |cx, struct_type_di_node| {
             (0..variant_layout.fields.count())
@@ -289,7 +290,7 @@ fn build_enum_variant_struct_type_di_node<'ll, 'tcx>(
                         &field_name,
                         (field_layout.size, field_layout.align.abi),
                         variant_layout.fields.offset(field_index),
-                        DIFlags::FlagZero,
+                        di_flags,
                         type_di_node(cx, field_layout.ty),
                     )
                 })
@@ -395,7 +396,7 @@ pub fn build_coroutine_variant_struct_type_di_node<'ll, 'tcx>(
                 })
                 .collect();
 
-            state_specific_fields.into_iter().chain(common_fields.into_iter()).collect()
+            state_specific_fields.into_iter().chain(common_fields).collect()
         },
         |cx| build_generic_type_param_di_nodes(cx, coroutine_type_and_layout.ty),
     )
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs
index 7eff52b85..cba4e3811 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs
@@ -7,8 +7,8 @@ use crate::{
             enums::tag_base_type,
             file_metadata, size_and_align_of, type_di_node,
             type_map::{self, Stub, StubInfo, UniqueTypeId},
-            unknown_file_metadata, DINodeCreationResult, SmallVec, NO_GENERICS,
-            UNKNOWN_LINE_NUMBER,
+            unknown_file_metadata, visibility_di_flags, DINodeCreationResult, SmallVec,
+            NO_GENERICS, UNKNOWN_LINE_NUMBER,
         },
         utils::{create_DIArray, get_namespace_for_item, DIB},
     },
@@ -63,6 +63,8 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>(
     let enum_type_and_layout = cx.layout_of(enum_type);
     let enum_type_name = compute_debuginfo_type_name(cx.tcx, enum_type, false);
 
+    let visibility_flags = visibility_di_flags(cx, enum_adt_def.did(), enum_adt_def.did());
+
     debug_assert!(!wants_c_like_enum_debuginfo(enum_type_and_layout));
 
     type_map::build_type_with_children(
@@ -74,7 +76,7 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>(
             &enum_type_name,
             size_and_align_of(enum_type_and_layout),
             Some(containing_scope),
-            DIFlags::FlagZero,
+            visibility_flags,
         ),
         |cx, enum_type_di_node| {
             // Build the struct type for each variant. These will be referenced by the
@@ -92,6 +94,7 @@ pub(super) fn build_enum_type_di_node<'ll, 'tcx>(
                         variant_index,
                         enum_adt_def.variant(variant_index),
                         enum_type_and_layout.for_variant(cx, variant_index),
+                        visibility_flags,
                     ),
                     source_info: None,
                 })
@@ -197,7 +200,7 @@ pub(super) fn build_coroutine_di_node<'ll, 'tcx>(
                                 coroutine_type_and_layout,
                                 coroutine_type_di_node,
                                 coroutine_layout,
-                                &common_upvar_names,
+                                common_upvar_names,
                             ),
                         source_info,
                     }
diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs
index 4832b147a..31631e8a8 100644
--- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs
+++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs
@@ -32,7 +32,7 @@ use rustc_middle::ty::{self, Instance, ParamEnv, Ty, TypeVisitableExt};
 use rustc_session::config::{self, DebugInfo};
 use rustc_session::Session;
 use rustc_span::symbol::Symbol;
-use rustc_span::{self, BytePos, Pos, SourceFile, SourceFileAndLine, SourceFileHash, Span};
+use rustc_span::{BytePos, Pos, SourceFile, SourceFileAndLine, SourceFileHash, Span};
 use rustc_target::abi::Size;
 
 use libc::c_uint;
@@ -112,7 +112,7 @@ impl<'ll, 'tcx> CodegenUnitDebugContext<'ll, 'tcx> {
                 llvm::LLVMRustAddModuleFlag(
                     self.llmod,
                     llvm::LLVMModFlagBehavior::Warning,
-                    "Dwarf Version\0".as_ptr().cast(),
+                    c"Dwarf Version".as_ptr().cast(),
                     dwarf_version,
                 );
             } else {
@@ -120,17 +120,16 @@ impl<'ll, 'tcx> CodegenUnitDebugContext<'ll, 'tcx> {
                 llvm::LLVMRustAddModuleFlag(
                     self.llmod,
                     llvm::LLVMModFlagBehavior::Warning,
-                    "CodeView\0".as_ptr().cast(),
+                    c"CodeView".as_ptr().cast(),
                     1,
                 )
             }
 
             // Prevent bitcode readers from deleting the debug info.
-            let ptr = "Debug Info Version\0".as_ptr();
             llvm::LLVMRustAddModuleFlag(
                 self.llmod,
                 llvm::LLVMModFlagBehavior::Warning,
-                ptr.cast(),
+                c"Debug Info Version".as_ptr().cast(),
                 llvm::LLVMRustDebugMetadataVersion(),
             );
         }
diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs
index 164b12cf8..78c0725a6 100644
--- a/compiler/rustc_codegen_llvm/src/declare.rs
+++ b/compiler/rustc_codegen_llvm/src/declare.rs
@@ -84,7 +84,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
         fn_type: &'ll Type,
     ) -> &'ll Value {
         // Declare C ABI functions with the visibility used by C by default.
-        let visibility = if self.tcx.sess.target.default_hidden_visibility {
+        let visibility = if self.tcx.sess.default_hidden_visibility() {
             llvm::Visibility::Hidden
         } else {
             llvm::Visibility::Default
@@ -107,7 +107,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
         unnamed: llvm::UnnamedAddr,
         fn_type: &'ll Type,
     ) -> &'ll Value {
-        let visibility = if self.tcx.sess.target.default_hidden_visibility {
+        let visibility = if self.tcx.sess.default_hidden_visibility() {
             llvm::Visibility::Hidden
         } else {
             llvm::Visibility::Default
diff --git a/compiler/rustc_codegen_llvm/src/errors.rs b/compiler/rustc_codegen_llvm/src/errors.rs
index 10ca5ad80..671a22525 100644
--- a/compiler/rustc_codegen_llvm/src/errors.rs
+++ b/compiler/rustc_codegen_llvm/src/errors.rs
@@ -5,7 +5,7 @@ use std::path::Path;
 use crate::fluent_generated as fluent;
 use rustc_data_structures::small_c_str::SmallCStr;
 use rustc_errors::{
-    DiagnosticBuilder, EmissionGuarantee, ErrorGuaranteed, Handler, IntoDiagnostic,
+    DiagCtxt, DiagnosticBuilder, EmissionGuarantee, ErrorGuaranteed, FatalError, IntoDiagnostic,
 };
 use rustc_macros::{Diagnostic, Subdiagnostic};
 use rustc_span::Span;
@@ -101,13 +101,13 @@ pub(crate) struct DynamicLinkingWithLTO;
 
 pub(crate) struct ParseTargetMachineConfig<'a>(pub LlvmError<'a>);
 
-impl<EM: EmissionGuarantee> IntoDiagnostic<'_, EM> for ParseTargetMachineConfig<'_> {
-    fn into_diagnostic(self, sess: &'_ Handler) -> DiagnosticBuilder<'_, EM> {
-        let diag: DiagnosticBuilder<'_, EM> = self.0.into_diagnostic(sess);
+impl IntoDiagnostic<'_, FatalError> for ParseTargetMachineConfig<'_> {
+    fn into_diagnostic(self, dcx: &'_ DiagCtxt) -> DiagnosticBuilder<'_, FatalError> {
+        let diag: DiagnosticBuilder<'_, FatalError> = self.0.into_diagnostic(dcx);
         let (message, _) = diag.styled_message().first().expect("`LlvmError` with no message");
-        let message = sess.eagerly_translate_to_string(message.clone(), diag.args());
+        let message = dcx.eagerly_translate_to_string(message.clone(), diag.args());
 
-        let mut diag = sess.struct_diagnostic(fluent::codegen_llvm_parse_target_machine_config);
+        let mut diag = dcx.struct_almost_fatal(fluent::codegen_llvm_parse_target_machine_config);
         diag.set_arg("error", message);
         diag
     }
@@ -124,8 +124,8 @@ pub(crate) struct TargetFeatureDisableOrEnable<'a> {
 pub(crate) struct MissingFeatures;
 
 impl IntoDiagnostic<'_, ErrorGuaranteed> for TargetFeatureDisableOrEnable<'_> {
-    fn into_diagnostic(self, sess: &'_ Handler) -> DiagnosticBuilder<'_, ErrorGuaranteed> {
-        let mut diag = sess.struct_err(fluent::codegen_llvm_target_feature_disable_or_enable);
+    fn into_diagnostic(self, dcx: &'_ DiagCtxt) -> DiagnosticBuilder<'_, ErrorGuaranteed> {
+        let mut diag = dcx.struct_err(fluent::codegen_llvm_target_feature_disable_or_enable);
         if let Some(span) = self.span {
             diag.set_span(span);
         };
@@ -183,8 +183,8 @@ pub enum LlvmError<'a> {
 
 pub(crate) struct WithLlvmError<'a>(pub LlvmError<'a>, pub String);
 
-impl<EM: EmissionGuarantee> IntoDiagnostic<'_, EM> for WithLlvmError<'_> {
-    fn into_diagnostic(self, sess: &'_ Handler) -> DiagnosticBuilder<'_, EM> {
+impl<G: EmissionGuarantee> IntoDiagnostic<'_, G> for WithLlvmError<'_> {
+    fn into_diagnostic(self, dcx: &'_ DiagCtxt) -> DiagnosticBuilder<'_, G> {
         use LlvmError::*;
         let msg_with_llvm_err = match &self.0 {
             WriteOutput { .. } => fluent::codegen_llvm_write_output_with_llvm_err,
@@ -201,7 +201,7 @@ impl<EM: EmissionGuarantee> IntoDiagnostic<'_, EM> for WithLlvmError<'_> {
             PrepareThinLtoModule => fluent::codegen_llvm_prepare_thin_lto_module_with_llvm_err,
             ParseBitcode => fluent::codegen_llvm_parse_bitcode_with_llvm_err,
         };
-        let mut diag = self.0.into_diagnostic(sess);
+        let mut diag = self.0.into_diagnostic(dcx);
         diag.set_primary_message(msg_with_llvm_err);
         diag.set_arg("llvm_err", self.1);
         diag
diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs
index cc7e78b9c..58e68a649 100644
--- a/compiler/rustc_codegen_llvm/src/intrinsic.rs
+++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs
@@ -10,7 +10,7 @@ use crate::value::Value;
 use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh};
 use rustc_codegen_ssa::common::{IntPredicate, TypeKind};
 use rustc_codegen_ssa::errors::{ExpectedPointerMutability, InvalidMonomorphization};
-use rustc_codegen_ssa::mir::operand::OperandRef;
+use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
 use rustc_codegen_ssa::mir::place::PlaceRef;
 use rustc_codegen_ssa::traits::*;
 use rustc_hir as hir;
@@ -946,6 +946,13 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         tcx.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), callee_ty.fn_sig(tcx));
     let arg_tys = sig.inputs();
 
+    // Vectors must be immediates (non-power-of-2 #[repr(packed)] are not)
+    for (ty, arg) in arg_tys.iter().zip(args) {
+        if ty.is_simd() && !matches!(arg.val, OperandValue::Immediate(_)) {
+            return_error!(InvalidMonomorphization::SimdArgument { span, name, ty: *ty });
+        }
+    }
+
     if name == sym::simd_select_bitmask {
         let (len, _) = require_simd!(arg_tys[1], SimdArgument);
 
@@ -1492,6 +1499,198 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         return Ok(v);
     }
 
+    if name == sym::simd_masked_load {
+        // simd_masked_load(mask: <N x i{M}>, pointer: *_ T, values: <N x T>) -> <N x T>
+        // * N: number of elements in the input vectors
+        // * T: type of the element to load
+        // * M: any integer width is supported, will be truncated to i1
+        // Loads contiguous elements from memory behind `pointer`, but only for
+        // those lanes whose `mask` bit is enabled.
+        // The memory addresses corresponding to the “off” lanes are not accessed.
+
+        // The element type of the "mask" argument must be a signed integer type of any width
+        let mask_ty = in_ty;
+        let (mask_len, mask_elem) = (in_len, in_elem);
+
+        // The second argument must be a pointer matching the element type
+        let pointer_ty = arg_tys[1];
+
+        // The last argument is a passthrough vector providing values for disabled lanes
+        let values_ty = arg_tys[2];
+        let (values_len, values_elem) = require_simd!(values_ty, SimdThird);
+
+        require_simd!(ret_ty, SimdReturn);
+
+        // Of the same length:
+        require!(
+            values_len == mask_len,
+            InvalidMonomorphization::ThirdArgumentLength {
+                span,
+                name,
+                in_len: mask_len,
+                in_ty: mask_ty,
+                arg_ty: values_ty,
+                out_len: values_len
+            }
+        );
+
+        // The return type must match the last argument type
+        require!(
+            ret_ty == values_ty,
+            InvalidMonomorphization::ExpectedReturnType { span, name, in_ty: values_ty, ret_ty }
+        );
+
+        require!(
+            matches!(
+                pointer_ty.kind(),
+                ty::RawPtr(p) if p.ty == values_elem && p.ty.kind() == values_elem.kind()
+            ),
+            InvalidMonomorphization::ExpectedElementType {
+                span,
+                name,
+                expected_element: values_elem,
+                second_arg: pointer_ty,
+                in_elem: values_elem,
+                in_ty: values_ty,
+                mutability: ExpectedPointerMutability::Not,
+            }
+        );
+
+        require!(
+            matches!(mask_elem.kind(), ty::Int(_)),
+            InvalidMonomorphization::ThirdArgElementType {
+                span,
+                name,
+                expected_element: values_elem,
+                third_arg: mask_ty,
+            }
+        );
+
+        // Alignment of T, must be a constant integer value:
+        let alignment_ty = bx.type_i32();
+        let alignment = bx.const_i32(bx.align_of(values_elem).bytes() as i32);
+
+        // Truncate the mask vector to a vector of i1s:
+        let (mask, mask_ty) = {
+            let i1 = bx.type_i1();
+            let i1xn = bx.type_vector(i1, mask_len);
+            (bx.trunc(args[0].immediate(), i1xn), i1xn)
+        };
+
+        let llvm_pointer = bx.type_ptr();
+
+        // Type of the vector of elements:
+        let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len);
+        let llvm_elem_vec_str = llvm_vector_str(bx, values_elem, values_len);
+
+        let llvm_intrinsic = format!("llvm.masked.load.{llvm_elem_vec_str}.p0");
+        let fn_ty = bx
+            .type_func(&[llvm_pointer, alignment_ty, mask_ty, llvm_elem_vec_ty], llvm_elem_vec_ty);
+        let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
+        let v = bx.call(
+            fn_ty,
+            None,
+            None,
+            f,
+            &[args[1].immediate(), alignment, mask, args[2].immediate()],
+            None,
+        );
+        return Ok(v);
+    }
+
+    if name == sym::simd_masked_store {
+        // simd_masked_store(mask: <N x i{M}>, pointer: *mut T, values: <N x T>) -> ()
+        // * N: number of elements in the input vectors
+        // * T: type of the element to load
+        // * M: any integer width is supported, will be truncated to i1
+        // Stores contiguous elements to memory behind `pointer`, but only for
+        // those lanes whose `mask` bit is enabled.
+        // The memory addresses corresponding to the “off” lanes are not accessed.
+
+        // The element type of the "mask" argument must be a signed integer type of any width
+        let mask_ty = in_ty;
+        let (mask_len, mask_elem) = (in_len, in_elem);
+
+        // The second argument must be a pointer matching the element type
+        let pointer_ty = arg_tys[1];
+
+        // The last argument specifies the values to store to memory
+        let values_ty = arg_tys[2];
+        let (values_len, values_elem) = require_simd!(values_ty, SimdThird);
+
+        // Of the same length:
+        require!(
+            values_len == mask_len,
+            InvalidMonomorphization::ThirdArgumentLength {
+                span,
+                name,
+                in_len: mask_len,
+                in_ty: mask_ty,
+                arg_ty: values_ty,
+                out_len: values_len
+            }
+        );
+
+        // The second argument must be a mutable pointer type matching the element type
+        require!(
+            matches!(
+                pointer_ty.kind(),
+                ty::RawPtr(p) if p.ty == values_elem && p.ty.kind() == values_elem.kind() && p.mutbl.is_mut()
+            ),
+            InvalidMonomorphization::ExpectedElementType {
+                span,
+                name,
+                expected_element: values_elem,
+                second_arg: pointer_ty,
+                in_elem: values_elem,
+                in_ty: values_ty,
+                mutability: ExpectedPointerMutability::Mut,
+            }
+        );
+
+        require!(
+            matches!(mask_elem.kind(), ty::Int(_)),
+            InvalidMonomorphization::ThirdArgElementType {
+                span,
+                name,
+                expected_element: values_elem,
+                third_arg: mask_ty,
+            }
+        );
+
+        // Alignment of T, must be a constant integer value:
+        let alignment_ty = bx.type_i32();
+        let alignment = bx.const_i32(bx.align_of(values_elem).bytes() as i32);
+
+        // Truncate the mask vector to a vector of i1s:
+        let (mask, mask_ty) = {
+            let i1 = bx.type_i1();
+            let i1xn = bx.type_vector(i1, in_len);
+            (bx.trunc(args[0].immediate(), i1xn), i1xn)
+        };
+
+        let ret_t = bx.type_void();
+
+        let llvm_pointer = bx.type_ptr();
+
+        // Type of the vector of elements:
+        let llvm_elem_vec_ty = llvm_vector_ty(bx, values_elem, values_len);
+        let llvm_elem_vec_str = llvm_vector_str(bx, values_elem, values_len);
+
+        let llvm_intrinsic = format!("llvm.masked.store.{llvm_elem_vec_str}.p0");
+        let fn_ty = bx.type_func(&[llvm_elem_vec_ty, llvm_pointer, alignment_ty, mask_ty], ret_t);
+        let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
+        let v = bx.call(
+            fn_ty,
+            None,
+            None,
+            f,
+            &[args[2].immediate(), args[1].immediate(), alignment, mask],
+            None,
+        );
+        return Ok(v);
+    }
+
     if name == sym::simd_scatter {
         // simd_scatter(values: <N x T>, pointers: <N x *mut T>,
         //             mask: <N x i{M}>) -> ()
diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs
index a4e027012..ed0d853e7 100644
--- a/compiler/rustc_codegen_llvm/src/lib.rs
+++ b/compiler/rustc_codegen_llvm/src/lib.rs
@@ -4,10 +4,11 @@
 //!
 //! This API is completely unstable and subject to change.
 
-#![cfg_attr(not(bootstrap), allow(internal_features))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
+#![allow(internal_features)]
+#![feature(rustdoc_internals)]
+#![doc(rust_logo)]
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
+#![feature(c_str_literals)]
 #![feature(exact_size_is_empty)]
 #![feature(extern_types)]
 #![feature(hash_raw_entry)]
@@ -17,7 +18,6 @@
 #![feature(never_type)]
 #![feature(impl_trait_in_assoc_type)]
 #![recursion_limit = "256"]
-#![allow(rustc::potential_query_instability)]
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
 
@@ -40,8 +40,7 @@ use rustc_codegen_ssa::traits::*;
 use rustc_codegen_ssa::ModuleCodegen;
 use rustc_codegen_ssa::{CodegenResults, CompiledModule};
 use rustc_data_structures::fx::FxIndexMap;
-use rustc_errors::{DiagnosticMessage, ErrorGuaranteed, FatalError, Handler, SubdiagnosticMessage};
-use rustc_fluent_macro::fluent_messages;
+use rustc_errors::{DiagCtxt, ErrorGuaranteed, FatalError};
 use rustc_metadata::EncodedMetadata;
 use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
 use rustc_middle::ty::TyCtxt;
@@ -93,7 +92,7 @@ mod type_of;
 mod va_arg;
 mod value;
 
-fluent_messages! { "../messages.ftl" }
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
 
 #[derive(Clone)]
 pub struct LlvmCodegenBackend(());
@@ -105,7 +104,7 @@ struct TimeTraceProfiler {
 impl TimeTraceProfiler {
     fn new(enabled: bool) -> Self {
         if enabled {
-            unsafe { llvm::LLVMTimeTraceProfilerInitialize() }
+            unsafe { llvm::LLVMRustTimeTraceProfilerInitialize() }
         }
         TimeTraceProfiler { enabled }
     }
@@ -114,7 +113,7 @@ impl TimeTraceProfiler {
 impl Drop for TimeTraceProfiler {
     fn drop(&mut self) {
         if self.enabled {
-            unsafe { llvm::LLVMTimeTraceProfilerFinishThread() }
+            unsafe { llvm::LLVMRustTimeTraceProfilerFinishThread() }
         }
     }
 }
@@ -201,10 +200,10 @@ impl WriteBackendMethods for LlvmCodegenBackend {
     }
     fn run_link(
         cgcx: &CodegenContext<Self>,
-        diag_handler: &Handler,
+        dcx: &DiagCtxt,
         modules: Vec<ModuleCodegen<Self::Module>>,
     ) -> Result<ModuleCodegen<Self::Module>, FatalError> {
-        back::write::link(cgcx, diag_handler, modules)
+        back::write::link(cgcx, dcx, modules)
     }
     fn run_fat_lto(
         cgcx: &CodegenContext<Self>,
@@ -222,18 +221,18 @@ impl WriteBackendMethods for LlvmCodegenBackend {
     }
     unsafe fn optimize(
         cgcx: &CodegenContext<Self>,
-        diag_handler: &Handler,
+        dcx: &DiagCtxt,
         module: &ModuleCodegen<Self::Module>,
         config: &ModuleConfig,
     ) -> Result<(), FatalError> {
-        back::write::optimize(cgcx, diag_handler, module, config)
+        back::write::optimize(cgcx, dcx, module, config)
     }
     fn optimize_fat(
         cgcx: &CodegenContext<Self>,
         module: &mut ModuleCodegen<Self::Module>,
     ) -> Result<(), FatalError> {
-        let diag_handler = cgcx.create_diag_handler();
-        back::lto::run_pass_manager(cgcx, &diag_handler, module, false)
+        let dcx = cgcx.create_dcx();
+        back::lto::run_pass_manager(cgcx, &dcx, module, false)
     }
     unsafe fn optimize_thin(
         cgcx: &CodegenContext<Self>,
@@ -243,11 +242,11 @@ impl WriteBackendMethods for LlvmCodegenBackend {
     }
     unsafe fn codegen(
         cgcx: &CodegenContext<Self>,
-        diag_handler: &Handler,
+        dcx: &DiagCtxt,
         module: ModuleCodegen<Self::Module>,
         config: &ModuleConfig,
     ) -> Result<CompiledModule, FatalError> {
-        back::write::codegen(cgcx, diag_handler, module, config)
+        back::write::codegen(cgcx, dcx, module, config)
     }
     fn prepare_thin(module: ModuleCodegen<Self::Module>) -> (String, Self::ThinBuffer) {
         back::lto::prepare_thin(module)
@@ -307,7 +306,9 @@ impl CodegenBackend for LlvmCodegenBackend {
             }
             PrintKind::TlsModels => {
                 writeln!(out, "Available TLS models:");
-                for name in &["global-dynamic", "local-dynamic", "initial-exec", "local-exec"] {
+                for name in
+                    &["global-dynamic", "local-dynamic", "initial-exec", "local-exec", "emulated"]
+                {
                     writeln!(out, "    {name}");
                 }
                 writeln!(out);
@@ -446,16 +447,16 @@ impl ModuleLlvm {
         cgcx: &CodegenContext<LlvmCodegenBackend>,
         name: &CStr,
         buffer: &[u8],
-        handler: &Handler,
+        dcx: &DiagCtxt,
     ) -> Result<Self, FatalError> {
         unsafe {
             let llcx = llvm::LLVMRustContextCreate(cgcx.fewer_names);
-            let llmod_raw = back::lto::parse_module(llcx, name, buffer, handler)?;
+            let llmod_raw = back::lto::parse_module(llcx, name, buffer, dcx)?;
             let tm_factory_config = TargetMachineFactoryConfig::new(cgcx, name.to_str().unwrap());
             let tm = match (cgcx.tm_factory)(tm_factory_config) {
                 Ok(m) => m,
                 Err(e) => {
-                    return Err(handler.emit_almost_fatal(ParseTargetMachineConfig(e)));
+                    return Err(dcx.emit_almost_fatal(ParseTargetMachineConfig(e)));
                 }
             };
 
diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
index 7fc02a95b..f936ba799 100644
--- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs
@@ -5,7 +5,7 @@ use super::debuginfo::{
     DIArray, DIBasicType, DIBuilder, DICompositeType, DIDerivedType, DIDescriptor, DIEnumerator,
     DIFile, DIFlags, DIGlobalVariableExpression, DILexicalBlock, DILocation, DINameSpace,
     DISPFlags, DIScope, DISubprogram, DISubrange, DITemplateTypeParameter, DIType, DIVariable,
-    DebugEmissionKind,
+    DebugEmissionKind, DebugNameTableKind,
 };
 
 use libc::{c_char, c_int, c_uint, size_t};
@@ -200,6 +200,7 @@ pub enum AttributeKind {
     AllocatedPointer = 38,
     AllocAlign = 39,
     SanitizeSafeStack = 40,
+    FnRetThunkExtern = 41,
 }
 
 /// LLVMIntPredicate
@@ -793,6 +794,15 @@ pub mod debuginfo {
             }
         }
     }
+
+    /// LLVMRustDebugNameTableKind
+    #[derive(Clone, Copy)]
+    #[repr(C)]
+    pub enum DebugNameTableKind {
+        Default,
+        Gnu,
+        None,
+    }
 }
 
 use bitflags::bitflags;
@@ -823,11 +833,7 @@ pub type GetSymbolsCallback = unsafe extern "C" fn(*mut c_void, *const c_char) -
 pub type GetSymbolsErrorCallback = unsafe extern "C" fn(*const c_char) -> *mut c_void;
 
 extern "C" {
-    pub fn LLVMRustInstallFatalErrorHandler();
-    pub fn LLVMRustDisableSystemDialogsOnCrash();
-
     // Create and destroy contexts.
-    pub fn LLVMRustContextCreate(shouldDiscardNames: bool) -> &'static mut Context;
     pub fn LLVMContextDispose(C: &'static mut Context);
     pub fn LLVMGetMDKindIDInContext(C: &Context, Name: *const c_char, SLen: c_uint) -> c_uint;
 
@@ -843,9 +849,6 @@ extern "C" {
     /// See Module::setModuleInlineAsm.
     pub fn LLVMAppendModuleInlineAsm(M: &Module, Asm: *const c_char, Len: size_t);
 
-    /// See llvm::LLVMTypeKind::getTypeID.
-    pub fn LLVMRustGetTypeKind(Ty: &Type) -> TypeKind;
-
     // Operations on integer types
     pub fn LLVMInt1TypeInContext(C: &Context) -> &Type;
     pub fn LLVMInt8TypeInContext(C: &Context) -> &Type;
@@ -879,7 +882,6 @@ extern "C" {
     ) -> &'a Type;
 
     // Operations on array, pointer, and vector types (sequence types)
-    pub fn LLVMRustArrayType(ElementType: &Type, ElementCount: u64) -> &Type;
     pub fn LLVMPointerTypeInContext(C: &Context, AddressSpace: c_uint) -> &Type;
     pub fn LLVMVectorType(ElementType: &Type, ElementCount: c_uint) -> &Type;
 
@@ -898,10 +900,7 @@ extern "C" {
     pub fn LLVMReplaceAllUsesWith<'a>(OldVal: &'a Value, NewVal: &'a Value);
     pub fn LLVMSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Node: &'a Value);
     pub fn LLVMGlobalSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata);
-    pub fn LLVMRustGlobalAddMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata);
     pub fn LLVMValueAsMetadata(Node: &Value) -> &Metadata;
-    pub fn LLVMIsAFunction(Val: &Value) -> Option<&Value>;
-    pub fn LLVMRustIsNonGVFunctionPointerTy(Val: &Value) -> bool;
 
     // Operations on constants of any type
     pub fn LLVMConstNull(Ty: &Type) -> &Value;
@@ -931,13 +930,6 @@ extern "C" {
     pub fn LLVMConstInt(IntTy: &Type, N: c_ulonglong, SignExtend: Bool) -> &Value;
     pub fn LLVMConstIntOfArbitraryPrecision(IntTy: &Type, Wn: c_uint, Ws: *const u64) -> &Value;
     pub fn LLVMConstReal(RealTy: &Type, N: f64) -> &Value;
-    pub fn LLVMRustConstIntGetZExtValue(ConstantVal: &ConstantInt, Value: &mut u64) -> bool;
-    pub fn LLVMRustConstInt128Get(
-        ConstantVal: &ConstantInt,
-        SExt: bool,
-        high: &mut u64,
-        low: &mut u64,
-    ) -> bool;
 
     // Operations on composite constants
     pub fn LLVMConstStringInContext(
@@ -972,17 +964,11 @@ extern "C" {
     pub fn LLVMConstPtrToInt<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value;
     pub fn LLVMConstIntToPtr<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value;
     pub fn LLVMConstBitCast<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value;
-    pub fn LLVMConstPointerCast<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value;
     pub fn LLVMGetAggregateElement(ConstantVal: &Value, Idx: c_uint) -> Option<&Value>;
 
     // Operations on global variables, functions, and aliases (globals)
     pub fn LLVMIsDeclaration(Global: &Value) -> Bool;
-    pub fn LLVMRustGetLinkage(Global: &Value) -> Linkage;
-    pub fn LLVMRustSetLinkage(Global: &Value, RustLinkage: Linkage);
     pub fn LLVMSetSection(Global: &Value, Section: *const c_char);
-    pub fn LLVMRustGetVisibility(Global: &Value) -> Visibility;
-    pub fn LLVMRustSetVisibility(Global: &Value, Viz: Visibility);
-    pub fn LLVMRustSetDSOLocal(Global: &Value, is_dso_local: bool);
     pub fn LLVMGetAlignment(Global: &Value) -> c_uint;
     pub fn LLVMSetAlignment(Global: &Value, Bytes: c_uint);
     pub fn LLVMSetDLLStorageClass(V: &Value, C: DLLStorageClass);
@@ -991,13 +977,6 @@ extern "C" {
     pub fn LLVMIsAGlobalVariable(GlobalVar: &Value) -> Option<&Value>;
     pub fn LLVMAddGlobal<'a>(M: &'a Module, Ty: &'a Type, Name: *const c_char) -> &'a Value;
     pub fn LLVMGetNamedGlobal(M: &Module, Name: *const c_char) -> Option<&Value>;
-    pub fn LLVMRustGetOrInsertGlobal<'a>(
-        M: &'a Module,
-        Name: *const c_char,
-        NameLen: size_t,
-        T: &'a Type,
-    ) -> &'a Value;
-    pub fn LLVMRustInsertPrivateGlobal<'a>(M: &'a Module, T: &'a Type) -> &'a Value;
     pub fn LLVMGetFirstGlobal(M: &Module) -> Option<&Value>;
     pub fn LLVMGetNextGlobal(GlobalVar: &Value) -> Option<&Value>;
     pub fn LLVMDeleteGlobal(GlobalVar: &Value);
@@ -1007,16 +986,9 @@ extern "C" {
     pub fn LLVMSetThreadLocalMode(GlobalVar: &Value, Mode: ThreadLocalMode);
     pub fn LLVMIsGlobalConstant(GlobalVar: &Value) -> Bool;
     pub fn LLVMSetGlobalConstant(GlobalVar: &Value, IsConstant: Bool);
-    pub fn LLVMRustGetNamedValue(
-        M: &Module,
-        Name: *const c_char,
-        NameLen: size_t,
-    ) -> Option<&Value>;
     pub fn LLVMSetTailCall(CallInst: &Value, IsTailCall: Bool);
-    pub fn LLVMRustSetTailCallKind(CallInst: &Value, TKC: TailCallKind);
 
     // Operations on attributes
-    pub fn LLVMRustCreateAttrNoValue(C: &Context, attr: AttributeKind) -> &Attribute;
     pub fn LLVMCreateStringAttribute(
         C: &Context,
         Name: *const c_char,
@@ -1024,31 +996,9 @@ extern "C" {
         Value: *const c_char,
         ValueLen: c_uint,
     ) -> &Attribute;
-    pub fn LLVMRustCreateAlignmentAttr(C: &Context, bytes: u64) -> &Attribute;
-    pub fn LLVMRustCreateDereferenceableAttr(C: &Context, bytes: u64) -> &Attribute;
-    pub fn LLVMRustCreateDereferenceableOrNullAttr(C: &Context, bytes: u64) -> &Attribute;
-    pub fn LLVMRustCreateByValAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute;
-    pub fn LLVMRustCreateStructRetAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute;
-    pub fn LLVMRustCreateElementTypeAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute;
-    pub fn LLVMRustCreateUWTableAttr(C: &Context, async_: bool) -> &Attribute;
-    pub fn LLVMRustCreateAllocSizeAttr(C: &Context, size_arg: u32) -> &Attribute;
-    pub fn LLVMRustCreateAllocKindAttr(C: &Context, size_arg: u64) -> &Attribute;
-    pub fn LLVMRustCreateMemoryEffectsAttr(C: &Context, effects: MemoryEffects) -> &Attribute;
 
     // Operations on functions
-    pub fn LLVMRustGetOrInsertFunction<'a>(
-        M: &'a Module,
-        Name: *const c_char,
-        NameLen: size_t,
-        FunctionTy: &'a Type,
-    ) -> &'a Value;
     pub fn LLVMSetFunctionCallConv(Fn: &Value, CC: c_uint);
-    pub fn LLVMRustAddFunctionAttributes<'a>(
-        Fn: &'a Value,
-        index: c_uint,
-        Attrs: *const &'a Attribute,
-        AttrsLen: size_t,
-    );
 
     // Operations on parameters
     pub fn LLVMIsAArgument(Val: &Value) -> Option<&Value>;
@@ -1069,12 +1019,6 @@ extern "C" {
 
     // Operations on call sites
     pub fn LLVMSetInstructionCallConv(Instr: &Value, CC: c_uint);
-    pub fn LLVMRustAddCallSiteAttributes<'a>(
-        Instr: &'a Value,
-        index: c_uint,
-        Attrs: *const &'a Attribute,
-        AttrsLen: size_t,
-    );
 
     // Operations on load/store instructions (only)
     pub fn LLVMSetVolatile(MemoryAccessInst: &Value, volatile: Bool);
@@ -1112,18 +1056,6 @@ extern "C" {
         Else: &'a BasicBlock,
         NumCases: c_uint,
     ) -> &'a Value;
-    pub fn LLVMRustBuildInvoke<'a>(
-        B: &Builder<'a>,
-        Ty: &'a Type,
-        Fn: &'a Value,
-        Args: *const &'a Value,
-        NumArgs: c_uint,
-        Then: &'a BasicBlock,
-        Catch: &'a BasicBlock,
-        OpBundles: *const &OperandBundleDef<'a>,
-        NumOpBundles: c_uint,
-        Name: *const c_char,
-    ) -> &'a Value;
     pub fn LLVMBuildLandingPad<'a>(
         B: &Builder<'a>,
         Ty: &'a Type,
@@ -1337,7 +1269,6 @@ extern "C" {
     pub fn LLVMBuildNeg<'a>(B: &Builder<'a>, V: &'a Value, Name: *const c_char) -> &'a Value;
     pub fn LLVMBuildFNeg<'a>(B: &Builder<'a>, V: &'a Value, Name: *const c_char) -> &'a Value;
     pub fn LLVMBuildNot<'a>(B: &Builder<'a>, V: &'a Value, Name: *const c_char) -> &'a Value;
-    pub fn LLVMRustSetFastMath(Instr: &Value);
 
     // Memory
     pub fn LLVMBuildAlloca<'a>(B: &Builder<'a>, Ty: &'a Type, Name: *const c_char) -> &'a Value;
@@ -1485,42 +1416,6 @@ extern "C" {
 
     // Miscellaneous instructions
     pub fn LLVMBuildPhi<'a>(B: &Builder<'a>, Ty: &'a Type, Name: *const c_char) -> &'a Value;
-    pub fn LLVMRustGetInstrProfIncrementIntrinsic(M: &Module) -> &Value;
-    pub fn LLVMRustBuildCall<'a>(
-        B: &Builder<'a>,
-        Ty: &'a Type,
-        Fn: &'a Value,
-        Args: *const &'a Value,
-        NumArgs: c_uint,
-        OpBundles: *const &OperandBundleDef<'a>,
-        NumOpBundles: c_uint,
-    ) -> &'a Value;
-    pub fn LLVMRustBuildMemCpy<'a>(
-        B: &Builder<'a>,
-        Dst: &'a Value,
-        DstAlign: c_uint,
-        Src: &'a Value,
-        SrcAlign: c_uint,
-        Size: &'a Value,
-        IsVolatile: bool,
-    ) -> &'a Value;
-    pub fn LLVMRustBuildMemMove<'a>(
-        B: &Builder<'a>,
-        Dst: &'a Value,
-        DstAlign: c_uint,
-        Src: &'a Value,
-        SrcAlign: c_uint,
-        Size: &'a Value,
-        IsVolatile: bool,
-    ) -> &'a Value;
-    pub fn LLVMRustBuildMemSet<'a>(
-        B: &Builder<'a>,
-        Dst: &'a Value,
-        DstAlign: c_uint,
-        Val: &'a Value,
-        Size: &'a Value,
-        IsVolatile: bool,
-    ) -> &'a Value;
     pub fn LLVMBuildSelect<'a>(
         B: &Builder<'a>,
         If: &'a Value,
@@ -1568,6 +1463,202 @@ extern "C" {
         Name: *const c_char,
     ) -> &'a Value;
 
+    // Atomic Operations
+    pub fn LLVMBuildAtomicCmpXchg<'a>(
+        B: &Builder<'a>,
+        LHS: &'a Value,
+        CMP: &'a Value,
+        RHS: &'a Value,
+        Order: AtomicOrdering,
+        FailureOrder: AtomicOrdering,
+        SingleThreaded: Bool,
+    ) -> &'a Value;
+
+    pub fn LLVMSetWeak(CmpXchgInst: &Value, IsWeak: Bool);
+
+    pub fn LLVMBuildAtomicRMW<'a>(
+        B: &Builder<'a>,
+        Op: AtomicRmwBinOp,
+        LHS: &'a Value,
+        RHS: &'a Value,
+        Order: AtomicOrdering,
+        SingleThreaded: Bool,
+    ) -> &'a Value;
+
+    pub fn LLVMBuildFence<'a>(
+        B: &Builder<'a>,
+        Order: AtomicOrdering,
+        SingleThreaded: Bool,
+        Name: *const c_char,
+    ) -> &'a Value;
+
+    /// Writes a module to the specified path. Returns 0 on success.
+    pub fn LLVMWriteBitcodeToFile(M: &Module, Path: *const c_char) -> c_int;
+
+    /// Creates a legacy pass manager -- only used for final codegen.
+    pub fn LLVMCreatePassManager<'a>() -> &'a mut PassManager<'a>;
+
+    pub fn LLVMAddAnalysisPasses<'a>(T: &'a TargetMachine, PM: &PassManager<'a>);
+
+    pub fn LLVMGetHostCPUFeatures() -> *mut c_char;
+
+    pub fn LLVMDisposeMessage(message: *mut c_char);
+
+    pub fn LLVMIsMultithreaded() -> Bool;
+
+    pub fn LLVMStructCreateNamed(C: &Context, Name: *const c_char) -> &Type;
+
+    pub fn LLVMStructSetBody<'a>(
+        StructTy: &'a Type,
+        ElementTypes: *const &'a Type,
+        ElementCount: c_uint,
+        Packed: Bool,
+    );
+
+    pub fn LLVMMetadataAsValue<'a>(C: &'a Context, MD: &'a Metadata) -> &'a Value;
+
+    pub fn LLVMSetUnnamedAddress(Global: &Value, UnnamedAddr: UnnamedAddr);
+
+    pub fn LLVMIsAConstantInt(value_ref: &Value) -> Option<&ConstantInt>;
+}
+
+#[link(name = "llvm-wrapper", kind = "static")]
+extern "C" {
+    pub fn LLVMRustInstallFatalErrorHandler();
+    pub fn LLVMRustDisableSystemDialogsOnCrash();
+
+    // Create and destroy contexts.
+    pub fn LLVMRustContextCreate(shouldDiscardNames: bool) -> &'static mut Context;
+
+    /// See llvm::LLVMTypeKind::getTypeID.
+    pub fn LLVMRustGetTypeKind(Ty: &Type) -> TypeKind;
+
+    // Operations on array, pointer, and vector types (sequence types)
+    pub fn LLVMRustArrayType(ElementType: &Type, ElementCount: u64) -> &Type;
+
+    // Operations on all values
+    pub fn LLVMRustGlobalAddMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata);
+    pub fn LLVMRustIsNonGVFunctionPointerTy(Val: &Value) -> bool;
+
+    // Operations on scalar constants
+    pub fn LLVMRustConstIntGetZExtValue(ConstantVal: &ConstantInt, Value: &mut u64) -> bool;
+    pub fn LLVMRustConstInt128Get(
+        ConstantVal: &ConstantInt,
+        SExt: bool,
+        high: &mut u64,
+        low: &mut u64,
+    ) -> bool;
+
+    // Operations on global variables, functions, and aliases (globals)
+    pub fn LLVMRustGetLinkage(Global: &Value) -> Linkage;
+    pub fn LLVMRustSetLinkage(Global: &Value, RustLinkage: Linkage);
+    pub fn LLVMRustGetVisibility(Global: &Value) -> Visibility;
+    pub fn LLVMRustSetVisibility(Global: &Value, Viz: Visibility);
+    pub fn LLVMRustSetDSOLocal(Global: &Value, is_dso_local: bool);
+
+    // Operations on global variables
+    pub fn LLVMRustGetOrInsertGlobal<'a>(
+        M: &'a Module,
+        Name: *const c_char,
+        NameLen: size_t,
+        T: &'a Type,
+    ) -> &'a Value;
+    pub fn LLVMRustInsertPrivateGlobal<'a>(M: &'a Module, T: &'a Type) -> &'a Value;
+    pub fn LLVMRustGetNamedValue(
+        M: &Module,
+        Name: *const c_char,
+        NameLen: size_t,
+    ) -> Option<&Value>;
+    pub fn LLVMRustSetTailCallKind(CallInst: &Value, TKC: TailCallKind);
+
+    // Operations on attributes
+    pub fn LLVMRustCreateAttrNoValue(C: &Context, attr: AttributeKind) -> &Attribute;
+    pub fn LLVMRustCreateAlignmentAttr(C: &Context, bytes: u64) -> &Attribute;
+    pub fn LLVMRustCreateDereferenceableAttr(C: &Context, bytes: u64) -> &Attribute;
+    pub fn LLVMRustCreateDereferenceableOrNullAttr(C: &Context, bytes: u64) -> &Attribute;
+    pub fn LLVMRustCreateByValAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute;
+    pub fn LLVMRustCreateStructRetAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute;
+    pub fn LLVMRustCreateElementTypeAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute;
+    pub fn LLVMRustCreateUWTableAttr(C: &Context, async_: bool) -> &Attribute;
+    pub fn LLVMRustCreateAllocSizeAttr(C: &Context, size_arg: u32) -> &Attribute;
+    pub fn LLVMRustCreateAllocKindAttr(C: &Context, size_arg: u64) -> &Attribute;
+    pub fn LLVMRustCreateMemoryEffectsAttr(C: &Context, effects: MemoryEffects) -> &Attribute;
+
+    // Operations on functions
+    pub fn LLVMRustGetOrInsertFunction<'a>(
+        M: &'a Module,
+        Name: *const c_char,
+        NameLen: size_t,
+        FunctionTy: &'a Type,
+    ) -> &'a Value;
+    pub fn LLVMRustAddFunctionAttributes<'a>(
+        Fn: &'a Value,
+        index: c_uint,
+        Attrs: *const &'a Attribute,
+        AttrsLen: size_t,
+    );
+
+    // Operations on call sites
+    pub fn LLVMRustAddCallSiteAttributes<'a>(
+        Instr: &'a Value,
+        index: c_uint,
+        Attrs: *const &'a Attribute,
+        AttrsLen: size_t,
+    );
+
+    pub fn LLVMRustBuildInvoke<'a>(
+        B: &Builder<'a>,
+        Ty: &'a Type,
+        Fn: &'a Value,
+        Args: *const &'a Value,
+        NumArgs: c_uint,
+        Then: &'a BasicBlock,
+        Catch: &'a BasicBlock,
+        OpBundles: *const &OperandBundleDef<'a>,
+        NumOpBundles: c_uint,
+        Name: *const c_char,
+    ) -> &'a Value;
+
+    pub fn LLVMRustSetFastMath(Instr: &Value);
+
+    // Miscellaneous instructions
+    pub fn LLVMRustGetInstrProfIncrementIntrinsic(M: &Module) -> &Value;
+    pub fn LLVMRustBuildCall<'a>(
+        B: &Builder<'a>,
+        Ty: &'a Type,
+        Fn: &'a Value,
+        Args: *const &'a Value,
+        NumArgs: c_uint,
+        OpBundles: *const &OperandBundleDef<'a>,
+        NumOpBundles: c_uint,
+    ) -> &'a Value;
+    pub fn LLVMRustBuildMemCpy<'a>(
+        B: &Builder<'a>,
+        Dst: &'a Value,
+        DstAlign: c_uint,
+        Src: &'a Value,
+        SrcAlign: c_uint,
+        Size: &'a Value,
+        IsVolatile: bool,
+    ) -> &'a Value;
+    pub fn LLVMRustBuildMemMove<'a>(
+        B: &Builder<'a>,
+        Dst: &'a Value,
+        DstAlign: c_uint,
+        Src: &'a Value,
+        SrcAlign: c_uint,
+        Size: &'a Value,
+        IsVolatile: bool,
+    ) -> &'a Value;
+    pub fn LLVMRustBuildMemSet<'a>(
+        B: &Builder<'a>,
+        Dst: &'a Value,
+        DstAlign: c_uint,
+        Val: &'a Value,
+        Size: &'a Value,
+        IsVolatile: bool,
+    ) -> &'a Value;
+
     pub fn LLVMRustBuildVectorReduceFAdd<'a>(
         B: &Builder<'a>,
         Acc: &'a Value,
@@ -1623,53 +1714,11 @@ extern "C" {
         Order: AtomicOrdering,
     ) -> &'a Value;
 
-    pub fn LLVMBuildAtomicCmpXchg<'a>(
-        B: &Builder<'a>,
-        LHS: &'a Value,
-        CMP: &'a Value,
-        RHS: &'a Value,
-        Order: AtomicOrdering,
-        FailureOrder: AtomicOrdering,
-        SingleThreaded: Bool,
-    ) -> &'a Value;
+    pub fn LLVMRustTimeTraceProfilerInitialize();
 
-    pub fn LLVMSetWeak(CmpXchgInst: &Value, IsWeak: Bool);
+    pub fn LLVMRustTimeTraceProfilerFinishThread();
 
-    pub fn LLVMBuildAtomicRMW<'a>(
-        B: &Builder<'a>,
-        Op: AtomicRmwBinOp,
-        LHS: &'a Value,
-        RHS: &'a Value,
-        Order: AtomicOrdering,
-        SingleThreaded: Bool,
-    ) -> &'a Value;
-
-    pub fn LLVMBuildFence<'a>(
-        B: &Builder<'a>,
-        Order: AtomicOrdering,
-        SingleThreaded: Bool,
-        Name: *const c_char,
-    ) -> &'a Value;
-
-    /// Writes a module to the specified path. Returns 0 on success.
-    pub fn LLVMWriteBitcodeToFile(M: &Module, Path: *const c_char) -> c_int;
-
-    /// Creates a legacy pass manager -- only used for final codegen.
-    pub fn LLVMCreatePassManager<'a>() -> &'a mut PassManager<'a>;
-
-    pub fn LLVMTimeTraceProfilerInitialize();
-
-    pub fn LLVMTimeTraceProfilerFinishThread();
-
-    pub fn LLVMTimeTraceProfilerFinish(FileName: *const c_char);
-
-    pub fn LLVMAddAnalysisPasses<'a>(T: &'a TargetMachine, PM: &PassManager<'a>);
-
-    pub fn LLVMGetHostCPUFeatures() -> *mut c_char;
-
-    pub fn LLVMDisposeMessage(message: *mut c_char);
-
-    pub fn LLVMIsMultithreaded() -> Bool;
+    pub fn LLVMRustTimeTraceProfilerFinish(FileName: *const c_char);
 
     /// Returns a string describing the last error caused by an LLVMRust* call.
     pub fn LLVMRustGetLastError() -> *const c_char;
@@ -1680,15 +1729,6 @@ extern "C" {
     /// Print the statistics since static dtors aren't picking them up.
     pub fn LLVMRustPrintStatistics(size: *const size_t) -> *const c_char;
 
-    pub fn LLVMStructCreateNamed(C: &Context, Name: *const c_char) -> &Type;
-
-    pub fn LLVMStructSetBody<'a>(
-        StructTy: &'a Type,
-        ElementTypes: *const &'a Type,
-        ElementCount: c_uint,
-        Packed: Bool,
-    );
-
     /// Prepares inline assembly.
     pub fn LLVMRustInlineAsm(
         Ty: &Type,
@@ -1761,8 +1801,6 @@ extern "C" {
     );
     pub fn LLVMRustHasModuleFlag(M: &Module, name: *const c_char, len: size_t) -> bool;
 
-    pub fn LLVMMetadataAsValue<'a>(C: &'a Context, MD: &'a Metadata) -> &'a Value;
-
     pub fn LLVMRustDIBuilderCreate(M: &Module) -> &mut DIBuilder<'_>;
 
     pub fn LLVMRustDIBuilderDispose<'a>(Builder: &'a mut DIBuilder<'a>);
@@ -1783,6 +1821,7 @@ extern "C" {
         kind: DebugEmissionKind,
         DWOId: u64,
         SplitDebugInlining: bool,
+        DebugNameTableKind: DebugNameTableKind,
     ) -> &'a DIDescriptor;
 
     pub fn LLVMRustDIBuilderCreateFile<'a>(
@@ -2052,8 +2091,6 @@ extern "C" {
         UniqueIdLen: size_t,
     ) -> &'a DIDerivedType;
 
-    pub fn LLVMSetUnnamedAddress(Global: &Value, UnnamedAddr: UnnamedAddr);
-
     pub fn LLVMRustDIBuilderCreateTemplateTypeParameter<'a>(
         Builder: &DIBuilder<'a>,
         Scope: Option<&'a DIScope>,
@@ -2092,8 +2129,6 @@ extern "C" {
     #[allow(improper_ctypes)]
     pub fn LLVMRustWriteValueToString(value_ref: &Value, s: &RustString);
 
-    pub fn LLVMIsAConstantInt(value_ref: &Value) -> Option<&ConstantInt>;
-
     pub fn LLVMRustHasFeature(T: &TargetMachine, s: *const c_char) -> bool;
 
     pub fn LLVMRustPrintTargetCPUs(
@@ -2134,7 +2169,7 @@ extern "C" {
         SplitDwarfFile: *const c_char,
         OutputObjFile: *const c_char,
         DebugInfoCompression: *const c_char,
-        ForceEmulatedTls: bool,
+        UseEmulatedTls: bool,
         ArgsCstrBuff: *const c_char,
         ArgsCstrBuffLen: usize,
     ) -> *mut TargetMachine;
@@ -2320,11 +2355,6 @@ extern "C" {
         len: usize,
         Identifier: *const c_char,
     ) -> Option<&Module>;
-    pub fn LLVMRustGetBitcodeSliceFromObjectData(
-        Data: *const u8,
-        len: usize,
-        out_len: &mut usize,
-    ) -> *const u8;
     pub fn LLVMRustGetSliceFromObjectDataByName(
         data: *const u8,
         len: usize,
diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs
index cc4ccaf19..08519723e 100644
--- a/compiler/rustc_codegen_llvm/src/llvm_util.rs
+++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs
@@ -5,9 +5,6 @@ use crate::errors::{
 };
 use crate::llvm;
 use libc::c_int;
-use rustc_codegen_ssa::target_features::{
-    supported_target_features, tied_target_features, RUSTC_SPECIFIC_FEATURES,
-};
 use rustc_codegen_ssa::traits::PrintBackendInfo;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::small_c_str::SmallCStr;
@@ -17,6 +14,7 @@ use rustc_session::config::{PrintKind, PrintRequest};
 use rustc_session::Session;
 use rustc_span::symbol::Symbol;
 use rustc_target::spec::{MergeFunctions, PanicStrategy};
+use rustc_target::target_features::RUSTC_SPECIFIC_FEATURES;
 
 use std::ffi::{c_char, c_void, CStr, CString};
 use std::path::Path;
@@ -121,7 +119,7 @@ unsafe fn configure_llvm(sess: &Session) {
     }
 
     if sess.opts.unstable_opts.llvm_time_trace {
-        llvm::LLVMTimeTraceProfilerInitialize();
+        llvm::LLVMRustTimeTraceProfilerInitialize();
     }
 
     rustc_llvm::initialize_available_targets();
@@ -132,7 +130,7 @@ unsafe fn configure_llvm(sess: &Session) {
 pub fn time_trace_profiler_finish(file_name: &Path) {
     unsafe {
         let file_name = path_to_c_string(file_name);
-        llvm::LLVMTimeTraceProfilerFinish(file_name.as_ptr());
+        llvm::LLVMRustTimeTraceProfilerFinish(file_name.as_ptr());
     }
 }
 
@@ -263,6 +261,10 @@ pub fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> LLVMFeature<'a> {
             "sve2-bitperm",
             TargetFeatureFoldStrength::EnableOnly("neon"),
         ),
+        // The unaligned-scalar-mem feature was renamed to fast-unaligned-access.
+        ("riscv32" | "riscv64", "fast-unaligned-access") if get_version().0 <= 17 => {
+            LLVMFeature::new("unaligned-scalar-mem")
+        }
         (_, s) => LLVMFeature::new(s),
     }
 }
@@ -274,7 +276,7 @@ pub fn check_tied_features(
     features: &FxHashMap<&str, bool>,
 ) -> Option<&'static [&'static str]> {
     if !features.is_empty() {
-        for tied in tied_target_features(sess) {
+        for tied in sess.target.tied_target_features() {
             // Tied features must be set to the same value, or not set at all
             let mut tied_iter = tied.iter();
             let enabled = features.get(tied_iter.next().unwrap());
@@ -290,10 +292,11 @@ pub fn check_tied_features(
 /// Must express features in the way Rust understands them
 pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec<Symbol> {
     let target_machine = create_informational_target_machine(sess);
-    supported_target_features(sess)
+    sess.target
+        .supported_target_features()
         .iter()
         .filter_map(|&(feature, gate)| {
-            if sess.is_nightly_build() || allow_unstable || gate.is_none() {
+            if sess.is_nightly_build() || allow_unstable || gate.is_stable() {
                 Some(feature)
             } else {
                 None
@@ -358,7 +361,9 @@ fn llvm_target_features(tm: &llvm::TargetMachine) -> Vec<(&str, &str)> {
 fn print_target_features(out: &mut dyn PrintBackendInfo, sess: &Session, tm: &llvm::TargetMachine) {
     let mut llvm_target_features = llvm_target_features(tm);
     let mut known_llvm_target_features = FxHashSet::<&'static str>::default();
-    let mut rustc_target_features = supported_target_features(sess)
+    let mut rustc_target_features = sess
+        .target
+        .supported_target_features()
         .iter()
         .map(|(feature, _gate)| {
             // LLVM asserts that these are sorted. LLVM and Rust both use byte comparison for these strings.
@@ -511,7 +516,7 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str
     );
 
     // -Ctarget-features
-    let supported_features = supported_target_features(sess);
+    let supported_features = sess.target.supported_target_features();
     let mut featsmap = FxHashMap::default();
     let feats = sess
         .opts
@@ -537,8 +542,7 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str
                 if feature_state.is_none() {
                     let rust_feature = supported_features.iter().find_map(|&(rust_feature, _)| {
                         let llvm_features = to_llvm_features(sess, rust_feature);
-                        if llvm_features.contains(&feature)
-                            && !llvm_features.contains(&rust_feature)
+                        if llvm_features.contains(feature) && !llvm_features.contains(rust_feature)
                         {
                             Some(rust_feature)
                         } else {
@@ -554,7 +558,8 @@ pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec<Str
                         UnknownCTargetFeature { feature, rust_feature: PossibleFeature::None }
                     };
                     sess.emit_warning(unknown_feature);
-                } else if feature_state.is_some_and(|(_name, feature_gate)| feature_gate.is_some())
+                } else if feature_state
+                    .is_some_and(|(_name, feature_gate)| !feature_gate.is_stable())
                 {
                     // An unstable feature. Warn about using it.
                     sess.emit_warning(UnstableCTargetFeature { feature });
diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs
index 06b770367..447c4ed1f 100644
--- a/compiler/rustc_codegen_llvm/src/type_.rs
+++ b/compiler/rustc_codegen_llvm/src/type_.rs
@@ -227,10 +227,6 @@ impl<'ll, 'tcx> BaseTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> {
 }
 
 impl Type {
-    pub fn i8_llcx(llcx: &llvm::Context) -> &Type {
-        unsafe { llvm::LLVMInt8TypeInContext(llcx) }
-    }
-
     /// Creates an integer type with the given number of bits, e.g., i24
     pub fn ix_llcx(llcx: &llvm::Context, num_bits: u64) -> &Type {
         unsafe { llvm::LLVMIntTypeInContext(llcx, num_bits as c_uint) }
diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs
index 712b6ed53..624ce6d88 100644
--- a/compiler/rustc_codegen_llvm/src/type_of.rs
+++ b/compiler/rustc_codegen_llvm/src/type_of.rs
@@ -9,7 +9,7 @@ use rustc_middle::ty::{self, Ty, TypeVisitableExt};
 use rustc_target::abi::HasDataLayout;
 use rustc_target::abi::{Abi, Align, FieldsShape};
 use rustc_target::abi::{Int, Pointer, F32, F64};
-use rustc_target::abi::{PointeeInfo, Scalar, Size, TyAbiInterface, Variants};
+use rustc_target::abi::{Scalar, Size, Variants};
 use smallvec::{smallvec, SmallVec};
 
 use std::fmt::Write;
@@ -184,7 +184,6 @@ pub trait LayoutLlvmExt<'tcx> {
         immediate: bool,
     ) -> &'a Type;
     fn llvm_field_index<'a>(&self, cx: &CodegenCx<'a, 'tcx>, index: usize) -> u64;
-    fn pointee_info_at<'a>(&self, cx: &CodegenCx<'a, 'tcx>, offset: Size) -> Option<PointeeInfo>;
     fn scalar_copy_llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> Option<&'a Type>;
 }
 
@@ -356,20 +355,6 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> {
         }
     }
 
-    // FIXME(eddyb) this having the same name as `TyAndLayout::pointee_info_at`
-    // (the inherent method, which is lacking this caching logic) can result in
-    // the uncached version being called - not wrong, but potentially inefficient.
-    fn pointee_info_at<'a>(&self, cx: &CodegenCx<'a, 'tcx>, offset: Size) -> Option<PointeeInfo> {
-        if let Some(&pointee) = cx.pointee_infos.borrow().get(&(self.ty, offset)) {
-            return pointee;
-        }
-
-        let result = Ty::ty_and_layout_pointee_info_at(*self, cx, offset);
-
-        cx.pointee_infos.borrow_mut().insert((self.ty, offset), result);
-        result
-    }
-
     fn scalar_copy_llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> Option<&'a Type> {
         debug_assert!(self.is_sized());
 
diff --git a/compiler/rustc_codegen_ssa/Cargo.toml b/compiler/rustc_codegen_ssa/Cargo.toml
index 4dae49f81..3f2ed257d 100644
--- a/compiler/rustc_codegen_ssa/Cargo.toml
+++ b/compiler/rustc_codegen_ssa/Cargo.toml
@@ -8,8 +8,8 @@ edition = "2021"
 ar_archive_writer = "0.1.5"
 bitflags = "1.2.1"
 cc = "1.0.69"
-itertools = "0.10.1"
-jobserver = "0.1.22"
+itertools = "0.11"
+jobserver = "0.1.27"
 pathdiff = "0.2.0"
 regex = "1.4"
 rustc_arena = { path = "../rustc_arena" }
@@ -46,7 +46,7 @@ libc = "0.2.50"
 # tidy-alphabetical-end
 
 [dependencies.object]
-version = "0.32.0"
+version = "0.32.1"
 default-features = false
 features = ["read_core", "elf", "macho", "pe", "xcoff", "unaligned", "archive", "write"]
 
diff --git a/compiler/rustc_codegen_ssa/src/assert_module_sources.rs b/compiler/rustc_codegen_ssa/src/assert_module_sources.rs
index 16bb7b12b..a5bd10ecb 100644
--- a/compiler/rustc_codegen_ssa/src/assert_module_sources.rs
+++ b/compiler/rustc_codegen_ssa/src/assert_module_sources.rs
@@ -149,7 +149,7 @@ impl<'tcx> AssertModuleSource<'tcx> {
 
         self.cgu_reuse_tracker.set_expectation(
             cgu_name,
-            &user_path,
+            user_path,
             attr.span,
             expected_reuse,
             comp_kind,
@@ -199,8 +199,8 @@ impl fmt::Display for CguReuse {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match *self {
             CguReuse::No => write!(f, "No"),
-            CguReuse::PreLto => write!(f, "PreLto "),
-            CguReuse::PostLto => write!(f, "PostLto "),
+            CguReuse::PreLto => write!(f, "PreLto"),
+            CguReuse::PostLto => write!(f, "PostLto"),
         }
     }
 }
@@ -278,13 +278,13 @@ impl CguReuseTracker {
 
                     if error {
                         let at_least = if at_least { 1 } else { 0 };
-                        errors::IncorrectCguReuseType {
+                        sess.emit_err(errors::IncorrectCguReuseType {
                             span: *error_span,
                             cgu_user_name,
                             actual_reuse,
                             expected_reuse,
                             at_least,
-                        };
+                        });
                     }
                 } else {
                     sess.emit_fatal(errors::CguNotRecorded { cgu_user_name, cgu_name });
diff --git a/compiler/rustc_codegen_ssa/src/back/archive.rs b/compiler/rustc_codegen_ssa/src/back/archive.rs
index 1c464b3ec..4dc28adf3 100644
--- a/compiler/rustc_codegen_ssa/src/back/archive.rs
+++ b/compiler/rustc_codegen_ssa/src/back/archive.rs
@@ -175,8 +175,7 @@ impl<'a> ArchiveBuilder<'a> for ArArchiveBuilder<'a> {
     ) -> io::Result<()> {
         let mut archive_path = archive_path.to_path_buf();
         if self.sess.target.llvm_target.contains("-apple-macosx") {
-            if let Some(new_archive_path) =
-                try_extract_macho_fat_archive(&self.sess, &archive_path)?
+            if let Some(new_archive_path) = try_extract_macho_fat_archive(self.sess, &archive_path)?
             {
                 archive_path = new_archive_path
             }
diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs
index dd9d277fb..b0d22ad0a 100644
--- a/compiler/rustc_codegen_ssa/src/back/link.rs
+++ b/compiler/rustc_codegen_ssa/src/back/link.rs
@@ -4,7 +4,7 @@ use rustc_data_structures::fx::FxHashSet;
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_data_structures::memmap::Mmap;
 use rustc_data_structures::temp_dir::MaybeTempDir;
-use rustc_errors::{ErrorGuaranteed, Handler};
+use rustc_errors::{DiagCtxt, ErrorGuaranteed};
 use rustc_fs_util::{fix_windows_verbatim_for_gcc, try_canonicalize};
 use rustc_hir::def_id::{CrateNum, LOCAL_CRATE};
 use rustc_metadata::find_native_static_library;
@@ -44,7 +44,7 @@ use tempfile::Builder as TempFileBuilder;
 use itertools::Itertools;
 use std::cell::OnceCell;
 use std::collections::BTreeSet;
-use std::ffi::OsString;
+use std::ffi::{OsStr, OsString};
 use std::fs::{read, File, OpenOptions};
 use std::io::{BufWriter, Write};
 use std::ops::Deref;
@@ -52,10 +52,10 @@ use std::path::{Path, PathBuf};
 use std::process::{ExitStatus, Output, Stdio};
 use std::{env, fmt, fs, io, mem, str};
 
-pub fn ensure_removed(diag_handler: &Handler, path: &Path) {
+pub fn ensure_removed(dcx: &DiagCtxt, path: &Path) {
     if let Err(e) = fs::remove_file(path) {
         if e.kind() != io::ErrorKind::NotFound {
-            diag_handler.err(format!("failed to remove {}: {}", path.display(), e));
+            dcx.err(format!("failed to remove {}: {}", path.display(), e));
         }
     }
 }
@@ -143,7 +143,7 @@ pub fn link_binary<'a>(
                 }
             }
             if sess.opts.json_artifact_notifications {
-                sess.parse_sess.span_diagnostic.emit_artifact_notification(&out_filename, "link");
+                sess.dcx().emit_artifact_notification(&out_filename, "link");
             }
 
             if sess.prof.enabled() {
@@ -183,13 +183,13 @@ pub fn link_binary<'a>(
             |preserve_objects: bool, preserve_dwarf_objects: bool, module: &CompiledModule| {
                 if !preserve_objects {
                     if let Some(ref obj) = module.object {
-                        ensure_removed(sess.diagnostic(), obj);
+                        ensure_removed(sess.dcx(), obj);
                     }
                 }
 
                 if !preserve_dwarf_objects {
                     if let Some(ref dwo_obj) = module.dwarf_object {
-                        ensure_removed(sess.diagnostic(), dwo_obj);
+                        ensure_removed(sess.dcx(), dwo_obj);
                     }
                 }
             };
@@ -208,7 +208,7 @@ pub fn link_binary<'a>(
 
         // Remove the temporary files if output goes to stdout
         for temp in tempfiles_for_stdout_output {
-            ensure_removed(sess.diagnostic(), &temp);
+            ensure_removed(sess.dcx(), &temp);
         }
 
         // If no requested outputs require linking, then the object temporaries should
@@ -277,7 +277,7 @@ pub fn each_linked_rlib(
         let crate_name = info.crate_name[&cnum];
         let used_crate_source = &info.used_crate_source[&cnum];
         if let Some((path, _)) = &used_crate_source.rlib {
-            f(cnum, &path);
+            f(cnum, path);
         } else {
             if used_crate_source.rmeta.is_some() {
                 return Err(errors::LinkRlibError::OnlyRmetaFound { crate_name });
@@ -524,7 +524,7 @@ fn link_staticlib<'a>(
                 && !ignored_for_lto(sess, &codegen_results.crate_info, cnum);
 
             let native_libs = codegen_results.crate_info.native_libraries[&cnum].iter();
-            let relevant = native_libs.clone().filter(|lib| relevant_lib(sess, &lib));
+            let relevant = native_libs.clone().filter(|lib| relevant_lib(sess, lib));
             let relevant_libs: FxHashSet<_> = relevant.filter_map(|lib| lib.filename).collect();
 
             let bundled_libs: FxHashSet<_> = native_libs.filter_map(|lib| lib.filename).collect();
@@ -689,7 +689,7 @@ fn link_dwarf_object<'a>(
         // Adding an executable is primarily done to make `thorin` check that all the referenced
         // dwarf objects are found in the end.
         package.add_executable(
-            &executable_out_filename,
+            executable_out_filename,
             thorin::MissingReferencedObjectBehaviour::Skip,
         )?;
 
@@ -928,7 +928,7 @@ fn link_natively<'a>(
                     command: &cmd,
                     escaped_output,
                 };
-                sess.diagnostic().emit_err(err);
+                sess.dcx().emit_err(err);
                 // If MSVC's `link.exe` was expected but the return code
                 // is not a Microsoft LNK error then suggest a way to fix or
                 // install the Visual Studio build tools.
@@ -945,7 +945,7 @@ fn link_natively<'a>(
                     {
                         let is_vs_installed = windows_registry::find_vs_version().is_ok();
                         let has_linker = windows_registry::find_tool(
-                            &sess.opts.target_triple.triple(),
+                            sess.opts.target_triple.triple(),
                             "link.exe",
                         )
                         .is_some();
@@ -1038,14 +1038,14 @@ fn link_natively<'a>(
     if sess.target.is_like_osx {
         match (strip, crate_type) {
             (Strip::Debuginfo, _) => {
-                strip_symbols_with_external_utility(sess, "strip", &out_filename, Some("-S"))
+                strip_symbols_with_external_utility(sess, "strip", out_filename, Some("-S"))
             }
             // Per the manpage, `-x` is the maximum safe strip level for dynamic libraries. (#93988)
             (Strip::Symbols, CrateType::Dylib | CrateType::Cdylib | CrateType::ProcMacro) => {
-                strip_symbols_with_external_utility(sess, "strip", &out_filename, Some("-x"))
+                strip_symbols_with_external_utility(sess, "strip", out_filename, Some("-x"))
             }
             (Strip::Symbols, _) => {
-                strip_symbols_with_external_utility(sess, "strip", &out_filename, None)
+                strip_symbols_with_external_utility(sess, "strip", out_filename, None)
             }
             (Strip::None, _) => {}
         }
@@ -1059,7 +1059,7 @@ fn link_natively<'a>(
         match strip {
             // Always preserve the symbol table (-x).
             Strip::Debuginfo => {
-                strip_symbols_with_external_utility(sess, stripcmd, &out_filename, Some("-x"))
+                strip_symbols_with_external_utility(sess, stripcmd, out_filename, Some("-x"))
             }
             // Strip::Symbols is handled via the --strip-all linker option.
             Strip::Symbols => {}
@@ -1245,13 +1245,13 @@ fn link_sanitizer_runtime(sess: &Session, linker: &mut dyn Linker, name: &str) {
         // rpath to the library as well (the rpath should be absolute, see
         // PR #41352 for details).
         let filename = format!("rustc{channel}_rt.{name}");
-        let path = find_sanitizer_runtime(&sess, &filename);
+        let path = find_sanitizer_runtime(sess, &filename);
         let rpath = path.to_str().expect("non-utf8 component in path");
         linker.args(&["-Wl,-rpath", "-Xlinker", rpath]);
         linker.link_dylib(&filename, false, true);
     } else {
         let filename = format!("librustc{channel}_rt.{name}.a");
-        let path = find_sanitizer_runtime(&sess, &filename).join(&filename);
+        let path = find_sanitizer_runtime(sess, &filename).join(&filename);
         linker.link_whole_rlib(&path);
     }
 }
@@ -1477,7 +1477,7 @@ fn print_native_static_libs(
                 sess.emit_note(errors::StaticLibraryNativeArtifacts);
                 // Prefix for greppability
                 // Note: This must not be translated as tools are allowed to depend on this exact string.
-                sess.note_without_error(format!("native-static-libs: {}", &lib_args.join(" ")));
+                sess.note(format!("native-static-libs: {}", &lib_args.join(" ")));
             }
         }
     }
@@ -1685,7 +1685,7 @@ fn link_output_kind(sess: &Session, crate_type: CrateType) -> LinkOutputKind {
 
 // Returns true if linker is located within sysroot
 fn detect_self_contained_mingw(sess: &Session) -> bool {
-    let (linker, _) = linker_and_flavor(&sess);
+    let (linker, _) = linker_and_flavor(sess);
     // Assume `-C linker=rust-lld` as self-contained mode
     if linker == Path::new("rust-lld") {
         return true;
@@ -1737,7 +1737,7 @@ fn self_contained_components(sess: &Session, crate_type: CrateType) -> LinkSelfC
                 LinkSelfContainedDefault::InferredForMingw => {
                     sess.host == sess.target
                         && sess.target.vendor != "uwp"
-                        && detect_self_contained_mingw(&sess)
+                        && detect_self_contained_mingw(sess)
                 }
             }
         };
@@ -2243,9 +2243,9 @@ fn linker_with_args<'a>(
     // ------------ Late order-dependent options ------------
 
     // Doesn't really make sense.
-    // FIXME: In practice built-in target specs use this for arbitrary order-independent options,
-    // introduce a target spec option for order-independent linker options, migrate built-in specs
-    // to it and remove the option.
+    // FIXME: In practice built-in target specs use this for arbitrary order-independent options.
+    // Introduce a target spec option for order-independent linker options, migrate built-in specs
+    // to it and remove the option. Currently the last holdout is wasm32-unknown-emscripten.
     add_post_link_args(cmd, sess, flavor);
 
     Ok(cmd.take_cmd())
@@ -2378,6 +2378,11 @@ fn add_order_independent_options(
         cmd.control_flow_guard();
     }
 
+    // OBJECT-FILES-NO, AUDIT-ORDER
+    if sess.opts.unstable_opts.ehcont_guard {
+        cmd.ehcont_guard();
+    }
+
     add_rpath_args(cmd, sess, codegen_results, out_filename);
 }
 
@@ -2432,7 +2437,7 @@ fn add_native_libs_from_crate(
         // If rlib contains native libs as archives, unpack them to tmpdir.
         let rlib = &codegen_results.crate_info.used_crate_source[&cnum].rlib.as_ref().unwrap().0;
         archive_builder_builder
-            .extract_bundled_libs(rlib, tmpdir, &bundled_libs)
+            .extract_bundled_libs(rlib, tmpdir, bundled_libs)
             .unwrap_or_else(|e| sess.emit_fatal(e));
     }
 
@@ -2485,7 +2490,7 @@ fn add_native_libs_from_crate(
                             cmd.link_whole_staticlib(
                                 name,
                                 verbatim,
-                                &search_paths.get_or_init(|| archive_search_paths(sess)),
+                                search_paths.get_or_init(|| archive_search_paths(sess)),
                             );
                         } else {
                             cmd.link_staticlib(name, verbatim)
@@ -2522,7 +2527,7 @@ fn add_native_libs_from_crate(
             NativeLibKind::WasmImportModule => {}
             NativeLibKind::LinkArg => {
                 if link_static {
-                    cmd.arg(name);
+                    cmd.linker_arg(OsStr::new(name), verbatim);
                 }
             }
         }
@@ -2719,7 +2724,7 @@ fn rehome_sysroot_lib_dir<'a>(sess: &'a Session, lib_dir: &Path) -> PathBuf {
         // already had `fix_windows_verbatim_for_gcc()` applied if needed.
         sysroot_lib_path
     } else {
-        fix_windows_verbatim_for_gcc(&lib_dir)
+        fix_windows_verbatim_for_gcc(lib_dir)
     }
 }
 
@@ -2756,7 +2761,7 @@ fn add_static_crate<'a>(
     let mut link_upstream = |path: &Path| {
         let rlib_path = if let Some(dir) = path.parent() {
             let file_name = path.file_name().expect("rlib path has no file name path component");
-            rehome_sysroot_lib_dir(sess, &dir).join(file_name)
+            rehome_sysroot_lib_dir(sess, dir).join(file_name)
         } else {
             fix_windows_verbatim_for_gcc(path)
         };
@@ -2793,7 +2798,7 @@ fn add_static_crate<'a>(
                 let canonical = f.replace('-', "_");
 
                 let is_rust_object =
-                    canonical.starts_with(&canonical_name) && looks_like_rust_object_file(&f);
+                    canonical.starts_with(&canonical_name) && looks_like_rust_object_file(f);
 
                 // If we're performing LTO and this is a rust-generated object
                 // file, then we don't need the object file as it's part of the
diff --git a/compiler/rustc_codegen_ssa/src/back/linker.rs b/compiler/rustc_codegen_ssa/src/back/linker.rs
index 09434513e..eeb57d4d0 100644
--- a/compiler/rustc_codegen_ssa/src/back/linker.rs
+++ b/compiler/rustc_codegen_ssa/src/back/linker.rs
@@ -45,7 +45,7 @@ pub fn get_linker<'a>(
     self_contained: bool,
     target_cpu: &'a str,
 ) -> Box<dyn Linker + 'a> {
-    let msvc_tool = windows_registry::find_tool(&sess.opts.target_triple.triple(), "link.exe");
+    let msvc_tool = windows_registry::find_tool(sess.opts.target_triple.triple(), "link.exe");
 
     // If our linker looks like a batch script on Windows then to execute this
     // we'll need to spawn `cmd` explicitly. This is primarily done to handle
@@ -78,7 +78,7 @@ pub fn get_linker<'a>(
     if matches!(flavor, LinkerFlavor::Msvc(..)) && t.vendor == "uwp" {
         if let Some(ref tool) = msvc_tool {
             let original_path = tool.path();
-            if let Some(ref root_lib_path) = original_path.ancestors().nth(4) {
+            if let Some(root_lib_path) = original_path.ancestors().nth(4) {
                 let arch = match t.arch.as_ref() {
                     "x86_64" => Some("x64"),
                     "x86" => Some("x86"),
@@ -185,6 +185,7 @@ pub trait Linker {
     fn optimize(&mut self);
     fn pgo_gen(&mut self);
     fn control_flow_guard(&mut self);
+    fn ehcont_guard(&mut self);
     fn debuginfo(&mut self, strip: Strip, natvis_debugger_visualizers: &[PathBuf]);
     fn no_crt_objects(&mut self);
     fn no_default_libraries(&mut self);
@@ -195,6 +196,14 @@ pub trait Linker {
     fn add_no_exec(&mut self) {}
     fn add_as_needed(&mut self) {}
     fn reset_per_library_state(&mut self) {}
+    fn linker_arg(&mut self, arg: &OsStr, verbatim: bool) {
+        self.linker_args(&[arg], verbatim);
+    }
+    fn linker_args(&mut self, args: &[&OsStr], _verbatim: bool) {
+        args.into_iter().for_each(|a| {
+            self.cmd().arg(a);
+        });
+    }
 }
 
 impl dyn Linker + '_ {
@@ -222,38 +231,12 @@ pub struct GccLinker<'a> {
 }
 
 impl<'a> GccLinker<'a> {
-    /// Passes an argument directly to the linker.
-    ///
-    /// When the linker is not ld-like such as when using a compiler as a linker, the argument is
-    /// prepended by `-Wl,`.
-    fn linker_arg(&mut self, arg: impl AsRef<OsStr>) -> &mut Self {
-        self.linker_args(&[arg]);
-        self
+    fn linker_arg(&mut self, arg: impl AsRef<OsStr>) {
+        Linker::linker_arg(self, arg.as_ref(), false);
     }
-
-    /// Passes a series of arguments directly to the linker.
-    ///
-    /// When the linker is ld-like, the arguments are simply appended to the command. When the
-    /// linker is not ld-like such as when using a compiler as a linker, the arguments are joined by
-    /// commas to form an argument that is then prepended with `-Wl`. In this situation, only a
-    /// single argument is appended to the command to ensure that the order of the arguments is
-    /// preserved by the compiler.
-    fn linker_args(&mut self, args: &[impl AsRef<OsStr>]) -> &mut Self {
-        if self.is_ld {
-            args.into_iter().for_each(|a| {
-                self.cmd.arg(a);
-            });
-        } else {
-            if !args.is_empty() {
-                let mut s = OsString::from("-Wl");
-                for a in args {
-                    s.push(",");
-                    s.push(a);
-                }
-                self.cmd.arg(s);
-            }
-        }
-        self
+    fn linker_args(&mut self, args: &[impl AsRef<OsStr>]) {
+        let args_vec: Vec<&OsStr> = args.iter().map(|x| x.as_ref()).collect();
+        Linker::linker_args(self, &args_vec, false);
     }
 
     fn takes_hints(&self) -> bool {
@@ -360,6 +343,30 @@ impl<'a> GccLinker<'a> {
 }
 
 impl<'a> Linker for GccLinker<'a> {
+    /// Passes a series of arguments directly to the linker.
+    ///
+    /// When the linker is ld-like, the arguments are simply appended to the command. When the
+    /// linker is not ld-like such as when using a compiler as a linker, the arguments are joined by
+    /// commas to form an argument that is then prepended with `-Wl`. In this situation, only a
+    /// single argument is appended to the command to ensure that the order of the arguments is
+    /// preserved by the compiler.
+    fn linker_args(&mut self, args: &[&OsStr], verbatim: bool) {
+        if self.is_ld || verbatim {
+            args.into_iter().for_each(|a| {
+                self.cmd.arg(a);
+            });
+        } else {
+            if !args.is_empty() {
+                let mut s = OsString::from("-Wl");
+                for a in args {
+                    s.push(",");
+                    s.push(a);
+                }
+                self.cmd.arg(s);
+            }
+        }
+    }
+
     fn cmd(&mut self) -> &mut Command {
         &mut self.cmd
     }
@@ -519,7 +526,7 @@ impl<'a> Linker for GccLinker<'a> {
             // -force_load is the macOS equivalent of --whole-archive, but it
             // involves passing the full path to the library to link.
             self.linker_arg("-force_load");
-            let lib = find_native_static_library(lib, verbatim, search_path, &self.sess);
+            let lib = find_native_static_library(lib, verbatim, search_path, self.sess);
             self.linker_arg(&lib);
         }
     }
@@ -530,7 +537,7 @@ impl<'a> Linker for GccLinker<'a> {
             self.linker_arg("-force_load");
             self.linker_arg(&lib);
         } else {
-            self.linker_arg("--whole-archive").cmd.arg(lib);
+            self.linker_args(&[OsString::from("--whole-archive"), lib.into()]);
             self.linker_arg("--no-whole-archive");
         }
     }
@@ -605,6 +612,8 @@ impl<'a> Linker for GccLinker<'a> {
 
     fn control_flow_guard(&mut self) {}
 
+    fn ehcont_guard(&mut self) {}
+
     fn debuginfo(&mut self, strip: Strip, _: &[PathBuf]) {
         // MacOS linker doesn't support stripping symbols directly anymore.
         if self.sess.target.is_like_osx {
@@ -914,6 +923,12 @@ impl<'a> Linker for MsvcLinker<'a> {
         self.cmd.arg("/guard:cf");
     }
 
+    fn ehcont_guard(&mut self) {
+        if self.sess.target.pointer_width == 64 {
+            self.cmd.arg("/guard:ehcont");
+        }
+    }
+
     fn debuginfo(&mut self, strip: Strip, natvis_debugger_visualizers: &[PathBuf]) {
         match strip {
             Strip::None => {
@@ -1127,6 +1142,8 @@ impl<'a> Linker for EmLinker<'a> {
 
     fn control_flow_guard(&mut self) {}
 
+    fn ehcont_guard(&mut self) {}
+
     fn debuginfo(&mut self, _strip: Strip, _: &[PathBuf]) {
         // Preserve names or generate source maps depending on debug info
         // For more information see https://emscripten.org/docs/tools_reference/emcc.html#emcc-g
@@ -1291,6 +1308,8 @@ impl<'a> Linker for WasmLd<'a> {
     }
 
     fn optimize(&mut self) {
+        // The -O flag is, as of late 2023, only used for merging of strings and debuginfo, and
+        // only differentiates -O0 and -O1. It does not apply to LTO.
         self.cmd.arg(match self.sess.opts.optimize {
             OptLevel::No => "-O0",
             OptLevel::Less => "-O1",
@@ -1319,6 +1338,8 @@ impl<'a> Linker for WasmLd<'a> {
 
     fn control_flow_guard(&mut self) {}
 
+    fn ehcont_guard(&mut self) {}
+
     fn no_crt_objects(&mut self) {}
 
     fn no_default_libraries(&mut self) {}
@@ -1341,7 +1362,31 @@ impl<'a> Linker for WasmLd<'a> {
     fn subsystem(&mut self, _subsystem: &str) {}
 
     fn linker_plugin_lto(&mut self) {
-        // Do nothing for now
+        match self.sess.opts.cg.linker_plugin_lto {
+            LinkerPluginLto::Disabled => {
+                // Nothing to do
+            }
+            LinkerPluginLto::LinkerPluginAuto => {
+                self.push_linker_plugin_lto_args();
+            }
+            LinkerPluginLto::LinkerPlugin(_) => {
+                self.push_linker_plugin_lto_args();
+            }
+        }
+    }
+}
+
+impl<'a> WasmLd<'a> {
+    fn push_linker_plugin_lto_args(&mut self) {
+        let opt_level = match self.sess.opts.optimize {
+            config::OptLevel::No => "O0",
+            config::OptLevel::Less => "O1",
+            config::OptLevel::Default => "O2",
+            config::OptLevel::Aggressive => "O3",
+            // wasm-ld only handles integer LTO opt levels. Use O2
+            config::OptLevel::Size | config::OptLevel::SizeMin => "O2",
+        };
+        self.cmd.arg(&format!("--lto-{opt_level}"));
     }
 }
 
@@ -1472,6 +1517,8 @@ impl<'a> Linker for L4Bender<'a> {
 
     fn control_flow_guard(&mut self) {}
 
+    fn ehcont_guard(&mut self) {}
+
     fn no_crt_objects(&mut self) {}
 }
 
@@ -1590,7 +1637,7 @@ impl<'a> Linker for AixLinker<'a> {
 
     fn link_whole_staticlib(&mut self, lib: &str, verbatim: bool, search_path: &[PathBuf]) {
         self.hint_static();
-        let lib = find_native_static_library(lib, verbatim, search_path, &self.sess);
+        let lib = find_native_static_library(lib, verbatim, search_path, self.sess);
         self.cmd.arg(format!("-bkeepfile:{}", lib.to_str().unwrap()));
     }
 
@@ -1613,6 +1660,8 @@ impl<'a> Linker for AixLinker<'a> {
 
     fn control_flow_guard(&mut self) {}
 
+    fn ehcont_guard(&mut self) {}
+
     fn debuginfo(&mut self, strip: Strip, _: &[PathBuf]) {
         match strip {
             Strip::None => {}
@@ -1699,7 +1748,9 @@ fn exported_symbols_for_non_proc_macro(tcx: TyCtxt<'_>, crate_type: CrateType) -
     let export_threshold = symbol_export::crates_export_threshold(&[crate_type]);
     for_each_exported_symbols_include_dep(tcx, crate_type, |symbol, info, cnum| {
         if info.level.is_below_threshold(export_threshold) {
-            symbols.push(symbol_export::symbol_name_for_instance_in_crate(tcx, symbol, cnum));
+            symbols.push(symbol_export::exporting_symbol_name_for_instance_in_crate(
+                tcx, symbol, cnum,
+            ));
         }
     });
 
@@ -1835,6 +1886,8 @@ impl<'a> Linker for PtxLinker<'a> {
 
     fn control_flow_guard(&mut self) {}
 
+    fn ehcont_guard(&mut self) {}
+
     fn export_symbols(&mut self, _tmpdir: &Path, _crate_type: CrateType, _symbols: &[String]) {}
 
     fn subsystem(&mut self, _subsystem: &str) {}
@@ -1931,6 +1984,8 @@ impl<'a> Linker for BpfLinker<'a> {
 
     fn control_flow_guard(&mut self) {}
 
+    fn ehcont_guard(&mut self) {}
+
     fn export_symbols(&mut self, tmpdir: &Path, _crate_type: CrateType, symbols: &[String]) {
         let path = tmpdir.join("symbols");
         let res: io::Result<()> = try {
diff --git a/compiler/rustc_codegen_ssa/src/back/metadata.rs b/compiler/rustc_codegen_ssa/src/back/metadata.rs
index cb60ed729..b683e1b45 100644
--- a/compiler/rustc_codegen_ssa/src/back/metadata.rs
+++ b/compiler/rustc_codegen_ssa/src/back/metadata.rs
@@ -12,9 +12,9 @@ use object::{
 
 use rustc_data_structures::memmap::Mmap;
 use rustc_data_structures::owned_slice::{try_slice_owned, OwnedSlice};
+use rustc_metadata::creader::MetadataLoader;
 use rustc_metadata::fs::METADATA_FILENAME;
 use rustc_metadata::EncodedMetadata;
-use rustc_session::cstore::MetadataLoader;
 use rustc_session::Session;
 use rustc_span::sym;
 use rustc_target::abi::Endian;
@@ -158,11 +158,12 @@ pub(super) fn get_metadata_xcoff<'a>(path: &Path, data: &'a [u8]) -> Result<&'a
         file.symbols().find(|sym| sym.name() == Ok(AIX_METADATA_SYMBOL_NAME))
     {
         let offset = metadata_symbol.address() as usize;
+        // The offset specifies the location of rustc metadata in the .info section of XCOFF.
+        // Each string stored in .info section of XCOFF is preceded by a 4-byte length field.
         if offset < 4 {
             return Err(format!("Invalid metadata symbol offset: {offset}"));
         }
-        // The offset specifies the location of rustc metadata in the comment section.
-        // The metadata is preceded by a 4-byte length field.
+        // XCOFF format uses big-endian byte order.
         let len = u32::from_be_bytes(info_data[(offset - 4)..offset].try_into().unwrap()) as usize;
         if offset + len > (info_data.len() as usize) {
             return Err(format!(
@@ -226,6 +227,10 @@ pub(crate) fn create_object_file(sess: &Session) -> Option<write::Object<'static
 
     let mut file = write::Object::new(binary_format, architecture, endianness);
     if sess.target.is_like_osx {
+        if macho_is_arm64e(&sess.target) {
+            file.set_macho_cpu_subtype(object::macho::CPU_SUBTYPE_ARM64E);
+        }
+
         file.set_macho_build_version(macho_object_build_version_for_target(&sess.target))
     }
     if binary_format == BinaryFormat::Coff {
@@ -385,6 +390,11 @@ fn macho_object_build_version_for_target(target: &Target) -> object::write::Mach
     build_version
 }
 
+/// Is Apple's CPU subtype `arm64e`s
+fn macho_is_arm64e(target: &Target) -> bool {
+    return target.llvm_target.starts_with("arm64e");
+}
+
 pub enum MetadataPosition {
     First,
     Last,
@@ -469,8 +479,11 @@ pub fn create_wrapper_file(
             file.add_section(Vec::new(), b".text".to_vec(), SectionKind::Text);
             file.section_mut(section).flags =
                 SectionFlags::Xcoff { s_flags: xcoff::STYP_INFO as u32 };
-
-            let len = data.len() as u32;
+            // Encode string stored in .info section of XCOFF.
+            // FIXME: The length of data here is not guaranteed to fit in a u32.
+            // We may have to split the data into multiple pieces in order to
+            // store in .info section.
+            let len: u32 = data.len().try_into().unwrap();
             let offset = file.append_section_data(section, &len.to_be_bytes(), 1);
             // Add a symbol referring to the data in .info section.
             file.add_symbol(Symbol {
@@ -515,7 +528,7 @@ pub fn create_compressed_metadata_file(
     symbol_name: &str,
 ) -> Vec<u8> {
     let mut packed_metadata = rustc_metadata::METADATA_HEADER.to_vec();
-    packed_metadata.write_all(&(metadata.raw_data().len() as u32).to_be_bytes()).unwrap();
+    packed_metadata.write_all(&(metadata.raw_data().len() as u64).to_le_bytes()).unwrap();
     packed_metadata.extend(metadata.raw_data());
 
     let Some(mut file) = create_object_file(sess) else {
@@ -590,7 +603,7 @@ pub fn create_compressed_metadata_file_for_xcoff(
         section: SymbolSection::Section(data_section),
         flags: SymbolFlags::None,
     });
-    let len = data.len() as u32;
+    let len: u32 = data.len().try_into().unwrap();
     let offset = file.append_section_data(section, &len.to_be_bytes(), 1);
     // Add a symbol referring to the rustc metadata.
     file.add_symbol(Symbol {
diff --git a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
index 9cd439410..ff667eecf 100644
--- a/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
+++ b/compiler/rustc_codegen_ssa/src/back/symbol_export.rs
@@ -16,7 +16,7 @@ use rustc_middle::ty::{self, SymbolName, TyCtxt};
 use rustc_middle::ty::{GenericArgKind, GenericArgsRef};
 use rustc_middle::util::Providers;
 use rustc_session::config::{CrateType, OomStrategy};
-use rustc_target::spec::SanitizerSet;
+use rustc_target::spec::{SanitizerSet, TlsModel};
 
 pub fn threshold(tcx: TyCtxt<'_>) -> SymbolExportLevel {
     crates_export_threshold(tcx.crate_types())
@@ -548,6 +548,12 @@ pub fn linking_symbol_name_for_instance_in_crate<'tcx>(
 
     let mut undecorated = symbol_name_for_instance_in_crate(tcx, symbol, instantiating_crate);
 
+    // thread local will not be a function call,
+    // so it is safe to return before windows symbol decoration check.
+    if let Some(name) = maybe_emutls_symbol_name(tcx, symbol, &undecorated) {
+        return name;
+    }
+
     let target = &tcx.sess.target;
     if !target.is_like_windows {
         // Mach-O has a global "_" suffix and `object` crate will handle it.
@@ -608,6 +614,32 @@ pub fn linking_symbol_name_for_instance_in_crate<'tcx>(
     format!("{prefix}{undecorated}{suffix}{args_in_bytes}")
 }
 
+pub fn exporting_symbol_name_for_instance_in_crate<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    symbol: ExportedSymbol<'tcx>,
+    cnum: CrateNum,
+) -> String {
+    let undecorated = symbol_name_for_instance_in_crate(tcx, symbol, cnum);
+    maybe_emutls_symbol_name(tcx, symbol, &undecorated).unwrap_or(undecorated)
+}
+
+fn maybe_emutls_symbol_name<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    symbol: ExportedSymbol<'tcx>,
+    undecorated: &str,
+) -> Option<String> {
+    if matches!(tcx.sess.tls_model(), TlsModel::Emulated)
+        && let ExportedSymbol::NonGeneric(def_id) = symbol
+        && tcx.is_thread_local_static(def_id)
+    {
+        // When using emutls, LLVM will add the `__emutls_v.` prefix to thread local symbols,
+        // and exported symbol name need to match this.
+        Some(format!("__emutls_v.{undecorated}"))
+    } else {
+        None
+    }
+}
+
 fn wasm_import_module_map(tcx: TyCtxt<'_>, cnum: CrateNum) -> FxHashMap<DefId, String> {
     // Build up a map from DefId to a `NativeLib` structure, where
     // `NativeLib` internally contains information about
@@ -621,7 +653,7 @@ fn wasm_import_module_map(tcx: TyCtxt<'_>, cnum: CrateNum) -> FxHashMap<DefId, S
 
     let mut ret = FxHashMap::default();
     for (def_id, lib) in tcx.foreign_modules(cnum).iter() {
-        let module = def_id_to_native_lib.get(&def_id).and_then(|s| s.wasm_import_module());
+        let module = def_id_to_native_lib.get(def_id).and_then(|s| s.wasm_import_module());
         let Some(module) = module else { continue };
         ret.extend(lib.foreign_items.iter().map(|id| {
             assert_eq!(id.krate, cnum);
diff --git a/compiler/rustc_codegen_ssa/src/back/write.rs b/compiler/rustc_codegen_ssa/src/back/write.rs
index 3d6a21243..d80ef1eba 100644
--- a/compiler/rustc_codegen_ssa/src/back/write.rs
+++ b/compiler/rustc_codegen_ssa/src/back/write.rs
@@ -14,7 +14,7 @@ use rustc_data_structures::memmap::Mmap;
 use rustc_data_structures::profiling::{SelfProfilerRef, VerboseTimingGuard};
 use rustc_data_structures::sync::Lrc;
 use rustc_errors::emitter::Emitter;
-use rustc_errors::{translation::Translate, DiagnosticId, FatalError, Handler, Level};
+use rustc_errors::{translation::Translate, DiagCtxt, DiagnosticId, FatalError, Level};
 use rustc_errors::{DiagnosticMessage, Style};
 use rustc_fs_util::link_or_copy;
 use rustc_hir::def_id::{CrateNum, LOCAL_CRATE};
@@ -355,7 +355,7 @@ pub struct CodegenContext<B: WriteBackendMethods> {
     /// how to call the compiler with the same arguments.
     pub expanded_args: Vec<String>,
 
-    /// Handler to use for diagnostics produced during codegen.
+    /// Emitter to use for diagnostics produced during codegen.
     pub diag_emitter: SharedEmitter,
     /// LLVM optimizations for which we want to print remarks.
     pub remark: Passes,
@@ -370,8 +370,8 @@ pub struct CodegenContext<B: WriteBackendMethods> {
 }
 
 impl<B: WriteBackendMethods> CodegenContext<B> {
-    pub fn create_diag_handler(&self) -> Handler {
-        Handler::with_emitter(Box::new(self.diag_emitter.clone()))
+    pub fn create_dcx(&self) -> DiagCtxt {
+        DiagCtxt::with_emitter(Box::new(self.diag_emitter.clone()))
     }
 
     pub fn config(&self, kind: ModuleKind) -> &ModuleConfig {
@@ -569,7 +569,7 @@ fn produce_final_output_artifacts(
             }
             if !sess.opts.cg.save_temps && !keep_numbered {
                 // The user just wants `foo.x`, not `foo.#module-name#.x`.
-                ensure_removed(sess.diagnostic(), &path);
+                ensure_removed(sess.dcx(), &path);
             }
         } else {
             let extension = crate_output
@@ -660,19 +660,19 @@ fn produce_final_output_artifacts(
         for module in compiled_modules.modules.iter() {
             if let Some(ref path) = module.object {
                 if !keep_numbered_objects {
-                    ensure_removed(sess.diagnostic(), path);
+                    ensure_removed(sess.dcx(), path);
                 }
             }
 
             if let Some(ref path) = module.dwarf_object {
                 if !keep_numbered_objects {
-                    ensure_removed(sess.diagnostic(), path);
+                    ensure_removed(sess.dcx(), path);
                 }
             }
 
             if let Some(ref path) = module.bytecode {
                 if !keep_numbered_bitcode {
-                    ensure_removed(sess.diagnostic(), path);
+                    ensure_removed(sess.dcx(), path);
                 }
             }
         }
@@ -680,7 +680,7 @@ fn produce_final_output_artifacts(
         if !user_wants_bitcode {
             if let Some(ref allocator_module) = compiled_modules.allocator_module {
                 if let Some(ref path) = allocator_module.bytecode {
-                    ensure_removed(sess.diagnostic(), path);
+                    ensure_removed(sess.dcx(), path);
                 }
             }
         }
@@ -836,10 +836,10 @@ fn execute_optimize_work_item<B: ExtraBackendMethods>(
     module: ModuleCodegen<B::Module>,
     module_config: &ModuleConfig,
 ) -> Result<WorkItemResult<B>, FatalError> {
-    let diag_handler = cgcx.create_diag_handler();
+    let dcx = cgcx.create_dcx();
 
     unsafe {
-        B::optimize(cgcx, &diag_handler, &module, module_config)?;
+        B::optimize(cgcx, &dcx, &module, module_config)?;
     }
 
     // After we've done the initial round of optimizations we need to
@@ -892,7 +892,7 @@ fn execute_copy_from_cache_work_item<B: ExtraBackendMethods>(
     let incr_comp_session_dir = cgcx.incr_comp_session_dir.as_ref().unwrap();
 
     let load_from_incr_comp_dir = |output_path: PathBuf, saved_path: &str| {
-        let source_file = in_incr_comp_dir(&incr_comp_session_dir, saved_path);
+        let source_file = in_incr_comp_dir(incr_comp_session_dir, saved_path);
         debug!(
             "copying preexisting module `{}` from {:?} to {}",
             module.name,
@@ -902,11 +902,7 @@ fn execute_copy_from_cache_work_item<B: ExtraBackendMethods>(
         match link_or_copy(&source_file, &output_path) {
             Ok(_) => Some(output_path),
             Err(error) => {
-                cgcx.create_diag_handler().emit_err(errors::CopyPathBuf {
-                    source_file,
-                    output_path,
-                    error,
-                });
+                cgcx.create_dcx().emit_err(errors::CopyPathBuf { source_file, output_path, error });
                 None
             }
         }
@@ -914,7 +910,7 @@ fn execute_copy_from_cache_work_item<B: ExtraBackendMethods>(
 
     let object = load_from_incr_comp_dir(
         cgcx.output_filenames.temp_path(OutputType::Object, Some(&module.name)),
-        &module.source.saved_files.get("o").expect("no saved object file in work product"),
+        module.source.saved_files.get("o").expect("no saved object file in work product"),
     );
     let dwarf_object =
         module.source.saved_files.get("dwo").as_ref().and_then(|saved_dwarf_object_file| {
@@ -924,7 +920,7 @@ fn execute_copy_from_cache_work_item<B: ExtraBackendMethods>(
                 .expect(
                     "saved dwarf object in work product but `split_dwarf_path` returned `None`",
                 );
-            load_from_incr_comp_dir(dwarf_obj_out, &saved_dwarf_object_file)
+            load_from_incr_comp_dir(dwarf_obj_out, saved_dwarf_object_file)
         });
 
     WorkItemResult::Finished(CompiledModule {
@@ -950,13 +946,13 @@ fn finish_intra_module_work<B: ExtraBackendMethods>(
     module: ModuleCodegen<B::Module>,
     module_config: &ModuleConfig,
 ) -> Result<WorkItemResult<B>, FatalError> {
-    let diag_handler = cgcx.create_diag_handler();
+    let dcx = cgcx.create_dcx();
 
     if !cgcx.opts.unstable_opts.combine_cgu
         || module.kind == ModuleKind::Metadata
         || module.kind == ModuleKind::Allocator
     {
-        let module = unsafe { B::codegen(cgcx, &diag_handler, module, module_config)? };
+        let module = unsafe { B::codegen(cgcx, &dcx, module, module_config)? };
         Ok(WorkItemResult::Finished(module))
     } else {
         Ok(WorkItemResult::NeedsLink(module))
@@ -1609,11 +1605,10 @@ fn start_executing_work<B: ExtraBackendMethods>(
         let needs_link = mem::take(&mut needs_link);
         if !needs_link.is_empty() {
             assert!(compiled_modules.is_empty());
-            let diag_handler = cgcx.create_diag_handler();
-            let module = B::run_link(&cgcx, &diag_handler, needs_link).map_err(|_| ())?;
+            let dcx = cgcx.create_dcx();
+            let module = B::run_link(&cgcx, &dcx, needs_link).map_err(|_| ())?;
             let module = unsafe {
-                B::codegen(&cgcx, &diag_handler, module, cgcx.config(ModuleKind::Regular))
-                    .map_err(|_| ())?
+                B::codegen(&cgcx, &dcx, module, cgcx.config(ModuleKind::Regular)).map_err(|_| ())?
             };
             compiled_modules.push(module);
         }
@@ -1856,13 +1851,13 @@ impl SharedEmitterMain {
 
             match message {
                 Ok(SharedEmitterMessage::Diagnostic(diag)) => {
-                    let handler = sess.diagnostic();
+                    let dcx = sess.dcx();
                     let mut d = rustc_errors::Diagnostic::new_with_messages(diag.lvl, diag.msg);
                     if let Some(code) = diag.code {
                         d.code(code);
                     }
                     d.replace_args(diag.args);
-                    handler.emit_diagnostic(&mut d);
+                    dcx.emit_diagnostic(d);
                 }
                 Ok(SharedEmitterMessage::InlineAsmError(cookie, msg, level, source)) => {
                     let msg = msg.strip_prefix("error: ").unwrap_or(&msg).to_string();
@@ -1870,7 +1865,7 @@ impl SharedEmitterMain {
                     let mut err = match level {
                         Level::Error { lint: false } => sess.struct_err(msg).forget_guarantee(),
                         Level::Warning(_) => sess.struct_warn(msg),
-                        Level::Note => sess.struct_note_without_error(msg),
+                        Level::Note => sess.struct_note(msg),
                         _ => bug!("Invalid inline asm diagnostic level"),
                     };
 
diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs
index 198e56963..1d5205ac6 100644
--- a/compiler/rustc_codegen_ssa/src/base.rs
+++ b/compiler/rustc_codegen_ssa/src/base.rs
@@ -148,10 +148,9 @@ pub fn unsized_info<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
         (&ty::Array(_, len), &ty::Slice(_)) => {
             cx.const_usize(len.eval_target_usize(cx.tcx(), ty::ParamEnv::reveal_all()))
         }
-        (
-            &ty::Dynamic(ref data_a, _, src_dyn_kind),
-            &ty::Dynamic(ref data_b, _, target_dyn_kind),
-        ) if src_dyn_kind == target_dyn_kind => {
+        (&ty::Dynamic(data_a, _, src_dyn_kind), &ty::Dynamic(data_b, _, target_dyn_kind))
+            if src_dyn_kind == target_dyn_kind =>
+        {
             let old_info =
                 old_info.expect("unsized_info: missing old info for trait upcasting coercion");
             if data_a.principal_def_id() == data_b.principal_def_id() {
@@ -322,8 +321,13 @@ pub fn cast_shift_expr_rhs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     if lhs_sz < rhs_sz {
         bx.trunc(rhs, lhs_llty)
     } else if lhs_sz > rhs_sz {
-        // FIXME (#1877: If in the future shifting by negative
-        // values is no longer undefined then this is wrong.
+        // We zero-extend even if the RHS is signed. So e.g. `(x: i32) << -1i8` will zero-extend the
+        // RHS to `255i32`. But then we mask the shift amount to be within the size of the LHS
+        // anyway so the result is `31` as it should be. All the extra bits introduced by zext
+        // are masked off so their value does not matter.
+        // FIXME: if we ever support 512bit integers, this will be wrong! For such large integers,
+        // the extra bits introduced by zext are *not* all masked away any more.
+        assert!(lhs_sz <= 256);
         bx.zext(rhs, lhs_llty)
     } else {
         rhs
@@ -453,8 +457,8 @@ pub fn maybe_create_entry_wrapper<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
         cx.set_frame_pointer_type(llfn);
         cx.apply_target_cpu_attr(llfn);
 
-        let llbb = Bx::append_block(&cx, llfn, "top");
-        let mut bx = Bx::build(&cx, llbb);
+        let llbb = Bx::append_block(cx, llfn, "top");
+        let mut bx = Bx::build(cx, llbb);
 
         bx.insert_reference_to_gdb_debug_scripts_section_global();
 
@@ -680,7 +684,7 @@ pub fn codegen_crate<B: ExtraBackendMethods>(
 
     // Calculate the CGU reuse
     let cgu_reuse = tcx.sess.time("find_cgu_reuse", || {
-        codegen_units.iter().map(|cgu| determine_cgu_reuse(tcx, &cgu)).collect::<Vec<_>>()
+        codegen_units.iter().map(|cgu| determine_cgu_reuse(tcx, cgu)).collect::<Vec<_>>()
     });
 
     crate::assert_module_sources::assert_module_sources(tcx, &|cgu_reuse_tracker| {
diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
index 2e0840f2d..e529956b1 100644
--- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
+++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs
@@ -57,7 +57,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
         );
     }
 
-    let attrs = tcx.hir().attrs(tcx.hir().local_def_id_to_hir_id(did));
+    let attrs = tcx.hir().attrs(tcx.local_def_id_to_hir_id(did));
     let mut codegen_fn_attrs = CodegenFnAttrs::new();
     if tcx.should_inherit_track_caller(did) {
         codegen_fn_attrs.flags |= CodegenFnAttrFlags::TRACK_CALLER;
@@ -91,7 +91,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
                 Some(tcx.fn_sig(did))
             } else {
                 tcx.sess
-                    .delay_span_bug(attr.span, "this attribute can only be applied to functions");
+                    .span_delayed_bug(attr.span, "this attribute can only be applied to functions");
                 None
             }
         };
@@ -386,7 +386,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
                                 [sym::arm, sym::a32] | [sym::arm, sym::t32] => {
                                     if !tcx.sess.target.has_thumb_interworking {
                                         struct_span_err!(
-                                            tcx.sess.diagnostic(),
+                                            tcx.sess.dcx(),
                                             attr.span,
                                             E0779,
                                             "target does not support `#[instruction_set]`"
@@ -403,7 +403,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
                                 }
                                 _ => {
                                     struct_span_err!(
-                                        tcx.sess.diagnostic(),
+                                        tcx.sess.dcx(),
                                         attr.span,
                                         E0779,
                                         "invalid instruction set specified",
@@ -415,7 +415,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
                         }
                         [] => {
                             struct_span_err!(
-                                tcx.sess.diagnostic(),
+                                tcx.sess.dcx(),
                                 attr.span,
                                 E0778,
                                 "`#[instruction_set]` requires an argument"
@@ -425,7 +425,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
                         }
                         _ => {
                             struct_span_err!(
-                                tcx.sess.diagnostic(),
+                                tcx.sess.dcx(),
                                 attr.span,
                                 E0779,
                                 "cannot specify more than one instruction set"
@@ -443,7 +443,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
                     rustc_attr::parse_alignment(&literal.kind)
                         .map_err(|msg| {
                             struct_span_err!(
-                                tcx.sess.diagnostic(),
+                                tcx.sess.dcx(),
                                 attr.span,
                                 E0589,
                                 "invalid `repr(align)` attribute: {}",
@@ -469,27 +469,17 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
             Some(MetaItemKind::List(ref items)) => {
                 inline_span = Some(attr.span);
                 if items.len() != 1 {
-                    struct_span_err!(
-                        tcx.sess.diagnostic(),
-                        attr.span,
-                        E0534,
-                        "expected one argument"
-                    )
-                    .emit();
+                    struct_span_err!(tcx.sess.dcx(), attr.span, E0534, "expected one argument")
+                        .emit();
                     InlineAttr::None
-                } else if list_contains_name(&items, sym::always) {
+                } else if list_contains_name(items, sym::always) {
                     InlineAttr::Always
-                } else if list_contains_name(&items, sym::never) {
+                } else if list_contains_name(items, sym::never) {
                     InlineAttr::Never
                 } else {
-                    struct_span_err!(
-                        tcx.sess.diagnostic(),
-                        items[0].span(),
-                        E0535,
-                        "invalid argument"
-                    )
-                    .help("valid inline arguments are `always` and `never`")
-                    .emit();
+                    struct_span_err!(tcx.sess.dcx(), items[0].span(), E0535, "invalid argument")
+                        .help("valid inline arguments are `always` and `never`")
+                        .emit();
 
                     InlineAttr::None
                 }
@@ -503,7 +493,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
         if !attr.has_name(sym::optimize) {
             return ia;
         }
-        let err = |sp, s| struct_span_err!(tcx.sess.diagnostic(), sp, E0722, "{}", s).emit();
+        let err = |sp, s| struct_span_err!(tcx.sess.dcx(), sp, E0722, "{}", s).emit();
         match attr.meta_kind() {
             Some(MetaItemKind::Word) => {
                 err(attr.span, "expected one argument");
@@ -514,9 +504,9 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
                 if items.len() != 1 {
                     err(attr.span, "expected one argument");
                     OptimizeAttr::None
-                } else if list_contains_name(&items, sym::size) {
+                } else if list_contains_name(items, sym::size) {
                     OptimizeAttr::Size
-                } else if list_contains_name(&items, sym::speed) {
+                } else if list_contains_name(items, sym::speed) {
                     OptimizeAttr::Speed
                 } else {
                     err(items[0].span(), "invalid argument");
@@ -572,13 +562,15 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
     if !codegen_fn_attrs.no_sanitize.is_empty() {
         if codegen_fn_attrs.inline == InlineAttr::Always {
             if let (Some(no_sanitize_span), Some(inline_span)) = (no_sanitize_span, inline_span) {
-                let hir_id = tcx.hir().local_def_id_to_hir_id(did);
+                let hir_id = tcx.local_def_id_to_hir_id(did);
                 tcx.struct_span_lint_hir(
                     lint::builtin::INLINE_NO_SANITIZE,
                     hir_id,
                     no_sanitize_span,
                     "`no_sanitize` will have no effect after inlining",
-                    |lint| lint.span_note(inline_span, "inlining requested here"),
+                    |lint| {
+                        lint.span_note(inline_span, "inlining requested here");
+                    },
                 )
             }
         }
diff --git a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs
index 1a85eb8dd..dda30046b 100644
--- a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs
+++ b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs
@@ -216,7 +216,7 @@ fn push_debuginfo_type_name<'tcx>(
                 output.push(']');
             }
         }
-        ty::Dynamic(ref trait_data, ..) => {
+        ty::Dynamic(trait_data, ..) => {
             let auto_traits: SmallVec<[DefId; 4]> = trait_data.auto_traits().collect();
 
             let has_enclosing_parens = if cpp_like_debuginfo {
@@ -249,7 +249,7 @@ fn push_debuginfo_type_name<'tcx>(
                     .projection_bounds()
                     .map(|bound| {
                         let ExistentialProjection { def_id: item_def_id, term, .. } =
-                            tcx.erase_late_bound_regions(bound);
+                            tcx.instantiate_bound_regions_with_erased(bound);
                         // FIXME(associated_const_equality): allow for consts here
                         (item_def_id, term.ty().unwrap())
                     })
@@ -566,6 +566,9 @@ fn coroutine_kind_label(coroutine_kind: Option<CoroutineKind>) -> &'static str {
         Some(CoroutineKind::Async(CoroutineSource::Block)) => "async_block",
         Some(CoroutineKind::Async(CoroutineSource::Closure)) => "async_closure",
         Some(CoroutineKind::Async(CoroutineSource::Fn)) => "async_fn",
+        Some(CoroutineKind::AsyncGen(CoroutineSource::Block)) => "async_gen_block",
+        Some(CoroutineKind::AsyncGen(CoroutineSource::Closure)) => "async_gen_closure",
+        Some(CoroutineKind::AsyncGen(CoroutineSource::Fn)) => "async_gen_fn",
         Some(CoroutineKind::Coroutine) => "coroutine",
         None => "closure",
     }
@@ -594,7 +597,7 @@ fn push_unqualified_item_name(
         DefPathData::CrateRoot => {
             output.push_str(tcx.crate_name(def_id.krate).as_str());
         }
-        DefPathData::ClosureExpr => {
+        DefPathData::Closure => {
             let label = coroutine_kind_label(tcx.coroutine_kind(def_id));
 
             push_disambiguated_special_name(
diff --git a/compiler/rustc_codegen_ssa/src/errors.rs b/compiler/rustc_codegen_ssa/src/errors.rs
index ed6ac9f9c..668d39afb 100644
--- a/compiler/rustc_codegen_ssa/src/errors.rs
+++ b/compiler/rustc_codegen_ssa/src/errors.rs
@@ -4,7 +4,7 @@ use crate::assert_module_sources::CguReuse;
 use crate::back::command::Command;
 use crate::fluent_generated as fluent;
 use rustc_errors::{
-    DiagnosticArgValue, DiagnosticBuilder, ErrorGuaranteed, Handler, IntoDiagnostic,
+    DiagCtxt, DiagnosticArgValue, DiagnosticBuilder, ErrorGuaranteed, IntoDiagnostic,
     IntoDiagnosticArg,
 };
 use rustc_macros::Diagnostic;
@@ -210,192 +210,191 @@ pub enum LinkRlibError {
 pub struct ThorinErrorWrapper(pub thorin::Error);
 
 impl IntoDiagnostic<'_> for ThorinErrorWrapper {
-    fn into_diagnostic(self, handler: &Handler) -> DiagnosticBuilder<'_, ErrorGuaranteed> {
+    fn into_diagnostic(self, dcx: &DiagCtxt) -> DiagnosticBuilder<'_, ErrorGuaranteed> {
         let mut diag;
         match self.0 {
             thorin::Error::ReadInput(_) => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_read_input_failure);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_read_input_failure);
                 diag
             }
             thorin::Error::ParseFileKind(_) => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_parse_input_file_kind);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_parse_input_file_kind);
                 diag
             }
             thorin::Error::ParseObjectFile(_) => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_parse_input_object_file);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_parse_input_object_file);
                 diag
             }
             thorin::Error::ParseArchiveFile(_) => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_parse_input_archive_file);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_parse_input_archive_file);
                 diag
             }
             thorin::Error::ParseArchiveMember(_) => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_parse_archive_member);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_parse_archive_member);
                 diag
             }
             thorin::Error::InvalidInputKind => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_invalid_input_kind);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_invalid_input_kind);
                 diag
             }
             thorin::Error::DecompressData(_) => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_decompress_data);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_decompress_data);
                 diag
             }
             thorin::Error::NamelessSection(_, offset) => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_section_without_name);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_section_without_name);
                 diag.set_arg("offset", format!("0x{offset:08x}"));
                 diag
             }
             thorin::Error::RelocationWithInvalidSymbol(section, offset) => {
-                diag =
-                    handler.struct_err(fluent::codegen_ssa_thorin_relocation_with_invalid_symbol);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_relocation_with_invalid_symbol);
                 diag.set_arg("section", section);
                 diag.set_arg("offset", format!("0x{offset:08x}"));
                 diag
             }
             thorin::Error::MultipleRelocations(section, offset) => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_multiple_relocations);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_multiple_relocations);
                 diag.set_arg("section", section);
                 diag.set_arg("offset", format!("0x{offset:08x}"));
                 diag
             }
             thorin::Error::UnsupportedRelocation(section, offset) => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_unsupported_relocation);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_unsupported_relocation);
                 diag.set_arg("section", section);
                 diag.set_arg("offset", format!("0x{offset:08x}"));
                 diag
             }
             thorin::Error::MissingDwoName(id) => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_missing_dwo_name);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_missing_dwo_name);
                 diag.set_arg("id", format!("0x{id:08x}"));
                 diag
             }
             thorin::Error::NoCompilationUnits => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_no_compilation_units);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_no_compilation_units);
                 diag
             }
             thorin::Error::NoDie => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_no_die);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_no_die);
                 diag
             }
             thorin::Error::TopLevelDieNotUnit => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_top_level_die_not_unit);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_top_level_die_not_unit);
                 diag
             }
             thorin::Error::MissingRequiredSection(section) => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_missing_required_section);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_missing_required_section);
                 diag.set_arg("section", section);
                 diag
             }
             thorin::Error::ParseUnitAbbreviations(_) => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_parse_unit_abbreviations);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_parse_unit_abbreviations);
                 diag
             }
             thorin::Error::ParseUnitAttribute(_) => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_parse_unit_attribute);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_parse_unit_attribute);
                 diag
             }
             thorin::Error::ParseUnitHeader(_) => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_parse_unit_header);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_parse_unit_header);
                 diag
             }
             thorin::Error::ParseUnit(_) => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_parse_unit);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_parse_unit);
                 diag
             }
             thorin::Error::IncompatibleIndexVersion(section, format, actual) => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_incompatible_index_version);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_incompatible_index_version);
                 diag.set_arg("section", section);
                 diag.set_arg("actual", actual);
                 diag.set_arg("format", format);
                 diag
             }
             thorin::Error::OffsetAtIndex(_, index) => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_offset_at_index);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_offset_at_index);
                 diag.set_arg("index", index);
                 diag
             }
             thorin::Error::StrAtOffset(_, offset) => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_str_at_offset);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_str_at_offset);
                 diag.set_arg("offset", format!("0x{offset:08x}"));
                 diag
             }
             thorin::Error::ParseIndex(_, section) => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_parse_index);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_parse_index);
                 diag.set_arg("section", section);
                 diag
             }
             thorin::Error::UnitNotInIndex(unit) => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_unit_not_in_index);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_unit_not_in_index);
                 diag.set_arg("unit", format!("0x{unit:08x}"));
                 diag
             }
             thorin::Error::RowNotInIndex(_, row) => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_row_not_in_index);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_row_not_in_index);
                 diag.set_arg("row", row);
                 diag
             }
             thorin::Error::SectionNotInRow => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_section_not_in_row);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_section_not_in_row);
                 diag
             }
             thorin::Error::EmptyUnit(unit) => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_empty_unit);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_empty_unit);
                 diag.set_arg("unit", format!("0x{unit:08x}"));
                 diag
             }
             thorin::Error::MultipleDebugInfoSection => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_multiple_debug_info_section);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_multiple_debug_info_section);
                 diag
             }
             thorin::Error::MultipleDebugTypesSection => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_multiple_debug_types_section);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_multiple_debug_types_section);
                 diag
             }
             thorin::Error::NotSplitUnit => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_not_split_unit);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_not_split_unit);
                 diag
             }
             thorin::Error::DuplicateUnit(unit) => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_duplicate_unit);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_duplicate_unit);
                 diag.set_arg("unit", format!("0x{unit:08x}"));
                 diag
             }
             thorin::Error::MissingReferencedUnit(unit) => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_missing_referenced_unit);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_missing_referenced_unit);
                 diag.set_arg("unit", format!("0x{unit:08x}"));
                 diag
             }
             thorin::Error::NoOutputObjectCreated => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_not_output_object_created);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_not_output_object_created);
                 diag
             }
             thorin::Error::MixedInputEncodings => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_mixed_input_encodings);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_mixed_input_encodings);
                 diag
             }
             thorin::Error::Io(e) => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_io);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_io);
                 diag.set_arg("error", format!("{e}"));
                 diag
             }
             thorin::Error::ObjectRead(e) => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_object_read);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_object_read);
                 diag.set_arg("error", format!("{e}"));
                 diag
             }
             thorin::Error::ObjectWrite(e) => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_object_write);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_object_write);
                 diag.set_arg("error", format!("{e}"));
                 diag
             }
             thorin::Error::GimliRead(e) => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_gimli_read);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_gimli_read);
                 diag.set_arg("error", format!("{e}"));
                 diag
             }
             thorin::Error::GimliWrite(e) => {
-                diag = handler.struct_err(fluent::codegen_ssa_thorin_gimli_write);
+                diag = dcx.struct_err(fluent::codegen_ssa_thorin_gimli_write);
                 diag.set_arg("error", format!("{e}"));
                 diag
             }
@@ -412,8 +411,8 @@ pub struct LinkingFailed<'a> {
 }
 
 impl IntoDiagnostic<'_> for LinkingFailed<'_> {
-    fn into_diagnostic(self, handler: &Handler) -> DiagnosticBuilder<'_, ErrorGuaranteed> {
-        let mut diag = handler.struct_err(fluent::codegen_ssa_linking_failed);
+    fn into_diagnostic(self, dcx: &DiagCtxt) -> DiagnosticBuilder<'_, ErrorGuaranteed> {
+        let mut diag = dcx.struct_err(fluent::codegen_ssa_linking_failed);
         diag.set_arg("linker_path", format!("{}", self.linker_path.display()));
         diag.set_arg("exit_status", format!("{}", self.exit_status));
 
diff --git a/compiler/rustc_codegen_ssa/src/lib.rs b/compiler/rustc_codegen_ssa/src/lib.rs
index 156c2904f..09fe138c6 100644
--- a/compiler/rustc_codegen_ssa/src/lib.rs
+++ b/compiler/rustc_codegen_ssa/src/lib.rs
@@ -1,7 +1,7 @@
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
-#![cfg_attr(not(bootstrap), allow(internal_features))]
+#![doc(rust_logo)]
+#![feature(rustdoc_internals)]
+#![allow(internal_features)]
 #![feature(associated_type_bounds)]
 #![feature(box_patterns)]
 #![feature(if_let_guard)]
@@ -27,8 +27,6 @@ extern crate rustc_middle;
 use rustc_ast as ast;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::sync::Lrc;
-use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
-use rustc_fluent_macro::fluent_messages;
 use rustc_hir::def_id::CrateNum;
 use rustc_middle::dep_graph::WorkProduct;
 use rustc_middle::middle::debugger_visualizer::DebuggerVisualizerFile;
@@ -53,14 +51,14 @@ pub mod codegen_attrs;
 pub mod common;
 pub mod debuginfo;
 pub mod errors;
-pub mod glue;
 pub mod meth;
 pub mod mir;
 pub mod mono_item;
+pub mod size_of_val;
 pub mod target_features;
 pub mod traits;
 
-fluent_messages! { "../messages.ftl" }
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
 
 pub struct ModuleCodegen<M> {
     /// The name of the module. When the crate may be saved between
@@ -218,6 +216,7 @@ impl CodegenResults {
         sess: &Session,
         rlink_file: &Path,
         codegen_results: &CodegenResults,
+        outputs: &OutputFilenames,
     ) -> Result<usize, io::Error> {
         let mut encoder = FileEncoder::new(rlink_file)?;
         encoder.emit_raw_bytes(RLINK_MAGIC);
@@ -226,10 +225,14 @@ impl CodegenResults {
         encoder.emit_raw_bytes(&RLINK_VERSION.to_be_bytes());
         encoder.emit_str(sess.cfg_version);
         Encodable::encode(codegen_results, &mut encoder);
-        encoder.finish()
+        Encodable::encode(outputs, &mut encoder);
+        encoder.finish().map_err(|(_path, err)| err)
     }
 
-    pub fn deserialize_rlink(sess: &Session, data: Vec<u8>) -> Result<Self, CodegenErrors> {
+    pub fn deserialize_rlink(
+        sess: &Session,
+        data: Vec<u8>,
+    ) -> Result<(Self, OutputFilenames), CodegenErrors> {
         // The Decodable machinery is not used here because it panics if the input data is invalid
         // and because its internal representation may change.
         if !data.starts_with(RLINK_MAGIC) {
@@ -258,6 +261,7 @@ impl CodegenResults {
         }
 
         let codegen_results = CodegenResults::decode(&mut decoder);
-        Ok(codegen_results)
+        let outputs = OutputFilenames::decode(&mut decoder);
+        Ok((codegen_results, outputs))
     }
 }
diff --git a/compiler/rustc_codegen_ssa/src/mir/analyze.rs b/compiler/rustc_codegen_ssa/src/mir/analyze.rs
index 2285e7f4e..c1de9b76f 100644
--- a/compiler/rustc_codegen_ssa/src/mir/analyze.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/analyze.rs
@@ -36,13 +36,13 @@ pub fn non_ssa_locals<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
 
     // Arguments get assigned to by means of the function being called
     for arg in mir.args_iter() {
-        analyzer.assign(arg, DefLocation::Argument);
+        analyzer.define(arg, DefLocation::Argument);
     }
 
     // If there exists a local definition that dominates all uses of that local,
     // the definition should be visited first. Traverse blocks in an order that
     // is a topological sort of dominance partial order.
-    for (bb, data) in traversal::reverse_postorder(&mir) {
+    for (bb, data) in traversal::reverse_postorder(mir) {
         analyzer.visit_basic_block_data(bb, data);
     }
 
@@ -74,7 +74,7 @@ struct LocalAnalyzer<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> {
 }
 
 impl<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> LocalAnalyzer<'mir, 'a, 'tcx, Bx> {
-    fn assign(&mut self, local: mir::Local, location: DefLocation) {
+    fn define(&mut self, local: mir::Local, location: DefLocation) {
         let kind = &mut self.locals[local];
         match *kind {
             LocalKind::ZST => {}
@@ -162,7 +162,7 @@ impl<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> Visitor<'tcx>
         debug!("visit_assign(place={:?}, rvalue={:?})", place, rvalue);
 
         if let Some(local) = place.as_local() {
-            self.assign(local, DefLocation::Body(location));
+            self.define(local, DefLocation::Assignment(location));
             if self.locals[local] != LocalKind::Memory {
                 let decl_span = self.fx.mir.local_decls[local].source_info.span;
                 if !self.fx.rvalue_creates_operand(rvalue, decl_span) {
@@ -183,9 +183,14 @@ impl<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> Visitor<'tcx>
 
     fn visit_local(&mut self, local: mir::Local, context: PlaceContext, location: Location) {
         match context {
-            PlaceContext::MutatingUse(MutatingUseContext::Call)
-            | PlaceContext::MutatingUse(MutatingUseContext::Yield) => {
-                self.assign(local, DefLocation::Body(location));
+            PlaceContext::MutatingUse(MutatingUseContext::Call) => {
+                let call = location.block;
+                let TerminatorKind::Call { target, .. } =
+                    self.fx.mir.basic_blocks[call].terminator().kind
+                else {
+                    bug!()
+                };
+                self.define(local, DefLocation::CallReturn { call, target });
             }
 
             PlaceContext::NonUse(_)
@@ -197,7 +202,7 @@ impl<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> Visitor<'tcx>
             ) => match &mut self.locals[local] {
                 LocalKind::ZST => {}
                 LocalKind::Memory => {}
-                LocalKind::SSA(def) if def.dominates(location, &self.dominators) => {}
+                LocalKind::SSA(def) if def.dominates(location, self.dominators) => {}
                 // Reads from uninitialized variables (e.g., in dead code, after
                 // optimizations) require locals to be in (uninitialized) memory.
                 // N.B., there can be uninitialized reads of a local visited after
@@ -237,6 +242,8 @@ impl<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> Visitor<'tcx>
                     }
                 }
             }
+
+            PlaceContext::MutatingUse(MutatingUseContext::Yield) => bug!(),
         }
     }
 }
diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs
index 3d2d8f8b5..a1662f25e 100644
--- a/compiler/rustc_codegen_ssa/src/mir/block.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/block.rs
@@ -47,7 +47,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
         &self,
         fx: &'b mut FunctionCx<'a, 'tcx, Bx>,
     ) -> Option<&'b Bx::Funclet> {
-        let cleanup_kinds = (&fx.cleanup_kinds).as_ref()?;
+        let cleanup_kinds = fx.cleanup_kinds.as_ref()?;
         let funclet_bb = cleanup_kinds[self.bb].funclet_bb(self.bb)?;
         // If `landing_pad_for` hasn't been called yet to create the `Funclet`,
         // it has to be now. This may not seem necessary, as RPO should lead
@@ -161,7 +161,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
     ) -> MergingSucc {
         // If there is a cleanup block and the function we're calling can unwind, then
         // do an invoke, otherwise do a call.
-        let fn_ty = bx.fn_decl_backend_type(&fn_abi);
+        let fn_ty = bx.fn_decl_backend_type(fn_abi);
 
         let fn_attrs = if bx.tcx().def_kind(fx.instance.def_id()).has_codegen_attrs() {
             Some(bx.tcx().codegen_fn_attrs(fx.instance.def_id()))
@@ -204,9 +204,9 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
             let invokeret = bx.invoke(
                 fn_ty,
                 fn_attrs,
-                Some(&fn_abi),
+                Some(fn_abi),
                 fn_ptr,
-                &llargs,
+                llargs,
                 ret_llbb,
                 unwind_block,
                 self.funclet(fx),
@@ -225,7 +225,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
             }
             MergingSucc::False
         } else {
-            let llret = bx.call(fn_ty, fn_attrs, Some(&fn_abi), fn_ptr, &llargs, self.funclet(fx));
+            let llret = bx.call(fn_ty, fn_attrs, Some(fn_abi), fn_ptr, llargs, self.funclet(fx));
             if fx.mir[self.bb].is_cleanup {
                 bx.apply_attrs_to_cleanup_callsite(llret);
             }
@@ -273,7 +273,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
 
             bx.codegen_inline_asm(
                 template,
-                &operands,
+                operands,
                 options,
                 line_spans,
                 instance,
@@ -281,7 +281,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
             );
             MergingSucc::False
         } else {
-            bx.codegen_inline_asm(template, &operands, options, line_spans, instance, None);
+            bx.codegen_inline_asm(template, operands, options, line_spans, instance, None);
 
             if let Some(target) = destination {
                 self.funclet_br(fx, bx, target, mergeable_succ)
@@ -318,7 +318,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         discr: &mir::Operand<'tcx>,
         targets: &SwitchTargets,
     ) {
-        let discr = self.codegen_operand(bx, &discr);
+        let discr = self.codegen_operand(bx, discr);
         let switch_ty = discr.layout.ty;
         let mut target_iter = targets.iter();
         if target_iter.len() == 1 {
@@ -498,7 +498,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     args = &args[..1];
                     (
                         meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE)
-                            .get_fn(bx, vtable, ty, &fn_abi),
+                            .get_fn(bx, vtable, ty, fn_abi),
                         fn_abi,
                     )
                 }
@@ -540,7 +540,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     debug!("args' = {:?}", args);
                     (
                         meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE)
-                            .get_fn(bx, meta.immediate(), ty, &fn_abi),
+                            .get_fn(bx, meta.immediate(), ty, fn_abi),
                         fn_abi,
                     )
                 }
@@ -864,7 +864,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         // promotes any complex rvalues to constants.
                         if i == 2 && intrinsic == sym::simd_shuffle {
                             if let mir::Operand::Constant(constant) = arg {
-                                let (llval, ty) = self.simd_shuffle_indices(&bx, constant);
+                                let (llval, ty) = self.simd_shuffle_indices(bx, constant);
                                 return OperandRef {
                                     val: Immediate(llval),
                                     layout: bx.layout_of(ty),
@@ -881,7 +881,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 Self::codegen_intrinsic_call(
                     bx,
                     *instance.as_ref().unwrap(),
-                    &fn_abi,
+                    fn_abi,
                     &args,
                     dest,
                     span,
@@ -937,7 +937,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                             bx,
                             meta,
                             op.layout.ty,
-                            &fn_abi,
+                            fn_abi,
                         ));
                         llargs.push(data_ptr);
                         continue 'make_args;
@@ -948,7 +948,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                             bx,
                             meta,
                             op.layout.ty,
-                            &fn_abi,
+                            fn_abi,
                         ));
                         llargs.push(data_ptr);
                         continue;
@@ -975,7 +975,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                             bx,
                             meta.immediate(),
                             op.layout.ty,
-                            &fn_abi,
+                            fn_abi,
                         ));
                         llargs.push(data_ptr.llval);
                         continue;
@@ -1587,9 +1587,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         self.set_debug_loc(&mut bx, mir::SourceInfo::outermost(self.mir.span));
 
         let (fn_abi, fn_ptr) = common::build_langcall(&bx, None, reason.lang_item());
-        let fn_ty = bx.fn_decl_backend_type(&fn_abi);
+        let fn_ty = bx.fn_decl_backend_type(fn_abi);
 
-        let llret = bx.call(fn_ty, None, Some(&fn_abi), fn_ptr, &[], funclet.as_ref());
+        let llret = bx.call(fn_ty, None, Some(fn_abi), fn_ptr, &[], funclet.as_ref());
         bx.apply_attrs_to_cleanup_callsite(llret);
 
         bx.unreachable();
@@ -1662,10 +1662,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 }
             }
         } else {
-            self.codegen_place(
-                bx,
-                mir::PlaceRef { local: dest.local, projection: &dest.projection },
-            )
+            self.codegen_place(bx, mir::PlaceRef { local: dest.local, projection: dest.projection })
         };
         if fn_ret.is_indirect() {
             if dest.align < dest.layout.align.abi {
@@ -1696,7 +1693,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
         match dest {
             Nothing => (),
-            Store(dst) => bx.store_arg(&ret_abi, llval, dst),
+            Store(dst) => bx.store_arg(ret_abi, llval, dst),
             IndirectOperand(tmp, index) => {
                 let op = bx.load_operand(tmp);
                 tmp.storage_dead(bx);
@@ -1708,7 +1705,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 let op = if let PassMode::Cast { .. } = ret_abi.mode {
                     let tmp = PlaceRef::alloca(bx, ret_abi.layout);
                     tmp.storage_live(bx);
-                    bx.store_arg(&ret_abi, llval, tmp);
+                    bx.store_arg(ret_abi, llval, tmp);
                     let op = bx.load_operand(tmp);
                     tmp.storage_dead(bx);
                     op
diff --git a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs
index 0dc30d21c..14915e816 100644
--- a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs
@@ -398,7 +398,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         let Some(dbg_loc) = self.dbg_loc(var.source_info) else { return };
 
         let DebugInfoOffset { direct_offset, indirect_offsets, result: _ } =
-            calculate_debuginfo_offset(bx, &var.projection, base.layout);
+            calculate_debuginfo_offset(bx, var.projection, base.layout);
 
         // When targeting MSVC, create extra allocas for arguments instead of pointing multiple
         // dbg_var_addr() calls into the same alloca with offsets. MSVC uses CodeView records
@@ -416,7 +416,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
         if should_create_individual_allocas {
             let DebugInfoOffset { direct_offset: _, indirect_offsets: _, result: place } =
-                calculate_debuginfo_offset(bx, &var.projection, base);
+                calculate_debuginfo_offset(bx, var.projection, base);
 
             // Create a variable which will be a pointer to the actual value
             let ptr_ty = Ty::new_ptr(
diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
index 136d06d56..a5bffc33d 100644
--- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
@@ -4,8 +4,8 @@ use super::FunctionCx;
 use crate::common::IntPredicate;
 use crate::errors;
 use crate::errors::InvalidMonomorphization;
-use crate::glue;
 use crate::meth;
+use crate::size_of_val;
 use crate::traits::*;
 use crate::MemFlags;
 
@@ -88,21 +88,17 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             sym::va_end => bx.va_end(args[0].immediate()),
             sym::size_of_val => {
                 let tp_ty = fn_args.type_at(0);
-                if let OperandValue::Pair(_, meta) = args[0].val {
-                    let (llsize, _) = glue::size_and_align_of_dst(bx, tp_ty, Some(meta));
-                    llsize
-                } else {
-                    bx.const_usize(bx.layout_of(tp_ty).size.bytes())
-                }
+                let meta =
+                    if let OperandValue::Pair(_, meta) = args[0].val { Some(meta) } else { None };
+                let (llsize, _) = size_of_val::size_and_align_of_dst(bx, tp_ty, meta);
+                llsize
             }
             sym::min_align_of_val => {
                 let tp_ty = fn_args.type_at(0);
-                if let OperandValue::Pair(_, meta) = args[0].val {
-                    let (_, llalign) = glue::size_and_align_of_dst(bx, tp_ty, Some(meta));
-                    llalign
-                } else {
-                    bx.const_usize(bx.layout_of(tp_ty).align.abi.bytes())
-                }
+                let meta =
+                    if let OperandValue::Pair(_, meta) = args[0].val { Some(meta) } else { None };
+                let (_, llalign) = size_of_val::size_and_align_of_dst(bx, tp_ty, meta);
+                llalign
             }
             sym::vtable_size | sym::vtable_align => {
                 let vtable = args[0].immediate();
diff --git a/compiler/rustc_codegen_ssa/src/mir/locals.rs b/compiler/rustc_codegen_ssa/src/mir/locals.rs
index 378c54013..7db260c9f 100644
--- a/compiler/rustc_codegen_ssa/src/mir/locals.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/locals.rs
@@ -43,7 +43,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     let local = mir::Local::from_usize(local);
                     let expected_ty = self.monomorphize(self.mir.local_decls[local].ty);
                     if expected_ty != op.layout.ty {
-                        warn!("Unexpected initial operand type. See the issues/114858");
+                        warn!(
+                            "Unexpected initial operand type: expected {expected_ty:?}, found {:?}.\
+                            See <https://github.com/rust-lang/rust/issues/114858>.",
+                            op.layout.ty
+                        );
                     }
                 }
             }
diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs
index d0b799e08..a6fcf1fd3 100644
--- a/compiler/rustc_codegen_ssa/src/mir/mod.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs
@@ -168,7 +168,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     let fn_abi = cx.fn_abi_of_instance(instance, ty::List::empty());
     debug!("fn_abi: {:?}", fn_abi);
 
-    let debug_context = cx.create_function_debug_context(instance, &fn_abi, llfn, &mir);
+    let debug_context = cx.create_function_debug_context(instance, fn_abi, llfn, mir);
 
     let start_llbb = Bx::append_block(cx, llfn, "start");
     let mut start_bx = Bx::build(cx, start_llbb);
@@ -180,7 +180,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     }
 
     let cleanup_kinds =
-        base::wants_new_eh_instructions(cx.tcx().sess).then(|| analyze::cleanup_kinds(&mir));
+        base::wants_new_eh_instructions(cx.tcx().sess).then(|| analyze::cleanup_kinds(mir));
 
     let cached_llbbs: IndexVec<mir::BasicBlock, CachedLlbb<Bx::BasicBlock>> =
         mir.basic_blocks
@@ -261,7 +261,7 @@ pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     drop(start_bx);
 
     // Codegen the body of each block using reverse postorder
-    for (bb, _) in traversal::reverse_postorder(&mir) {
+    for (bb, _) in traversal::reverse_postorder(mir) {
         fx.codegen_block(bb);
     }
 }
diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs
index 0ab2b7ecd..794cbd315 100644
--- a/compiler/rustc_codegen_ssa/src/mir/operand.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs
@@ -2,7 +2,7 @@ use super::place::PlaceRef;
 use super::{FunctionCx, LocalRef};
 
 use crate::base;
-use crate::glue;
+use crate::size_of_val;
 use crate::traits::*;
 use crate::MemFlags;
 
@@ -105,7 +105,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
                     bug!("from_const: invalid ScalarPair layout: {:#?}", layout);
                 };
                 let a = Scalar::from_pointer(
-                    Pointer::new(bx.tcx().reserve_and_set_memory_alloc(data), Size::ZERO),
+                    Pointer::new(bx.tcx().reserve_and_set_memory_alloc(data).into(), Size::ZERO),
                     &bx.tcx(),
                 );
                 let a_llval = bx.scalar_to_backend(
@@ -132,7 +132,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
         offset: Size,
     ) -> Self {
         let alloc_align = alloc.inner().align;
-        assert_eq!(alloc_align, layout.align.abi);
+        assert!(alloc_align >= layout.align.abi);
 
         let read_scalar = |start, size, s: abi::Scalar, ty| {
             match alloc.0.read_scalar(
@@ -155,7 +155,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
             Abi::Scalar(s @ abi::Scalar::Initialized { .. }) => {
                 let size = s.size(bx);
                 assert_eq!(size, layout.size, "abi::Scalar size does not match layout size");
-                let val = read_scalar(offset, size, s, bx.backend_type(layout));
+                let val = read_scalar(offset, size, s, bx.immediate_backend_type(layout));
                 OperandRef { val: OperandValue::Immediate(val), layout }
             }
             Abi::ScalarPair(
@@ -414,6 +414,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandValue<V> {
                 // value is through `undef`/`poison`, and the store itself is useless.
             }
             OperandValue::Ref(r, None, source_align) => {
+                assert!(dest.layout.is_sized(), "cannot directly store unsized values");
                 if flags.contains(MemFlags::NONTEMPORAL) {
                     // HACK(nox): This is inefficient but there is no nontemporal memcpy.
                     let ty = bx.backend_type(dest.layout);
@@ -465,13 +466,13 @@ impl<'a, 'tcx, V: CodegenObject> OperandValue<V> {
             .ty;
 
         let OperandValue::Ref(llptr, Some(llextra), _) = self else {
-            bug!("store_unsized called with a sized value")
+            bug!("store_unsized called with a sized value (or with an extern type)")
         };
 
         // Allocate an appropriate region on the stack, and copy the value into it. Since alloca
         // doesn't support dynamic alignment, we allocate an extra align - 1 bytes, and align the
         // pointer manually.
-        let (size, align) = glue::size_and_align_of_dst(bx, unsized_ty, Some(llextra));
+        let (size, align) = size_of_val::size_and_align_of_dst(bx, unsized_ty, Some(llextra));
         let one = bx.const_usize(1);
         let align_minus_1 = bx.sub(align, one);
         let size_extra = bx.add(size, align_minus_1);
diff --git a/compiler/rustc_codegen_ssa/src/mir/place.rs b/compiler/rustc_codegen_ssa/src/mir/place.rs
index eb590a45a..c0bb3ac56 100644
--- a/compiler/rustc_codegen_ssa/src/mir/place.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/place.rs
@@ -2,7 +2,7 @@ use super::operand::OperandValue;
 use super::{FunctionCx, LocalRef};
 
 use crate::common::IntPredicate;
-use crate::glue;
+use crate::size_of_val;
 use crate::traits::*;
 
 use rustc_middle::mir;
@@ -99,6 +99,8 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
         let offset = self.layout.fields.offset(ix);
         let effective_field_align = self.align.restrict_for_offset(offset);
 
+        // `simple` is called when we don't need to adjust the offset to
+        // the dynamic alignment of the field.
         let mut simple = || {
             let llval = match self.layout.abi {
                 _ if offset.bytes() == 0 => {
@@ -141,35 +143,21 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
         };
 
         // Simple cases, which don't need DST adjustment:
-        //   * no metadata available - just log the case
-        //   * known alignment - sized types, `[T]`, `str` or a foreign type
-        //   * packed struct - there is no alignment padding
+        //   * known alignment - sized types, `[T]`, `str`
+        //   * offset 0 -- rounding up to alignment cannot change the offset
+        // Note that looking at `field.align` is incorrect since that is not necessarily equal
+        // to the dynamic alignment of the type.
         match field.ty.kind() {
-            _ if self.llextra.is_none() => {
-                debug!(
-                    "unsized field `{}`, of `{:?}` has no metadata for adjustment",
-                    ix, self.llval
-                );
-                return simple();
-            }
             _ if field.is_sized() => return simple(),
-            ty::Slice(..) | ty::Str | ty::Foreign(..) => return simple(),
-            ty::Adt(def, _) => {
-                if def.repr().packed() {
-                    // FIXME(eddyb) generalize the adjustment when we
-                    // start supporting packing to larger alignments.
-                    assert_eq!(self.layout.align.abi.bytes(), 1);
-                    return simple();
-                }
-            }
+            ty::Slice(..) | ty::Str => return simple(),
+            _ if offset.bytes() == 0 => return simple(),
             _ => {}
         }
 
         // We need to get the pointer manually now.
         // We do this by casting to a `*i8`, then offsetting it by the appropriate amount.
         // We do this instead of, say, simply adjusting the pointer from the result of a GEP
-        // because the field may have an arbitrary alignment in the LLVM representation
-        // anyway.
+        // because the field may have an arbitrary alignment in the LLVM representation.
         //
         // To demonstrate:
         //
@@ -186,7 +174,16 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
         let unaligned_offset = bx.cx().const_usize(offset.bytes());
 
         // Get the alignment of the field
-        let (_, unsized_align) = glue::size_and_align_of_dst(bx, field.ty, meta);
+        let (_, mut unsized_align) = size_of_val::size_and_align_of_dst(bx, field.ty, meta);
+
+        // For packed types, we need to cap alignment.
+        if let ty::Adt(def, _) = self.layout.ty.kind()
+            && let Some(packed) = def.repr().pack
+        {
+            let packed = bx.const_usize(packed.bytes());
+            let cmp = bx.icmp(IntPredicate::IntULT, unsized_align, packed);
+            unsized_align = bx.select(cmp, unsized_align, packed)
+        }
 
         // Bump the unaligned offset up to the appropriate alignment
         let offset = round_up_const_value_to_alignment(bx, unaligned_offset, unsized_align);
@@ -474,27 +471,25 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     cg_base.project_index(bx, llindex)
                 }
                 mir::ProjectionElem::ConstantIndex { offset, from_end: false, min_length: _ } => {
-                    let lloffset = bx.cx().const_usize(offset as u64);
+                    let lloffset = bx.cx().const_usize(offset);
                     cg_base.project_index(bx, lloffset)
                 }
                 mir::ProjectionElem::ConstantIndex { offset, from_end: true, min_length: _ } => {
-                    let lloffset = bx.cx().const_usize(offset as u64);
+                    let lloffset = bx.cx().const_usize(offset);
                     let lllen = cg_base.len(bx.cx());
                     let llindex = bx.sub(lllen, lloffset);
                     cg_base.project_index(bx, llindex)
                 }
                 mir::ProjectionElem::Subslice { from, to, from_end } => {
-                    let mut subslice = cg_base.project_index(bx, bx.cx().const_usize(from as u64));
+                    let mut subslice = cg_base.project_index(bx, bx.cx().const_usize(from));
                     let projected_ty =
                         PlaceTy::from_ty(cg_base.layout.ty).projection_ty(tcx, *elem).ty;
                     subslice.layout = bx.cx().layout_of(self.monomorphize(projected_ty));
 
                     if subslice.layout.is_unsized() {
                         assert!(from_end, "slice subslices should be `from_end`");
-                        subslice.llextra = Some(bx.sub(
-                            cg_base.llextra.unwrap(),
-                            bx.cx().const_usize((from as u64) + (to as u64)),
-                        ));
+                        subslice.llextra =
+                            Some(bx.sub(cg_base.llextra.unwrap(), bx.cx().const_usize(from + to)));
                     }
 
                     subslice
diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
index 8e5019967..02b51dfe5 100644
--- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
@@ -702,7 +702,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     };
                     let fn_ptr = bx.get_fn_addr(instance);
                     let fn_abi = bx.fn_abi_of_instance(instance, ty::List::empty());
-                    let fn_ty = bx.fn_decl_backend_type(&fn_abi);
+                    let fn_ty = bx.fn_decl_backend_type(fn_abi);
                     let fn_attrs = if bx.tcx().def_kind(instance.def_id()).has_codegen_attrs() {
                         Some(bx.tcx().codegen_fn_attrs(instance.def_id()))
                     } else {
diff --git a/compiler/rustc_codegen_ssa/src/mir/statement.rs b/compiler/rustc_codegen_ssa/src/mir/statement.rs
index 899e41265..a158fc6e2 100644
--- a/compiler/rustc_codegen_ssa/src/mir/statement.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/statement.rs
@@ -3,7 +3,6 @@ use rustc_middle::mir::NonDivergingIntrinsic;
 
 use super::FunctionCx;
 use super::LocalRef;
-use crate::traits::BuilderMethods;
 use crate::traits::*;
 
 impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
diff --git a/compiler/rustc_codegen_ssa/src/mono_item.rs b/compiler/rustc_codegen_ssa/src/mono_item.rs
index 6fbf992ed..295e27691 100644
--- a/compiler/rustc_codegen_ssa/src/mono_item.rs
+++ b/compiler/rustc_codegen_ssa/src/mono_item.rs
@@ -34,7 +34,7 @@ impl<'a, 'tcx: 'a> MonoItemExt<'a, 'tcx> for MonoItem<'tcx> {
             }
             MonoItem::GlobalAsm(item_id) => {
                 let item = cx.tcx().hir().item(item_id);
-                if let hir::ItemKind::GlobalAsm(ref asm) = item.kind {
+                if let hir::ItemKind::GlobalAsm(asm) = item.kind {
                     let operands: Vec<_> = asm
                         .operands
                         .iter()
@@ -88,7 +88,7 @@ impl<'a, 'tcx: 'a> MonoItemExt<'a, 'tcx> for MonoItem<'tcx> {
                 }
             }
             MonoItem::Fn(instance) => {
-                base::codegen_instance::<Bx>(&cx, instance);
+                base::codegen_instance::<Bx>(cx, instance);
             }
         }
 
@@ -119,10 +119,10 @@ impl<'a, 'tcx: 'a> MonoItemExt<'a, 'tcx> for MonoItem<'tcx> {
 
         match *self {
             MonoItem::Static(def_id) => {
-                cx.predefine_static(def_id, linkage, visibility, &symbol_name);
+                cx.predefine_static(def_id, linkage, visibility, symbol_name);
             }
             MonoItem::Fn(instance) => {
-                cx.predefine_fn(instance, linkage, visibility, &symbol_name);
+                cx.predefine_fn(instance, linkage, visibility, symbol_name);
             }
             MonoItem::GlobalAsm(..) => {}
         }
diff --git a/compiler/rustc_codegen_ssa/src/glue.rs b/compiler/rustc_codegen_ssa/src/size_of_val.rs
index c34f1dbf8..087836ca3 100644
--- a/compiler/rustc_codegen_ssa/src/glue.rs
+++ b/compiler/rustc_codegen_ssa/src/size_of_val.rs
@@ -1,10 +1,11 @@
-//!
-//
-// Code relating to drop glue.
+//! Computing the size and alignment of a value.
 
+use crate::common;
 use crate::common::IntPredicate;
 use crate::meth;
 use crate::traits::*;
+use rustc_hir::LangItem;
+use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths};
 use rustc_middle::ty::{self, Ty};
 use rustc_target::abi::WrappingRange;
 
@@ -14,7 +15,7 @@ pub fn size_and_align_of_dst<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     info: Option<Bx::Value>,
 ) -> (Bx::Value, Bx::Value) {
     let layout = bx.layout_of(t);
-    debug!("size_and_align_of_dst(ty={}, info={:?}): layout: {:?}", t, info, layout);
+    trace!("size_and_align_of_dst(ty={}, info={:?}): layout: {:?}", t, info, layout);
     if layout.is_sized() {
         let size = bx.const_usize(layout.size.bytes());
         let align = bx.const_usize(layout.align.abi.bytes());
@@ -51,7 +52,31 @@ pub fn size_and_align_of_dst<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
                 bx.const_usize(unit.align.abi.bytes()),
             )
         }
-        _ => {
+        ty::Foreign(_) => {
+            // `extern` type. We cannot compute the size, so panic.
+            let msg_str = with_no_visible_paths!({
+                with_no_trimmed_paths!({
+                    format!("attempted to compute the size or alignment of extern type `{t}`")
+                })
+            });
+            let msg = bx.const_str(&msg_str);
+
+            // Obtain the panic entry point.
+            let (fn_abi, llfn) = common::build_langcall(bx, None, LangItem::PanicNounwind);
+
+            // Generate the call.
+            // Cannot use `do_call` since we don't have a MIR terminator so we can't create a `TerminationCodegenHelper`.
+            // (But we are in good company, this code is duplicated plenty of times.)
+            let fn_ty = bx.fn_decl_backend_type(fn_abi);
+
+            bx.call(fn_ty, /* fn_attrs */ None, Some(fn_abi), llfn, &[msg.0, msg.1], None);
+
+            // This function does not return so we can now return whatever we want.
+            let size = bx.const_usize(layout.size.bytes());
+            let align = bx.const_usize(layout.align.abi.bytes());
+            (size, align)
+        }
+        ty::Adt(..) | ty::Tuple(..) => {
             // First get the size of all statically known fields.
             // Don't use size_of because it also rounds up to alignment, which we
             // want to avoid, as the unsized field's alignment could be smaller.
@@ -59,10 +84,13 @@ pub fn size_and_align_of_dst<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
             debug!("DST {} layout: {:?}", t, layout);
 
             let i = layout.fields.count() - 1;
-            let sized_size = layout.fields.offset(i).bytes();
+            let unsized_offset_unadjusted = layout.fields.offset(i).bytes();
             let sized_align = layout.align.abi.bytes();
-            debug!("DST {} statically sized prefix size: {} align: {}", t, sized_size, sized_align);
-            let sized_size = bx.const_usize(sized_size);
+            debug!(
+                "DST {} offset of dyn field: {}, statically sized align: {}",
+                t, unsized_offset_unadjusted, sized_align
+            );
+            let unsized_offset_unadjusted = bx.const_usize(unsized_offset_unadjusted);
             let sized_align = bx.const_usize(sized_align);
 
             // Recurse to get the size of the dynamically sized field (must be
@@ -70,26 +98,26 @@ pub fn size_and_align_of_dst<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
             let field_ty = layout.field(bx, i).ty;
             let (unsized_size, mut unsized_align) = size_and_align_of_dst(bx, field_ty, info);
 
-            // FIXME (#26403, #27023): We should be adding padding
-            // to `sized_size` (to accommodate the `unsized_align`
-            // required of the unsized field that follows) before
-            // summing it with `sized_size`. (Note that since #26403
-            // is unfixed, we do not yet add the necessary padding
-            // here. But this is where the add would go.)
+            // # First compute the dynamic alignment
 
-            // Return the sum of sizes and max of aligns.
-            let size = bx.add(sized_size, unsized_size);
-
-            // Packed types ignore the alignment of their fields.
-            if let ty::Adt(def, _) = t.kind() {
-                if def.repr().packed() {
-                    unsized_align = sized_align;
+            // For packed types, we need to cap the alignment.
+            if let ty::Adt(def, _) = t.kind()
+                && let Some(packed) = def.repr().pack
+            {
+                if packed.bytes() == 1 {
+                    // We know this will be capped to 1.
+                    unsized_align = bx.const_usize(1);
+                } else {
+                    // We have to dynamically compute `min(unsized_align, packed)`.
+                    let packed = bx.const_usize(packed.bytes());
+                    let cmp = bx.icmp(IntPredicate::IntULT, unsized_align, packed);
+                    unsized_align = bx.select(cmp, unsized_align, packed);
                 }
             }
 
             // Choose max of two known alignments (combined value must
             // be aligned according to more restrictive of the two).
-            let align = match (
+            let full_align = match (
                 bx.const_to_opt_u128(sized_align, false),
                 bx.const_to_opt_u128(unsized_align, false),
             ) {
@@ -104,6 +132,19 @@ pub fn size_and_align_of_dst<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
                 }
             };
 
+            // # Then compute the dynamic size
+
+            // The full formula for the size would be:
+            // let unsized_offset_adjusted = unsized_offset_unadjusted.align_to(unsized_align);
+            // let full_size = (unsized_offset_adjusted + unsized_size).align_to(full_align);
+            // However, `unsized_size` is a multiple of `unsized_align`.
+            // Therefore, we can equivalently do the `align_to(unsized_align)` *after* adding `unsized_size`:
+            // let full_size = (unsized_offset_unadjusted + unsized_size).align_to(unsized_align).align_to(full_align);
+            // Furthermore, `align >= unsized_align`, and therefore we only need to do:
+            // let full_size = (unsized_offset_unadjusted + unsized_size).align_to(full_align);
+
+            let full_size = bx.add(unsized_offset_unadjusted, unsized_size);
+
             // Issue #27023: must add any necessary padding to `size`
             // (to make it a multiple of `align`) before returning it.
             //
@@ -115,12 +156,13 @@ pub fn size_and_align_of_dst<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
             //
             //   `(size + (align-1)) & -align`
             let one = bx.const_usize(1);
-            let addend = bx.sub(align, one);
-            let add = bx.add(size, addend);
-            let neg = bx.neg(align);
-            let size = bx.and(add, neg);
+            let addend = bx.sub(full_align, one);
+            let add = bx.add(full_size, addend);
+            let neg = bx.neg(full_align);
+            let full_size = bx.and(add, neg);
 
-            (size, align)
+            (full_size, full_align)
         }
+        _ => bug!("size_and_align_of_dst: {t} not supported"),
     }
 }
diff --git a/compiler/rustc_codegen_ssa/src/target_features.rs b/compiler/rustc_codegen_ssa/src/target_features.rs
index 2936f1de3..0b9b08c6a 100644
--- a/compiler/rustc_codegen_ssa/src/target_features.rs
+++ b/compiler/rustc_codegen_ssa/src/target_features.rs
@@ -11,403 +11,10 @@ use rustc_hir::def_id::LOCAL_CRATE;
 use rustc_middle::query::Providers;
 use rustc_middle::ty::TyCtxt;
 use rustc_session::parse::feature_err;
-use rustc_session::Session;
 use rustc_span::symbol::sym;
 use rustc_span::symbol::Symbol;
 use rustc_span::Span;
 
-/// Features that control behaviour of rustc, rather than the codegen.
-pub const RUSTC_SPECIFIC_FEATURES: &[&str] = &["crt-static"];
-
-// When adding features to the below lists
-// check whether they're named already elsewhere in rust
-// e.g. in stdarch and whether the given name matches LLVM's
-// if it doesn't, to_llvm_feature in llvm_util in rustc_codegen_llvm needs to be adapted
-//
-// When adding a new feature, be particularly mindful of features that affect function ABIs. Those
-// need to be treated very carefully to avoid introducing unsoundness! This often affects features
-// that enable/disable hardfloat support (see https://github.com/rust-lang/rust/issues/116344 for an
-// example of this going wrong), but features enabling new SIMD registers are also a concern (see
-// https://github.com/rust-lang/rust/issues/116558 for an example of this going wrong).
-//
-// Stabilizing a target feature (setting the 2nd component of the pair to `None`) requires t-lang
-// approval.
-
-const ARM_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
-    // tidy-alphabetical-start
-    ("aclass", Some(sym::arm_target_feature)),
-    ("aes", Some(sym::arm_target_feature)),
-    ("crc", Some(sym::arm_target_feature)),
-    ("d32", Some(sym::arm_target_feature)),
-    ("dotprod", Some(sym::arm_target_feature)),
-    ("dsp", Some(sym::arm_target_feature)),
-    ("fp-armv8", Some(sym::arm_target_feature)),
-    ("i8mm", Some(sym::arm_target_feature)),
-    ("mclass", Some(sym::arm_target_feature)),
-    ("neon", Some(sym::arm_target_feature)),
-    ("rclass", Some(sym::arm_target_feature)),
-    ("sha2", Some(sym::arm_target_feature)),
-    // This is needed for inline assembly, but shouldn't be stabilized as-is
-    // since it should be enabled per-function using #[instruction_set], not
-    // #[target_feature].
-    ("thumb-mode", Some(sym::arm_target_feature)),
-    ("thumb2", Some(sym::arm_target_feature)),
-    ("trustzone", Some(sym::arm_target_feature)),
-    ("v5te", Some(sym::arm_target_feature)),
-    ("v6", Some(sym::arm_target_feature)),
-    ("v6k", Some(sym::arm_target_feature)),
-    ("v6t2", Some(sym::arm_target_feature)),
-    ("v7", Some(sym::arm_target_feature)),
-    ("v8", Some(sym::arm_target_feature)),
-    ("vfp2", Some(sym::arm_target_feature)),
-    ("vfp3", Some(sym::arm_target_feature)),
-    ("vfp4", Some(sym::arm_target_feature)),
-    ("virtualization", Some(sym::arm_target_feature)),
-    // tidy-alphabetical-end
-];
-
-const AARCH64_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
-    // tidy-alphabetical-start
-    // FEAT_AES
-    ("aes", None),
-    // FEAT_BF16
-    ("bf16", None),
-    // FEAT_BTI
-    ("bti", None),
-    // FEAT_CRC
-    ("crc", None),
-    // FEAT_DIT
-    ("dit", None),
-    // FEAT_DotProd
-    ("dotprod", None),
-    // FEAT_DPB
-    ("dpb", None),
-    // FEAT_DPB2
-    ("dpb2", None),
-    // FEAT_F32MM
-    ("f32mm", None),
-    // FEAT_F64MM
-    ("f64mm", None),
-    // FEAT_FCMA
-    ("fcma", None),
-    // FEAT_FHM
-    ("fhm", None),
-    // FEAT_FLAGM
-    ("flagm", None),
-    // FEAT_FP16
-    ("fp16", None),
-    // FEAT_FRINTTS
-    ("frintts", None),
-    // FEAT_I8MM
-    ("i8mm", None),
-    // FEAT_JSCVT
-    ("jsconv", None),
-    // FEAT_LOR
-    ("lor", None),
-    // FEAT_LSE
-    ("lse", None),
-    // FEAT_MTE
-    ("mte", None),
-    // FEAT_AdvSimd & FEAT_FP
-    ("neon", None),
-    // FEAT_PAUTH (address authentication)
-    ("paca", None),
-    // FEAT_PAUTH (generic authentication)
-    ("pacg", None),
-    // FEAT_PAN
-    ("pan", None),
-    // FEAT_PMUv3
-    ("pmuv3", None),
-    // FEAT_RAND
-    ("rand", None),
-    // FEAT_RAS
-    ("ras", None),
-    // FEAT_RCPC
-    ("rcpc", None),
-    // FEAT_RCPC2
-    ("rcpc2", None),
-    // FEAT_RDM
-    ("rdm", None),
-    // FEAT_SB
-    ("sb", None),
-    // FEAT_SHA1 & FEAT_SHA256
-    ("sha2", None),
-    // FEAT_SHA512 & FEAT_SHA3
-    ("sha3", None),
-    // FEAT_SM3 & FEAT_SM4
-    ("sm4", None),
-    // FEAT_SPE
-    ("spe", None),
-    // FEAT_SSBS
-    ("ssbs", None),
-    // FEAT_SVE
-    ("sve", None),
-    // FEAT_SVE2
-    ("sve2", None),
-    // FEAT_SVE2_AES
-    ("sve2-aes", None),
-    // FEAT_SVE2_BitPerm
-    ("sve2-bitperm", None),
-    // FEAT_SVE2_SHA3
-    ("sve2-sha3", None),
-    // FEAT_SVE2_SM4
-    ("sve2-sm4", None),
-    // FEAT_TME
-    ("tme", None),
-    ("v8.1a", Some(sym::aarch64_ver_target_feature)),
-    ("v8.2a", Some(sym::aarch64_ver_target_feature)),
-    ("v8.3a", Some(sym::aarch64_ver_target_feature)),
-    ("v8.4a", Some(sym::aarch64_ver_target_feature)),
-    ("v8.5a", Some(sym::aarch64_ver_target_feature)),
-    ("v8.6a", Some(sym::aarch64_ver_target_feature)),
-    ("v8.7a", Some(sym::aarch64_ver_target_feature)),
-    // FEAT_VHE
-    ("vh", None),
-    // tidy-alphabetical-end
-];
-
-const AARCH64_TIED_FEATURES: &[&[&str]] = &[
-    &["paca", "pacg"], // Together these represent `pauth` in LLVM
-];
-
-const X86_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
-    // tidy-alphabetical-start
-    ("adx", None),
-    ("aes", None),
-    ("avx", None),
-    ("avx2", None),
-    ("avx512bf16", Some(sym::avx512_target_feature)),
-    ("avx512bitalg", Some(sym::avx512_target_feature)),
-    ("avx512bw", Some(sym::avx512_target_feature)),
-    ("avx512cd", Some(sym::avx512_target_feature)),
-    ("avx512dq", Some(sym::avx512_target_feature)),
-    ("avx512er", Some(sym::avx512_target_feature)),
-    ("avx512f", Some(sym::avx512_target_feature)),
-    ("avx512ifma", Some(sym::avx512_target_feature)),
-    ("avx512pf", Some(sym::avx512_target_feature)),
-    ("avx512vbmi", Some(sym::avx512_target_feature)),
-    ("avx512vbmi2", Some(sym::avx512_target_feature)),
-    ("avx512vl", Some(sym::avx512_target_feature)),
-    ("avx512vnni", Some(sym::avx512_target_feature)),
-    ("avx512vp2intersect", Some(sym::avx512_target_feature)),
-    ("avx512vpopcntdq", Some(sym::avx512_target_feature)),
-    ("bmi1", None),
-    ("bmi2", None),
-    ("cmpxchg16b", None),
-    ("ermsb", Some(sym::ermsb_target_feature)),
-    ("f16c", None),
-    ("fma", None),
-    ("fxsr", None),
-    ("gfni", Some(sym::avx512_target_feature)),
-    ("lzcnt", None),
-    ("movbe", None),
-    ("pclmulqdq", None),
-    ("popcnt", None),
-    ("rdrand", None),
-    ("rdseed", None),
-    ("rtm", Some(sym::rtm_target_feature)),
-    ("sha", None),
-    ("sse", None),
-    ("sse2", None),
-    ("sse3", None),
-    ("sse4.1", None),
-    ("sse4.2", None),
-    ("sse4a", Some(sym::sse4a_target_feature)),
-    ("ssse3", None),
-    ("tbm", Some(sym::tbm_target_feature)),
-    ("vaes", Some(sym::avx512_target_feature)),
-    ("vpclmulqdq", Some(sym::avx512_target_feature)),
-    ("xsave", None),
-    ("xsavec", None),
-    ("xsaveopt", None),
-    ("xsaves", None),
-    // tidy-alphabetical-end
-];
-
-const HEXAGON_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
-    // tidy-alphabetical-start
-    ("hvx", Some(sym::hexagon_target_feature)),
-    ("hvx-length128b", Some(sym::hexagon_target_feature)),
-    // tidy-alphabetical-end
-];
-
-const POWERPC_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
-    // tidy-alphabetical-start
-    ("altivec", Some(sym::powerpc_target_feature)),
-    ("power10-vector", Some(sym::powerpc_target_feature)),
-    ("power8-altivec", Some(sym::powerpc_target_feature)),
-    ("power8-vector", Some(sym::powerpc_target_feature)),
-    ("power9-altivec", Some(sym::powerpc_target_feature)),
-    ("power9-vector", Some(sym::powerpc_target_feature)),
-    ("vsx", Some(sym::powerpc_target_feature)),
-    // tidy-alphabetical-end
-];
-
-const MIPS_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
-    // tidy-alphabetical-start
-    ("fp64", Some(sym::mips_target_feature)),
-    ("msa", Some(sym::mips_target_feature)),
-    ("virt", Some(sym::mips_target_feature)),
-    // tidy-alphabetical-end
-];
-
-const RISCV_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
-    // tidy-alphabetical-start
-    ("a", None),
-    ("c", None),
-    ("d", Some(sym::riscv_target_feature)),
-    ("e", Some(sym::riscv_target_feature)),
-    ("f", Some(sym::riscv_target_feature)),
-    ("m", None),
-    ("relax", Some(sym::riscv_target_feature)),
-    ("unaligned-scalar-mem", Some(sym::riscv_target_feature)),
-    ("v", Some(sym::riscv_target_feature)),
-    ("zba", None),
-    ("zbb", None),
-    ("zbc", None),
-    ("zbkb", None),
-    ("zbkc", None),
-    ("zbkx", None),
-    ("zbs", None),
-    ("zdinx", Some(sym::riscv_target_feature)),
-    ("zfh", Some(sym::riscv_target_feature)),
-    ("zfhmin", Some(sym::riscv_target_feature)),
-    ("zfinx", Some(sym::riscv_target_feature)),
-    ("zhinx", Some(sym::riscv_target_feature)),
-    ("zhinxmin", Some(sym::riscv_target_feature)),
-    ("zk", None),
-    ("zkn", None),
-    ("zknd", None),
-    ("zkne", None),
-    ("zknh", None),
-    ("zkr", None),
-    ("zks", None),
-    ("zksed", None),
-    ("zksh", None),
-    ("zkt", None),
-    // tidy-alphabetical-end
-];
-
-const WASM_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
-    // tidy-alphabetical-start
-    ("atomics", Some(sym::wasm_target_feature)),
-    ("bulk-memory", Some(sym::wasm_target_feature)),
-    ("exception-handling", Some(sym::wasm_target_feature)),
-    ("multivalue", Some(sym::wasm_target_feature)),
-    ("mutable-globals", Some(sym::wasm_target_feature)),
-    ("nontrapping-fptoint", Some(sym::wasm_target_feature)),
-    ("reference-types", Some(sym::wasm_target_feature)),
-    ("relaxed-simd", Some(sym::wasm_target_feature)),
-    ("sign-ext", Some(sym::wasm_target_feature)),
-    ("simd128", None),
-    // tidy-alphabetical-end
-];
-
-const BPF_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[("alu32", Some(sym::bpf_target_feature))];
-
-const CSKY_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
-    // tidy-alphabetical-start
-    ("10e60", Some(sym::csky_target_feature)),
-    ("2e3", Some(sym::csky_target_feature)),
-    ("3e3r1", Some(sym::csky_target_feature)),
-    ("3e3r2", Some(sym::csky_target_feature)),
-    ("3e3r3", Some(sym::csky_target_feature)),
-    ("3e7", Some(sym::csky_target_feature)),
-    ("7e10", Some(sym::csky_target_feature)),
-    ("cache", Some(sym::csky_target_feature)),
-    ("doloop", Some(sym::csky_target_feature)),
-    ("dsp1e2", Some(sym::csky_target_feature)),
-    ("dspe60", Some(sym::csky_target_feature)),
-    ("e1", Some(sym::csky_target_feature)),
-    ("e2", Some(sym::csky_target_feature)),
-    ("edsp", Some(sym::csky_target_feature)),
-    ("elrw", Some(sym::csky_target_feature)),
-    ("float1e2", Some(sym::csky_target_feature)),
-    ("float1e3", Some(sym::csky_target_feature)),
-    ("float3e4", Some(sym::csky_target_feature)),
-    ("float7e60", Some(sym::csky_target_feature)),
-    ("floate1", Some(sym::csky_target_feature)),
-    ("hard-tp", Some(sym::csky_target_feature)),
-    ("high-registers", Some(sym::csky_target_feature)),
-    ("hwdiv", Some(sym::csky_target_feature)),
-    ("mp", Some(sym::csky_target_feature)),
-    ("mp1e2", Some(sym::csky_target_feature)),
-    ("nvic", Some(sym::csky_target_feature)),
-    ("trust", Some(sym::csky_target_feature)),
-    ("vdsp2e60f", Some(sym::csky_target_feature)),
-    ("vdspv1", Some(sym::csky_target_feature)),
-    ("vdspv2", Some(sym::csky_target_feature)),
-    // tidy-alphabetical-end
-    //fpu
-    // tidy-alphabetical-start
-    ("fdivdu", Some(sym::csky_target_feature)),
-    ("fpuv2_df", Some(sym::csky_target_feature)),
-    ("fpuv2_sf", Some(sym::csky_target_feature)),
-    ("fpuv3_df", Some(sym::csky_target_feature)),
-    ("fpuv3_hf", Some(sym::csky_target_feature)),
-    ("fpuv3_hi", Some(sym::csky_target_feature)),
-    ("fpuv3_sf", Some(sym::csky_target_feature)),
-    ("hard-float", Some(sym::csky_target_feature)),
-    ("hard-float-abi", Some(sym::csky_target_feature)),
-    // tidy-alphabetical-end
-];
-
-const LOONGARCH_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
-    // tidy-alphabetical-start
-    ("d", Some(sym::loongarch_target_feature)),
-    ("f", Some(sym::loongarch_target_feature)),
-    ("lasx", Some(sym::loongarch_target_feature)),
-    ("lbt", Some(sym::loongarch_target_feature)),
-    ("lsx", Some(sym::loongarch_target_feature)),
-    ("lvz", Some(sym::loongarch_target_feature)),
-    ("ual", Some(sym::loongarch_target_feature)),
-    // tidy-alphabetical-end
-];
-
-/// When rustdoc is running, provide a list of all known features so that all their respective
-/// primitives may be documented.
-///
-/// IMPORTANT: If you're adding another feature list above, make sure to add it to this iterator!
-pub fn all_known_features() -> impl Iterator<Item = (&'static str, Option<Symbol>)> {
-    std::iter::empty()
-        .chain(ARM_ALLOWED_FEATURES.iter())
-        .chain(AARCH64_ALLOWED_FEATURES.iter())
-        .chain(X86_ALLOWED_FEATURES.iter())
-        .chain(HEXAGON_ALLOWED_FEATURES.iter())
-        .chain(POWERPC_ALLOWED_FEATURES.iter())
-        .chain(MIPS_ALLOWED_FEATURES.iter())
-        .chain(RISCV_ALLOWED_FEATURES.iter())
-        .chain(WASM_ALLOWED_FEATURES.iter())
-        .chain(BPF_ALLOWED_FEATURES.iter())
-        .chain(CSKY_ALLOWED_FEATURES)
-        .chain(LOONGARCH_ALLOWED_FEATURES)
-        .cloned()
-}
-
-pub fn supported_target_features(sess: &Session) -> &'static [(&'static str, Option<Symbol>)] {
-    match &*sess.target.arch {
-        "arm" => ARM_ALLOWED_FEATURES,
-        "aarch64" => AARCH64_ALLOWED_FEATURES,
-        "x86" | "x86_64" => X86_ALLOWED_FEATURES,
-        "hexagon" => HEXAGON_ALLOWED_FEATURES,
-        "mips" | "mips32r6" | "mips64" | "mips64r6" => MIPS_ALLOWED_FEATURES,
-        "powerpc" | "powerpc64" => POWERPC_ALLOWED_FEATURES,
-        "riscv32" | "riscv64" => RISCV_ALLOWED_FEATURES,
-        "wasm32" | "wasm64" => WASM_ALLOWED_FEATURES,
-        "bpf" => BPF_ALLOWED_FEATURES,
-        "csky" => CSKY_ALLOWED_FEATURES,
-        "loongarch64" => LOONGARCH_ALLOWED_FEATURES,
-        _ => &[],
-    }
-}
-
-pub fn tied_target_features(sess: &Session) -> &'static [&'static [&'static str]] {
-    match &*sess.target.arch {
-        "aarch64" => AARCH64_TIED_FEATURES,
-        _ => &[],
-    }
-}
-
 pub fn from_target_feature(
     tcx: TyCtxt<'_>,
     attr: &ast::Attribute,
@@ -529,11 +136,15 @@ pub(crate) fn provide(providers: &mut Providers) {
             if tcx.sess.opts.actually_rustdoc {
                 // rustdoc needs to be able to document functions that use all the features, so
                 // whitelist them all
-                all_known_features().map(|(a, b)| (a.to_string(), b)).collect()
+                rustc_target::target_features::all_known_features()
+                    .map(|(a, b)| (a.to_string(), b.as_feature_name()))
+                    .collect()
             } else {
-                supported_target_features(tcx.sess)
+                tcx.sess
+                    .target
+                    .supported_target_features()
                     .iter()
-                    .map(|&(a, b)| (a.to_string(), b))
+                    .map(|&(a, b)| (a.to_string(), b.as_feature_name()))
                     .collect()
             }
         },
diff --git a/compiler/rustc_codegen_ssa/src/traits/backend.rs b/compiler/rustc_codegen_ssa/src/traits/backend.rs
index 35744d9a1..8e9907ed8 100644
--- a/compiler/rustc_codegen_ssa/src/traits/backend.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/backend.rs
@@ -9,6 +9,7 @@ use rustc_ast::expand::allocator::AllocatorKind;
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_data_structures::sync::{DynSend, DynSync};
 use rustc_errors::ErrorGuaranteed;
+use rustc_metadata::creader::MetadataLoaderDyn;
 use rustc_metadata::EncodedMetadata;
 use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
 use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, LayoutOf, TyAndLayout};
@@ -16,7 +17,6 @@ use rustc_middle::ty::{Ty, TyCtxt};
 use rustc_middle::util::Providers;
 use rustc_session::{
     config::{self, OutputFilenames, PrintRequest},
-    cstore::MetadataLoaderDyn,
     Session,
 };
 use rustc_span::symbol::Symbol;
diff --git a/compiler/rustc_codegen_ssa/src/traits/write.rs b/compiler/rustc_codegen_ssa/src/traits/write.rs
index ecf5095d8..048540894 100644
--- a/compiler/rustc_codegen_ssa/src/traits/write.rs
+++ b/compiler/rustc_codegen_ssa/src/traits/write.rs
@@ -2,7 +2,7 @@ use crate::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule};
 use crate::back::write::{CodegenContext, FatLtoInput, ModuleConfig};
 use crate::{CompiledModule, ModuleCodegen};
 
-use rustc_errors::{FatalError, Handler};
+use rustc_errors::{DiagCtxt, FatalError};
 use rustc_middle::dep_graph::WorkProduct;
 
 pub trait WriteBackendMethods: 'static + Sized + Clone {
@@ -16,7 +16,7 @@ pub trait WriteBackendMethods: 'static + Sized + Clone {
     /// Merge all modules into main_module and returning it
     fn run_link(
         cgcx: &CodegenContext<Self>,
-        diag_handler: &Handler,
+        dcx: &DiagCtxt,
         modules: Vec<ModuleCodegen<Self::Module>>,
     ) -> Result<ModuleCodegen<Self::Module>, FatalError>;
     /// Performs fat LTO by merging all modules into a single one and returning it
@@ -38,7 +38,7 @@ pub trait WriteBackendMethods: 'static + Sized + Clone {
     fn print_statistics(&self);
     unsafe fn optimize(
         cgcx: &CodegenContext<Self>,
-        diag_handler: &Handler,
+        dcx: &DiagCtxt,
         module: &ModuleCodegen<Self::Module>,
         config: &ModuleConfig,
     ) -> Result<(), FatalError>;
@@ -52,7 +52,7 @@ pub trait WriteBackendMethods: 'static + Sized + Clone {
     ) -> Result<ModuleCodegen<Self::Module>, FatalError>;
     unsafe fn codegen(
         cgcx: &CodegenContext<Self>,
-        diag_handler: &Handler,
+        dcx: &DiagCtxt,
         module: ModuleCodegen<Self::Module>,
         config: &ModuleConfig,
     ) -> Result<CompiledModule, FatalError>;
diff --git a/compiler/rustc_const_eval/messages.ftl b/compiler/rustc_const_eval/messages.ftl
index f926da464..4a426ed16 100644
--- a/compiler/rustc_const_eval/messages.ftl
+++ b/compiler/rustc_const_eval/messages.ftl
@@ -461,6 +461,9 @@ const_eval_validation_uninhabited_val = {$front_matter}: encountered a value of
 const_eval_validation_uninit = {$front_matter}: encountered uninitialized memory, but {$expected}
 const_eval_validation_unsafe_cell = {$front_matter}: encountered `UnsafeCell` in a `const`
 
+const_eval_write_through_immutable_pointer =
+    writing through a pointer that was derived from a shared (immutable) reference
+
 const_eval_write_to_read_only =
     writing to {$allocation} which is read-only
 const_eval_zst_pointer_out_of_bounds =
diff --git a/compiler/rustc_const_eval/src/const_eval/error.rs b/compiler/rustc_const_eval/src/const_eval/error.rs
index bf1e0a370..8f18cd78d 100644
--- a/compiler/rustc_const_eval/src/const_eval/error.rs
+++ b/compiler/rustc_const_eval/src/const_eval/error.rs
@@ -1,14 +1,16 @@
 use std::mem;
 
 use rustc_errors::{DiagnosticArgValue, DiagnosticMessage, IntoDiagnostic, IntoDiagnosticArg};
+use rustc_hir::CRATE_HIR_ID;
 use rustc_middle::mir::AssertKind;
+use rustc_middle::query::TyCtxtAt;
 use rustc_middle::ty::TyCtxt;
 use rustc_middle::ty::{layout::LayoutError, ConstInt};
 use rustc_span::{ErrorGuaranteed, Span, Symbol, DUMMY_SP};
 
-use super::InterpCx;
+use super::{CompileTimeInterpreter, InterpCx};
 use crate::errors::{self, FrameNote, ReportErrorExt};
-use crate::interpret::{ErrorHandled, InterpError, InterpErrorInfo, Machine, MachineStopType};
+use crate::interpret::{ErrorHandled, InterpError, InterpErrorInfo, MachineStopType};
 
 /// The CTFE machine has some custom error kinds.
 #[derive(Clone, Debug)]
@@ -57,16 +59,20 @@ impl<'tcx> Into<InterpErrorInfo<'tcx>> for ConstEvalErrKind {
     }
 }
 
-pub fn get_span_and_frames<'tcx, 'mir, M: Machine<'mir, 'tcx>>(
-    ecx: &InterpCx<'mir, 'tcx, M>,
+pub fn get_span_and_frames<'tcx, 'mir>(
+    tcx: TyCtxtAt<'tcx>,
+    machine: &CompileTimeInterpreter<'mir, 'tcx>,
 ) -> (Span, Vec<errors::FrameNote>)
 where
     'tcx: 'mir,
 {
-    let mut stacktrace = ecx.generate_stacktrace();
+    let mut stacktrace =
+        InterpCx::<CompileTimeInterpreter<'mir, 'tcx>>::generate_stacktrace_from_stack(
+            &machine.stack,
+        );
     // Filter out `requires_caller_location` frames.
-    stacktrace.retain(|frame| !frame.instance.def.requires_caller_location(*ecx.tcx));
-    let span = stacktrace.first().map(|f| f.span).unwrap_or(ecx.tcx.span);
+    stacktrace.retain(|frame| !frame.instance.def.requires_caller_location(*tcx));
+    let span = stacktrace.first().map(|f| f.span).unwrap_or(tcx.span);
 
     let mut frames = Vec::new();
 
@@ -87,7 +93,7 @@ where
 
         let mut last_frame: Option<errors::FrameNote> = None;
         for frame_info in &stacktrace {
-            let frame = frame_info.as_note(*ecx.tcx);
+            let frame = frame_info.as_note(*tcx);
             match last_frame.as_mut() {
                 Some(last_frame)
                     if last_frame.span == frame.span
@@ -148,7 +154,7 @@ where
             let mut err = tcx.sess.create_err(err);
 
             let msg = error.diagnostic_message();
-            error.add_args(&tcx.sess.parse_sess.span_diagnostic, &mut err);
+            error.add_args(tcx.sess.dcx(), &mut err);
 
             // Use *our* span to label the interp error
             err.span_label(our_span, msg);
@@ -156,3 +162,25 @@ where
         }
     }
 }
+
+/// Emit a lint from a const-eval situation.
+// Even if this is unused, please don't remove it -- chances are we will need to emit a lint during const-eval again in the future!
+pub(super) fn lint<'tcx, 'mir, L>(
+    tcx: TyCtxtAt<'tcx>,
+    machine: &CompileTimeInterpreter<'mir, 'tcx>,
+    lint: &'static rustc_session::lint::Lint,
+    decorator: impl FnOnce(Vec<errors::FrameNote>) -> L,
+) where
+    L: for<'a> rustc_errors::DecorateLint<'a, ()>,
+{
+    let (span, frames) = get_span_and_frames(tcx, machine);
+
+    tcx.emit_spanned_lint(
+        lint,
+        // We use the root frame for this so the crate that defines the const defines whether the
+        // lint is emitted.
+        machine.stack.first().and_then(|frame| frame.lint_root()).unwrap_or(CRATE_HIR_ID),
+        span,
+        decorator(frames),
+    );
+}
diff --git a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs
index 13937a941..9d22df50d 100644
--- a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs
+++ b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs
@@ -155,8 +155,8 @@ pub(super) fn op_to_const<'tcx>(
     match immediate {
         Left(ref mplace) => {
             // We know `offset` is relative to the allocation, so we can use `into_parts`.
-            let (alloc_id, offset) = mplace.ptr().into_parts();
-            let alloc_id = alloc_id.expect("cannot have `fake` place fot non-ZST type");
+            let (prov, offset) = mplace.ptr().into_parts();
+            let alloc_id = prov.expect("cannot have `fake` place for non-ZST type").alloc_id();
             ConstValue::Indirect { alloc_id, offset }
         }
         // see comment on `let force_as_immediate` above
@@ -178,8 +178,8 @@ pub(super) fn op_to_const<'tcx>(
                 );
                 let msg = "`op_to_const` on an immediate scalar pair must only be used on slice references to the beginning of an actual allocation";
                 // We know `offset` is relative to the allocation, so we can use `into_parts`.
-                let (alloc_id, offset) = a.to_pointer(ecx).expect(msg).into_parts();
-                let alloc_id = alloc_id.expect(msg);
+                let (prov, offset) = a.to_pointer(ecx).expect(msg).into_parts();
+                let alloc_id = prov.expect(msg).alloc_id();
                 let data = ecx.tcx.global_alloc(alloc_id).unwrap_memory();
                 assert!(offset == abi::Size::ZERO, "{}", msg);
                 let meta = b.to_target_usize(ecx).expect(msg);
@@ -314,7 +314,7 @@ pub fn eval_in_interpreter<'mir, 'tcx>(
     is_static: bool,
 ) -> ::rustc_middle::mir::interpret::EvalToAllocationRawResult<'tcx> {
     let res = ecx.load_mir(cid.instance.def, cid.promoted);
-    match res.and_then(|body| eval_body_using_ecx(&mut ecx, cid, &body)) {
+    match res.and_then(|body| eval_body_using_ecx(&mut ecx, cid, body)) {
         Err(error) => {
             let (error, backtrace) = error.into_parts();
             backtrace.print_backtrace();
@@ -338,7 +338,7 @@ pub fn eval_in_interpreter<'mir, 'tcx>(
                 *ecx.tcx,
                 error,
                 None,
-                || super::get_span_and_frames(&ecx),
+                || super::get_span_and_frames(ecx.tcx, &ecx.machine),
                 |span, frames| ConstEvalError {
                     span,
                     error_kind: kind,
@@ -353,7 +353,7 @@ pub fn eval_in_interpreter<'mir, 'tcx>(
             let validation =
                 const_validate_mplace(&ecx, &mplace, is_static, cid.promoted.is_some());
 
-            let alloc_id = mplace.ptr().provenance.unwrap();
+            let alloc_id = mplace.ptr().provenance.unwrap().alloc_id();
 
             // Validation failed, report an error.
             if let Err(error) = validation {
@@ -419,7 +419,7 @@ pub fn const_report_error<'mir, 'tcx>(
         *ecx.tcx,
         error,
         None,
-        || crate::const_eval::get_span_and_frames(ecx),
+        || crate::const_eval::get_span_and_frames(ecx.tcx, &ecx.machine),
         move |span, frames| errors::UndefinedBehavior { span, ub_note, frames, raw_bytes },
     )
 }
diff --git a/compiler/rustc_const_eval/src/const_eval/fn_queries.rs b/compiler/rustc_const_eval/src/const_eval/fn_queries.rs
index 9e992637f..dbc29e607 100644
--- a/compiler/rustc_const_eval/src/const_eval/fn_queries.rs
+++ b/compiler/rustc_const_eval/src/const_eval/fn_queries.rs
@@ -33,7 +33,7 @@ pub fn is_parent_const_impl_raw(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
 /// return if it has a `const` modifier. If it is an intrinsic, report whether said intrinsic
 /// has a `rustc_const_{un,}stable` attribute. Otherwise, return `Constness::NotConst`.
 fn constness(tcx: TyCtxt<'_>, def_id: LocalDefId) -> hir::Constness {
-    let node = tcx.hir().get_by_def_id(def_id);
+    let node = tcx.hir_node_by_def_id(def_id);
 
     match node {
         hir::Node::Ctor(_)
diff --git a/compiler/rustc_const_eval/src/const_eval/machine.rs b/compiler/rustc_const_eval/src/const_eval/machine.rs
index 4b447229c..9aaf6c510 100644
--- a/compiler/rustc_const_eval/src/const_eval/machine.rs
+++ b/compiler/rustc_const_eval/src/const_eval/machine.rs
@@ -1,30 +1,30 @@
-use rustc_hir::def::DefKind;
-use rustc_hir::LangItem;
-use rustc_middle::mir;
-use rustc_middle::mir::interpret::PointerArithmetic;
-use rustc_middle::ty::layout::{FnAbiOf, TyAndLayout};
-use rustc_middle::ty::{self, TyCtxt};
-use rustc_span::Span;
 use std::borrow::Borrow;
+use std::fmt;
 use std::hash::Hash;
 use std::ops::ControlFlow;
 
+use rustc_ast::Mutability;
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_data_structures::fx::IndexEntry;
-use std::fmt;
-
-use rustc_ast::Mutability;
+use rustc_hir::def::DefKind;
 use rustc_hir::def_id::DefId;
+use rustc_hir::LangItem;
+use rustc_middle::mir;
 use rustc_middle::mir::AssertMessage;
+use rustc_middle::query::TyCtxtAt;
+use rustc_middle::ty;
+use rustc_middle::ty::layout::{FnAbiOf, TyAndLayout};
+use rustc_session::lint::builtin::WRITES_THROUGH_IMMUTABLE_POINTER;
 use rustc_span::symbol::{sym, Symbol};
+use rustc_span::Span;
 use rustc_target::abi::{Align, Size};
 use rustc_target::spec::abi::Abi as CallAbi;
 
 use crate::errors::{LongRunning, LongRunningWarn};
 use crate::fluent_generated as fluent;
 use crate::interpret::{
-    self, compile_time_machine, AllocId, ConstAllocation, FnArg, FnVal, Frame, ImmTy, InterpCx,
-    InterpResult, OpTy, PlaceTy, Pointer, Scalar,
+    self, compile_time_machine, AllocId, AllocRange, ConstAllocation, CtfeProvenance, FnArg, FnVal,
+    Frame, ImmTy, InterpCx, InterpResult, OpTy, PlaceTy, Pointer, PointerArithmetic, Scalar,
 };
 
 use super::error::*;
@@ -49,7 +49,7 @@ pub struct CompileTimeInterpreter<'mir, 'tcx> {
     pub(super) num_evaluated_steps: usize,
 
     /// The virtual call stack.
-    pub(super) stack: Vec<Frame<'mir, 'tcx, AllocId, ()>>,
+    pub(super) stack: Vec<Frame<'mir, 'tcx>>,
 
     /// We need to make sure consts never point to anything mutable, even recursively. That is
     /// relied on for pattern matching on consts with references.
@@ -108,6 +108,14 @@ impl<K: Hash + Eq, V> interpret::AllocMap<K, V> for FxIndexMap<K, V> {
     }
 
     #[inline(always)]
+    fn contains_key_ref<Q: ?Sized + Hash + Eq>(&self, k: &Q) -> bool
+    where
+        K: Borrow<Q>,
+    {
+        FxIndexMap::contains_key(self, k)
+    }
+
+    #[inline(always)]
     fn insert(&mut self, k: K, v: V) -> Option<V> {
         FxIndexMap::insert(self, k, v)
     }
@@ -192,7 +200,7 @@ impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> {
                 &caller
                     .file
                     .name
-                    .for_scope(&self.tcx.sess, RemapPathScopeComponents::DIAGNOSTICS)
+                    .for_scope(self.tcx.sess, RemapPathScopeComponents::DIAGNOSTICS)
                     .to_string_lossy(),
             ),
             u32::try_from(caller.line).unwrap(),
@@ -383,7 +391,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
                 if ecx.tcx.is_ctfe_mir_available(def) {
                     Ok(ecx.tcx.mir_for_ctfe(def))
                 } else if ecx.tcx.def_kind(def) == DefKind::AssocConst {
-                    let guar = ecx.tcx.sess.delay_span_bug(
+                    let guar = ecx.tcx.sess.span_delayed_bug(
                         rustc_span::DUMMY_SP,
                         "This is likely a const item that is missing from its impl",
                     );
@@ -485,7 +493,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
                 };
 
                 let ptr = ecx.allocate_ptr(
-                    Size::from_bytes(size as u64),
+                    Size::from_bytes(size),
                     align,
                     interpret::MemoryKind::Machine(MemoryKind::Heap),
                 )?;
@@ -614,7 +622,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
                     let guard = ecx
                         .tcx
                         .sess
-                        .delay_span_bug(span, "The deny lint should have already errored");
+                        .span_delayed_bug(span, "The deny lint should have already errored");
                     throw_inval!(AlreadyReported(guard.into()));
                 }
             } else if new_steps > start && new_steps.is_power_of_two() {
@@ -630,10 +638,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
     }
 
     #[inline(always)]
-    fn expose_ptr(
-        _ecx: &mut InterpCx<'mir, 'tcx, Self>,
-        _ptr: Pointer<AllocId>,
-    ) -> InterpResult<'tcx> {
+    fn expose_ptr(_ecx: &mut InterpCx<'mir, 'tcx, Self>, _ptr: Pointer) -> InterpResult<'tcx> {
         // This is only reachable with -Zunleash-the-miri-inside-of-you.
         throw_unsup_format!("exposing pointers is not possible at compile-time")
     }
@@ -666,7 +671,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
     }
 
     fn before_access_global(
-        _tcx: TyCtxt<'tcx>,
+        _tcx: TyCtxtAt<'tcx>,
         machine: &Self,
         alloc_id: AllocId,
         alloc: ConstAllocation<'tcx>,
@@ -703,6 +708,48 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
             }
         }
     }
+
+    fn retag_ptr_value(
+        ecx: &mut InterpCx<'mir, 'tcx, Self>,
+        _kind: mir::RetagKind,
+        val: &ImmTy<'tcx, CtfeProvenance>,
+    ) -> InterpResult<'tcx, ImmTy<'tcx, CtfeProvenance>> {
+        // If it's a frozen shared reference that's not already immutable, make it immutable.
+        // (Do nothing on `None` provenance, that cannot store immutability anyway.)
+        if let ty::Ref(_, ty, mutbl) = val.layout.ty.kind()
+            && *mutbl == Mutability::Not
+            && val.to_scalar_and_meta().0.to_pointer(ecx)?.provenance.is_some_and(|p| !p.immutable())
+            // That next check is expensive, that's why we have all the guards above.
+            && ty.is_freeze(*ecx.tcx, ecx.param_env)
+        {
+            let place = ecx.ref_to_mplace(val)?;
+            let new_place = place.map_provenance(|p| p.map(CtfeProvenance::as_immutable));
+            Ok(ImmTy::from_immediate(new_place.to_ref(ecx), val.layout))
+        } else {
+            Ok(val.clone())
+        }
+    }
+
+    fn before_memory_write(
+        tcx: TyCtxtAt<'tcx>,
+        machine: &mut Self,
+        _alloc_extra: &mut Self::AllocExtra,
+        (_alloc_id, immutable): (AllocId, bool),
+        range: AllocRange,
+    ) -> InterpResult<'tcx> {
+        if range.size == Size::ZERO {
+            // Nothing to check.
+            return Ok(());
+        }
+        // Reject writes through immutable pointers.
+        if immutable {
+            super::lint(tcx, machine, WRITES_THROUGH_IMMUTABLE_POINTER, |frames| {
+                crate::errors::WriteThroughImmutablePointer { frames }
+            });
+        }
+        // Everything else is fine.
+        Ok(())
+    }
 }
 
 // Please do not add any code below the above `Machine` trait impl. I (oli-obk) plan more cleanups
diff --git a/compiler/rustc_const_eval/src/const_eval/valtrees.rs b/compiler/rustc_const_eval/src/const_eval/valtrees.rs
index ed2d81727..854fe9a07 100644
--- a/compiler/rustc_const_eval/src/const_eval/valtrees.rs
+++ b/compiler/rustc_const_eval/src/const_eval/valtrees.rs
@@ -11,7 +11,7 @@ use rustc_middle::mir;
 use rustc_middle::ty::layout::{LayoutCx, LayoutOf, TyAndLayout};
 use rustc_middle::ty::{self, ScalarInt, Ty, TyCtxt};
 use rustc_span::DUMMY_SP;
-use rustc_target::abi::VariantIdx;
+use rustc_target::abi::{Abi, VariantIdx};
 
 #[instrument(skip(ecx), level = "debug")]
 fn branches<'tcx>(
@@ -101,11 +101,16 @@ pub(crate) fn const_to_valtree_inner<'tcx>(
             // Not all raw pointers are allowed, as we cannot properly test them for
             // equality at compile-time (see `ptr_guaranteed_cmp`).
             // However we allow those that are just integers in disguise.
-            // (We could allow wide raw pointers where both sides are integers in the future,
-            // but for now we reject them.)
-            let Ok(val) = ecx.read_scalar(place) else {
+            // First, get the pointer. Remember it might be wide!
+            let Ok(val) = ecx.read_immediate(place) else {
                 return Err(ValTreeCreationError::Other);
             };
+            // We could allow wide raw pointers where both sides are integers in the future,
+            // but for now we reject them.
+            if matches!(val.layout.abi, Abi::ScalarPair(..)) {
+                return Err(ValTreeCreationError::Other);
+            }
+            let val = val.to_scalar();
             // We are in the CTFE machine, so ptr-to-int casts will fail.
             // This can only be `Ok` if `val` already is an integer.
             let Ok(val) = val.try_to_int() else {
@@ -382,7 +387,7 @@ fn valtree_into_mplace<'tcx>(
 
                 debug!(?place_inner);
                 valtree_into_mplace(ecx, &place_inner, *inner_valtree);
-                dump_place(&ecx, &place_inner);
+                dump_place(ecx, &place_inner);
             }
 
             debug!("dump of place_adjusted:");
diff --git a/compiler/rustc_const_eval/src/errors.rs b/compiler/rustc_const_eval/src/errors.rs
index cc8f33872..adce1f543 100644
--- a/compiler/rustc_const_eval/src/errors.rs
+++ b/compiler/rustc_const_eval/src/errors.rs
@@ -1,5 +1,5 @@
 use rustc_errors::{
-    DiagnosticArgValue, DiagnosticBuilder, DiagnosticMessage, EmissionGuarantee, Handler,
+    DiagCtxt, DiagnosticArgValue, DiagnosticBuilder, DiagnosticMessage, EmissionGuarantee,
     IntoDiagnostic,
 };
 use rustc_hir::ConstContext;
@@ -402,6 +402,13 @@ pub struct ConstEvalError {
     pub frame_notes: Vec<FrameNote>,
 }
 
+#[derive(LintDiagnostic)]
+#[diag(const_eval_write_through_immutable_pointer)]
+pub struct WriteThroughImmutablePointer {
+    #[subdiagnostic]
+    pub frames: Vec<FrameNote>,
+}
+
 #[derive(Diagnostic)]
 #[diag(const_eval_nullary_intrinsic_fail)]
 pub struct NullaryIntrinsicError {
@@ -425,11 +432,7 @@ pub struct UndefinedBehavior {
 pub trait ReportErrorExt {
     /// Returns the diagnostic message for this error.
     fn diagnostic_message(&self) -> DiagnosticMessage;
-    fn add_args<G: EmissionGuarantee>(
-        self,
-        handler: &Handler,
-        builder: &mut DiagnosticBuilder<'_, G>,
-    );
+    fn add_args<G: EmissionGuarantee>(self, dcx: &DiagCtxt, builder: &mut DiagnosticBuilder<'_, G>);
 
     fn debug(self) -> String
     where
@@ -437,17 +440,17 @@ pub trait ReportErrorExt {
     {
         ty::tls::with(move |tcx| {
             let mut builder = tcx.sess.struct_allow(DiagnosticMessage::Str(String::new().into()));
-            let handler = &tcx.sess.parse_sess.span_diagnostic;
+            let dcx = tcx.sess.dcx();
             let message = self.diagnostic_message();
-            self.add_args(handler, &mut builder);
-            let s = handler.eagerly_translate_to_string(message, builder.args());
+            self.add_args(dcx, &mut builder);
+            let s = dcx.eagerly_translate_to_string(message, builder.args());
             builder.cancel();
             s
         })
     }
 }
 
-fn bad_pointer_message(msg: CheckInAllocMsg, handler: &Handler) -> String {
+fn bad_pointer_message(msg: CheckInAllocMsg, dcx: &DiagCtxt) -> String {
     use crate::fluent_generated::*;
 
     let msg = match msg {
@@ -457,7 +460,7 @@ fn bad_pointer_message(msg: CheckInAllocMsg, handler: &Handler) -> String {
         CheckInAllocMsg::InboundsTest => const_eval_in_bounds_test,
     };
 
-    handler.eagerly_translate_to_string(msg, [].into_iter())
+    dcx.eagerly_translate_to_string(msg, [].into_iter())
 }
 
 impl<'a> ReportErrorExt for UndefinedBehaviorInfo<'a> {
@@ -507,7 +510,7 @@ impl<'a> ReportErrorExt for UndefinedBehaviorInfo<'a> {
 
     fn add_args<G: EmissionGuarantee>(
         self,
-        handler: &Handler,
+        dcx: &DiagCtxt,
         builder: &mut DiagnosticBuilder<'_, G>,
     ) {
         use UndefinedBehaviorInfo::*;
@@ -518,7 +521,7 @@ impl<'a> ReportErrorExt for UndefinedBehaviorInfo<'a> {
                     builder.set_arg(name, value);
                 });
             }
-            ValidationError(e) => e.add_args(handler, builder),
+            ValidationError(e) => e.add_args(dcx, builder),
 
             Unreachable
             | DivisionByZero
@@ -542,7 +545,7 @@ impl<'a> ReportErrorExt for UndefinedBehaviorInfo<'a> {
             PointerUseAfterFree(alloc_id, msg) => {
                 builder
                     .set_arg("alloc_id", alloc_id)
-                    .set_arg("bad_pointer_message", bad_pointer_message(msg, handler));
+                    .set_arg("bad_pointer_message", bad_pointer_message(msg, dcx));
             }
             PointerOutOfBounds { alloc_id, alloc_size, ptr_offset, ptr_size, msg } => {
                 builder
@@ -550,14 +553,14 @@ impl<'a> ReportErrorExt for UndefinedBehaviorInfo<'a> {
                     .set_arg("alloc_size", alloc_size.bytes())
                     .set_arg("ptr_offset", ptr_offset)
                     .set_arg("ptr_size", ptr_size.bytes())
-                    .set_arg("bad_pointer_message", bad_pointer_message(msg, handler));
+                    .set_arg("bad_pointer_message", bad_pointer_message(msg, dcx));
             }
             DanglingIntPointer(ptr, msg) => {
                 if ptr != 0 {
                     builder.set_arg("pointer", format!("{ptr:#x}[noalloc]"));
                 }
 
-                builder.set_arg("bad_pointer_message", bad_pointer_message(msg, handler));
+                builder.set_arg("bad_pointer_message", bad_pointer_message(msg, dcx));
             }
             AlignmentCheckFailed(Misalignment { required, has }, msg) => {
                 builder.set_arg("required", required.bytes());
@@ -671,7 +674,7 @@ impl<'tcx> ReportErrorExt for ValidationErrorInfo<'tcx> {
         }
     }
 
-    fn add_args<G: EmissionGuarantee>(self, handler: &Handler, err: &mut DiagnosticBuilder<'_, G>) {
+    fn add_args<G: EmissionGuarantee>(self, dcx: &DiagCtxt, err: &mut DiagnosticBuilder<'_, G>) {
         use crate::fluent_generated as fluent;
         use rustc_middle::mir::interpret::ValidationErrorKind::*;
 
@@ -681,12 +684,12 @@ impl<'tcx> ReportErrorExt for ValidationErrorInfo<'tcx> {
         }
 
         let message = if let Some(path) = self.path {
-            handler.eagerly_translate_to_string(
+            dcx.eagerly_translate_to_string(
                 fluent::const_eval_validation_front_matter_invalid_value_with_path,
                 [("path".into(), DiagnosticArgValue::Str(path.into()))].iter().map(|(a, b)| (a, b)),
             )
         } else {
-            handler.eagerly_translate_to_string(
+            dcx.eagerly_translate_to_string(
                 fluent::const_eval_validation_front_matter_invalid_value,
                 [].into_iter(),
             )
@@ -697,7 +700,7 @@ impl<'tcx> ReportErrorExt for ValidationErrorInfo<'tcx> {
         fn add_range_arg<G: EmissionGuarantee>(
             r: WrappingRange,
             max_hi: u128,
-            handler: &Handler,
+            dcx: &DiagCtxt,
             err: &mut DiagnosticBuilder<'_, G>,
         ) {
             let WrappingRange { start: lo, end: hi } = r;
@@ -721,7 +724,7 @@ impl<'tcx> ReportErrorExt for ValidationErrorInfo<'tcx> {
                 ("hi".into(), DiagnosticArgValue::Str(hi.to_string().into())),
             ];
             let args = args.iter().map(|(a, b)| (a, b));
-            let message = handler.eagerly_translate_to_string(msg, args);
+            let message = dcx.eagerly_translate_to_string(msg, args);
             err.set_arg("in_range", message);
         }
 
@@ -743,7 +746,7 @@ impl<'tcx> ReportErrorExt for ValidationErrorInfo<'tcx> {
                     ExpectedKind::EnumTag => fluent::const_eval_validation_expected_enum_tag,
                     ExpectedKind::Str => fluent::const_eval_validation_expected_str,
                 };
-                let msg = handler.eagerly_translate_to_string(msg, [].into_iter());
+                let msg = dcx.eagerly_translate_to_string(msg, [].into_iter());
                 err.set_arg("expected", msg);
             }
             InvalidEnumTag { value }
@@ -754,11 +757,11 @@ impl<'tcx> ReportErrorExt for ValidationErrorInfo<'tcx> {
                 err.set_arg("value", value);
             }
             NullablePtrOutOfRange { range, max_value } | PtrOutOfRange { range, max_value } => {
-                add_range_arg(range, max_value, handler, err)
+                add_range_arg(range, max_value, dcx, err)
             }
             OutOfRange { range, max_value, value } => {
                 err.set_arg("value", value);
-                add_range_arg(range, max_value, handler, err);
+                add_range_arg(range, max_value, dcx, err);
             }
             UnalignedPtr { required_bytes, found_bytes, .. } => {
                 err.set_arg("required_bytes", required_bytes);
@@ -797,7 +800,7 @@ impl ReportErrorExt for UnsupportedOpInfo {
             UnsupportedOpInfo::ReadExternStatic(_) => const_eval_read_extern_static,
         }
     }
-    fn add_args<G: EmissionGuarantee>(self, _: &Handler, builder: &mut DiagnosticBuilder<'_, G>) {
+    fn add_args<G: EmissionGuarantee>(self, _: &DiagCtxt, builder: &mut DiagnosticBuilder<'_, G>) {
         use crate::fluent_generated::*;
 
         use UnsupportedOpInfo::*;
@@ -832,14 +835,14 @@ impl<'tcx> ReportErrorExt for InterpError<'tcx> {
     }
     fn add_args<G: EmissionGuarantee>(
         self,
-        handler: &Handler,
+        dcx: &DiagCtxt,
         builder: &mut DiagnosticBuilder<'_, G>,
     ) {
         match self {
-            InterpError::UndefinedBehavior(ub) => ub.add_args(handler, builder),
-            InterpError::Unsupported(e) => e.add_args(handler, builder),
-            InterpError::InvalidProgram(e) => e.add_args(handler, builder),
-            InterpError::ResourceExhaustion(e) => e.add_args(handler, builder),
+            InterpError::UndefinedBehavior(ub) => ub.add_args(dcx, builder),
+            InterpError::Unsupported(e) => e.add_args(dcx, builder),
+            InterpError::InvalidProgram(e) => e.add_args(dcx, builder),
+            InterpError::ResourceExhaustion(e) => e.add_args(dcx, builder),
             InterpError::MachineStop(e) => e.add_args(&mut |name, value| {
                 builder.set_arg(name, value);
             }),
@@ -864,7 +867,7 @@ impl<'tcx> ReportErrorExt for InvalidProgramInfo<'tcx> {
     }
     fn add_args<G: EmissionGuarantee>(
         self,
-        handler: &Handler,
+        dcx: &DiagCtxt,
         builder: &mut DiagnosticBuilder<'_, G>,
     ) {
         match self {
@@ -872,7 +875,7 @@ impl<'tcx> ReportErrorExt for InvalidProgramInfo<'tcx> {
             | InvalidProgramInfo::AlreadyReported(_)
             | InvalidProgramInfo::ConstPropNonsense => {}
             InvalidProgramInfo::Layout(e) => {
-                let diag: DiagnosticBuilder<'_, ()> = e.into_diagnostic().into_diagnostic(handler);
+                let diag: DiagnosticBuilder<'_, ()> = e.into_diagnostic().into_diagnostic(dcx);
                 for (name, val) in diag.args() {
                     builder.set_arg(name.clone(), val.clone());
                 }
@@ -897,5 +900,5 @@ impl ReportErrorExt for ResourceExhaustionInfo {
             ResourceExhaustionInfo::AddressSpaceFull => const_eval_address_space_full,
         }
     }
-    fn add_args<G: EmissionGuarantee>(self, _: &Handler, _: &mut DiagnosticBuilder<'_, G>) {}
+    fn add_args<G: EmissionGuarantee>(self, _: &DiagCtxt, _: &mut DiagnosticBuilder<'_, G>) {}
 }
diff --git a/compiler/rustc_const_eval/src/interpret/cast.rs b/compiler/rustc_const_eval/src/interpret/cast.rs
index f4cb12c8d..d296ff592 100644
--- a/compiler/rustc_const_eval/src/interpret/cast.rs
+++ b/compiler/rustc_const_eval/src/interpret/cast.rs
@@ -256,7 +256,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         let addr = addr.to_target_usize(self)?;
 
         // Then turn address into pointer.
-        let ptr = M::ptr_from_addr_cast(&self, addr)?;
+        let ptr = M::ptr_from_addr_cast(self, addr)?;
         Ok(ImmTy::from_scalar(Scalar::from_maybe_pointer(ptr, self), cast_to))
     }
 
diff --git a/compiler/rustc_const_eval/src/interpret/discriminant.rs b/compiler/rustc_const_eval/src/interpret/discriminant.rs
index fd1736703..d9f583c1d 100644
--- a/compiler/rustc_const_eval/src/interpret/discriminant.rs
+++ b/compiler/rustc_const_eval/src/interpret/discriminant.rs
@@ -119,7 +119,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                     if matches!(ty.kind(), ty::Adt(def, ..) if def.variants().is_empty()) {
                         throw_ub!(UninhabitedEnumVariantRead(index))
                     }
-                    // For consisteny with `write_discriminant`, and to make sure that
+                    // For consistency with `write_discriminant`, and to make sure that
                     // `project_downcast` cannot fail due to strange layouts, we declare immediate UB
                     // for uninhabited variants.
                     if op.layout().for_variant(self, index).abi.is_uninhabited() {
@@ -236,7 +236,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 variant
             }
         };
-        // For consisteny with `write_discriminant`, and to make sure that `project_downcast` cannot fail due to strange layouts, we declare immediate UB for uninhabited variants.
+        // For consistency with `write_discriminant`, and to make sure that `project_downcast` cannot fail due to strange layouts, we declare immediate UB for uninhabited variants.
         if op.layout().for_variant(self, index).abi.is_uninhabited() {
             throw_ub!(UninhabitedEnumVariantRead(index))
         }
diff --git a/compiler/rustc_const_eval/src/interpret/eval_context.rs b/compiler/rustc_const_eval/src/interpret/eval_context.rs
index 07cab5e34..af8e5e7d1 100644
--- a/compiler/rustc_const_eval/src/interpret/eval_context.rs
+++ b/compiler/rustc_const_eval/src/interpret/eval_context.rs
@@ -7,7 +7,9 @@ use hir::CRATE_HIR_ID;
 use rustc_hir::{self as hir, def_id::DefId, definitions::DefPathData};
 use rustc_index::IndexVec;
 use rustc_middle::mir;
-use rustc_middle::mir::interpret::{ErrorHandled, InvalidMetaKind, ReportedErrorInfo};
+use rustc_middle::mir::interpret::{
+    CtfeProvenance, ErrorHandled, InvalidMetaKind, ReportedErrorInfo,
+};
 use rustc_middle::query::TyCtxtAt;
 use rustc_middle::ty::layout::{
     self, FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOf, LayoutOfHelpers,
@@ -20,9 +22,9 @@ use rustc_span::Span;
 use rustc_target::abi::{call::FnAbi, Align, HasDataLayout, Size, TargetDataLayout};
 
 use super::{
-    AllocId, GlobalId, Immediate, InterpErrorInfo, InterpResult, MPlaceTy, Machine, MemPlace,
-    MemPlaceMeta, Memory, MemoryKind, OpTy, Operand, Place, PlaceTy, Pointer, PointerArithmetic,
-    Projectable, Provenance, Scalar, StackPopJump,
+    GlobalId, Immediate, InterpErrorInfo, InterpResult, MPlaceTy, Machine, MemPlace, MemPlaceMeta,
+    Memory, MemoryKind, OpTy, Operand, Place, PlaceTy, Pointer, PointerArithmetic, Projectable,
+    Provenance, Scalar, StackPopJump,
 };
 use crate::errors;
 use crate::util;
@@ -84,7 +86,7 @@ impl Drop for SpanGuard {
 }
 
 /// A stack frame.
-pub struct Frame<'mir, 'tcx, Prov: Provenance = AllocId, Extra = ()> {
+pub struct Frame<'mir, 'tcx, Prov: Provenance = CtfeProvenance, Extra = ()> {
     ////////////////////////////////////////////////////////////////////////////////
     // Function and callsite information
     ////////////////////////////////////////////////////////////////////////////////
@@ -156,7 +158,7 @@ pub enum StackPopCleanup {
 
 /// State of a local variable including a memoized layout
 #[derive(Clone)]
-pub struct LocalState<'tcx, Prov: Provenance = AllocId> {
+pub struct LocalState<'tcx, Prov: Provenance = CtfeProvenance> {
     value: LocalValue<Prov>,
     /// Don't modify if `Some`, this is only used to prevent computing the layout twice.
     /// Avoids computing the layout of locals that are never actually initialized.
@@ -173,8 +175,11 @@ impl<Prov: Provenance> std::fmt::Debug for LocalState<'_, Prov> {
 }
 
 /// Current value of a local variable
+///
+/// This does not store the type of the local; the type is given by `body.local_decls` and can never
+/// change, so by not storing here we avoid having to maintain that as an invariant.
 #[derive(Copy, Clone, Debug)] // Miri debug-prints these
-pub(super) enum LocalValue<Prov: Provenance = AllocId> {
+pub(super) enum LocalValue<Prov: Provenance = CtfeProvenance> {
     /// This local is not currently alive, and cannot be used at all.
     Dead,
     /// A normal, live local.
@@ -279,14 +284,12 @@ impl<'mir, 'tcx, Prov: Provenance, Extra> Frame<'mir, 'tcx, Prov, Extra> {
 impl<'tcx> fmt::Display for FrameInfo<'tcx> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         ty::tls::with(|tcx| {
-            if tcx.def_key(self.instance.def_id()).disambiguated_data.data
-                == DefPathData::ClosureExpr
-            {
+            if tcx.def_key(self.instance.def_id()).disambiguated_data.data == DefPathData::Closure {
                 write!(f, "inside closure")
             } else {
-                // Note: this triggers a `good_path_bug` state, which means that if we ever get here
-                // we must emit a diagnostic. We should never display a `FrameInfo` unless we
-                // actually want to emit a warning or error to the user.
+                // Note: this triggers a `good_path_delayed_bug` state, which means that if we ever
+                // get here we must emit a diagnostic. We should never display a `FrameInfo` unless
+                // we actually want to emit a warning or error to the user.
                 write!(f, "inside `{}`", self.instance)
             }
         })
@@ -296,12 +299,12 @@ impl<'tcx> fmt::Display for FrameInfo<'tcx> {
 impl<'tcx> FrameInfo<'tcx> {
     pub fn as_note(&self, tcx: TyCtxt<'tcx>) -> errors::FrameNote {
         let span = self.span;
-        if tcx.def_key(self.instance.def_id()).disambiguated_data.data == DefPathData::ClosureExpr {
+        if tcx.def_key(self.instance.def_id()).disambiguated_data.data == DefPathData::Closure {
             errors::FrameNote { where_: "closure", span, instance: String::new(), times: 0 }
         } else {
             let instance = format!("{}", self.instance);
-            // Note: this triggers a `good_path_bug` state, which means that if we ever get here
-            // we must emit a diagnostic. We should never display a `FrameInfo` unless we
+            // Note: this triggers a `good_path_delayed_bug` state, which means that if we ever get
+            // here we must emit a diagnostic. We should never display a `FrameInfo` unless we
             // actually want to emit a warning or error to the user.
             errors::FrameNote { where_: "instance", span, instance, times: 0 }
         }
@@ -456,7 +459,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         self.stack()
             .iter()
             .find_map(|frame| frame.body.source.def_id().as_local())
-            .map_or(CRATE_HIR_ID, |def_id| self.tcx.hir().local_def_id_to_hir_id(def_id))
+            .map_or(CRATE_HIR_ID, |def_id| self.tcx.local_def_id_to_hir_id(def_id))
     }
 
     /// Turn the given error into a human-readable string. Expects the string to be printed, so if
@@ -470,12 +473,12 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         backtrace.print_backtrace();
         // FIXME(fee1-dead), HACK: we want to use the error as title therefore we can just extract the
         // label and arguments from the InterpError.
-        let handler = &self.tcx.sess.parse_sess.span_diagnostic;
+        let dcx = self.tcx.sess.dcx();
         #[allow(rustc::untranslatable_diagnostic)]
         let mut diag = self.tcx.sess.struct_allow("");
         let msg = e.diagnostic_message();
-        e.add_args(handler, &mut diag);
-        let s = handler.eagerly_translate_to_string(msg, diag.args());
+        e.add_args(dcx, &mut diag);
+        let s = dcx.eagerly_translate_to_string(msg, diag.args());
         diag.cancel();
         s
     }
@@ -683,14 +686,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 assert!(layout.fields.count() > 0);
                 trace!("DST layout: {:?}", layout);
 
-                let sized_size = layout.fields.offset(layout.fields.count() - 1);
+                let unsized_offset_unadjusted = layout.fields.offset(layout.fields.count() - 1);
                 let sized_align = layout.align.abi;
-                trace!(
-                    "DST {} statically sized prefix size: {:?} align: {:?}",
-                    layout.ty,
-                    sized_size,
-                    sized_align
-                );
 
                 // Recurse to get the size of the dynamically sized field (must be
                 // the last field). Can't have foreign types here, how would we
@@ -704,36 +701,35 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                     return Ok(None);
                 };
 
-                // FIXME (#26403, #27023): We should be adding padding
-                // to `sized_size` (to accommodate the `unsized_align`
-                // required of the unsized field that follows) before
-                // summing it with `sized_size`. (Note that since #26403
-                // is unfixed, we do not yet add the necessary padding
-                // here. But this is where the add would go.)
-
-                // Return the sum of sizes and max of aligns.
-                let size = sized_size + unsized_size; // `Size` addition
+                // # First compute the dynamic alignment
 
-                // Packed types ignore the alignment of their fields.
+                // Packed type alignment needs to be capped.
                 if let ty::Adt(def, _) = layout.ty.kind() {
-                    if def.repr().packed() {
-                        unsized_align = sized_align;
+                    if let Some(packed) = def.repr().pack {
+                        unsized_align = unsized_align.min(packed);
                     }
                 }
 
                 // Choose max of two known alignments (combined value must
                 // be aligned according to more restrictive of the two).
-                let align = sized_align.max(unsized_align);
+                let full_align = sized_align.max(unsized_align);
 
-                // Issue #27023: must add any necessary padding to `size`
-                // (to make it a multiple of `align`) before returning it.
-                let size = size.align_to(align);
+                // # Then compute the dynamic size
+
+                let unsized_offset_adjusted = unsized_offset_unadjusted.align_to(unsized_align);
+                let full_size = (unsized_offset_adjusted + unsized_size).align_to(full_align);
+
+                // Just for our sanitiy's sake, assert that this is equal to what codegen would compute.
+                assert_eq!(
+                    full_size,
+                    (unsized_offset_unadjusted + unsized_size).align_to(full_align)
+                );
 
                 // Check if this brought us over the size limit.
-                if size > self.max_size_of_val() {
+                if full_size > self.max_size_of_val() {
                     throw_ub!(InvalidMeta(InvalidMetaKind::TooBig));
                 }
-                Ok(Some((size, align)))
+                Ok(Some((full_size, full_align)))
             }
             ty::Dynamic(_, _, ty::Dyn) => {
                 let vtable = metadata.unwrap_meta().to_pointer(self)?;
diff --git a/compiler/rustc_const_eval/src/interpret/intern.rs b/compiler/rustc_const_eval/src/interpret/intern.rs
index 3d90e95c0..7931789e4 100644
--- a/compiler/rustc_const_eval/src/interpret/intern.rs
+++ b/compiler/rustc_const_eval/src/interpret/intern.rs
@@ -18,7 +18,7 @@ use super::validity::RefTracking;
 use rustc_data_structures::fx::{FxIndexMap, FxIndexSet};
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir as hir;
-use rustc_middle::mir::interpret::InterpResult;
+use rustc_middle::mir::interpret::{CtfeProvenance, InterpResult};
 use rustc_middle::ty::{self, layout::TyAndLayout, Ty};
 
 use rustc_ast::Mutability;
@@ -34,7 +34,7 @@ pub trait CompileTimeMachine<'mir, 'tcx: 'mir, T> = Machine<
         'mir,
         'tcx,
         MemoryKind = T,
-        Provenance = AllocId,
+        Provenance = CtfeProvenance,
         ExtraFnVal = !,
         FrameExtra = (),
         AllocExtra = (),
@@ -94,9 +94,9 @@ fn intern_shallow<'rt, 'mir, 'tcx, M: CompileTimeMachine<'mir, 'tcx, const_eval:
         // If the pointer is dangling (neither in local nor global memory), we leave it
         // to validation to error -- it has the much better error messages, pointing out where
         // in the value the dangling reference lies.
-        // The `delay_span_bug` ensures that we don't forget such a check in validation.
+        // The `span_delayed_bug` ensures that we don't forget such a check in validation.
         if tcx.try_get_global_alloc(alloc_id).is_none() {
-            tcx.sess.delay_span_bug(ecx.tcx.span, "tried to intern dangling pointer");
+            tcx.sess.span_delayed_bug(ecx.tcx.span, "tried to intern dangling pointer");
         }
         // treat dangling pointers like other statics
         // just to stop trying to recurse into them
@@ -135,7 +135,7 @@ fn intern_shallow<'rt, 'mir, 'tcx, M: CompileTimeMachine<'mir, 'tcx, const_eval:
         alloc.mutability = Mutability::Not;
     };
     // link the alloc id to the actual allocation
-    leftover_allocations.extend(alloc.provenance().ptrs().iter().map(|&(_, alloc_id)| alloc_id));
+    leftover_allocations.extend(alloc.provenance().ptrs().iter().map(|&(_, prov)| prov.alloc_id()));
     let alloc = tcx.mk_const_alloc(alloc);
     tcx.set_alloc_id_memory(alloc_id, alloc);
     None
@@ -178,20 +178,20 @@ impl<'rt, 'mir, 'tcx: 'mir, M: CompileTimeMachine<'mir, 'tcx, const_eval::Memory
                 tcx.struct_tail_erasing_lifetimes(referenced_ty, self.ecx.param_env).kind()
             {
                 let ptr = mplace.meta().unwrap_meta().to_pointer(&tcx)?;
-                if let Some(alloc_id) = ptr.provenance {
+                if let Some(prov) = ptr.provenance {
                     // Explicitly choose const mode here, since vtables are immutable, even
                     // if the reference of the fat pointer is mutable.
-                    self.intern_shallow(alloc_id, InternMode::Const, None);
+                    self.intern_shallow(prov.alloc_id(), InternMode::Const, None);
                 } else {
                     // Validation will error (with a better message) on an invalid vtable pointer.
                     // Let validation show the error message, but make sure it *does* error.
                     tcx.sess
-                        .delay_span_bug(tcx.span, "vtables pointers cannot be integer pointers");
+                        .span_delayed_bug(tcx.span, "vtables pointers cannot be integer pointers");
                 }
             }
             // Check if we have encountered this pointer+layout combination before.
             // Only recurse for allocation-backed pointers.
-            if let Some(alloc_id) = mplace.ptr().provenance {
+            if let Some(prov) = mplace.ptr().provenance {
                 // Compute the mode with which we intern this. Our goal here is to make as many
                 // statics as we can immutable so they can be placed in read-only memory by LLVM.
                 let ref_mode = match self.mode {
@@ -234,7 +234,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: CompileTimeMachine<'mir, 'tcx, const_eval::Memory
                         InternMode::Const
                     }
                 };
-                match self.intern_shallow(alloc_id, ref_mode, Some(referenced_ty)) {
+                match self.intern_shallow(prov.alloc_id(), ref_mode, Some(referenced_ty)) {
                     // No need to recurse, these are interned already and statics may have
                     // cycles, so we don't want to recurse there
                     Some(IsStaticOrFn) => {}
@@ -259,7 +259,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: CompileTimeMachine<'mir, 'tcx, const_eval::Memory
                 // to avoid could be expensive: on the potentially larger types, arrays and slices,
                 // rather than on all aggregates unconditionally.
                 if matches!(mplace.layout.ty.kind(), ty::Array(..) | ty::Slice(..)) {
-                    let Some((size, _align)) = self.ecx.size_and_align_of_mplace(&mplace)? else {
+                    let Some((size, _align)) = self.ecx.size_and_align_of_mplace(mplace)? else {
                         // We do the walk if we can't determine the size of the mplace: we may be
                         // dealing with extern types here in the future.
                         return Ok(true);
@@ -353,7 +353,7 @@ pub fn intern_const_alloc_recursive<
         leftover_allocations,
         // The outermost allocation must exist, because we allocated it with
         // `Memory::allocate`.
-        ret.ptr().provenance.unwrap(),
+        ret.ptr().provenance.unwrap().alloc_id(),
         base_intern_mode,
         Some(ret.layout.ty),
     );
@@ -375,7 +375,7 @@ pub fn intern_const_alloc_recursive<
         match res {
             Ok(()) => {}
             Err(error) => {
-                ecx.tcx.sess.delay_span_bug(
+                ecx.tcx.sess.span_delayed_bug(
                     ecx.tcx.span,
                     format!(
                         "error during interning should later cause validation failure: {}",
@@ -431,7 +431,8 @@ pub fn intern_const_alloc_recursive<
             }
             let alloc = tcx.mk_const_alloc(alloc);
             tcx.set_alloc_id_memory(alloc_id, alloc);
-            for &(_, alloc_id) in alloc.inner().provenance().ptrs().iter() {
+            for &(_, prov) in alloc.inner().provenance().ptrs().iter() {
+                let alloc_id = prov.alloc_id();
                 if leftover_allocations.insert(alloc_id) {
                     todo.push(alloc_id);
                 }
@@ -503,10 +504,11 @@ impl<'mir, 'tcx: 'mir, M: super::intern::CompileTimeMachine<'mir, 'tcx, !>>
         // `allocate` picks a fresh AllocId that we will associate with its data below.
         let dest = self.allocate(layout, MemoryKind::Stack)?;
         f(self, &dest.clone().into())?;
-        let mut alloc = self.memory.alloc_map.remove(&dest.ptr().provenance.unwrap()).unwrap().1;
+        let mut alloc =
+            self.memory.alloc_map.remove(&dest.ptr().provenance.unwrap().alloc_id()).unwrap().1;
         alloc.mutability = Mutability::Not;
         let alloc = self.tcx.mk_const_alloc(alloc);
-        let alloc_id = dest.ptr().provenance.unwrap(); // this was just allocated, it must have provenance
+        let alloc_id = dest.ptr().provenance.unwrap().alloc_id(); // this was just allocated, it must have provenance
         self.tcx.set_alloc_id_memory(alloc_id, alloc);
         Ok(alloc_id)
     }
diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs
index b23cafc19..c29f23b91 100644
--- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs
+++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs
@@ -3,17 +3,22 @@
 //! and miri.
 
 use rustc_hir::def_id::DefId;
-use rustc_middle::mir::{
-    self,
-    interpret::{Allocation, ConstAllocation, GlobalId, InterpResult, PointerArithmetic, Scalar},
-    BinOp, ConstValue, NonDivergingIntrinsic,
-};
 use rustc_middle::ty;
 use rustc_middle::ty::layout::{LayoutOf as _, ValidityRequirement};
 use rustc_middle::ty::GenericArgsRef;
 use rustc_middle::ty::{Ty, TyCtxt};
+use rustc_middle::{
+    mir::{
+        self,
+        interpret::{
+            Allocation, ConstAllocation, GlobalId, InterpResult, PointerArithmetic, Scalar,
+        },
+        BinOp, ConstValue, NonDivergingIntrinsic,
+    },
+    ty::layout::TyAndLayout,
+};
 use rustc_span::symbol::{sym, Symbol};
-use rustc_target::abi::{Abi, Primitive, Size};
+use rustc_target::abi::Size;
 
 use super::{
     util::ensure_monomorphic_enough, CheckInAllocMsg, ImmTy, InterpCx, Machine, OpTy, PlaceTy,
@@ -22,23 +27,6 @@ use super::{
 
 use crate::fluent_generated as fluent;
 
-fn numeric_intrinsic<Prov>(name: Symbol, bits: u128, kind: Primitive) -> Scalar<Prov> {
-    let size = match kind {
-        Primitive::Int(integer, _) => integer.size(),
-        _ => bug!("invalid `{}` argument: {:?}", name, bits),
-    };
-    let extra = 128 - u128::from(size.bits());
-    let bits_out = match name {
-        sym::ctpop => u128::from(bits.count_ones()),
-        sym::ctlz => u128::from(bits.leading_zeros()) - extra,
-        sym::cttz => u128::from((bits << extra).trailing_zeros()) - extra,
-        sym::bswap => (bits << extra).swap_bytes(),
-        sym::bitreverse => (bits << extra).reverse_bits(),
-        _ => bug!("not a numeric intrinsic: {}", name),
-    };
-    Scalar::from_uint(bits_out, size)
-}
-
 /// Directly returns an `Allocation` containing an absolute path representation of the given type.
 pub(crate) fn alloc_type_name<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> ConstAllocation<'tcx> {
     let path = crate::util::type_name(tcx, ty);
@@ -179,30 +167,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             | sym::bswap
             | sym::bitreverse => {
                 let ty = instance_args.type_at(0);
-                let layout_of = self.layout_of(ty)?;
+                let layout = self.layout_of(ty)?;
                 let val = self.read_scalar(&args[0])?;
-                let bits = val.to_bits(layout_of.size)?;
-                let kind = match layout_of.abi {
-                    Abi::Scalar(scalar) => scalar.primitive(),
-                    _ => span_bug!(
-                        self.cur_span(),
-                        "{} called on invalid type {:?}",
-                        intrinsic_name,
-                        ty
-                    ),
-                };
-                let (nonzero, actual_intrinsic_name) = match intrinsic_name {
-                    sym::cttz_nonzero => (true, sym::cttz),
-                    sym::ctlz_nonzero => (true, sym::ctlz),
-                    other => (false, other),
-                };
-                if nonzero && bits == 0 {
-                    throw_ub_custom!(
-                        fluent::const_eval_call_nonzero_intrinsic,
-                        name = intrinsic_name,
-                    );
-                }
-                let out_val = numeric_intrinsic(actual_intrinsic_name, bits, kind);
+                let out_val = self.numeric_intrinsic(intrinsic_name, val, layout)?;
                 self.write_scalar(out_val, dest)?;
             }
             sym::saturating_add | sym::saturating_sub => {
@@ -493,6 +460,29 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         }
     }
 
+    pub fn numeric_intrinsic(
+        &self,
+        name: Symbol,
+        val: Scalar<M::Provenance>,
+        layout: TyAndLayout<'tcx>,
+    ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
+        assert!(layout.ty.is_integral(), "invalid type for numeric intrinsic: {}", layout.ty);
+        let bits = val.to_bits(layout.size)?;
+        let extra = 128 - u128::from(layout.size.bits());
+        let bits_out = match name {
+            sym::ctpop => u128::from(bits.count_ones()),
+            sym::ctlz_nonzero | sym::cttz_nonzero if bits == 0 => {
+                throw_ub_custom!(fluent::const_eval_call_nonzero_intrinsic, name = name,);
+            }
+            sym::ctlz | sym::ctlz_nonzero => u128::from(bits.leading_zeros()) - extra,
+            sym::cttz | sym::cttz_nonzero => u128::from((bits << extra).trailing_zeros()) - extra,
+            sym::bswap => (bits << extra).swap_bytes(),
+            sym::bitreverse => (bits << extra).reverse_bits(),
+            _ => bug!("not a numeric intrinsic: {}", name),
+        };
+        Ok(Scalar::from_uint(bits_out, layout.size))
+    }
+
     pub fn exact_div(
         &mut self,
         a: &ImmTy<'tcx, M::Provenance>,
@@ -505,7 +495,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         // Performs an exact division, resulting in undefined behavior where
         // `x % y != 0` or `y == 0` or `x == T::MIN && y == -1`.
         // First, check x % y != 0 (or if that computation overflows).
-        let (res, overflow) = self.overflowing_binary_op(BinOp::Rem, &a, &b)?;
+        let (res, overflow) = self.overflowing_binary_op(BinOp::Rem, a, b)?;
         assert!(!overflow); // All overflow is UB, so this should never return on overflow.
         if res.to_scalar().assert_bits(a.layout.size) != 0 {
             throw_ub_custom!(
@@ -515,7 +505,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             )
         }
         // `Rem` says this is all right, so we can let `Div` do its job.
-        self.binop_ignore_overflow(BinOp::Div, &a, &b, dest)
+        self.binop_ignore_overflow(BinOp::Div, a, b, dest)
     }
 
     pub fn saturating_arith(
diff --git a/compiler/rustc_const_eval/src/interpret/machine.rs b/compiler/rustc_const_eval/src/interpret/machine.rs
index 61fe9151d..5e6996551 100644
--- a/compiler/rustc_const_eval/src/interpret/machine.rs
+++ b/compiler/rustc_const_eval/src/interpret/machine.rs
@@ -9,15 +9,17 @@ use std::hash::Hash;
 use rustc_apfloat::{Float, FloatConvert};
 use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
 use rustc_middle::mir;
+use rustc_middle::query::TyCtxtAt;
+use rustc_middle::ty;
 use rustc_middle::ty::layout::TyAndLayout;
-use rustc_middle::ty::{self, TyCtxt};
 use rustc_span::def_id::DefId;
-use rustc_target::abi::Size;
+use rustc_target::abi::{Align, Size};
 use rustc_target::spec::abi::Abi as CallAbi;
 
 use super::{
-    AllocBytes, AllocId, AllocRange, Allocation, ConstAllocation, FnArg, Frame, ImmTy, InterpCx,
-    InterpResult, MPlaceTy, MemoryKind, OpTy, PlaceTy, Pointer, Provenance,
+    AllocBytes, AllocId, AllocKind, AllocRange, Allocation, ConstAllocation, CtfeProvenance, FnArg,
+    Frame, ImmTy, InterpCx, InterpResult, MPlaceTy, MemoryKind, Misalignment, OpTy, PlaceTy,
+    Pointer, Provenance,
 };
 
 /// Data returned by Machine::stack_pop,
@@ -49,6 +51,14 @@ pub trait AllocMap<K: Hash + Eq, V> {
     where
         K: Borrow<Q>;
 
+    /// Callers should prefer [`AllocMap::contains_key`] when it is possible to call because it may
+    /// be more efficient. This function exists for callers that only have a shared reference
+    /// (which might make it slightly less efficient than `contains_key`, e.g. if
+    /// the data is stored inside a `RefCell`).
+    fn contains_key_ref<Q: ?Sized + Hash + Eq>(&self, k: &Q) -> bool
+    where
+        K: Borrow<Q>;
+
     /// Inserts a new entry into the map.
     fn insert(&mut self, k: K, v: V) -> Option<V>;
 
@@ -135,11 +145,18 @@ pub trait Machine<'mir, 'tcx: 'mir>: Sized {
     /// Whether memory accesses should be alignment-checked.
     fn enforce_alignment(ecx: &InterpCx<'mir, 'tcx, Self>) -> bool;
 
-    /// Whether, when checking alignment, we should look at the actual address and thus support
-    /// custom alignment logic based on whatever the integer address happens to be.
-    ///
-    /// If this returns true, Provenance::OFFSET_IS_ADDR must be true.
-    fn use_addr_for_alignment_check(ecx: &InterpCx<'mir, 'tcx, Self>) -> bool;
+    /// Gives the machine a chance to detect more misalignment than the built-in checks would catch.
+    #[inline(always)]
+    fn alignment_check(
+        _ecx: &InterpCx<'mir, 'tcx, Self>,
+        _alloc_id: AllocId,
+        _alloc_align: Align,
+        _alloc_kind: AllocKind,
+        _offset: Size,
+        _align: Align,
+    ) -> Option<Misalignment> {
+        None
+    }
 
     /// Whether to enforce the validity invariant for a specific layout.
     fn enforce_validity(ecx: &InterpCx<'mir, 'tcx, Self>, layout: TyAndLayout<'tcx>) -> bool;
@@ -277,7 +294,7 @@ pub trait Machine<'mir, 'tcx: 'mir>: Sized {
     /// `def_id` is `Some` if this is the "lazy" allocation of a static.
     #[inline]
     fn before_access_global(
-        _tcx: TyCtxt<'tcx>,
+        _tcx: TyCtxtAt<'tcx>,
         _machine: &Self,
         _alloc_id: AllocId,
         _allocation: ConstAllocation<'tcx>,
@@ -372,7 +389,7 @@ pub trait Machine<'mir, 'tcx: 'mir>: Sized {
     /// need to mutate.
     #[inline(always)]
     fn before_memory_read(
-        _tcx: TyCtxt<'tcx>,
+        _tcx: TyCtxtAt<'tcx>,
         _machine: &Self,
         _alloc_extra: &Self::AllocExtra,
         _prov: (AllocId, Self::ProvenanceExtra),
@@ -384,7 +401,7 @@ pub trait Machine<'mir, 'tcx: 'mir>: Sized {
     /// Hook for performing extra checks on a memory write access.
     #[inline(always)]
     fn before_memory_write(
-        _tcx: TyCtxt<'tcx>,
+        _tcx: TyCtxtAt<'tcx>,
         _machine: &mut Self,
         _alloc_extra: &mut Self::AllocExtra,
         _prov: (AllocId, Self::ProvenanceExtra),
@@ -396,7 +413,7 @@ pub trait Machine<'mir, 'tcx: 'mir>: Sized {
     /// Hook for performing extra operations on a memory deallocation.
     #[inline(always)]
     fn before_memory_deallocation(
-        _tcx: TyCtxt<'tcx>,
+        _tcx: TyCtxtAt<'tcx>,
         _machine: &mut Self,
         _alloc_extra: &mut Self::AllocExtra,
         _prov: (AllocId, Self::ProvenanceExtra),
@@ -498,8 +515,8 @@ pub trait Machine<'mir, 'tcx: 'mir>: Sized {
 /// A lot of the flexibility above is just needed for `Miri`, but all "compile-time" machines
 /// (CTFE and ConstProp) use the same instance. Here, we share that code.
 pub macro compile_time_machine(<$mir: lifetime, $tcx: lifetime>) {
-    type Provenance = AllocId;
-    type ProvenanceExtra = ();
+    type Provenance = CtfeProvenance;
+    type ProvenanceExtra = bool; // the "immutable" flag
 
     type ExtraFnVal = !;
 
@@ -512,12 +529,6 @@ pub macro compile_time_machine(<$mir: lifetime, $tcx: lifetime>) {
     type Bytes = Box<[u8]>;
 
     #[inline(always)]
-    fn use_addr_for_alignment_check(_ecx: &InterpCx<$mir, $tcx, Self>) -> bool {
-        // We do not support `use_addr`.
-        false
-    }
-
-    #[inline(always)]
     fn ignore_optional_overflow_checks(_ecx: &InterpCx<$mir, $tcx, Self>) -> bool {
         false
     }
@@ -558,14 +569,14 @@ pub macro compile_time_machine(<$mir: lifetime, $tcx: lifetime>) {
         def_id: DefId,
     ) -> InterpResult<$tcx, Pointer> {
         // Use the `AllocId` associated with the `DefId`. Any actual *access* will fail.
-        Ok(Pointer::new(ecx.tcx.reserve_and_set_static_alloc(def_id), Size::ZERO))
+        Ok(Pointer::new(ecx.tcx.reserve_and_set_static_alloc(def_id).into(), Size::ZERO))
     }
 
     #[inline(always)]
     fn adjust_alloc_base_pointer(
         _ecx: &InterpCx<$mir, $tcx, Self>,
-        ptr: Pointer<AllocId>,
-    ) -> InterpResult<$tcx, Pointer<AllocId>> {
+        ptr: Pointer<CtfeProvenance>,
+    ) -> InterpResult<$tcx, Pointer<CtfeProvenance>> {
         Ok(ptr)
     }
 
@@ -573,7 +584,7 @@ pub macro compile_time_machine(<$mir: lifetime, $tcx: lifetime>) {
     fn ptr_from_addr_cast(
         _ecx: &InterpCx<$mir, $tcx, Self>,
         addr: u64,
-    ) -> InterpResult<$tcx, Pointer<Option<AllocId>>> {
+    ) -> InterpResult<$tcx, Pointer<Option<CtfeProvenance>>> {
         // Allow these casts, but make the pointer not dereferenceable.
         // (I.e., they behave like transmutation.)
         // This is correct because no pointers can ever be exposed in compile-time evaluation.
@@ -583,10 +594,10 @@ pub macro compile_time_machine(<$mir: lifetime, $tcx: lifetime>) {
     #[inline(always)]
     fn ptr_get_alloc(
         _ecx: &InterpCx<$mir, $tcx, Self>,
-        ptr: Pointer<AllocId>,
+        ptr: Pointer<CtfeProvenance>,
     ) -> Option<(AllocId, Size, Self::ProvenanceExtra)> {
         // We know `offset` is relative to the allocation, so we can use `into_parts`.
-        let (alloc_id, offset) = ptr.into_parts();
-        Some((alloc_id, offset, ()))
+        let (prov, offset) = ptr.into_parts();
+        Some((prov.alloc_id(), offset, prov.immutable()))
     }
 }
diff --git a/compiler/rustc_const_eval/src/interpret/memory.rs b/compiler/rustc_const_eval/src/interpret/memory.rs
index 16905e93b..3fde6ae9b 100644
--- a/compiler/rustc_const_eval/src/interpret/memory.rs
+++ b/compiler/rustc_const_eval/src/interpret/memory.rs
@@ -22,8 +22,8 @@ use crate::fluent_generated as fluent;
 
 use super::{
     alloc_range, AllocBytes, AllocId, AllocMap, AllocRange, Allocation, CheckAlignMsg,
-    CheckInAllocMsg, GlobalAlloc, InterpCx, InterpResult, Machine, MayLeak, Misalignment, Pointer,
-    PointerArithmetic, Provenance, Scalar,
+    CheckInAllocMsg, CtfeProvenance, GlobalAlloc, InterpCx, InterpResult, Machine, MayLeak,
+    Misalignment, Pointer, PointerArithmetic, Provenance, Scalar,
 };
 
 #[derive(Debug, PartialEq, Copy, Clone)]
@@ -58,6 +58,7 @@ impl<T: fmt::Display> fmt::Display for MemoryKind<T> {
 }
 
 /// The return value of `get_alloc_info` indicates the "kind" of the allocation.
+#[derive(Copy, Clone, PartialEq, Debug)]
 pub enum AllocKind {
     /// A regular live data allocation.
     LiveData,
@@ -158,9 +159,9 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
     #[inline]
     pub fn global_base_pointer(
         &self,
-        ptr: Pointer<AllocId>,
+        ptr: Pointer<CtfeProvenance>,
     ) -> InterpResult<'tcx, Pointer<M::Provenance>> {
-        let alloc_id = ptr.provenance;
+        let alloc_id = ptr.provenance.alloc_id();
         // We need to handle `extern static`.
         match self.tcx.try_get_global_alloc(alloc_id) {
             Some(GlobalAlloc::Static(def_id)) if self.tcx.is_thread_local_static(def_id) => {
@@ -338,7 +339,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         // Let the machine take some extra action
         let size = alloc.size();
         M::before_memory_deallocation(
-            *self.tcx,
+            self.tcx,
             &mut self.machine,
             &mut alloc.extra,
             (alloc_id, prov),
@@ -473,8 +474,12 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         match self.ptr_try_get_alloc_id(ptr) {
             Err(addr) => offset_misalignment(addr, align),
             Ok((alloc_id, offset, _prov)) => {
-                let (_size, alloc_align, _kind) = self.get_alloc_info(alloc_id);
-                if M::use_addr_for_alignment_check(self) {
+                let (_size, alloc_align, kind) = self.get_alloc_info(alloc_id);
+                if let Some(misalign) =
+                    M::alignment_check(self, alloc_id, alloc_align, kind, offset, align)
+                {
+                    Some(misalign)
+                } else if M::Provenance::OFFSET_IS_ADDR {
                     // `use_addr_for_alignment_check` can only be true if `OFFSET_IS_ADDR` is true.
                     offset_misalignment(ptr.addr().bytes(), align)
                 } else {
@@ -501,6 +506,17 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
     }
 }
 
+impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
+    /// This function is used by Miri's provenance GC to remove unreachable entries from the dead_alloc_map.
+    pub fn remove_unreachable_allocs(&mut self, reachable_allocs: &FxHashSet<AllocId>) {
+        // Unlike all the other GC helpers where we check if an `AllocId` is found in the interpreter or
+        // is live, here all the IDs in the map are for dead allocations so we don't
+        // need to check for liveness.
+        #[allow(rustc::potential_query_instability)] // Only used from Miri, not queries.
+        self.memory.dead_alloc_map.retain(|id, _| reachable_allocs.contains(id));
+    }
+}
+
 /// Allocation accessors
 impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
     /// Helper function to obtain a global (tcx) allocation.
@@ -545,7 +561,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 (val, Some(def_id))
             }
         };
-        M::before_access_global(*self.tcx, &self.machine, id, alloc, def_id, is_write)?;
+        M::before_access_global(self.tcx, &self.machine, id, alloc, def_id, is_write)?;
         // We got tcx memory. Let the machine initialize its "extra" stuff.
         M::adjust_allocation(
             self,
@@ -610,7 +626,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         )?;
         if let Some((alloc_id, offset, prov, alloc)) = ptr_and_alloc {
             let range = alloc_range(offset, size);
-            M::before_memory_read(*self.tcx, &self.machine, &alloc.extra, (alloc_id, prov), range)?;
+            M::before_memory_read(self.tcx, &self.machine, &alloc.extra, (alloc_id, prov), range)?;
             Ok(Some(AllocRef { alloc, range, tcx: *self.tcx, alloc_id }))
         } else {
             // Even in this branch we have to be sure that we actually access the allocation, in
@@ -671,13 +687,13 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
     {
         let parts = self.get_ptr_access(ptr, size)?;
         if let Some((alloc_id, offset, prov)) = parts {
-            let tcx = *self.tcx;
+            let tcx = self.tcx;
             // FIXME: can we somehow avoid looking up the allocation twice here?
             // We cannot call `get_raw_mut` inside `check_and_deref_ptr` as that would duplicate `&mut self`.
             let (alloc, machine) = self.get_alloc_raw_mut(alloc_id)?;
             let range = alloc_range(offset, size);
             M::before_memory_write(tcx, machine, &mut alloc.extra, (alloc_id, prov), range)?;
-            Ok(Some(AllocRefMut { alloc, range, tcx, alloc_id }))
+            Ok(Some(AllocRefMut { alloc, range, tcx: *tcx, alloc_id }))
         } else {
             Ok(None)
         }
@@ -692,6 +708,15 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         Ok((&mut alloc.extra, machine))
     }
 
+    /// Check whether an allocation is live. This is faster than calling
+    /// [`InterpCx::get_alloc_info`] if all you need to check is whether the kind is
+    /// [`AllocKind::Dead`] because it doesn't have to look up the type and layout of statics.
+    pub fn is_alloc_live(&self, id: AllocId) -> bool {
+        self.tcx.try_get_global_alloc(id).is_some()
+            || self.memory.alloc_map.contains_key_ref(&id)
+            || self.memory.extra_fn_ptr_map.contains_key(&id)
+    }
+
     /// Obtain the size and alignment of an allocation, even if that allocation has
     /// been deallocated.
     pub fn get_alloc_info(&self, id: AllocId) -> (Size, Align, AllocKind) {
@@ -1108,7 +1133,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         let src_alloc = self.get_alloc_raw(src_alloc_id)?;
         let src_range = alloc_range(src_offset, size);
         M::before_memory_read(
-            *tcx,
+            tcx,
             &self.machine,
             &src_alloc.extra,
             (src_alloc_id, src_prov),
@@ -1138,7 +1163,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         let (dest_alloc, extra) = self.get_alloc_raw_mut(dest_alloc_id)?;
         let dest_range = alloc_range(dest_offset, size * num_copies);
         M::before_memory_write(
-            *tcx,
+            tcx,
             extra,
             &mut dest_alloc.extra,
             (dest_alloc_id, dest_prov),
diff --git a/compiler/rustc_const_eval/src/interpret/operand.rs b/compiler/rustc_const_eval/src/interpret/operand.rs
index 255dd1eba..b39b219b4 100644
--- a/compiler/rustc_const_eval/src/interpret/operand.rs
+++ b/compiler/rustc_const_eval/src/interpret/operand.rs
@@ -13,9 +13,9 @@ use rustc_middle::{mir, ty};
 use rustc_target::abi::{self, Abi, HasDataLayout, Size};
 
 use super::{
-    alloc_range, from_known_layout, mir_assign_valid_types, AllocId, Frame, InterpCx, InterpResult,
-    MPlaceTy, Machine, MemPlace, MemPlaceMeta, OffsetMode, PlaceTy, Pointer, Projectable,
-    Provenance, Scalar,
+    alloc_range, from_known_layout, mir_assign_valid_types, CtfeProvenance, Frame, InterpCx,
+    InterpResult, MPlaceTy, Machine, MemPlace, MemPlaceMeta, OffsetMode, PlaceTy, Pointer,
+    Projectable, Provenance, Scalar,
 };
 
 /// An `Immediate` represents a single immediate self-contained Rust value.
@@ -26,7 +26,7 @@ use super::{
 /// In particular, thanks to `ScalarPair`, arithmetic operations and casts can be entirely
 /// defined on `Immediate`, and do not have to work with a `Place`.
 #[derive(Copy, Clone, Debug)]
-pub enum Immediate<Prov: Provenance = AllocId> {
+pub enum Immediate<Prov: Provenance = CtfeProvenance> {
     /// A single scalar value (must have *initialized* `Scalar` ABI).
     Scalar(Scalar<Prov>),
     /// A pair of two scalar value (must have `ScalarPair` ABI where both fields are
@@ -93,12 +93,23 @@ impl<Prov: Provenance> Immediate<Prov> {
             Immediate::Uninit => bug!("Got uninit where a scalar pair was expected"),
         }
     }
+
+    /// Returns the scalar from the first component and optionally the 2nd component as metadata.
+    #[inline]
+    #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
+    pub fn to_scalar_and_meta(self) -> (Scalar<Prov>, MemPlaceMeta<Prov>) {
+        match self {
+            Immediate::ScalarPair(val1, val2) => (val1, MemPlaceMeta::Meta(val2)),
+            Immediate::Scalar(val) => (val, MemPlaceMeta::None),
+            Immediate::Uninit => bug!("Got uninit where a scalar or scalar pair was expected"),
+        }
+    }
 }
 
 // ScalarPair needs a type to interpret, so we often have an immediate and a type together
 // as input for binary and cast operations.
 #[derive(Clone)]
-pub struct ImmTy<'tcx, Prov: Provenance = AllocId> {
+pub struct ImmTy<'tcx, Prov: Provenance = CtfeProvenance> {
     imm: Immediate<Prov>,
     pub layout: TyAndLayout<'tcx>,
 }
@@ -334,13 +345,13 @@ impl<'tcx, Prov: Provenance> Projectable<'tcx, Prov> for ImmTy<'tcx, Prov> {
 /// or still in memory. The latter is an optimization, to delay reading that chunk of
 /// memory and to avoid having to store arbitrary-sized data here.
 #[derive(Copy, Clone, Debug)]
-pub(super) enum Operand<Prov: Provenance = AllocId> {
+pub(super) enum Operand<Prov: Provenance = CtfeProvenance> {
     Immediate(Immediate<Prov>),
     Indirect(MemPlace<Prov>),
 }
 
 #[derive(Clone)]
-pub struct OpTy<'tcx, Prov: Provenance = AllocId> {
+pub struct OpTy<'tcx, Prov: Provenance = CtfeProvenance> {
     op: Operand<Prov>, // Keep this private; it helps enforce invariants.
     pub layout: TyAndLayout<'tcx>,
 }
@@ -750,17 +761,19 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         let layout = from_known_layout(self.tcx, self.param_env, layout, || self.layout_of(ty))?;
         let imm = match val_val {
             mir::ConstValue::Indirect { alloc_id, offset } => {
-                // We rely on mutability being set correctly in that allocation to prevent writes
-                // where none should happen.
-                let ptr = self.global_base_pointer(Pointer::new(alloc_id, offset))?;
+                // This is const data, no mutation allowed.
+                let ptr = self.global_base_pointer(Pointer::new(
+                    CtfeProvenance::from(alloc_id).as_immutable(),
+                    offset,
+                ))?;
                 return Ok(self.ptr_to_mplace(ptr.into(), layout).into());
             }
             mir::ConstValue::Scalar(x) => adjust_scalar(x)?.into(),
             mir::ConstValue::ZeroSized => Immediate::Uninit,
             mir::ConstValue::Slice { data, meta } => {
-                // We rely on mutability being set correctly in `data` to prevent writes
-                // where none should happen.
-                let ptr = Pointer::new(self.tcx.reserve_and_set_memory_alloc(data), Size::ZERO);
+                // This is const data, no mutation allowed.
+                let alloc_id = self.tcx.reserve_and_set_memory_alloc(data);
+                let ptr = Pointer::new(CtfeProvenance::from(alloc_id).as_immutable(), Size::ZERO);
                 Immediate::new_slice(self.global_base_pointer(ptr)?.into(), meta, self)
             }
         };
diff --git a/compiler/rustc_const_eval/src/interpret/operator.rs b/compiler/rustc_const_eval/src/interpret/operator.rs
index a3ba9530f..eef154257 100644
--- a/compiler/rustc_const_eval/src/interpret/operator.rs
+++ b/compiler/rustc_const_eval/src/interpret/operator.rs
@@ -20,7 +20,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         right: &ImmTy<'tcx, M::Provenance>,
         dest: &PlaceTy<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx> {
-        let (val, overflowed) = self.overflowing_binary_op(op, &left, &right)?;
+        let (val, overflowed) = self.overflowing_binary_op(op, left, right)?;
         debug_assert_eq!(
             Ty::new_tup(self.tcx.tcx, &[val.layout.ty, self.tcx.types.bool]),
             dest.layout.ty,
@@ -114,9 +114,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         use rustc_middle::mir::BinOp::*;
 
         // Performs appropriate non-deterministic adjustments of NaN results.
-        let adjust_nan = |f: F| -> F {
-            if f.is_nan() { M::generate_nan(self, &[l, r]) } else { f }
-        };
+        let adjust_nan =
+            |f: F| -> F { if f.is_nan() { M::generate_nan(self, &[l, r]) } else { f } };
 
         let val = match bin_op {
             Eq => ImmTy::from_bool(l == r, *self.tcx),
@@ -157,41 +156,35 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
 
         // Shift ops can have an RHS with a different numeric type.
         if matches!(bin_op, Shl | ShlUnchecked | Shr | ShrUnchecked) {
-            let size = u128::from(left_layout.size.bits());
-            // Even if `r` is signed, we treat it as if it was unsigned (i.e., we use its
-            // zero-extended form). This matches the codegen backend:
-            // <https://github.com/rust-lang/rust/blob/c274e4969f058b1c644243181ece9f829efa7594/compiler/rustc_codegen_ssa/src/base.rs#L315-L317>.
-            // The overflow check is also ignorant to the sign:
-            // <https://github.com/rust-lang/rust/blob/c274e4969f058b1c644243181ece9f829efa7594/compiler/rustc_codegen_ssa/src/mir/rvalue.rs#L728>.
-            // This would behave rather strangely if we had integer types of size 256: a shift by
-            // -1i8 would actually shift by 255, but that would *not* be considered overflowing. A
-            // shift by -1i16 though would be considered overflowing. If we had integers of size
-            // 512, then a shift by -1i8 would even produce a different result than one by -1i16:
-            // the first shifts by 255, the latter by u16::MAX % 512 = 511. Lucky enough, our
-            // integers are maximally 128bits wide, so negative shifts *always* overflow and we have
-            // consistent results for the same value represented at different bit widths.
-            assert!(size <= 128);
-            let original_r = r;
-            let overflow = r >= size;
-            // The shift offset is implicitly masked to the type size, to make sure this operation
-            // is always defined. This is the one MIR operator that does *not* directly map to a
-            // single LLVM operation. See
-            // <https://github.com/rust-lang/rust/blob/c274e4969f058b1c644243181ece9f829efa7594/compiler/rustc_codegen_ssa/src/common.rs#L131-L158>
-            // for the corresponding truncation in our codegen backends.
-            let r = r % size;
-            let r = u32::try_from(r).unwrap(); // we masked so this will always fit
+            let size = left_layout.size.bits();
+            // The shift offset is implicitly masked to the type size. (This is the one MIR operator
+            // that does *not* directly map to a single LLVM operation.) Compute how much we
+            // actually shift and whether there was an overflow due to shifting too much.
+            let (shift_amount, overflow) = if right_layout.abi.is_signed() {
+                let shift_amount = self.sign_extend(r, right_layout) as i128;
+                let overflow = shift_amount < 0 || shift_amount >= i128::from(size);
+                let masked_amount = (shift_amount as u128) % u128::from(size);
+                debug_assert_eq!(overflow, shift_amount != (masked_amount as i128));
+                (masked_amount, overflow)
+            } else {
+                let shift_amount = r;
+                let masked_amount = shift_amount % u128::from(size);
+                (masked_amount, shift_amount != masked_amount)
+            };
+            let shift_amount = u32::try_from(shift_amount).unwrap(); // we masked so this will always fit
+            // Compute the shifted result.
             let result = if left_layout.abi.is_signed() {
                 let l = self.sign_extend(l, left_layout) as i128;
                 let result = match bin_op {
-                    Shl | ShlUnchecked => l.checked_shl(r).unwrap(),
-                    Shr | ShrUnchecked => l.checked_shr(r).unwrap(),
+                    Shl | ShlUnchecked => l.checked_shl(shift_amount).unwrap(),
+                    Shr | ShrUnchecked => l.checked_shr(shift_amount).unwrap(),
                     _ => bug!(),
                 };
                 result as u128
             } else {
                 match bin_op {
-                    Shl | ShlUnchecked => l.checked_shl(r).unwrap(),
-                    Shr | ShrUnchecked => l.checked_shr(r).unwrap(),
+                    Shl | ShlUnchecked => l.checked_shl(shift_amount).unwrap(),
+                    Shr | ShrUnchecked => l.checked_shr(shift_amount).unwrap(),
                     _ => bug!(),
                 }
             };
@@ -200,7 +193,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             if overflow && let Some(intrinsic_name) = throw_ub_on_overflow {
                 throw_ub_custom!(
                     fluent::const_eval_overflow_shift,
-                    val = original_r,
+                    val = if right_layout.abi.is_signed() {
+                        (self.sign_extend(r, right_layout) as i128).to_string()
+                    } else {
+                        r.to_string()
+                    },
                     name = intrinsic_name
                 );
             }
diff --git a/compiler/rustc_const_eval/src/interpret/place.rs b/compiler/rustc_const_eval/src/interpret/place.rs
index 09ffdec7d..639b269ac 100644
--- a/compiler/rustc_const_eval/src/interpret/place.rs
+++ b/compiler/rustc_const_eval/src/interpret/place.rs
@@ -7,22 +7,21 @@ use std::assert_matches::assert_matches;
 use either::{Either, Left, Right};
 
 use rustc_ast::Mutability;
-use rustc_index::IndexSlice;
 use rustc_middle::mir;
 use rustc_middle::ty;
 use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
 use rustc_middle::ty::Ty;
-use rustc_target::abi::{Abi, Align, FieldIdx, HasDataLayout, Size, FIRST_VARIANT};
+use rustc_target::abi::{Abi, Align, HasDataLayout, Size};
 
 use super::{
-    alloc_range, mir_assign_valid_types, AllocId, AllocRef, AllocRefMut, CheckAlignMsg, ImmTy,
-    Immediate, InterpCx, InterpResult, Machine, MemoryKind, Misalignment, OffsetMode, OpTy,
+    alloc_range, mir_assign_valid_types, AllocRef, AllocRefMut, CheckAlignMsg, CtfeProvenance,
+    ImmTy, Immediate, InterpCx, InterpResult, Machine, MemoryKind, Misalignment, OffsetMode, OpTy,
     Operand, Pointer, PointerArithmetic, Projectable, Provenance, Readable, Scalar,
 };
 
 #[derive(Copy, Clone, Hash, PartialEq, Eq, Debug)]
 /// Information required for the sound usage of a `MemPlace`.
-pub enum MemPlaceMeta<Prov: Provenance = AllocId> {
+pub enum MemPlaceMeta<Prov: Provenance = CtfeProvenance> {
     /// The unsized payload (e.g. length for slices or vtable pointer for trait objects).
     Meta(Scalar<Prov>),
     /// `Sized` types or unsized `extern type`
@@ -50,7 +49,7 @@ impl<Prov: Provenance> MemPlaceMeta<Prov> {
 }
 
 #[derive(Copy, Clone, Hash, PartialEq, Eq, Debug)]
-pub(super) struct MemPlace<Prov: Provenance = AllocId> {
+pub(super) struct MemPlace<Prov: Provenance = CtfeProvenance> {
     /// The pointer can be a pure integer, with the `None` provenance.
     pub ptr: Pointer<Option<Prov>>,
     /// Metadata for unsized places. Interpretation is up to the type.
@@ -101,7 +100,7 @@ impl<Prov: Provenance> MemPlace<Prov> {
 
 /// A MemPlace with its layout. Constructing it is only possible in this module.
 #[derive(Clone, Hash, Eq, PartialEq)]
-pub struct MPlaceTy<'tcx, Prov: Provenance = AllocId> {
+pub struct MPlaceTy<'tcx, Prov: Provenance = CtfeProvenance> {
     mplace: MemPlace<Prov>,
     pub layout: TyAndLayout<'tcx>,
 }
@@ -180,7 +179,7 @@ impl<'tcx, Prov: Provenance> Projectable<'tcx, Prov> for MPlaceTy<'tcx, Prov> {
 }
 
 #[derive(Copy, Clone, Debug)]
-pub(super) enum Place<Prov: Provenance = AllocId> {
+pub(super) enum Place<Prov: Provenance = CtfeProvenance> {
     /// A place referring to a value allocated in the `Memory` system.
     Ptr(MemPlace<Prov>),
 
@@ -196,7 +195,7 @@ pub(super) enum Place<Prov: Provenance = AllocId> {
 }
 
 #[derive(Clone)]
-pub struct PlaceTy<'tcx, Prov: Provenance = AllocId> {
+pub struct PlaceTy<'tcx, Prov: Provenance = CtfeProvenance> {
     place: Place<Prov>, // Keep this private; it helps enforce invariants.
     pub layout: TyAndLayout<'tcx>,
 }
@@ -407,11 +406,7 @@ where
         let pointee_type =
             val.layout.ty.builtin_deref(true).expect("`ref_to_mplace` called on non-ptr type").ty;
         let layout = self.layout_of(pointee_type)?;
-        let (ptr, meta) = match **val {
-            Immediate::Scalar(ptr) => (ptr, MemPlaceMeta::None),
-            Immediate::ScalarPair(ptr, meta) => (ptr, MemPlaceMeta::Meta(meta)),
-            Immediate::Uninit => throw_ub!(InvalidUninitBytes(None)),
-        };
+        let (ptr, meta) = val.to_scalar_and_meta();
 
         // `ref_to_mplace` is called on raw pointers even if they don't actually get dereferenced;
         // we hence can't call `size_and_align_of` since that asserts more validity than we want.
@@ -456,7 +451,7 @@ where
     ) -> InterpResult<'tcx, Option<AllocRef<'_, 'tcx, M::Provenance, M::AllocExtra, M::Bytes>>>
     {
         let (size, _align) = self
-            .size_and_align_of_mplace(&mplace)?
+            .size_and_align_of_mplace(mplace)?
             .unwrap_or((mplace.layout.size, mplace.layout.align.abi));
         // We check alignment separately, and *after* checking everything else.
         // If an access is both OOB and misaligned, we want to see the bounds error.
@@ -472,7 +467,7 @@ where
     ) -> InterpResult<'tcx, Option<AllocRefMut<'_, 'tcx, M::Provenance, M::AllocExtra, M::Bytes>>>
     {
         let (size, _align) = self
-            .size_and_align_of_mplace(&mplace)?
+            .size_and_align_of_mplace(mplace)?
             .unwrap_or((mplace.layout.size, mplace.layout.align.abi));
         // We check alignment separately, and raise that error *after* checking everything else.
         // If an access is both OOB and misaligned, we want to see the bounds error.
@@ -977,34 +972,6 @@ where
         Ok(self.ptr_with_meta_to_mplace(ptr.into(), MemPlaceMeta::Meta(meta), layout))
     }
 
-    /// Writes the aggregate to the destination.
-    #[instrument(skip(self), level = "trace")]
-    pub fn write_aggregate(
-        &mut self,
-        kind: &mir::AggregateKind<'tcx>,
-        operands: &IndexSlice<FieldIdx, mir::Operand<'tcx>>,
-        dest: &PlaceTy<'tcx, M::Provenance>,
-    ) -> InterpResult<'tcx> {
-        self.write_uninit(dest)?;
-        let (variant_index, variant_dest, active_field_index) = match *kind {
-            mir::AggregateKind::Adt(_, variant_index, _, _, active_field_index) => {
-                let variant_dest = self.project_downcast(dest, variant_index)?;
-                (variant_index, variant_dest, active_field_index)
-            }
-            _ => (FIRST_VARIANT, dest.clone(), None),
-        };
-        if active_field_index.is_some() {
-            assert_eq!(operands.len(), 1);
-        }
-        for (field_index, operand) in operands.iter_enumerated() {
-            let field_index = active_field_index.unwrap_or(field_index);
-            let field_dest = self.project_field(&variant_dest, field_index.as_usize())?;
-            let op = self.eval_operand(operand, Some(field_dest.layout))?;
-            self.copy_op(&op, &field_dest, /*allow_transmute*/ false)?;
-        }
-        self.write_discriminant(variant_index, dest)
-    }
-
     pub fn raw_const_to_mplace(
         &self,
         raw: mir::ConstAlloc<'tcx>,
diff --git a/compiler/rustc_const_eval/src/interpret/projection.rs b/compiler/rustc_const_eval/src/interpret/projection.rs
index 6694c43c9..9a034ba22 100644
--- a/compiler/rustc_const_eval/src/interpret/projection.rs
+++ b/compiler/rustc_const_eval/src/interpret/projection.rs
@@ -163,13 +163,26 @@ where
             // With custom DSTS, this *will* execute user-defined code, but the same
             // happens at run-time so that's okay.
             match self.size_and_align_of(&base_meta, &field_layout)? {
-                Some((_, align)) => (base_meta, offset.align_to(align)),
-                None => {
-                    // For unsized types with an extern type tail we perform no adjustments.
-                    // NOTE: keep this in sync with `PlaceRef::project_field` in the codegen backend.
-                    assert!(matches!(base_meta, MemPlaceMeta::None));
+                Some((_, align)) => {
+                    // For packed types, we need to cap alignment.
+                    let align = if let ty::Adt(def, _) = base.layout().ty.kind()
+                        && let Some(packed) = def.repr().pack
+                    {
+                        align.min(packed)
+                    } else {
+                        align
+                    };
+                    (base_meta, offset.align_to(align))
+                }
+                None if offset == Size::ZERO => {
+                    // If the offset is 0, then rounding it up to alignment wouldn't change anything,
+                    // so we can do this even for types where we cannot determine the alignment.
                     (base_meta, offset)
                 }
+                None => {
+                    // We don't know the alignment of this field, so we cannot adjust.
+                    throw_unsup_format!("`extern type` does not have a known offset")
+                }
             }
         } else {
             // base_meta could be present; we might be accessing a sized field of an unsized
@@ -195,6 +208,24 @@ where
         if layout.abi.is_uninhabited() {
             // `read_discriminant` should have excluded uninhabited variants... but ConstProp calls
             // us on dead code.
+            // In the future we might want to allow this to permit code like this:
+            // (this is a Rust/MIR pseudocode mix)
+            // ```
+            // enum Option2 {
+            //   Some(i32, !),
+            //   None,
+            // }
+            //
+            // fn panic() -> ! { panic!() }
+            //
+            // let x: Option2;
+            // x.Some.0 = 42;
+            // x.Some.1 = panic();
+            // SetDiscriminant(x, Some);
+            // ```
+            // However, for now we don't generate such MIR, and this check here *has* found real
+            // bugs (see https://github.com/rust-lang/rust/issues/115145), so we will keep rejecting
+            // it.
             throw_inval!(ConstPropNonsense)
         }
         // This cannot be `transmute` as variants *can* have a smaller size than the entire enum.
@@ -256,13 +287,13 @@ where
     }
 
     /// Iterates over all fields of an array. Much more efficient than doing the
-    /// same by repeatedly calling `operand_index`.
+    /// same by repeatedly calling `project_index`.
     pub fn project_array_fields<'a, P: Projectable<'tcx, M::Provenance>>(
         &self,
         base: &'a P,
     ) -> InterpResult<'tcx, ArrayIterator<'tcx, 'a, M::Provenance, P>> {
         let abi::FieldsShape::Array { stride, .. } = base.layout().fields else {
-            span_bug!(self.cur_span(), "operand_array_fields: expected an array layout");
+            span_bug!(self.cur_span(), "project_array_fields: expected an array layout");
         };
         let len = base.len(self)?;
         let field_layout = base.layout().field(self, 0);
diff --git a/compiler/rustc_const_eval/src/interpret/step.rs b/compiler/rustc_const_eval/src/interpret/step.rs
index b6993d939..d48329b6c 100644
--- a/compiler/rustc_const_eval/src/interpret/step.rs
+++ b/compiler/rustc_const_eval/src/interpret/step.rs
@@ -4,11 +4,12 @@
 
 use either::Either;
 
+use rustc_index::IndexSlice;
 use rustc_middle::mir;
-use rustc_middle::mir::interpret::{InterpResult, Scalar};
 use rustc_middle::ty::layout::LayoutOf;
+use rustc_target::abi::{FieldIdx, FIRST_VARIANT};
 
-use super::{ImmTy, InterpCx, Machine, Projectable};
+use super::{ImmTy, InterpCx, InterpResult, Machine, PlaceTy, Projectable, Scalar};
 use crate::util;
 
 impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
@@ -187,34 +188,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             }
 
             Repeat(ref operand, _) => {
-                let src = self.eval_operand(operand, None)?;
-                assert!(src.layout.is_sized());
-                let dest = self.force_allocation(&dest)?;
-                let length = dest.len(self)?;
-
-                if length == 0 {
-                    // Nothing to copy... but let's still make sure that `dest` as a place is valid.
-                    self.get_place_alloc_mut(&dest)?;
-                } else {
-                    // Write the src to the first element.
-                    let first = self.project_index(&dest, 0)?;
-                    self.copy_op(&src, &first, /*allow_transmute*/ false)?;
-
-                    // This is performance-sensitive code for big static/const arrays! So we
-                    // avoid writing each operand individually and instead just make many copies
-                    // of the first element.
-                    let elem_size = first.layout.size;
-                    let first_ptr = first.ptr();
-                    let rest_ptr = first_ptr.offset(elem_size, self)?;
-                    // No alignment requirement since `copy_op` above already checked it.
-                    self.mem_copy_repeatedly(
-                        first_ptr,
-                        rest_ptr,
-                        elem_size,
-                        length - 1,
-                        /*nonoverlapping:*/ true,
-                    )?;
-                }
+                self.write_repeat(operand, &dest)?;
             }
 
             Len(place) => {
@@ -307,6 +281,73 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         Ok(())
     }
 
+    /// Writes the aggregate to the destination.
+    #[instrument(skip(self), level = "trace")]
+    fn write_aggregate(
+        &mut self,
+        kind: &mir::AggregateKind<'tcx>,
+        operands: &IndexSlice<FieldIdx, mir::Operand<'tcx>>,
+        dest: &PlaceTy<'tcx, M::Provenance>,
+    ) -> InterpResult<'tcx> {
+        self.write_uninit(dest)?; // make sure all the padding ends up as uninit
+        let (variant_index, variant_dest, active_field_index) = match *kind {
+            mir::AggregateKind::Adt(_, variant_index, _, _, active_field_index) => {
+                let variant_dest = self.project_downcast(dest, variant_index)?;
+                (variant_index, variant_dest, active_field_index)
+            }
+            _ => (FIRST_VARIANT, dest.clone(), None),
+        };
+        if active_field_index.is_some() {
+            assert_eq!(operands.len(), 1);
+        }
+        for (field_index, operand) in operands.iter_enumerated() {
+            let field_index = active_field_index.unwrap_or(field_index);
+            let field_dest = self.project_field(&variant_dest, field_index.as_usize())?;
+            let op = self.eval_operand(operand, Some(field_dest.layout))?;
+            self.copy_op(&op, &field_dest, /*allow_transmute*/ false)?;
+        }
+        self.write_discriminant(variant_index, dest)
+    }
+
+    /// Repeats `operand` into the destination. `dest` must have array type, and that type
+    /// determines how often `operand` is repeated.
+    fn write_repeat(
+        &mut self,
+        operand: &mir::Operand<'tcx>,
+        dest: &PlaceTy<'tcx, M::Provenance>,
+    ) -> InterpResult<'tcx> {
+        let src = self.eval_operand(operand, None)?;
+        assert!(src.layout.is_sized());
+        let dest = self.force_allocation(&dest)?;
+        let length = dest.len(self)?;
+
+        if length == 0 {
+            // Nothing to copy... but let's still make sure that `dest` as a place is valid.
+            self.get_place_alloc_mut(&dest)?;
+        } else {
+            // Write the src to the first element.
+            let first = self.project_index(&dest, 0)?;
+            self.copy_op(&src, &first, /*allow_transmute*/ false)?;
+
+            // This is performance-sensitive code for big static/const arrays! So we
+            // avoid writing each operand individually and instead just make many copies
+            // of the first element.
+            let elem_size = first.layout.size;
+            let first_ptr = first.ptr();
+            let rest_ptr = first_ptr.offset(elem_size, self)?;
+            // No alignment requirement since `copy_op` above already checked it.
+            self.mem_copy_repeatedly(
+                first_ptr,
+                rest_ptr,
+                elem_size,
+                length - 1,
+                /*nonoverlapping:*/ true,
+            )?;
+        }
+
+        Ok(())
+    }
+
     /// Evaluate the given terminator. Will also adjust the stack frame and statement position accordingly.
     fn terminator(&mut self, terminator: &mir::Terminator<'tcx>) -> InterpResult<'tcx> {
         info!("{:?}", terminator.kind);
diff --git a/compiler/rustc_const_eval/src/interpret/terminator.rs b/compiler/rustc_const_eval/src/interpret/terminator.rs
index b54c66814..2358caffc 100644
--- a/compiler/rustc_const_eval/src/interpret/terminator.rs
+++ b/compiler/rustc_const_eval/src/interpret/terminator.rs
@@ -18,14 +18,14 @@ use rustc_target::abi::{
 use rustc_target::spec::abi::Abi;
 
 use super::{
-    AllocId, FnVal, ImmTy, InterpCx, InterpResult, MPlaceTy, Machine, OpTy, PlaceTy, Projectable,
-    Provenance, Scalar, StackPopCleanup,
+    CtfeProvenance, FnVal, ImmTy, InterpCx, InterpResult, MPlaceTy, Machine, OpTy, PlaceTy,
+    Projectable, Provenance, Scalar, StackPopCleanup,
 };
 use crate::fluent_generated as fluent;
 
 /// An argment passed to a function.
 #[derive(Clone, Debug)]
-pub enum FnArg<'tcx, Prov: Provenance = AllocId> {
+pub enum FnArg<'tcx, Prov: Provenance = CtfeProvenance> {
     /// Pass a copy of the given operand.
     Copy(OpTy<'tcx, Prov>),
     /// Allow for the argument to be passed in-place: destroy the value originally stored at that place and
@@ -51,7 +51,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
     ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
         match arg {
             FnArg::Copy(op) => Ok(op.clone()),
-            FnArg::InPlace(place) => self.place_to_op(&place),
+            FnArg::InPlace(place) => self.place_to_op(place),
         }
     }
 
@@ -384,10 +384,12 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         }
 
         // Compatible integer types (in particular, usize vs ptr-sized-u32/u64).
+        // `char` counts as `u32.`
         let int_ty = |ty: Ty<'tcx>| {
             Some(match ty.kind() {
                 ty::Int(ity) => (Integer::from_int_ty(&self.tcx, *ity), /* signed */ true),
                 ty::Uint(uty) => (Integer::from_uint_ty(&self.tcx, *uty), /* signed */ false),
+                ty::Char => (Integer::I32, /* signed */ false),
                 _ => return None,
             })
         };
@@ -410,7 +412,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         // so we implement a type-based check that reflects the guaranteed rules for ABI compatibility.
         if self.layout_compat(caller_abi.layout, callee_abi.layout)? {
             // Ensure that our checks imply actual ABI compatibility for this concrete call.
-            assert!(caller_abi.eq_abi(&callee_abi));
+            assert!(caller_abi.eq_abi(callee_abi));
             return Ok(true);
         } else {
             trace!(
@@ -464,7 +466,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         // We work with a copy of the argument for now; if this is in-place argument passing, we
         // will later protect the source it comes from. This means the callee cannot observe if we
         // did in-place of by-copy argument passing, except for pointer equality tests.
-        let caller_arg_copy = self.copy_fn_arg(&caller_arg)?;
+        let caller_arg_copy = self.copy_fn_arg(caller_arg)?;
         if !already_live {
             let local = callee_arg.as_local().unwrap();
             let meta = caller_arg_copy.meta();
diff --git a/compiler/rustc_const_eval/src/interpret/validity.rs b/compiler/rustc_const_eval/src/interpret/validity.rs
index d21fef58f..07500f744 100644
--- a/compiler/rustc_const_eval/src/interpret/validity.rs
+++ b/compiler/rustc_const_eval/src/interpret/validity.rs
@@ -227,7 +227,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValidityVisitor<'rt, 'mir, '
                         // Sometimes the index is beyond the number of upvars (seen
                         // for a coroutine).
                         let var_hir_id = captured_place.get_root_variable();
-                        let node = self.ecx.tcx.hir().get(var_hir_id);
+                        let node = self.ecx.tcx.hir_node(var_hir_id);
                         if let hir::Node::Pat(pat) = node {
                             if let hir::PatKind::Binding(_, _, ident, _) = pat.kind {
                                 name = Some(ident.name);
@@ -896,7 +896,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         let mut visitor = ValidityVisitor { path, ref_tracking, ctfe_mode, ecx: self };
 
         // Run it.
-        match visitor.visit_value(&op) {
+        match visitor.visit_value(op) {
             Ok(()) => Ok(()),
             // Pass through validation failures and "invalid program" issues.
             Err(err)
diff --git a/compiler/rustc_const_eval/src/interpret/visitor.rs b/compiler/rustc_const_eval/src/interpret/visitor.rs
index fc21ad1f1..de0590a4b 100644
--- a/compiler/rustc_const_eval/src/interpret/visitor.rs
+++ b/compiler/rustc_const_eval/src/interpret/visitor.rs
@@ -21,7 +21,7 @@ pub trait ValueVisitor<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>>: Sized {
     /// `read_discriminant` can be hooked for better error messages.
     #[inline(always)]
     fn read_discriminant(&mut self, v: &Self::V) -> InterpResult<'tcx, VariantIdx> {
-        Ok(self.ecx().read_discriminant(&v.to_op(self.ecx())?)?)
+        self.ecx().read_discriminant(&v.to_op(self.ecx())?)
     }
 
     /// This function provides the chance to reorder the order in which fields are visited for
@@ -97,14 +97,14 @@ pub trait ValueVisitor<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>>: Sized {
                 let inner_mplace = self.ecx().unpack_dyn_trait(&dest)?.0;
                 trace!("walk_value: dyn object layout: {:#?}", inner_mplace.layout);
                 // recurse with the inner type
-                return self.visit_field(&v, 0, &inner_mplace.into());
+                return self.visit_field(v, 0, &inner_mplace.into());
             }
             ty::Dynamic(_, _, ty::DynStar) => {
                 // DynStar types. Very different from a dyn type (but strangely part of the
                 // same variant in `TyKind`): These are pairs where the 2nd component is the
                 // vtable, and the first component is the data (which must be ptr-sized).
                 let data = self.ecx().unpack_dyn_star(v)?.0;
-                return self.visit_field(&v, 0, &data);
+                return self.visit_field(v, 0, &data);
             }
             // Slices do not need special handling here: they have `Array` field
             // placement with length 0, so we enter the `Array` case below which
diff --git a/compiler/rustc_const_eval/src/lib.rs b/compiler/rustc_const_eval/src/lib.rs
index 1e21c4940..d6c36ccc5 100644
--- a/compiler/rustc_const_eval/src/lib.rs
+++ b/compiler/rustc_const_eval/src/lib.rs
@@ -4,9 +4,9 @@ Rust MIR: a lowered representation of Rust.
 
 */
 
-#![cfg_attr(not(bootstrap), allow(internal_features))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
+#![allow(internal_features)]
+#![feature(rustdoc_internals)]
+#![doc(rust_logo)]
 #![deny(rustc::untranslatable_diagnostic)]
 #![feature(assert_matches)]
 #![feature(box_patterns)]
@@ -39,11 +39,9 @@ pub mod util;
 
 pub use errors::ReportErrorExt;
 
-use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
-use rustc_fluent_macro::fluent_messages;
 use rustc_middle::{ty, util::Providers};
 
-fluent_messages! { "../messages.ftl" }
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
 
 pub fn provide(providers: &mut Providers) {
     const_eval::provide(providers);
diff --git a/compiler/rustc_const_eval/src/transform/check_consts/check.rs b/compiler/rustc_const_eval/src/transform/check_consts/check.rs
index 76116e339..949606ed6 100644
--- a/compiler/rustc_const_eval/src/transform/check_consts/check.rs
+++ b/compiler/rustc_const_eval/src/transform/check_consts/check.rs
@@ -12,7 +12,7 @@ use rustc_middle::traits::BuiltinImplSource;
 use rustc_middle::ty::GenericArgs;
 use rustc_middle::ty::{self, adjustment::PointerCoercion, Instance, InstanceDef, Ty, TyCtxt};
 use rustc_middle::ty::{TraitRef, TypeVisitableExt};
-use rustc_mir_dataflow::{self, Analysis};
+use rustc_mir_dataflow::Analysis;
 use rustc_span::{sym, Span, Symbol};
 use rustc_trait_selection::traits::error_reporting::TypeErrCtxtExt as _;
 use rustc_trait_selection::traits::{self, ObligationCauseCode, ObligationCtxt, SelectionContext};
@@ -22,7 +22,7 @@ use std::mem;
 use std::ops::{ControlFlow, Deref};
 
 use super::ops::{self, NonConstOp, Status};
-use super::qualifs::{self, CustomEq, HasMutInterior, NeedsDrop};
+use super::qualifs::{self, CustomEq, HasMutInterior, NeedsDrop, NeedsNonConstDrop};
 use super::resolver::FlowSensitiveAnalysis;
 use super::{ConstCx, Qualif};
 use crate::const_eval::is_unstable_const_fn;
@@ -35,7 +35,7 @@ type QualifResults<'mir, 'tcx, Q> =
 pub struct Qualifs<'mir, 'tcx> {
     has_mut_interior: Option<QualifResults<'mir, 'tcx, HasMutInterior>>,
     needs_drop: Option<QualifResults<'mir, 'tcx, NeedsDrop>>,
-    // needs_non_const_drop: Option<QualifResults<'mir, 'tcx, NeedsNonConstDrop>>,
+    needs_non_const_drop: Option<QualifResults<'mir, 'tcx, NeedsNonConstDrop>>,
 }
 
 impl<'mir, 'tcx> Qualifs<'mir, 'tcx> {
@@ -60,9 +60,9 @@ impl<'mir, 'tcx> Qualifs<'mir, 'tcx> {
             let ConstCx { tcx, body, .. } = *ccx;
 
             FlowSensitiveAnalysis::new(NeedsDrop, ccx)
-                .into_engine(tcx, &body)
+                .into_engine(tcx, body)
                 .iterate_to_fixpoint()
-                .into_results_cursor(&body)
+                .into_results_cursor(body)
         });
 
         needs_drop.seek_before_primary_effect(location);
@@ -78,27 +78,25 @@ impl<'mir, 'tcx> Qualifs<'mir, 'tcx> {
         local: Local,
         location: Location,
     ) -> bool {
-        // FIXME(effects) replace with `NeedsNonconstDrop` after const traits work again
-        /*
         let ty = ccx.body.local_decls[local].ty;
-        if !NeedsDrop::in_any_value_of_ty(ccx, ty) {
+        // Peeking into opaque types causes cycles if the current function declares said opaque
+        // type. Thus we avoid short circuiting on the type and instead run the more expensive
+        // analysis that looks at the actual usage within this function
+        if !ty.has_opaque_types() && !NeedsNonConstDrop::in_any_value_of_ty(ccx, ty) {
             return false;
         }
 
         let needs_non_const_drop = self.needs_non_const_drop.get_or_insert_with(|| {
             let ConstCx { tcx, body, .. } = *ccx;
 
-            FlowSensitiveAnalysis::new(NeedsDrop, ccx)
-                .into_engine(tcx, &body)
+            FlowSensitiveAnalysis::new(NeedsNonConstDrop, ccx)
+                .into_engine(tcx, body)
                 .iterate_to_fixpoint()
-                .into_results_cursor(&body)
+                .into_results_cursor(body)
         });
 
         needs_non_const_drop.seek_before_primary_effect(location);
         needs_non_const_drop.get().contains(local)
-        */
-
-        self.needs_drop(ccx, local, location)
     }
 
     /// Returns `true` if `local` is `HasMutInterior` at the given `Location`.
@@ -122,9 +120,9 @@ impl<'mir, 'tcx> Qualifs<'mir, 'tcx> {
             let ConstCx { tcx, body, .. } = *ccx;
 
             FlowSensitiveAnalysis::new(HasMutInterior, ccx)
-                .into_engine(tcx, &body)
+                .into_engine(tcx, body)
                 .iterate_to_fixpoint()
-                .into_results_cursor(&body)
+                .into_results_cursor(body)
         });
 
         has_mut_interior.seek_before_primary_effect(location);
@@ -170,9 +168,9 @@ impl<'mir, 'tcx> Qualifs<'mir, 'tcx> {
 
             hir::ConstContext::Const { .. } | hir::ConstContext::Static(_) => {
                 let mut cursor = FlowSensitiveAnalysis::new(CustomEq, ccx)
-                    .into_engine(ccx.tcx, &ccx.body)
+                    .into_engine(ccx.tcx, ccx.body)
                     .iterate_to_fixpoint()
-                    .into_results_cursor(&ccx.body);
+                    .into_results_cursor(ccx.body);
 
                 cursor.seek_after_primary_effect(return_loc);
                 cursor.get().contains(RETURN_PLACE)
@@ -225,7 +223,7 @@ impl<'mir, 'tcx> Deref for Checker<'mir, 'tcx> {
     type Target = ConstCx<'mir, 'tcx>;
 
     fn deref(&self) -> &Self::Target {
-        &self.ccx
+        self.ccx
     }
 }
 
@@ -248,7 +246,7 @@ impl<'mir, 'tcx> Checker<'mir, 'tcx> {
         // `async` functions cannot be `const fn`. This is checked during AST lowering, so there's
         // no need to emit duplicate errors here.
         if self.ccx.is_async() || body.coroutine.is_some() {
-            tcx.sess.delay_span_bug(body.span, "`async` functions cannot be `const fn`");
+            tcx.sess.span_delayed_bug(body.span, "`async` functions cannot be `const fn`");
             return;
         }
 
@@ -272,15 +270,15 @@ impl<'mir, 'tcx> Checker<'mir, 'tcx> {
         }
 
         if !tcx.has_attr(def_id, sym::rustc_do_not_const_check) {
-            self.visit_body(&body);
+            self.visit_body(body);
         }
 
         // If we got through const-checking without emitting any "primary" errors, emit any
         // "secondary" errors if they occurred.
         let secondary_errors = mem::take(&mut self.secondary_errors);
         if self.error_emitted.is_none() {
-            for mut error in secondary_errors {
-                self.tcx.sess.diagnostic().emit_diagnostic(&mut error);
+            for error in secondary_errors {
+                self.tcx.sess.dcx().emit_diagnostic(error);
             }
         } else {
             assert!(self.tcx.sess.has_errors().is_some());
@@ -357,7 +355,9 @@ impl<'mir, 'tcx> Checker<'mir, 'tcx> {
 
     fn check_static(&mut self, def_id: DefId, span: Span) {
         if self.tcx.is_thread_local_static(def_id) {
-            self.tcx.sess.delay_span_bug(span, "tls access is checked in `Rvalue::ThreadLocalRef`");
+            self.tcx
+                .sess
+                .span_delayed_bug(span, "tls access is checked in `Rvalue::ThreadLocalRef`");
         }
         self.check_op_spanned(ops::StaticAccess, span)
     }
@@ -503,7 +503,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
             Rvalue::Ref(_, BorrowKind::Shared | BorrowKind::Fake, place)
             | Rvalue::AddressOf(Mutability::Not, place) => {
                 let borrowed_place_has_mut_interior = qualifs::in_place::<HasMutInterior, _>(
-                    &self.ccx,
+                    self.ccx,
                     &mut |local| self.qualifs.has_mut_interior(self.ccx, local, location),
                     place.as_ref(),
                 );
@@ -1011,9 +1011,8 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
                 let mut err_span = self.span;
                 let ty_of_dropped_place = dropped_place.ty(self.body, self.tcx).ty;
 
-                // FIXME(effects) replace with `NeedsNonConstDrop` once we fix const traits
                 let ty_needs_non_const_drop =
-                    qualifs::NeedsDrop::in_any_value_of_ty(self.ccx, ty_of_dropped_place);
+                    qualifs::NeedsNonConstDrop::in_any_value_of_ty(self.ccx, ty_of_dropped_place);
 
                 debug!(?ty_of_dropped_place, ?ty_needs_non_const_drop);
 
diff --git a/compiler/rustc_const_eval/src/transform/check_consts/mod.rs b/compiler/rustc_const_eval/src/transform/check_consts/mod.rs
index e51082e1e..4f7e165c5 100644
--- a/compiler/rustc_const_eval/src/transform/check_consts/mod.rs
+++ b/compiler/rustc_const_eval/src/transform/check_consts/mod.rs
@@ -82,8 +82,8 @@ pub fn rustc_allow_const_fn_unstable(
     def_id: LocalDefId,
     feature_gate: Symbol,
 ) -> bool {
-    let attrs = tcx.hir().attrs(tcx.hir().local_def_id_to_hir_id(def_id));
-    attr::rustc_allow_const_fn_unstable(&tcx.sess, attrs).any(|name| name == feature_gate)
+    let attrs = tcx.hir().attrs(tcx.local_def_id_to_hir_id(def_id));
+    attr::rustc_allow_const_fn_unstable(tcx.sess, attrs).any(|name| name == feature_gate)
 }
 
 /// Returns `true` if the given `const fn` is "const-stable".
@@ -112,7 +112,7 @@ pub fn is_const_stable_const_fn(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
         None if is_parent_const_stable_trait(tcx, def_id) => {
             // Remove this when `#![feature(const_trait_impl)]` is stabilized,
             // returning `true` unconditionally.
-            tcx.sess.delay_span_bug(
+            tcx.sess.span_delayed_bug(
                 tcx.def_span(def_id),
                 "trait implementations cannot be const stable yet",
             );
diff --git a/compiler/rustc_const_eval/src/transform/check_consts/ops.rs b/compiler/rustc_const_eval/src/transform/check_consts/ops.rs
index 40183bacc..2de6362b9 100644
--- a/compiler/rustc_const_eval/src/transform/check_consts/ops.rs
+++ b/compiler/rustc_const_eval/src/transform/check_consts/ops.rs
@@ -119,8 +119,7 @@ impl<'tcx> NonConstOp<'tcx> for FnCallNonConst<'tcx> {
             match self_ty.kind() {
                 Param(param_ty) => {
                     debug!(?param_ty);
-                    let caller_hir_id = tcx.hir().local_def_id_to_hir_id(caller);
-                    if let Some(generics) = tcx.hir().get(caller_hir_id).generics() {
+                    if let Some(generics) = tcx.hir_node_by_def_id(caller).generics() {
                         let constraint = with_no_trimmed_paths!(format!(
                             "~const {}",
                             trait_ref.print_only_trait_path()
@@ -129,7 +128,7 @@ impl<'tcx> NonConstOp<'tcx> for FnCallNonConst<'tcx> {
                             tcx,
                             generics,
                             err,
-                            &param_ty.name.as_str(),
+                            param_ty.name.as_str(),
                             &constraint,
                             None,
                             None,
diff --git a/compiler/rustc_const_eval/src/transform/check_consts/post_drop_elaboration.rs b/compiler/rustc_const_eval/src/transform/check_consts/post_drop_elaboration.rs
index aff256b3e..5cd13783c 100644
--- a/compiler/rustc_const_eval/src/transform/check_consts/post_drop_elaboration.rs
+++ b/compiler/rustc_const_eval/src/transform/check_consts/post_drop_elaboration.rs
@@ -5,7 +5,7 @@ use rustc_span::{symbol::sym, Span};
 
 use super::check::Qualifs;
 use super::ops::{self, NonConstOp};
-use super::qualifs::{NeedsDrop, Qualif};
+use super::qualifs::{NeedsNonConstDrop, Qualif};
 use super::ConstCx;
 
 /// Returns `true` if we should use the more precise live drop checker that runs after drop
@@ -54,7 +54,7 @@ impl<'mir, 'tcx> std::ops::Deref for CheckLiveDrops<'mir, 'tcx> {
     type Target = ConstCx<'mir, 'tcx>;
 
     fn deref(&self) -> &Self::Target {
-        &self.ccx
+        self.ccx
     }
 }
 
@@ -83,8 +83,7 @@ impl<'tcx> Visitor<'tcx> for CheckLiveDrops<'_, 'tcx> {
             mir::TerminatorKind::Drop { place: dropped_place, .. } => {
                 let dropped_ty = dropped_place.ty(self.body, self.tcx).ty;
 
-                // FIXME(effects) use `NeedsNonConstDrop`
-                if !NeedsDrop::in_any_value_of_ty(self.ccx, dropped_ty) {
+                if !NeedsNonConstDrop::in_any_value_of_ty(self.ccx, dropped_ty) {
                     // Instead of throwing a bug, we just return here. This is because we have to
                     // run custom `const Drop` impls.
                     return;
diff --git a/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs b/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs
index de3186a53..7e1cbfe66 100644
--- a/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs
+++ b/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs
@@ -23,8 +23,7 @@ pub fn in_any_value_of_ty<'tcx>(
     ConstQualifs {
         has_mut_interior: HasMutInterior::in_any_value_of_ty(cx, ty),
         needs_drop: NeedsDrop::in_any_value_of_ty(cx, ty),
-        // FIXME(effects)
-        needs_non_const_drop: NeedsDrop::in_any_value_of_ty(cx, ty),
+        needs_non_const_drop: NeedsNonConstDrop::in_any_value_of_ty(cx, ty),
         custom_eq: CustomEq::in_any_value_of_ty(cx, ty),
         tainted_by_errors,
     }
@@ -155,12 +154,27 @@ impl Qualif for NeedsNonConstDrop {
             return false;
         }
 
-        // FIXME(effects) constness
+        // FIXME(effects): If `destruct` is not a `const_trait`,
+        // or effects are disabled in this crate, then give up.
+        let destruct_def_id = cx.tcx.require_lang_item(LangItem::Destruct, Some(cx.body.span));
+        if cx.tcx.generics_of(destruct_def_id).host_effect_index.is_none()
+            || !cx.tcx.features().effects
+        {
+            return NeedsDrop::in_any_value_of_ty(cx, ty);
+        }
+
         let obligation = Obligation::new(
             cx.tcx,
             ObligationCause::dummy_with_span(cx.body.span),
             cx.param_env,
-            ty::TraitRef::from_lang_item(cx.tcx, LangItem::Destruct, cx.body.span, [ty]),
+            ty::TraitRef::new(
+                cx.tcx,
+                destruct_def_id,
+                [
+                    ty::GenericArg::from(ty),
+                    ty::GenericArg::from(cx.tcx.expected_host_effect_param_for_body(cx.def_id())),
+                ],
+            ),
         );
 
         let infcx = cx.tcx.infer_ctxt().build();
diff --git a/compiler/rustc_const_eval/src/transform/promote_consts.rs b/compiler/rustc_const_eval/src/transform/promote_consts.rs
index 32af537e2..8b2ea2dc2 100644
--- a/compiler/rustc_const_eval/src/transform/promote_consts.rs
+++ b/compiler/rustc_const_eval/src/transform/promote_consts.rs
@@ -188,7 +188,7 @@ impl<'a, 'tcx> std::ops::Deref for Validator<'a, 'tcx> {
     type Target = ConstCx<'a, 'tcx>;
 
     fn deref(&self) -> &Self::Target {
-        &self.ccx
+        self.ccx
     }
 }
 
@@ -229,7 +229,7 @@ impl<'tcx> Validator<'_, 'tcx> {
                 let statement = &self.body[loc.block].statements[loc.statement_index];
                 match &statement.kind {
                     StatementKind::Assign(box (_, rhs)) => qualifs::in_rvalue::<Q, _>(
-                        &self.ccx,
+                        self.ccx,
                         &mut |l| self.qualif_local::<Q>(l),
                         rhs,
                     ),
@@ -246,7 +246,7 @@ impl<'tcx> Validator<'_, 'tcx> {
                 match &terminator.kind {
                     TerminatorKind::Call { .. } => {
                         let return_ty = self.body.local_decls[local].ty;
-                        Q::in_any_value_of_ty(&self.ccx, return_ty)
+                        Q::in_any_value_of_ty(self.ccx, return_ty)
                     }
                     kind => {
                         span_bug!(terminator.source_info.span, "{:?} not promotable", kind);
@@ -864,6 +864,7 @@ impl<'a, 'tcx> Promoter<'a, 'tcx> {
             };
 
             // Use the underlying local for this (necessarily interior) borrow.
+            debug_assert!(region.is_erased());
             let ty = local_decls[place.local].ty;
             let span = statement.source_info.span;
 
@@ -873,8 +874,6 @@ impl<'a, 'tcx> Promoter<'a, 'tcx> {
                 ty::TypeAndMut { ty, mutbl: borrow_kind.to_mutbl_lossy() },
             );
 
-            *region = tcx.lifetimes.re_erased;
-
             let mut projection = vec![PlaceElem::Deref];
             projection.extend(place.projection);
             place.projection = tcx.mk_place_elems(&projection);
@@ -1024,36 +1023,3 @@ pub fn promote_candidates<'tcx>(
 
     promotions
 }
-
-/// This function returns `true` if the function being called in the array
-/// repeat expression is a `const` function.
-pub fn is_const_fn_in_array_repeat_expression<'tcx>(
-    ccx: &ConstCx<'_, 'tcx>,
-    place: &Place<'tcx>,
-    body: &Body<'tcx>,
-) -> bool {
-    match place.as_local() {
-        // rule out cases such as: `let my_var = some_fn(); [my_var; N]`
-        Some(local) if body.local_decls[local].is_user_variable() => return false,
-        None => return false,
-        _ => {}
-    }
-
-    for block in body.basic_blocks.iter() {
-        if let Some(Terminator { kind: TerminatorKind::Call { func, destination, .. }, .. }) =
-            &block.terminator
-        {
-            if let Operand::Constant(box ConstOperand { const_, .. }) = func {
-                if let ty::FnDef(def_id, _) = *const_.ty().kind() {
-                    if destination == place {
-                        if ccx.tcx.is_const_fn(def_id) {
-                            return true;
-                        }
-                    }
-                }
-            }
-        }
-    }
-
-    false
-}
diff --git a/compiler/rustc_const_eval/src/transform/validate.rs b/compiler/rustc_const_eval/src/transform/validate.rs
index 5922922d4..cca5b90ab 100644
--- a/compiler/rustc_const_eval/src/transform/validate.rs
+++ b/compiler/rustc_const_eval/src/transform/validate.rs
@@ -128,9 +128,9 @@ impl<'a, 'tcx> CfgChecker<'a, 'tcx> {
     #[track_caller]
     fn fail(&self, location: Location, msg: impl AsRef<str>) {
         let span = self.body.source_info(location).span;
-        // We use `delay_span_bug` as we might see broken MIR when other errors have already
+        // We use `span_delayed_bug` as we might see broken MIR when other errors have already
         // occurred.
-        self.tcx.sess.diagnostic().delay_span_bug(
+        self.tcx.sess.dcx().span_delayed_bug(
             span,
             format!(
                 "broken MIR in {:?} ({}) at {:?}:\n{}",
@@ -285,6 +285,12 @@ impl<'a, 'tcx> CfgChecker<'a, 'tcx> {
             UnwindAction::Unreachable | UnwindAction::Terminate(UnwindTerminateReason::Abi) => (),
         }
     }
+
+    fn is_critical_call_edge(&self, target: Option<BasicBlock>, unwind: UnwindAction) -> bool {
+        let Some(target) = target else { return false };
+        matches!(unwind, UnwindAction::Cleanup(_) | UnwindAction::Terminate(_))
+            && self.body.basic_blocks.predecessors()[target].len() > 1
+    }
 }
 
 impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> {
@@ -425,6 +431,22 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> {
                 }
                 self.check_unwind_edge(location, *unwind);
 
+                // The code generation assumes that there are no critical call edges. The assumption
+                // is used to simplify inserting code that should be executed along the return edge
+                // from the call. FIXME(tmiasko): Since this is a strictly code generation concern,
+                // the code generation should be responsible for handling it.
+                if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Optimized)
+                    && self.is_critical_call_edge(*target, *unwind)
+                {
+                    self.fail(
+                        location,
+                        format!(
+                            "encountered critical edge in `Call` terminator {:?}",
+                            terminator.kind,
+                        ),
+                    );
+                }
+
                 // The call destination place and Operand::Move place used as an argument might be
                 // passed by a reference to the callee. Consequently they must be non-overlapping
                 // and cannot be packed. Currently this simply checks for duplicate places.
@@ -549,7 +571,7 @@ impl<'a, 'tcx> Visitor<'tcx> for CfgChecker<'a, 'tcx> {
 
     fn visit_source_scope(&mut self, scope: SourceScope) {
         if self.body.source_scopes.get(scope).is_none() {
-            self.tcx.sess.diagnostic().delay_span_bug(
+            self.tcx.sess.dcx().span_delayed_bug(
                 self.body.span,
                 format!(
                     "broken MIR in {:?} ({}):\ninvalid source scope {:?}",
diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml
index 8d91c4c43..4732783a1 100644
--- a/compiler/rustc_data_structures/Cargo.toml
+++ b/compiler/rustc_data_structures/Cargo.toml
@@ -10,8 +10,8 @@ bitflags = "1.2.1"
 elsa = "=1.7.1"
 ena = "0.14.2"
 indexmap = { version = "2.0.0" }
-itertools = "0.10.1"
-jobserver_crate = { version = "0.1.13", package = "jobserver" }
+itertools = "0.11"
+jobserver_crate = { version = "0.1.27", package = "jobserver" }
 libc = "0.2"
 measureme = "10.0.0"
 rustc-hash = "1.1.0"
diff --git a/compiler/rustc_data_structures/src/fingerprint/tests.rs b/compiler/rustc_data_structures/src/fingerprint/tests.rs
index 09ec2622a..e04af19ab 100644
--- a/compiler/rustc_data_structures/src/fingerprint/tests.rs
+++ b/compiler/rustc_data_structures/src/fingerprint/tests.rs
@@ -1,5 +1,4 @@
 use super::*;
-use crate::stable_hasher::Hash64;
 
 // Check that `combine_commutative` is order independent.
 #[test]
diff --git a/compiler/rustc_data_structures/src/graph/dominators/mod.rs b/compiler/rustc_data_structures/src/graph/dominators/mod.rs
index 5dd414cfd..4b819e1cb 100644
--- a/compiler/rustc_data_structures/src/graph/dominators/mod.rs
+++ b/compiler/rustc_data_structures/src/graph/dominators/mod.rs
@@ -23,6 +23,7 @@ struct PreOrderFrame<Iter> {
 }
 
 rustc_index::newtype_index! {
+    #[orderable]
     struct PreorderIndex {}
 }
 
diff --git a/compiler/rustc_data_structures/src/graph/implementation/tests.rs b/compiler/rustc_data_structures/src/graph/implementation/tests.rs
index dc1ce1747..3ae5f5868 100644
--- a/compiler/rustc_data_structures/src/graph/implementation/tests.rs
+++ b/compiler/rustc_data_structures/src/graph/implementation/tests.rs
@@ -1,5 +1,4 @@
 use crate::graph::implementation::*;
-use std::fmt::Debug;
 
 type TestGraph = Graph<&'static str, &'static str>;
 
diff --git a/compiler/rustc_data_structures/src/graph/scc/mod.rs b/compiler/rustc_data_structures/src/graph/scc/mod.rs
index cf9312ea8..b54d75f7e 100644
--- a/compiler/rustc_data_structures/src/graph/scc/mod.rs
+++ b/compiler/rustc_data_structures/src/graph/scc/mod.rs
@@ -492,7 +492,7 @@ where
             let returned_walk =
                 return_value.take().into_iter().map(|walk| (*successor_node, Some(walk)));
 
-            let successor_walk = successors.by_ref().map(|successor_node| {
+            let successor_walk = successors.map(|successor_node| {
                 debug!(?node, ?successor_node);
                 (successor_node, self.inspect_node(successor_node))
             });
diff --git a/compiler/rustc_data_structures/src/intern/tests.rs b/compiler/rustc_data_structures/src/intern/tests.rs
index 09810a085..a85cd480a 100644
--- a/compiler/rustc_data_structures/src/intern/tests.rs
+++ b/compiler/rustc_data_structures/src/intern/tests.rs
@@ -1,5 +1,4 @@
 use super::*;
-use std::cmp::Ordering;
 
 #[derive(Debug)]
 struct S(u32);
diff --git a/compiler/rustc_data_structures/src/jobserver.rs b/compiler/rustc_data_structures/src/jobserver.rs
index 09baa3095..412e33aaa 100644
--- a/compiler/rustc_data_structures/src/jobserver.rs
+++ b/compiler/rustc_data_structures/src/jobserver.rs
@@ -1,40 +1,78 @@
 pub use jobserver_crate::Client;
-use std::sync::LazyLock;
-
-// We can only call `from_env` once per process
-
-// Note that this is unsafe because it may misinterpret file descriptors
-// on Unix as jobserver file descriptors. We hopefully execute this near
-// the beginning of the process though to ensure we don't get false
-// positives, or in other words we try to execute this before we open
-// any file descriptors ourselves.
-//
-// Pick a "reasonable maximum" if we don't otherwise have
-// a jobserver in our environment, capping out at 32 so we
-// don't take everything down by hogging the process run queue.
-// The fixed number is used to have deterministic compilation
-// across machines.
-//
-// Also note that we stick this in a global because there could be
-// multiple rustc instances in this process, and the jobserver is
-// per-process.
-static GLOBAL_CLIENT: LazyLock<Client> = LazyLock::new(|| unsafe {
-    Client::from_env().unwrap_or_else(|| {
-        let client = Client::new(32).expect("failed to create jobserver");
-        // Acquire a token for the main thread which we can release later
-        client.acquire_raw().ok();
-        client
-    })
+
+use jobserver_crate::{FromEnv, FromEnvErrorKind};
+
+use std::sync::{LazyLock, OnceLock};
+
+// We can only call `from_env_ext` once per process
+
+// We stick this in a global because there could be multiple rustc instances
+// in this process, and the jobserver is per-process.
+static GLOBAL_CLIENT: LazyLock<Result<Client, String>> = LazyLock::new(|| {
+    // Note that this is unsafe because it may misinterpret file descriptors
+    // on Unix as jobserver file descriptors. We hopefully execute this near
+    // the beginning of the process though to ensure we don't get false
+    // positives, or in other words we try to execute this before we open
+    // any file descriptors ourselves.
+    let FromEnv { client, var } = unsafe { Client::from_env_ext(true) };
+
+    let error = match client {
+        Ok(client) => return Ok(client),
+        Err(e) => e,
+    };
+
+    if matches!(
+        error.kind(),
+        FromEnvErrorKind::NoEnvVar | FromEnvErrorKind::NoJobserver | FromEnvErrorKind::Unsupported
+    ) {
+        return Ok(default_client());
+    }
+
+    // Environment specifies jobserver, but it looks incorrect.
+    // Safety: `error.kind()` should be `NoEnvVar` if `var == None`.
+    let (name, value) = var.unwrap();
+    Err(format!(
+        "failed to connect to jobserver from environment variable `{name}={:?}`: {error}",
+        value
+    ))
 });
 
+// Create a new jobserver if there's no inherited one.
+fn default_client() -> Client {
+    // Pick a "reasonable maximum" capping out at 32
+    // so we don't take everything down by hogging the process run queue.
+    // The fixed number is used to have deterministic compilation across machines.
+    let client = Client::new(32).expect("failed to create jobserver");
+
+    // Acquire a token for the main thread which we can release later
+    client.acquire_raw().ok();
+
+    client
+}
+
+static GLOBAL_CLIENT_CHECKED: OnceLock<Client> = OnceLock::new();
+
+pub fn initialize_checked(report_warning: impl FnOnce(&'static str)) {
+    let client_checked = match &*GLOBAL_CLIENT {
+        Ok(client) => client.clone(),
+        Err(e) => {
+            report_warning(e);
+            default_client()
+        }
+    };
+    GLOBAL_CLIENT_CHECKED.set(client_checked).ok();
+}
+
+const ACCESS_ERROR: &str = "jobserver check should have been called earlier";
+
 pub fn client() -> Client {
-    GLOBAL_CLIENT.clone()
+    GLOBAL_CLIENT_CHECKED.get().expect(ACCESS_ERROR).clone()
 }
 
 pub fn acquire_thread() {
-    GLOBAL_CLIENT.acquire_raw().ok();
+    GLOBAL_CLIENT_CHECKED.get().expect(ACCESS_ERROR).acquire_raw().ok();
 }
 
 pub fn release_thread() {
-    GLOBAL_CLIENT.release_raw().ok();
+    GLOBAL_CLIENT_CHECKED.get().expect(ACCESS_ERROR).release_raw().ok();
 }
diff --git a/compiler/rustc_data_structures/src/lib.rs b/compiler/rustc_data_structures/src/lib.rs
index d09c026c4..3ef87684f 100644
--- a/compiler/rustc_data_structures/src/lib.rs
+++ b/compiler/rustc_data_structures/src/lib.rs
@@ -10,12 +10,11 @@
 #![allow(internal_features)]
 #![allow(rustc::default_hash_types)]
 #![allow(rustc::potential_query_instability)]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
 #![deny(rustc::diagnostic_outside_of_impl)]
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(unsafe_op_in_unsafe_fn)]
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
+#![doc(rust_logo)]
 #![feature(allocator_api)]
 #![feature(array_windows)]
 #![feature(auto_traits)]
@@ -34,6 +33,7 @@
 #![feature(never_type)]
 #![feature(ptr_alignment_type)]
 #![feature(rustc_attrs)]
+#![feature(rustdoc_internals)]
 #![feature(strict_provenance)]
 #![feature(test)]
 #![feature(thread_id_value)]
diff --git a/compiler/rustc_data_structures/src/obligation_forest/tests.rs b/compiler/rustc_data_structures/src/obligation_forest/tests.rs
index bc252f772..d09c8e544 100644
--- a/compiler/rustc_data_structures/src/obligation_forest/tests.rs
+++ b/compiler/rustc_data_structures/src/obligation_forest/tests.rs
@@ -1,7 +1,6 @@
 use super::*;
 
 use std::fmt;
-use std::marker::PhantomData;
 
 impl<'a> super::ForestObligation for &'a str {
     type CacheKey = &'a str;
diff --git a/compiler/rustc_data_structures/src/sharded.rs b/compiler/rustc_data_structures/src/sharded.rs
index 29516fffd..162dbd234 100644
--- a/compiler/rustc_data_structures/src/sharded.rs
+++ b/compiler/rustc_data_structures/src/sharded.rs
@@ -50,7 +50,7 @@ impl<T> Sharded<T> {
     #[inline]
     pub fn get_shard_by_value<K: Hash + ?Sized>(&self, _val: &K) -> &Lock<T> {
         match self {
-            Self::Single(single) => &single,
+            Self::Single(single) => single,
             #[cfg(parallel_compiler)]
             Self::Shards(..) => self.get_shard_by_hash(make_hash(_val)),
         }
@@ -64,7 +64,7 @@ impl<T> Sharded<T> {
     #[inline]
     pub fn get_shard_by_index(&self, _i: usize) -> &Lock<T> {
         match self {
-            Self::Single(single) => &single,
+            Self::Single(single) => single,
             #[cfg(parallel_compiler)]
             Self::Shards(shards) => {
                 // SAFETY: The index gets ANDed with the shard mask, ensuring it is always inbounds.
@@ -79,7 +79,7 @@ impl<T> Sharded<T> {
     pub fn lock_shard_by_value<K: Hash + ?Sized>(&self, _val: &K) -> LockGuard<'_, T> {
         match self {
             Self::Single(single) => {
-                // Syncronization is disabled so use the `lock_assume_no_sync` method optimized
+                // Synchronization is disabled so use the `lock_assume_no_sync` method optimized
                 // for that case.
 
                 // SAFETY: We know `is_dyn_thread_safe` was false when creating the lock thus
@@ -102,7 +102,7 @@ impl<T> Sharded<T> {
     pub fn lock_shard_by_index(&self, _i: usize) -> LockGuard<'_, T> {
         match self {
             Self::Single(single) => {
-                // Syncronization is disabled so use the `lock_assume_no_sync` method optimized
+                // Synchronization is disabled so use the `lock_assume_no_sync` method optimized
                 // for that case.
 
                 // SAFETY: We know `is_dyn_thread_safe` was false when creating the lock thus
@@ -111,7 +111,7 @@ impl<T> Sharded<T> {
             }
             #[cfg(parallel_compiler)]
             Self::Shards(shards) => {
-                // Syncronization is enabled so use the `lock_assume_sync` method optimized
+                // Synchronization is enabled so use the `lock_assume_sync` method optimized
                 // for that case.
 
                 // SAFETY (get_unchecked): The index gets ANDed with the shard mask, ensuring it is
diff --git a/compiler/rustc_data_structures/src/sip128/tests.rs b/compiler/rustc_data_structures/src/sip128/tests.rs
index cc6d3b0f4..e9dd0f117 100644
--- a/compiler/rustc_data_structures/src/sip128/tests.rs
+++ b/compiler/rustc_data_structures/src/sip128/tests.rs
@@ -1,6 +1,6 @@
 use super::*;
 
-use std::hash::{Hash, Hasher};
+use std::hash::Hash;
 
 // Hash just the bytes of the slice, without length prefix
 struct Bytes<'a>(&'a [u8]);
diff --git a/compiler/rustc_data_structures/src/sorted_map.rs b/compiler/rustc_data_structures/src/sorted_map.rs
index 60b343afb..ed2e558bf 100644
--- a/compiler/rustc_data_structures/src/sorted_map.rs
+++ b/compiler/rustc_data_structures/src/sorted_map.rs
@@ -198,7 +198,7 @@ impl<K: Ord, V> SortedMap<K, V> {
                 if index == self.data.len() || elements.last().unwrap().0 < self.data[index].0 {
                     // We can copy the whole range without having to mix with
                     // existing elements.
-                    self.data.splice(index..index, elements.into_iter());
+                    self.data.splice(index..index, elements);
                     return;
                 }
 
diff --git a/compiler/rustc_data_structures/src/sync/lock.rs b/compiler/rustc_data_structures/src/sync/lock.rs
index 339aebbf8..040a8aa6b 100644
--- a/compiler/rustc_data_structures/src/sync/lock.rs
+++ b/compiler/rustc_data_structures/src/sync/lock.rs
@@ -38,7 +38,7 @@ mod maybe_sync {
         lock: &'a Lock<T>,
         marker: PhantomData<&'a mut T>,
 
-        /// The syncronization mode of the lock. This is explicitly passed to let LLVM relate it
+        /// The synchronization mode of the lock. This is explicitly passed to let LLVM relate it
         /// to the original lock operation.
         mode: Mode,
     }
@@ -142,7 +142,7 @@ mod maybe_sync {
             .then(|| LockGuard { lock: self, marker: PhantomData, mode })
         }
 
-        /// This acquires the lock assuming syncronization is in a specific mode.
+        /// This acquires the lock assuming synchronization is in a specific mode.
         ///
         /// Safety
         /// This method must only be called with `Mode::Sync` if `might_be_dyn_thread_safe` was
diff --git a/compiler/rustc_data_structures/src/sync/worker_local.rs b/compiler/rustc_data_structures/src/sync/worker_local.rs
index ffafdba13..b34d3dd90 100644
--- a/compiler/rustc_data_structures/src/sync/worker_local.rs
+++ b/compiler/rustc_data_structures/src/sync/worker_local.rs
@@ -1,6 +1,7 @@
 use parking_lot::Mutex;
 use std::cell::Cell;
 use std::cell::OnceCell;
+use std::num::NonZeroUsize;
 use std::ops::Deref;
 use std::ptr;
 use std::sync::Arc;
@@ -30,7 +31,7 @@ impl RegistryId {
 }
 
 struct RegistryData {
-    thread_limit: usize,
+    thread_limit: NonZeroUsize,
     threads: Mutex<usize>,
 }
 
@@ -60,7 +61,7 @@ thread_local! {
 
 impl Registry {
     /// Creates a registry which can hold up to `thread_limit` threads.
-    pub fn new(thread_limit: usize) -> Self {
+    pub fn new(thread_limit: NonZeroUsize) -> Self {
         Registry(Arc::new(RegistryData { thread_limit, threads: Mutex::new(0) }))
     }
 
@@ -73,7 +74,7 @@ impl Registry {
     /// Panics if the thread limit is hit or if the thread already has an associated registry.
     pub fn register(&self) {
         let mut threads = self.0.threads.lock();
-        if *threads < self.0.thread_limit {
+        if *threads < self.0.thread_limit.get() {
             REGISTRY.with(|registry| {
                 if registry.get().is_some() {
                     drop(threads);
@@ -126,7 +127,9 @@ impl<T> WorkerLocal<T> {
         {
             let registry = Registry::current();
             WorkerLocal {
-                locals: (0..registry.0.thread_limit).map(|i| CacheAligned(initial(i))).collect(),
+                locals: (0..registry.0.thread_limit.get())
+                    .map(|i| CacheAligned(initial(i)))
+                    .collect(),
                 registry,
             }
         }
diff --git a/compiler/rustc_data_structures/src/tagged_ptr/impl_tag.rs b/compiler/rustc_data_structures/src/tagged_ptr/impl_tag.rs
index cb7f7d318..cafa91c8b 100644
--- a/compiler/rustc_data_structures/src/tagged_ptr/impl_tag.rs
+++ b/compiler/rustc_data_structures/src/tagged_ptr/impl_tag.rs
@@ -81,6 +81,7 @@
 ///     E::A,
 /// }
 /// ```
+#[cfg(bootstrap)]
 #[macro_export]
 macro_rules! impl_tag {
     (
@@ -140,5 +141,148 @@ macro_rules! impl_tag {
     };
 }
 
+/// Implements [`Tag`] for a given type.
+///
+/// You can use `impl_tag` on structs and enums.
+/// You need to specify the type and all its possible values,
+/// which can only be paths with optional fields.
+///
+/// [`Tag`]: crate::tagged_ptr::Tag
+///
+/// # Examples
+///
+/// Basic usage:
+///
+/// ```
+/// #![feature(macro_metavar_expr)]
+/// use rustc_data_structures::{impl_tag, tagged_ptr::Tag};
+///
+/// #[derive(Copy, Clone, PartialEq, Debug)]
+/// enum SomeTag {
+///     A,
+///     B,
+///     X { v: bool },
+///     Y(bool, bool),
+/// }
+///
+/// impl_tag! {
+///     // The type for which the `Tag` will be implemented
+///     impl Tag for SomeTag;
+///     // You need to specify all possible tag values:
+///     SomeTag::A, // 0
+///     SomeTag::B, // 1
+///     // For variants with fields, you need to specify the fields:
+///     SomeTag::X { v: true  }, // 2
+///     SomeTag::X { v: false }, // 3
+///     // For tuple variants use named syntax:
+///     SomeTag::Y { 0: true,  1: true  }, // 4
+///     SomeTag::Y { 0: false, 1: true  }, // 5
+///     SomeTag::Y { 0: true,  1: false }, // 6
+///     SomeTag::Y { 0: false, 1: false }, // 7
+/// }
+///
+/// // Tag values are assigned in order:
+/// assert_eq!(SomeTag::A.into_usize(), 0);
+/// assert_eq!(SomeTag::X { v: false }.into_usize(), 3);
+/// assert_eq!(SomeTag::Y(false, true).into_usize(), 5);
+///
+/// assert_eq!(unsafe { SomeTag::from_usize(1) }, SomeTag::B);
+/// assert_eq!(unsafe { SomeTag::from_usize(2) }, SomeTag::X { v: true });
+/// assert_eq!(unsafe { SomeTag::from_usize(7) }, SomeTag::Y(false, false));
+/// ```
+///
+/// Structs are supported:
+///
+/// ```
+/// #![feature(macro_metavar_expr)]
+/// # use rustc_data_structures::impl_tag;
+/// #[derive(Copy, Clone)]
+/// struct Flags { a: bool, b: bool }
+///
+/// impl_tag! {
+///     impl Tag for Flags;
+///     Flags { a: true,  b: true  },
+///     Flags { a: false, b: true  },
+///     Flags { a: true,  b: false },
+///     Flags { a: false, b: false },
+/// }
+/// ```
+///
+/// Not specifying all values results in a compile error:
+///
+/// ```compile_fail,E0004
+/// #![feature(macro_metavar_expr)]
+/// # use rustc_data_structures::impl_tag;
+/// #[derive(Copy, Clone)]
+/// enum E {
+///     A,
+///     B,
+/// }
+///
+/// impl_tag! {
+///     impl Tag for E;
+///     E::A,
+/// }
+/// ```
+#[cfg(not(bootstrap))]
+#[macro_export]
+macro_rules! impl_tag {
+    (
+        impl Tag for $Self:ty;
+        $(
+            $($path:ident)::* $( { $( $fields:tt )* })?,
+        )*
+    ) => {
+        // Safety:
+        // `bits_for_tags` is called on the same `${index()}`-es as
+        // `into_usize` returns, thus `BITS` constant is correct.
+        unsafe impl $crate::tagged_ptr::Tag for $Self {
+            const BITS: u32 = $crate::tagged_ptr::bits_for_tags(&[
+                $(
+                    ${index()},
+                    $( ${ignore($path)} )*
+                )*
+            ]);
+
+            #[inline]
+            fn into_usize(self) -> usize {
+                // This forbids use of repeating patterns (`Enum::V`&`Enum::V`, etc)
+                // (or at least it should, see <https://github.com/rust-lang/rust/issues/110613>)
+                #[forbid(unreachable_patterns)]
+                match self {
+                    // `match` is doing heavy lifting here, by requiring exhaustiveness
+                    $(
+                        $($path)::* $( { $( $fields )* } )? => ${index()},
+                    )*
+                }
+            }
+
+            #[inline]
+            unsafe fn from_usize(tag: usize) -> Self {
+                match tag {
+                    $(
+                        ${index()} => $($path)::* $( { $( $fields )* } )?,
+                    )*
+
+                    // Safety:
+                    // `into_usize` only returns `${index()}` of the same
+                    // repetition as we are filtering above, thus if this is
+                    // reached, the safety contract of this function was
+                    // already breached.
+                    _ => unsafe {
+                        debug_assert!(
+                            false,
+                            "invalid tag: {tag}\
+                             (this is a bug in the caller of `from_usize`)"
+                        );
+                        std::hint::unreachable_unchecked()
+                    },
+                }
+            }
+
+        }
+    };
+}
+
 #[cfg(test)]
 mod tests;
diff --git a/compiler/rustc_driver/src/lib.rs b/compiler/rustc_driver/src/lib.rs
index 27f08fe7e..acd93b0b2 100644
--- a/compiler/rustc_driver/src/lib.rs
+++ b/compiler/rustc_driver/src/lib.rs
@@ -1,8 +1,8 @@
 // This crate is intentionally empty and a re-export of `rustc_driver_impl` to allow the code in
 // `rustc_driver_impl` to be compiled in parallel with other crates.
 
-#![cfg_attr(not(bootstrap), allow(internal_features))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
+#![allow(internal_features)]
+#![feature(rustdoc_internals)]
+#![doc(rust_logo)]
 
 pub use rustc_driver_impl::*;
diff --git a/compiler/rustc_driver_impl/Cargo.toml b/compiler/rustc_driver_impl/Cargo.toml
index da7c2440f..490429845 100644
--- a/compiler/rustc_driver_impl/Cargo.toml
+++ b/compiler/rustc_driver_impl/Cargo.toml
@@ -16,7 +16,6 @@ rustc_codegen_ssa = { path = "../rustc_codegen_ssa" }
 rustc_const_eval = { path = "../rustc_const_eval" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_error_codes = { path = "../rustc_error_codes" }
-rustc_error_messages = { path = "../rustc_error_messages" }
 rustc_errors = { path = "../rustc_errors" }
 rustc_expand = { path = "../rustc_expand" }
 rustc_feature = { path = "../rustc_feature" }
@@ -39,10 +38,12 @@ rustc_mir_transform = { path = "../rustc_mir_transform" }
 rustc_monomorphize = { path = "../rustc_monomorphize" }
 rustc_parse = { path = "../rustc_parse" }
 rustc_passes = { path = "../rustc_passes" }
+rustc_pattern_analysis = { path = "../rustc_pattern_analysis" }
 rustc_privacy = { path = "../rustc_privacy" }
 rustc_query_system = { path = "../rustc_query_system" }
 rustc_resolve = { path = "../rustc_resolve" }
 rustc_session = { path = "../rustc_session" }
+rustc_smir ={ path = "../rustc_smir" }
 rustc_span = { path = "../rustc_span" }
 rustc_symbol_mangling = { path = "../rustc_symbol_mangling" }
 rustc_target = { path = "../rustc_target" }
diff --git a/compiler/rustc_driver_impl/src/args.rs b/compiler/rustc_driver_impl/src/args.rs
index 654d7636d..dc546da73 100644
--- a/compiler/rustc_driver_impl/src/args.rs
+++ b/compiler/rustc_driver_impl/src/args.rs
@@ -3,7 +3,7 @@ use std::fmt;
 use std::fs;
 use std::io;
 
-use rustc_session::EarlyErrorHandler;
+use rustc_session::EarlyDiagCtxt;
 
 fn arg_expand(arg: String) -> Result<Vec<String>, Error> {
     if let Some(path) = arg.strip_prefix('@') {
@@ -23,12 +23,12 @@ fn arg_expand(arg: String) -> Result<Vec<String>, Error> {
 /// **Note:** This function doesn't interpret argument 0 in any special way.
 /// If this function is intended to be used with command line arguments,
 /// `argv[0]` must be removed prior to calling it manually.
-pub fn arg_expand_all(handler: &EarlyErrorHandler, at_args: &[String]) -> Vec<String> {
+pub fn arg_expand_all(early_dcx: &EarlyDiagCtxt, at_args: &[String]) -> Vec<String> {
     let mut args = Vec::new();
     for arg in at_args {
         match arg_expand(arg.clone()) {
             Ok(arg) => args.extend(arg),
-            Err(err) => handler.early_error(format!("Failed to load argument file: {err}")),
+            Err(err) => early_dcx.early_error(format!("Failed to load argument file: {err}")),
         }
     }
     args
diff --git a/compiler/rustc_driver_impl/src/lib.rs b/compiler/rustc_driver_impl/src/lib.rs
index 84ae45d6a..d67fea7e9 100644
--- a/compiler/rustc_driver_impl/src/lib.rs
+++ b/compiler/rustc_driver_impl/src/lib.rs
@@ -5,15 +5,14 @@
 //! This API is completely unstable and subject to change.
 
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
-#![cfg_attr(not(bootstrap), allow(internal_features))]
+#![doc(rust_logo)]
+#![feature(rustdoc_internals)]
+#![allow(internal_features)]
 #![feature(decl_macro)]
 #![feature(lazy_cell)]
 #![feature(let_chains)]
 #![feature(panic_update_hook)]
 #![recursion_limit = "256"]
-#![allow(rustc::potential_query_instability)]
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
 
@@ -28,19 +27,18 @@ use rustc_data_structures::profiling::{
 use rustc_data_structures::sync::SeqCst;
 use rustc_errors::registry::{InvalidErrorCode, Registry};
 use rustc_errors::{markdown, ColorConfig};
-use rustc_errors::{DiagnosticMessage, ErrorGuaranteed, Handler, PResult, SubdiagnosticMessage};
+use rustc_errors::{DiagCtxt, ErrorGuaranteed, PResult};
 use rustc_feature::find_gated_cfg;
-use rustc_fluent_macro::fluent_messages;
 use rustc_interface::util::{self, collect_crate_types, get_codegen_backend};
 use rustc_interface::{interface, Queries};
-use rustc_lint::{unerased_lint_store, LintStore};
+use rustc_lint::unerased_lint_store;
+use rustc_metadata::creader::MetadataLoader;
 use rustc_metadata::locator;
 use rustc_session::config::{nightly_options, CG_OPTIONS, Z_OPTIONS};
 use rustc_session::config::{ErrorOutputType, Input, OutFileName, OutputType, TrimmedDefPaths};
-use rustc_session::cstore::MetadataLoader;
 use rustc_session::getopts::{self, Matches};
 use rustc_session::lint::{Lint, LintId};
-use rustc_session::{config, EarlyErrorHandler, Session};
+use rustc_session::{config, EarlyDiagCtxt, Session};
 use rustc_span::def_id::LOCAL_CRATE;
 use rustc_span::source_map::FileLoader;
 use rustc_span::symbol::sym;
@@ -102,7 +100,7 @@ use crate::session_diagnostics::{
     RLinkWrongFileType, RlinkNotAFile, RlinkUnableToRead,
 };
 
-fluent_messages! { "../messages.ftl" }
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
 
 pub static DEFAULT_LOCALE_RESOURCES: &[&str] = &[
     // tidy-alphabetical-start
@@ -114,7 +112,6 @@ pub static DEFAULT_LOCALE_RESOURCES: &[&str] = &[
     rustc_builtin_macros::DEFAULT_LOCALE_RESOURCE,
     rustc_codegen_ssa::DEFAULT_LOCALE_RESOURCE,
     rustc_const_eval::DEFAULT_LOCALE_RESOURCE,
-    rustc_error_messages::DEFAULT_LOCALE_RESOURCE,
     rustc_errors::DEFAULT_LOCALE_RESOURCE,
     rustc_expand::DEFAULT_LOCALE_RESOURCE,
     rustc_hir_analysis::DEFAULT_LOCALE_RESOURCE,
@@ -131,6 +128,7 @@ pub static DEFAULT_LOCALE_RESOURCES: &[&str] = &[
     rustc_monomorphize::DEFAULT_LOCALE_RESOURCE,
     rustc_parse::DEFAULT_LOCALE_RESOURCE,
     rustc_passes::DEFAULT_LOCALE_RESOURCE,
+    rustc_pattern_analysis::DEFAULT_LOCALE_RESOURCE,
     rustc_privacy::DEFAULT_LOCALE_RESOURCE,
     rustc_query_system::DEFAULT_LOCALE_RESOURCE,
     rustc_resolve::DEFAULT_LOCALE_RESOURCE,
@@ -293,7 +291,7 @@ fn run_compiler(
     >,
     using_internal_features: Arc<std::sync::atomic::AtomicBool>,
 ) -> interface::Result<()> {
-    let mut early_error_handler = EarlyErrorHandler::new(ErrorOutputType::default());
+    let mut default_early_dcx = EarlyDiagCtxt::new(ErrorOutputType::default());
 
     // Throw away the first argument, the name of the binary.
     // In case of at_args being empty, as might be the case by
@@ -305,14 +303,14 @@ fn run_compiler(
     // the compiler with @empty_file as argv[0] and no more arguments.
     let at_args = at_args.get(1..).unwrap_or_default();
 
-    let args = args::arg_expand_all(&early_error_handler, at_args);
+    let args = args::arg_expand_all(&default_early_dcx, at_args);
 
-    let Some(matches) = handle_options(&early_error_handler, &args) else { return Ok(()) };
+    let Some(matches) = handle_options(&default_early_dcx, &args) else { return Ok(()) };
 
-    let sopts = config::build_session_options(&mut early_error_handler, &matches);
+    let sopts = config::build_session_options(&mut default_early_dcx, &matches);
 
     if let Some(ref code) = matches.opt_str("explain") {
-        handle_explain(&early_error_handler, diagnostics_registry(), code, sopts.color);
+        handle_explain(&default_early_dcx, diagnostics_registry(), code, sopts.color);
         return Ok(());
     }
 
@@ -338,71 +336,56 @@ fn run_compiler(
         expanded_args: args,
     };
 
-    match make_input(&early_error_handler, &matches.free) {
+    let has_input = match make_input(&default_early_dcx, &matches.free) {
         Err(reported) => return Err(reported),
         Ok(Some(input)) => {
             config.input = input;
-
-            callbacks.config(&mut config);
+            true // has input: normal compilation
         }
         Ok(None) => match matches.free.len() {
-            0 => {
-                callbacks.config(&mut config);
-
-                early_error_handler.abort_if_errors();
-
-                interface::run_compiler(config, |compiler| {
-                    let sopts = &compiler.session().opts;
-                    let handler = EarlyErrorHandler::new(sopts.error_format);
-
-                    if sopts.describe_lints {
-                        let mut lint_store =
-                            rustc_lint::new_lint_store(compiler.session().enable_internal_lints());
-                        let registered_lints =
-                            if let Some(register_lints) = compiler.register_lints() {
-                                register_lints(compiler.session(), &mut lint_store);
-                                true
-                            } else {
-                                false
-                            };
-                        describe_lints(compiler.session(), &lint_store, registered_lints);
-                        return;
-                    }
-                    let should_stop = print_crate_info(
-                        &handler,
-                        &**compiler.codegen_backend(),
-                        compiler.session(),
-                        false,
-                    );
-
-                    if should_stop == Compilation::Stop {
-                        return;
-                    }
-                    handler.early_error("no input filename given")
-                });
-                return Ok(());
-            }
+            0 => false, // no input: we will exit early
             1 => panic!("make_input should have provided valid inputs"),
-            _ => early_error_handler.early_error(format!(
+            _ => default_early_dcx.early_error(format!(
                 "multiple input filenames provided (first two filenames are `{}` and `{}`)",
                 matches.free[0], matches.free[1],
             )),
         },
     };
 
-    early_error_handler.abort_if_errors();
+    callbacks.config(&mut config);
+
+    default_early_dcx.abort_if_errors();
+    drop(default_early_dcx);
 
     interface::run_compiler(config, |compiler| {
-        let sess = compiler.session();
-        let handler = EarlyErrorHandler::new(sess.opts.error_format);
+        let sess = &compiler.sess;
+        let codegen_backend = &*compiler.codegen_backend;
+
+        // This implements `-Whelp`. It should be handled very early, like
+        // `--help`/`-Zhelp`/`-Chelp`. This is the earliest it can run, because
+        // it must happen after lints are registered, during session creation.
+        if sess.opts.describe_lints {
+            describe_lints(sess);
+            return sess.compile_status();
+        }
 
-        let should_stop = print_crate_info(&handler, &**compiler.codegen_backend(), sess, true)
-            .and_then(|| {
-                list_metadata(&handler, sess, &*compiler.codegen_backend().metadata_loader())
-            })
-            .and_then(|| try_process_rlink(sess, compiler));
+        let early_dcx = EarlyDiagCtxt::new(sess.opts.error_format);
 
-        if should_stop == Compilation::Stop {
+        if print_crate_info(&early_dcx, codegen_backend, sess, has_input) == Compilation::Stop {
+            return sess.compile_status();
+        }
+
+        if !has_input {
+            early_dcx.early_error("no input filename given"); // this is fatal
+        }
+
+        if !sess.opts.unstable_opts.ls.is_empty() {
+            list_metadata(&early_dcx, sess, &*codegen_backend.metadata_loader());
+            return sess.compile_status();
+        }
+
+        if sess.opts.unstable_opts.link_only {
+            process_rlink(sess, compiler);
             return sess.compile_status();
         }
 
@@ -418,9 +401,7 @@ fn run_compiler(
                         Ok(())
                     })?;
 
-                    // Make sure the `output_filenames` query is run for its side
-                    // effects of writing the dep-info and reporting errors.
-                    queries.global_ctxt()?.enter(|tcx| tcx.output_filenames(()));
+                    queries.write_dep_info()?;
                 } else {
                     let krate = queries.parse()?;
                     pretty::print(
@@ -441,13 +422,6 @@ fn run_compiler(
                 return early_exit();
             }
 
-            if sess.opts.describe_lints {
-                queries
-                    .global_ctxt()?
-                    .enter(|tcx| describe_lints(sess, unerased_lint_store(tcx), true));
-                return early_exit();
-            }
-
             // Make sure name resolution and macro expansion is run.
             queries.global_ctxt()?.enter(|tcx| tcx.resolver_for_lowering(()));
 
@@ -455,9 +429,7 @@ fn run_compiler(
                 return early_exit();
             }
 
-            // Make sure the `output_filenames` query is run for its side
-            // effects of writing the dep-info and reporting errors.
-            queries.global_ctxt()?.enter(|tcx| tcx.output_filenames(()));
+            queries.write_dep_info()?;
 
             if sess.opts.output_types.contains_key(&OutputType::DepInfo)
                 && sess.opts.output_types.len() == 1
@@ -475,8 +447,10 @@ fn run_compiler(
                 return early_exit();
             }
 
-            let ongoing_codegen = queries.ongoing_codegen()?;
+            let linker = queries.codegen_and_build_linker()?;
 
+            // This must run after monomorphization so that all generic types
+            // have been instantiated.
             if sess.opts.unstable_opts.print_type_sizes {
                 sess.code_stats.print_type_sizes();
             }
@@ -487,17 +461,14 @@ fn run_compiler(
                 sess.code_stats.print_vtable_sizes(crate_name);
             }
 
-            let linker = queries.linker(ongoing_codegen)?;
             Ok(Some(linker))
         })?;
 
+        // Linking is done outside the `compiler.enter()` so that the
+        // `GlobalCtxt` within `Queries` can be freed as early as possible.
         if let Some(linker) = linker {
             let _timer = sess.timer("link");
-            linker.link()?
-        }
-
-        if sess.opts.unstable_opts.perf_stats {
-            sess.print_perf_stats();
+            linker.link(sess, codegen_backend)?
         }
 
         if sess.opts.unstable_opts.print_fuel.is_some() {
@@ -524,7 +495,7 @@ fn make_output(matches: &getopts::Matches) -> (Option<PathBuf>, Option<OutFileNa
 
 // Extract input (string or file and optional path) from matches.
 fn make_input(
-    handler: &EarlyErrorHandler,
+    early_dcx: &EarlyDiagCtxt,
     free_matches: &[String],
 ) -> Result<Option<Input>, ErrorGuaranteed> {
     if free_matches.len() == 1 {
@@ -534,7 +505,7 @@ fn make_input(
             if io::stdin().read_to_string(&mut src).is_err() {
                 // Immediately stop compilation if there was an issue reading
                 // the input (for example if the input stream is not UTF-8).
-                let reported = handler.early_error_no_abort(
+                let reported = early_dcx.early_error_no_abort(
                     "couldn't read from stdin, as it did not contain valid UTF-8",
                 );
                 return Err(reported);
@@ -566,16 +537,7 @@ pub enum Compilation {
     Continue,
 }
 
-impl Compilation {
-    fn and_then<F: FnOnce() -> Compilation>(self, next: F) -> Compilation {
-        match self {
-            Compilation::Stop => Compilation::Stop,
-            Compilation::Continue => next(),
-        }
-    }
-}
-
-fn handle_explain(handler: &EarlyErrorHandler, registry: Registry, code: &str, color: ColorConfig) {
+fn handle_explain(early_dcx: &EarlyDiagCtxt, registry: Registry, code: &str, color: ColorConfig) {
     let upper_cased_code = code.to_ascii_uppercase();
     let normalised =
         if upper_cased_code.starts_with('E') { upper_cased_code } else { format!("E{code:0>4}") };
@@ -605,7 +567,7 @@ fn handle_explain(handler: &EarlyErrorHandler, registry: Registry, code: &str, c
             }
         }
         Err(InvalidErrorCode) => {
-            handler.early_error(format!("{code} is not a valid error code"));
+            early_dcx.early_error(format!("{code} is not a valid error code"));
         }
     }
 }
@@ -625,10 +587,8 @@ fn show_md_content_with_pager(content: &str, color: ColorConfig) {
     let mut print_formatted = if pager_name == "less" {
         cmd.arg("-r");
         true
-    } else if ["bat", "catbat", "delta"].iter().any(|v| *v == pager_name) {
-        true
     } else {
-        false
+        ["bat", "catbat", "delta"].iter().any(|v| *v == pager_name)
     };
 
     if color == ColorConfig::Never {
@@ -679,74 +639,59 @@ fn show_md_content_with_pager(content: &str, color: ColorConfig) {
     }
 }
 
-fn try_process_rlink(sess: &Session, compiler: &interface::Compiler) -> Compilation {
-    if sess.opts.unstable_opts.link_only {
-        if let Input::File(file) = &sess.io.input {
-            let outputs = compiler.build_output_filenames(sess, &[]);
-            let rlink_data = fs::read(file).unwrap_or_else(|err| {
-                sess.emit_fatal(RlinkUnableToRead { err });
-            });
-            let codegen_results = match CodegenResults::deserialize_rlink(sess, rlink_data) {
-                Ok(codegen) => codegen,
-                Err(err) => {
-                    match err {
-                        CodegenErrors::WrongFileType => sess.emit_fatal(RLinkWrongFileType),
-                        CodegenErrors::EmptyVersionNumber => {
-                            sess.emit_fatal(RLinkEmptyVersionNumber)
-                        }
-                        CodegenErrors::EncodingVersionMismatch { version_array, rlink_version } => {
-                            sess.emit_fatal(RLinkEncodingVersionMismatch {
-                                version_array,
-                                rlink_version,
-                            })
-                        }
-                        CodegenErrors::RustcVersionMismatch { rustc_version } => {
-                            sess.emit_fatal(RLinkRustcVersionMismatch {
-                                rustc_version,
-                                current_version: sess.cfg_version,
-                            })
-                        }
-                    };
-                }
-            };
-            let result = compiler.codegen_backend().link(sess, codegen_results, &outputs);
-            abort_on_err(result, sess);
-        } else {
-            sess.emit_fatal(RlinkNotAFile {})
-        }
-        Compilation::Stop
+fn process_rlink(sess: &Session, compiler: &interface::Compiler) {
+    assert!(sess.opts.unstable_opts.link_only);
+    if let Input::File(file) = &sess.io.input {
+        let rlink_data = fs::read(file).unwrap_or_else(|err| {
+            sess.emit_fatal(RlinkUnableToRead { err });
+        });
+        let (codegen_results, outputs) = match CodegenResults::deserialize_rlink(sess, rlink_data) {
+            Ok((codegen, outputs)) => (codegen, outputs),
+            Err(err) => {
+                match err {
+                    CodegenErrors::WrongFileType => sess.emit_fatal(RLinkWrongFileType),
+                    CodegenErrors::EmptyVersionNumber => sess.emit_fatal(RLinkEmptyVersionNumber),
+                    CodegenErrors::EncodingVersionMismatch { version_array, rlink_version } => sess
+                        .emit_fatal(RLinkEncodingVersionMismatch { version_array, rlink_version }),
+                    CodegenErrors::RustcVersionMismatch { rustc_version } => {
+                        sess.emit_fatal(RLinkRustcVersionMismatch {
+                            rustc_version,
+                            current_version: sess.cfg_version,
+                        })
+                    }
+                };
+            }
+        };
+        let result = compiler.codegen_backend.link(sess, codegen_results, &outputs);
+        abort_on_err(result, sess);
     } else {
-        Compilation::Continue
+        sess.emit_fatal(RlinkNotAFile {})
     }
 }
 
-fn list_metadata(
-    handler: &EarlyErrorHandler,
-    sess: &Session,
-    metadata_loader: &dyn MetadataLoader,
-) -> Compilation {
-    let ls_kinds = &sess.opts.unstable_opts.ls;
-    if !ls_kinds.is_empty() {
-        match sess.io.input {
-            Input::File(ref ifile) => {
-                let path = &(*ifile);
-                let mut v = Vec::new();
-                locator::list_file_metadata(&sess.target, path, metadata_loader, &mut v, ls_kinds)
-                    .unwrap();
-                safe_println!("{}", String::from_utf8(v).unwrap());
-            }
-            Input::Str { .. } => {
-                handler.early_error("cannot list metadata for stdin");
-            }
+fn list_metadata(early_dcx: &EarlyDiagCtxt, sess: &Session, metadata_loader: &dyn MetadataLoader) {
+    match sess.io.input {
+        Input::File(ref ifile) => {
+            let path = &(*ifile);
+            let mut v = Vec::new();
+            locator::list_file_metadata(
+                &sess.target,
+                path,
+                metadata_loader,
+                &mut v,
+                &sess.opts.unstable_opts.ls,
+            )
+            .unwrap();
+            safe_println!("{}", String::from_utf8(v).unwrap());
+        }
+        Input::Str { .. } => {
+            early_dcx.early_error("cannot list metadata for stdin");
         }
-        return Compilation::Stop;
     }
-
-    Compilation::Continue
 }
 
 fn print_crate_info(
-    handler: &EarlyErrorHandler,
+    early_dcx: &EarlyDiagCtxt,
     codegen_backend: &dyn CodegenBackend,
     sess: &Session,
     parse_attrs: bool,
@@ -893,7 +838,7 @@ fn print_crate_info(
                         .expect("unknown Apple target OS");
                     println_info!("deployment_target={}", format!("{major}.{minor}"))
                 } else {
-                    handler
+                    early_dcx
                         .early_error("only Apple targets currently support deployment version info")
                 }
             }
@@ -907,12 +852,12 @@ fn print_crate_info(
 /// Prints version information
 ///
 /// NOTE: this is a macro to support drivers built at a different time than the main `rustc_driver` crate.
-pub macro version($handler: expr, $binary: literal, $matches: expr) {
+pub macro version($early_dcx: expr, $binary: literal, $matches: expr) {
     fn unw(x: Option<&str>) -> &str {
         x.unwrap_or("unknown")
     }
     $crate::version_at_macro_invocation(
-        $handler,
+        $early_dcx,
         $binary,
         $matches,
         unw(option_env!("CFG_VERSION")),
@@ -924,7 +869,7 @@ pub macro version($handler: expr, $binary: literal, $matches: expr) {
 
 #[doc(hidden)] // use the macro instead
 pub fn version_at_macro_invocation(
-    handler: &EarlyErrorHandler,
+    early_dcx: &EarlyDiagCtxt,
     binary: &str,
     matches: &getopts::Matches,
     version: &str,
@@ -945,7 +890,7 @@ pub fn version_at_macro_invocation(
 
         let debug_flags = matches.opt_strs("Z");
         let backend_name = debug_flags.iter().find_map(|x| x.strip_prefix("codegen-backend="));
-        get_codegen_backend(handler, &None, backend_name).print_version();
+        get_codegen_backend(early_dcx, &None, backend_name).print_version();
     }
 }
 
@@ -995,7 +940,7 @@ the command line flag directly.
 }
 
 /// Write to stdout lint command options, together with a list of all available lints
-pub fn describe_lints(sess: &Session, lint_store: &LintStore, loaded_lints: bool) {
+pub fn describe_lints(sess: &Session) {
     safe_println!(
         "
 Available lint options:
@@ -1021,6 +966,7 @@ Available lint options:
         lints
     }
 
+    let lint_store = unerased_lint_store(sess);
     let (loaded, builtin): (Vec<_>, _) =
         lint_store.get_lints().iter().cloned().partition(|&lint| lint.is_loaded);
     let loaded = sort_lints(sess, loaded);
@@ -1098,7 +1044,7 @@ Available lint options:
 
     print_lint_groups(builtin_groups, true);
 
-    match (loaded_lints, loaded.len(), loaded_groups.len()) {
+    match (sess.registered_lints, loaded.len(), loaded_groups.len()) {
         (false, 0, _) | (false, _, 0) => {
             safe_println!("Lint tools like Clippy can load additional lints and lint groups.");
         }
@@ -1122,7 +1068,7 @@ Available lint options:
 /// Show help for flag categories shared between rustdoc and rustc.
 ///
 /// Returns whether a help option was printed.
-pub fn describe_flag_categories(handler: &EarlyErrorHandler, matches: &Matches) -> bool {
+pub fn describe_flag_categories(early_dcx: &EarlyDiagCtxt, matches: &Matches) -> bool {
     // Handle the special case of -Wall.
     let wall = matches.opt_strs("W");
     if wall.iter().any(|x| *x == "all") {
@@ -1144,12 +1090,12 @@ pub fn describe_flag_categories(handler: &EarlyErrorHandler, matches: &Matches)
     }
 
     if cg_flags.iter().any(|x| *x == "no-stack-check") {
-        handler.early_warn("the --no-stack-check flag is deprecated and does nothing");
+        early_dcx.early_warn("the --no-stack-check flag is deprecated and does nothing");
     }
 
     if cg_flags.iter().any(|x| *x == "passes=list") {
         let backend_name = debug_flags.iter().find_map(|x| x.strip_prefix("codegen-backend="));
-        get_codegen_backend(handler, &None, backend_name).print_passes();
+        get_codegen_backend(early_dcx, &None, backend_name).print_passes();
         return true;
     }
 
@@ -1210,7 +1156,7 @@ fn print_flag_list<T>(
 /// This does not need to be `pub` for rustc itself, but @chaosite needs it to
 /// be public when using rustc as a library, see
 /// <https://github.com/rust-lang/rust/commit/2b4c33817a5aaecabf4c6598d41e190080ec119e>
-pub fn handle_options(handler: &EarlyErrorHandler, args: &[String]) -> Option<getopts::Matches> {
+pub fn handle_options(early_dcx: &EarlyDiagCtxt, args: &[String]) -> Option<getopts::Matches> {
     if args.is_empty() {
         // user did not write `-v` nor `-Z unstable-options`, so do not
         // include that extra information.
@@ -1236,7 +1182,7 @@ pub fn handle_options(handler: &EarlyErrorHandler, args: &[String]) -> Option<ge
                 .map(|(flag, _)| format!("{e}. Did you mean `-{flag} {opt}`?")),
             _ => None,
         };
-        handler.early_error(msg.unwrap_or_else(|| e.to_string()));
+        early_dcx.early_error(msg.unwrap_or_else(|| e.to_string()));
     });
 
     // For all options we just parsed, we check a few aspects:
@@ -1250,7 +1196,7 @@ pub fn handle_options(handler: &EarlyErrorHandler, args: &[String]) -> Option<ge
     //   we're good to go.
     // * Otherwise, if we're an unstable option then we generate an error
     //   (unstable option being used on stable)
-    nightly_options::check_nightly_options(handler, &matches, &config::rustc_optgroups());
+    nightly_options::check_nightly_options(early_dcx, &matches, &config::rustc_optgroups());
 
     if matches.opt_present("h") || matches.opt_present("help") {
         // Only show unstable options in --help if we accept unstable options.
@@ -1260,12 +1206,12 @@ pub fn handle_options(handler: &EarlyErrorHandler, args: &[String]) -> Option<ge
         return None;
     }
 
-    if describe_flag_categories(handler, &matches) {
+    if describe_flag_categories(early_dcx, &matches) {
         return None;
     }
 
     if matches.opt_present("version") {
-        version!(handler, "rustc", &matches);
+        version!(early_dcx, "rustc", &matches);
         return None;
     }
 
@@ -1360,7 +1306,10 @@ fn ice_path() -> &'static Option<PathBuf> {
 /// internal features.
 ///
 /// A custom rustc driver can skip calling this to set up a custom ICE hook.
-pub fn install_ice_hook(bug_report_url: &'static str, extra_info: fn(&Handler)) -> Arc<AtomicBool> {
+pub fn install_ice_hook(
+    bug_report_url: &'static str,
+    extra_info: fn(&DiagCtxt),
+) -> Arc<AtomicBool> {
     // If the user has not explicitly overridden "RUST_BACKTRACE", then produce
     // full backtraces. When a compiler ICE happens, we want to gather
     // as much information as possible to present in the issue opened
@@ -1383,8 +1332,8 @@ pub fn install_ice_hook(bug_report_url: &'static str, extra_info: fn(&Handler))
                 if msg.starts_with("failed printing to stdout: ") && msg.ends_with("(os error 232)")
                 {
                     // the error code is already going to be reported when the panic unwinds up the stack
-                    let handler = EarlyErrorHandler::new(ErrorOutputType::default());
-                    let _ = handler.early_error_no_abort(msg.clone());
+                    let early_dcx = EarlyDiagCtxt::new(ErrorOutputType::default());
+                    let _ = early_dcx.early_error_no_abort(msg.clone());
                     return;
                 }
             };
@@ -1438,7 +1387,7 @@ pub fn install_ice_hook(bug_report_url: &'static str, extra_info: fn(&Handler))
 fn report_ice(
     info: &panic::PanicInfo<'_>,
     bug_report_url: &str,
-    extra_info: fn(&Handler),
+    extra_info: fn(&DiagCtxt),
     using_internal_features: &AtomicBool,
 ) {
     let fallback_bundle =
@@ -1447,20 +1396,20 @@ fn report_ice(
         rustc_errors::ColorConfig::Auto,
         fallback_bundle,
     ));
-    let handler = rustc_errors::Handler::with_emitter(emitter);
+    let dcx = rustc_errors::DiagCtxt::with_emitter(emitter);
 
     // a .span_bug or .bug call has already printed what
     // it wants to print.
     if !info.payload().is::<rustc_errors::ExplicitBug>()
         && !info.payload().is::<rustc_errors::DelayedBugPanic>()
     {
-        handler.emit_err(session_diagnostics::Ice);
+        dcx.emit_err(session_diagnostics::Ice);
     }
 
     if using_internal_features.load(std::sync::atomic::Ordering::Relaxed) {
-        handler.emit_note(session_diagnostics::IceBugReportInternalFeature);
+        dcx.emit_note(session_diagnostics::IceBugReportInternalFeature);
     } else {
-        handler.emit_note(session_diagnostics::IceBugReport { bug_report_url });
+        dcx.emit_note(session_diagnostics::IceBugReport { bug_report_url });
     }
 
     let version = util::version_str!().unwrap_or("unknown_version");
@@ -1472,7 +1421,7 @@ fn report_ice(
         // Create the ICE dump target file.
         match crate::fs::File::options().create(true).append(true).open(&path) {
             Ok(mut file) => {
-                handler.emit_note(session_diagnostics::IcePath { path: path.clone() });
+                dcx.emit_note(session_diagnostics::IcePath { path: path.clone() });
                 if FIRST_PANIC.swap(false, Ordering::SeqCst) {
                     let _ = write!(file, "\n\nrustc version: {version}\nplatform: {triple}");
                 }
@@ -1480,26 +1429,26 @@ fn report_ice(
             }
             Err(err) => {
                 // The path ICE couldn't be written to disk, provide feedback to the user as to why.
-                handler.emit_warning(session_diagnostics::IcePathError {
+                dcx.emit_warning(session_diagnostics::IcePathError {
                     path: path.clone(),
                     error: err.to_string(),
                     env_var: std::env::var_os("RUSTC_ICE")
                         .map(PathBuf::from)
                         .map(|env_var| session_diagnostics::IcePathErrorEnv { env_var }),
                 });
-                handler.emit_note(session_diagnostics::IceVersion { version, triple });
+                dcx.emit_note(session_diagnostics::IceVersion { version, triple });
                 None
             }
         }
     } else {
-        handler.emit_note(session_diagnostics::IceVersion { version, triple });
+        dcx.emit_note(session_diagnostics::IceVersion { version, triple });
         None
     };
 
     if let Some((flags, excluded_cargo_defaults)) = rustc_session::utils::extra_compiler_flags() {
-        handler.emit_note(session_diagnostics::IceFlags { flags: flags.join(" ") });
+        dcx.emit_note(session_diagnostics::IceFlags { flags: flags.join(" ") });
         if excluded_cargo_defaults {
-            handler.emit_note(session_diagnostics::IceExcludeCargoDefaults);
+            dcx.emit_note(session_diagnostics::IceExcludeCargoDefaults);
         }
     }
 
@@ -1508,11 +1457,11 @@ fn report_ice(
 
     let num_frames = if backtrace { None } else { Some(2) };
 
-    interface::try_print_query_stack(&handler, num_frames, file);
+    interface::try_print_query_stack(&dcx, num_frames, file);
 
     // We don't trust this callback not to panic itself, so run it at the end after we're sure we've
     // printed all the relevant info.
-    extra_info(&handler);
+    extra_info(&dcx);
 
     #[cfg(windows)]
     if env::var("RUSTC_BREAK_ON_ICE").is_ok() {
@@ -1523,16 +1472,16 @@ fn report_ice(
 
 /// This allows tools to enable rust logging without having to magically match rustc's
 /// tracing crate version.
-pub fn init_rustc_env_logger(handler: &EarlyErrorHandler) {
-    init_env_logger(handler, "RUSTC_LOG");
+pub fn init_rustc_env_logger(early_dcx: &EarlyDiagCtxt) {
+    init_logger(early_dcx, rustc_log::LoggerConfig::from_env("RUSTC_LOG"));
 }
 
 /// This allows tools to enable rust logging without having to magically match rustc's
-/// tracing crate version. In contrast to `init_rustc_env_logger` it allows you to choose an env var
-/// other than `RUSTC_LOG`.
-pub fn init_env_logger(handler: &EarlyErrorHandler, env: &str) {
-    if let Err(error) = rustc_log::init_env_logger(env) {
-        handler.early_error(error.to_string());
+/// tracing crate version. In contrast to `init_rustc_env_logger` it allows you to choose
+/// the values directly rather than having to set an environment variable.
+pub fn init_logger(early_dcx: &EarlyDiagCtxt, cfg: rustc_log::LoggerConfig) {
+    if let Err(error) = rustc_log::init_logger(cfg) {
+        early_dcx.early_error(error.to_string());
     }
 }
 
@@ -1540,9 +1489,9 @@ pub fn main() -> ! {
     let start_time = Instant::now();
     let start_rss = get_resident_set_size();
 
-    let handler = EarlyErrorHandler::new(ErrorOutputType::default());
+    let early_dcx = EarlyDiagCtxt::new(ErrorOutputType::default());
 
-    init_rustc_env_logger(&handler);
+    init_rustc_env_logger(&early_dcx);
     signal_handler::install();
     let mut callbacks = TimePassesCallbacks::default();
     let using_internal_features = install_ice_hook(DEFAULT_BUG_REPORT_URL, |_| ());
@@ -1551,7 +1500,7 @@ pub fn main() -> ! {
             .enumerate()
             .map(|(i, arg)| {
                 arg.into_string().unwrap_or_else(|arg| {
-                    handler.early_error(format!("argument {i} is not valid Unicode: {arg:?}"))
+                    early_dcx.early_error(format!("argument {i} is not valid Unicode: {arg:?}"))
                 })
             })
             .collect::<Vec<_>>();
diff --git a/compiler/rustc_driver_impl/src/pretty.rs b/compiler/rustc_driver_impl/src/pretty.rs
index cc533b994..7cd63bc64 100644
--- a/compiler/rustc_driver_impl/src/pretty.rs
+++ b/compiler/rustc_driver_impl/src/pretty.rs
@@ -9,6 +9,7 @@ use rustc_middle::mir::{write_mir_graphviz, write_mir_pretty};
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_session::config::{OutFileName, PpHirMode, PpMode, PpSourceMode};
 use rustc_session::Session;
+use rustc_smir::rustc_internal::pretty::write_smir_pretty;
 use rustc_span::symbol::Ident;
 use rustc_span::FileName;
 
@@ -325,6 +326,11 @@ pub fn print<'tcx>(sess: &Session, ppm: PpMode, ex: PrintExtra<'tcx>) {
             write_mir_graphviz(ex.tcx(), None, &mut out).unwrap();
             String::from_utf8(out).unwrap()
         }
+        StableMir => {
+            let mut out = Vec::new();
+            write_smir_pretty(ex.tcx(), &mut out).unwrap();
+            String::from_utf8(out).unwrap()
+        }
         ThirTree => {
             let tcx = ex.tcx();
             let mut out = String::new();
diff --git a/compiler/rustc_error_codes/src/error_codes.rs b/compiler/rustc_error_codes/src/error_codes.rs
index 6680e8875..1028d43f9 100644
--- a/compiler/rustc_error_codes/src/error_codes.rs
+++ b/compiler/rustc_error_codes/src/error_codes.rs
@@ -653,3 +653,4 @@ E0795: include_str!("./error_codes/E0795.md"),
 //  E0721, // `await` keyword
 //  E0723, // unstable feature in `const` context
 //  E0738, // Removed; errored on `#[track_caller] fn`s in `extern "Rust" { ... }`.
+//  E0744, // merged into E0728
diff --git a/compiler/rustc_error_codes/src/error_codes/E0705.md b/compiler/rustc_error_codes/src/error_codes/E0705.md
index eb76d1836..317f3a47e 100644
--- a/compiler/rustc_error_codes/src/error_codes/E0705.md
+++ b/compiler/rustc_error_codes/src/error_codes/E0705.md
@@ -1,9 +1,11 @@
+#### Note: this error code is no longer emitted by the compiler.
+
 A `#![feature]` attribute was declared for a feature that is stable in the
 current edition, but not in all editions.
 
 Erroneous code example:
 
-```rust2018,compile_fail,E0705
+```compile_fail
 #![feature(rust_2018_preview)]
 #![feature(test_2018_feature)] // error: the feature
                                // `test_2018_feature` is
diff --git a/compiler/rustc_error_codes/src/error_codes/E0744.md b/compiler/rustc_error_codes/src/error_codes/E0744.md
index 9a8ef3b84..e56c45db1 100644
--- a/compiler/rustc_error_codes/src/error_codes/E0744.md
+++ b/compiler/rustc_error_codes/src/error_codes/E0744.md
@@ -1,8 +1,10 @@
+#### Note: this error code is no longer emitted by the compiler.
+
 An unsupported expression was used inside a const context.
 
 Erroneous code example:
 
-```compile_fail,edition2018,E0744
+```ignore (removed error code)
 const _: i32 = {
     async { 0 }.await
 };
diff --git a/compiler/rustc_error_codes/src/error_codes/E0761.md b/compiler/rustc_error_codes/src/error_codes/E0761.md
index 760c58976..975f967d0 100644
--- a/compiler/rustc_error_codes/src/error_codes/E0761.md
+++ b/compiler/rustc_error_codes/src/error_codes/E0761.md
@@ -15,7 +15,7 @@ fn foo() {}
 
 mod ambiguous_module; // error: file for module `ambiguous_module`
                       // found at both ambiguous_module.rs and
-                      // ambiguous_module.rs/mod.rs
+                      // ambiguous_module/mod.rs
 ```
 
 Please remove this ambiguity by deleting/renaming one of the candidate files.
diff --git a/compiler/rustc_error_codes/src/lib.rs b/compiler/rustc_error_codes/src/lib.rs
index 81ad66128..dd2818116 100644
--- a/compiler/rustc_error_codes/src/lib.rs
+++ b/compiler/rustc_error_codes/src/lib.rs
@@ -1,6 +1,6 @@
-#![cfg_attr(not(bootstrap), allow(internal_features))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
+#![allow(internal_features)]
+#![feature(rustdoc_internals)]
+#![doc(rust_logo)]
 #![deny(rustdoc::invalid_codeblock_attributes)]
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
diff --git a/compiler/rustc_error_messages/Cargo.toml b/compiler/rustc_error_messages/Cargo.toml
index 1969feed4..5b6b8b3f1 100644
--- a/compiler/rustc_error_messages/Cargo.toml
+++ b/compiler/rustc_error_messages/Cargo.toml
@@ -13,7 +13,6 @@ icu_provider_adapters = "1.2"
 intl-memoizer = "0.5.1"
 rustc_baked_icu_data = { path = "../rustc_baked_icu_data" }
 rustc_data_structures = { path = "../rustc_data_structures" }
-rustc_fluent_macro = { path = "../rustc_fluent_macro" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_span = { path = "../rustc_span" }
diff --git a/compiler/rustc_error_messages/messages.ftl b/compiler/rustc_error_messages/messages.ftl
deleted file mode 100644
index e62923744..000000000
--- a/compiler/rustc_error_messages/messages.ftl
+++ /dev/null
@@ -1 +0,0 @@
-# satisfy tidy lint by having a line in this file
diff --git a/compiler/rustc_error_messages/src/lib.rs b/compiler/rustc_error_messages/src/lib.rs
index 6249c1e79..fc544c31e 100644
--- a/compiler/rustc_error_messages/src/lib.rs
+++ b/compiler/rustc_error_messages/src/lib.rs
@@ -1,5 +1,5 @@
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
+#![doc(rust_logo)]
+#![feature(rustdoc_internals)]
 #![feature(let_chains)]
 #![feature(lazy_cell)]
 #![feature(rustc_attrs)]
@@ -15,7 +15,6 @@ use fluent_bundle::FluentResource;
 use fluent_syntax::parser::ParserError;
 use icu_provider_adapters::fallback::{LocaleFallbackProvider, LocaleFallbacker};
 use rustc_data_structures::sync::{IntoDynSyncSend, Lrc};
-use rustc_fluent_macro::fluent_messages;
 use rustc_macros::{Decodable, Encodable};
 use rustc_span::Span;
 use std::borrow::Cow;
@@ -38,8 +37,6 @@ use intl_memoizer::IntlLangMemoizer;
 pub use fluent_bundle::{self, types::FluentType, FluentArgs, FluentError, FluentValue};
 pub use unic_langid::{langid, LanguageIdentifier};
 
-fluent_messages! { "../messages.ftl" }
-
 pub type FluentBundle =
     IntoDynSyncSend<fluent_bundle::bundle::FluentBundle<FluentResource, IntlLangMemoizer>>;
 
diff --git a/compiler/rustc_errors/src/annotate_snippet_emitter_writer.rs b/compiler/rustc_errors/src/annotate_snippet_emitter_writer.rs
index 203e52912..da266bf9c 100644
--- a/compiler/rustc_errors/src/annotate_snippet_emitter_writer.rs
+++ b/compiler/rustc_errors/src/annotate_snippet_emitter_writer.rs
@@ -138,7 +138,7 @@ impl AnnotateSnippetEmitterWriter {
         let message = self.translate_messages(messages, args);
         if let Some(source_map) = &self.source_map {
             // Make sure our primary file comes first
-            let primary_lo = if let Some(ref primary_span) = msp.primary_span().as_ref() {
+            let primary_lo = if let Some(primary_span) = msp.primary_span().as_ref() {
                 if primary_span.is_dummy() {
                     // FIXME(#59346): Not sure when this is the case and what
                     // should be done if it happens
@@ -203,7 +203,7 @@ impl AnnotateSnippetEmitterWriter {
                         Slice {
                             source,
                             line_start: *line_index,
-                            origin: Some(&file_name),
+                            origin: Some(file_name),
                             // FIXME(#59346): Not really sure when `fold` should be true or false
                             fold: false,
                             annotations: annotations
diff --git a/compiler/rustc_errors/src/diagnostic.rs b/compiler/rustc_errors/src/diagnostic.rs
index 470f318eb..be5068060 100644
--- a/compiler/rustc_errors/src/diagnostic.rs
+++ b/compiler/rustc_errors/src/diagnostic.rs
@@ -91,10 +91,7 @@ where
 #[rustc_diagnostic_item = "DecorateLint"]
 pub trait DecorateLint<'a, G: EmissionGuarantee> {
     /// Decorate and emit a lint.
-    fn decorate_lint<'b>(
-        self,
-        diag: &'b mut DiagnosticBuilder<'a, G>,
-    ) -> &'b mut DiagnosticBuilder<'a, G>;
+    fn decorate_lint<'b>(self, diag: &'b mut DiagnosticBuilder<'a, G>);
 
     fn msg(&self) -> DiagnosticMessage;
 }
@@ -239,7 +236,7 @@ impl Diagnostic {
     }
 
     #[track_caller]
-    pub fn new_with_code<M: Into<DiagnosticMessage>>(
+    pub(crate) fn new_with_code<M: Into<DiagnosticMessage>>(
         level: Level,
         code: Option<DiagnosticId>,
         message: M,
@@ -281,7 +278,7 @@ impl Diagnostic {
         }
     }
 
-    pub fn update_unstable_expectation_id(
+    pub(crate) fn update_unstable_expectation_id(
         &mut self,
         unstable_to_stable: &FxHashMap<LintExpectationId, LintExpectationId>,
     ) {
@@ -307,14 +304,14 @@ impl Diagnostic {
     }
 
     /// Indicates whether this diagnostic should show up in cargo's future breakage report.
-    pub fn has_future_breakage(&self) -> bool {
+    pub(crate) fn has_future_breakage(&self) -> bool {
         match self.code {
             Some(DiagnosticId::Lint { has_future_breakage, .. }) => has_future_breakage,
             _ => false,
         }
     }
 
-    pub fn is_force_warn(&self) -> bool {
+    pub(crate) fn is_force_warn(&self) -> bool {
         match self.code {
             Some(DiagnosticId::Lint { is_force_warn, .. }) => is_force_warn,
             _ => false,
@@ -391,29 +388,6 @@ impl Diagnostic {
         self.note_expected_found_extra(expected_label, expected, found_label, found, &"", &"")
     }
 
-    pub fn note_unsuccessful_coercion(
-        &mut self,
-        expected: DiagnosticStyledString,
-        found: DiagnosticStyledString,
-    ) -> &mut Self {
-        let mut msg: Vec<_> =
-            vec![(Cow::from("required when trying to coerce from type `"), Style::NoStyle)];
-        msg.extend(expected.0.iter().map(|x| match *x {
-            StringPart::Normal(ref s) => (Cow::from(s.clone()), Style::NoStyle),
-            StringPart::Highlighted(ref s) => (Cow::from(s.clone()), Style::Highlight),
-        }));
-        msg.push((Cow::from("` to type '"), Style::NoStyle));
-        msg.extend(found.0.iter().map(|x| match *x {
-            StringPart::Normal(ref s) => (Cow::from(s.clone()), Style::NoStyle),
-            StringPart::Highlighted(ref s) => (Cow::from(s.clone()), Style::Highlight),
-        }));
-        msg.push((Cow::from("`"), Style::NoStyle));
-
-        // For now, just attach these as notes
-        self.highlighted_note(msg);
-        self
-    }
-
     pub fn note_expected_found_extra(
         &mut self,
         expected_label: &dyn fmt::Display,
@@ -475,7 +449,7 @@ impl Diagnostic {
         self
     }
 
-    pub fn highlighted_note<M: Into<SubdiagnosticMessage>>(
+    fn highlighted_note<M: Into<SubdiagnosticMessage>>(
         &mut self,
         msg: Vec<(M, Style)>,
     ) -> &mut Self {
@@ -572,14 +546,6 @@ impl Diagnostic {
         self
     }
 
-    /// Clear any existing suggestions.
-    pub fn clear_suggestions(&mut self) -> &mut Self {
-        if let Ok(suggestions) = &mut self.suggestions {
-            suggestions.clear();
-        }
-        self
-    }
-
     /// Helper for pushing to `self.suggestions`, if available (not disable).
     fn push_suggestion(&mut self, suggestion: CodeSuggestion) {
         if let Ok(suggestions) = &mut self.suggestions {
@@ -622,17 +588,18 @@ impl Diagnostic {
     pub fn multipart_suggestion_with_style(
         &mut self,
         msg: impl Into<SubdiagnosticMessage>,
-        suggestion: Vec<(Span, String)>,
+        mut suggestion: Vec<(Span, String)>,
         applicability: Applicability,
         style: SuggestionStyle,
     ) -> &mut Self {
-        let mut parts = suggestion
+        suggestion.sort_unstable();
+        suggestion.dedup();
+
+        let parts = suggestion
             .into_iter()
             .map(|(span, snippet)| SubstitutionPart { snippet, span })
             .collect::<Vec<_>>();
 
-        parts.sort_unstable_by_key(|part| part.span);
-
         assert!(!parts.is_empty());
         debug_assert_eq!(
             parts.iter().find(|part| part.span.is_empty() && part.snippet.is_empty()),
@@ -777,17 +744,15 @@ impl Diagnostic {
         applicability: Applicability,
         style: SuggestionStyle,
     ) -> &mut Self {
-        let mut suggestions: Vec<_> = suggestions.into_iter().collect();
-        suggestions.sort();
-
-        debug_assert!(
-            !(sp.is_empty() && suggestions.iter().any(|suggestion| suggestion.is_empty())),
-            "Span must not be empty and have no suggestion"
-        );
-
         let substitutions = suggestions
             .into_iter()
-            .map(|snippet| Substitution { parts: vec![SubstitutionPart { snippet, span: sp }] })
+            .map(|snippet| {
+                debug_assert!(
+                    !(sp.is_empty() && snippet.is_empty()),
+                    "Span must not be empty and have no suggestion"
+                );
+                Substitution { parts: vec![SubstitutionPart { snippet, span: sp }] }
+            })
             .collect();
         self.push_suggestion(CodeSuggestion {
             substitutions,
@@ -921,13 +886,13 @@ impl Diagnostic {
     /// interpolated variables).
     pub fn eager_subdiagnostic(
         &mut self,
-        handler: &crate::Handler,
+        dcx: &crate::DiagCtxt,
         subdiagnostic: impl AddToDiagnostic,
     ) -> &mut Self {
         subdiagnostic.add_to_diagnostic_with(self, |diag, msg| {
             let args = diag.args();
             let msg = diag.subdiagnostic_message_to_diagnostic_message(msg);
-            handler.eagerly_translate(msg, args)
+            dcx.eagerly_translate(msg, args)
         });
         self
     }
@@ -994,7 +959,7 @@ impl Diagnostic {
     /// Helper function that takes a `SubdiagnosticMessage` and returns a `DiagnosticMessage` by
     /// combining it with the primary message of the diagnostic (if translatable, otherwise it just
     /// passes the user's string along).
-    pub(crate) fn subdiagnostic_message_to_diagnostic_message(
+    fn subdiagnostic_message_to_diagnostic_message(
         &self,
         attr: impl Into<SubdiagnosticMessage>,
     ) -> DiagnosticMessage {
diff --git a/compiler/rustc_errors/src/diagnostic_builder.rs b/compiler/rustc_errors/src/diagnostic_builder.rs
index 85acf8ab5..3f66af1fc 100644
--- a/compiler/rustc_errors/src/diagnostic_builder.rs
+++ b/compiler/rustc_errors/src/diagnostic_builder.rs
@@ -1,9 +1,9 @@
 use crate::diagnostic::IntoDiagnosticArg;
+use crate::{DiagCtxt, Level, MultiSpan, StashKey};
 use crate::{
     Diagnostic, DiagnosticId, DiagnosticMessage, DiagnosticStyledString, ErrorGuaranteed,
     ExplicitBug, SubdiagnosticMessage,
 };
-use crate::{Handler, Level, MultiSpan, StashKey};
 use rustc_lint_defs::Applicability;
 use rustc_span::source_map::Spanned;
 
@@ -18,19 +18,19 @@ use std::thread::panicking;
 /// Trait implemented by error types. This should not be implemented manually. Instead, use
 /// `#[derive(Diagnostic)]` -- see [rustc_macros::Diagnostic].
 #[rustc_diagnostic_item = "IntoDiagnostic"]
-pub trait IntoDiagnostic<'a, T: EmissionGuarantee = ErrorGuaranteed> {
-    /// Write out as a diagnostic out of `Handler`.
+pub trait IntoDiagnostic<'a, G: EmissionGuarantee = ErrorGuaranteed> {
+    /// Write out as a diagnostic out of `DiagCtxt`.
     #[must_use]
-    fn into_diagnostic(self, handler: &'a Handler) -> DiagnosticBuilder<'a, T>;
+    fn into_diagnostic(self, dcx: &'a DiagCtxt) -> DiagnosticBuilder<'a, G>;
 }
 
-impl<'a, T, E> IntoDiagnostic<'a, E> for Spanned<T>
+impl<'a, T, G> IntoDiagnostic<'a, G> for Spanned<T>
 where
-    T: IntoDiagnostic<'a, E>,
-    E: EmissionGuarantee,
+    T: IntoDiagnostic<'a, G>,
+    G: EmissionGuarantee,
 {
-    fn into_diagnostic(self, handler: &'a Handler) -> DiagnosticBuilder<'a, E> {
-        let mut diag = self.node.into_diagnostic(handler);
+    fn into_diagnostic(self, dcx: &'a DiagCtxt) -> DiagnosticBuilder<'a, G> {
+        let mut diag = self.node.into_diagnostic(dcx);
         diag.set_span(self.span);
         diag
     }
@@ -40,7 +40,7 @@ where
 ///
 /// If there is some state in a downstream crate you would like to
 /// access in the methods of `DiagnosticBuilder` here, consider
-/// extending `HandlerFlags`, accessed via `self.handler.flags`.
+/// extending `DiagCtxtFlags`.
 #[must_use]
 #[derive(Clone)]
 pub struct DiagnosticBuilder<'a, G: EmissionGuarantee> {
@@ -74,8 +74,8 @@ struct DiagnosticBuilderInner<'a> {
 enum DiagnosticBuilderState<'a> {
     /// Initial state of a `DiagnosticBuilder`, before `.emit()` or `.cancel()`.
     ///
-    /// The `Diagnostic` will be emitted through this `Handler`.
-    Emittable(&'a Handler),
+    /// The `Diagnostic` will be emitted through this `DiagCtxt`.
+    Emittable(&'a DiagCtxt),
 
     /// State of a `DiagnosticBuilder`, after `.emit()` or *during* `.cancel()`.
     ///
@@ -95,7 +95,7 @@ enum DiagnosticBuilderState<'a> {
 // `DiagnosticBuilderState` should be pointer-sized.
 rustc_data_structures::static_assert_size!(
     DiagnosticBuilderState<'_>,
-    std::mem::size_of::<&Handler>()
+    std::mem::size_of::<&DiagCtxt>()
 );
 
 /// Trait for types that `DiagnosticBuilder::emit` can return as a "guarantee"
@@ -110,32 +110,12 @@ pub trait EmissionGuarantee: Sized {
     /// Creates a new `DiagnosticBuilder` that will return this type of guarantee.
     #[track_caller]
     fn make_diagnostic_builder(
-        handler: &Handler,
+        dcx: &DiagCtxt,
         msg: impl Into<DiagnosticMessage>,
     ) -> DiagnosticBuilder<'_, Self>;
 }
 
 impl<'a> DiagnosticBuilder<'a, ErrorGuaranteed> {
-    /// Convenience function for internal use, clients should use one of the
-    /// `struct_*` methods on [`Handler`].
-    #[track_caller]
-    pub(crate) fn new_guaranteeing_error<M: Into<DiagnosticMessage>>(
-        handler: &'a Handler,
-        message: M,
-    ) -> Self {
-        Self {
-            inner: DiagnosticBuilderInner {
-                state: DiagnosticBuilderState::Emittable(handler),
-                diagnostic: Box::new(Diagnostic::new_with_code(
-                    Level::Error { lint: false },
-                    None,
-                    message,
-                )),
-            },
-            _marker: PhantomData,
-        }
-    }
-
     /// Discard the guarantee `.emit()` would return, in favor of having the
     /// type `DiagnosticBuilder<'a, ()>`. This may be necessary whenever there
     /// is a common codepath handling both errors and warnings.
@@ -148,11 +128,11 @@ impl<'a> DiagnosticBuilder<'a, ErrorGuaranteed> {
 impl EmissionGuarantee for ErrorGuaranteed {
     fn diagnostic_builder_emit_producing_guarantee(db: &mut DiagnosticBuilder<'_, Self>) -> Self {
         match db.inner.state {
-            // First `.emit()` call, the `&Handler` is still available.
-            DiagnosticBuilderState::Emittable(handler) => {
+            // First `.emit()` call, the `&DiagCtxt` is still available.
+            DiagnosticBuilderState::Emittable(dcx) => {
                 db.inner.state = DiagnosticBuilderState::AlreadyEmittedOrDuringCancellation;
 
-                let guar = handler.emit_diagnostic(&mut db.inner.diagnostic);
+                let guar = dcx.emit_diagnostic_without_consuming(&mut db.inner.diagnostic);
 
                 // Only allow a guarantee if the `level` wasn't switched to a
                 // non-error - the field isn't `pub`, but the whole `Diagnostic`
@@ -186,38 +166,10 @@ impl EmissionGuarantee for ErrorGuaranteed {
 
     #[track_caller]
     fn make_diagnostic_builder(
-        handler: &Handler,
+        dcx: &DiagCtxt,
         msg: impl Into<DiagnosticMessage>,
     ) -> DiagnosticBuilder<'_, Self> {
-        DiagnosticBuilder::new_guaranteeing_error(handler, msg)
-    }
-}
-
-impl<'a> DiagnosticBuilder<'a, ()> {
-    /// Convenience function for internal use, clients should use one of the
-    /// `struct_*` methods on [`Handler`].
-    #[track_caller]
-    pub(crate) fn new<M: Into<DiagnosticMessage>>(
-        handler: &'a Handler,
-        level: Level,
-        message: M,
-    ) -> Self {
-        let diagnostic = Diagnostic::new_with_code(level, None, message);
-        Self::new_diagnostic(handler, diagnostic)
-    }
-
-    /// Creates a new `DiagnosticBuilder` with an already constructed
-    /// diagnostic.
-    #[track_caller]
-    pub(crate) fn new_diagnostic(handler: &'a Handler, diagnostic: Diagnostic) -> Self {
-        debug!("Created new diagnostic");
-        Self {
-            inner: DiagnosticBuilderInner {
-                state: DiagnosticBuilderState::Emittable(handler),
-                diagnostic: Box::new(diagnostic),
-            },
-            _marker: PhantomData,
-        }
+        DiagnosticBuilder::new(dcx, Level::Error { lint: false }, msg)
     }
 }
 
@@ -225,11 +177,11 @@ impl<'a> DiagnosticBuilder<'a, ()> {
 impl EmissionGuarantee for () {
     fn diagnostic_builder_emit_producing_guarantee(db: &mut DiagnosticBuilder<'_, Self>) -> Self {
         match db.inner.state {
-            // First `.emit()` call, the `&Handler` is still available.
-            DiagnosticBuilderState::Emittable(handler) => {
+            // First `.emit()` call, the `&DiagCtxt` is still available.
+            DiagnosticBuilderState::Emittable(dcx) => {
                 db.inner.state = DiagnosticBuilderState::AlreadyEmittedOrDuringCancellation;
 
-                handler.emit_diagnostic(&mut db.inner.diagnostic);
+                dcx.emit_diagnostic_without_consuming(&mut db.inner.diagnostic);
             }
             // `.emit()` was previously called, disallowed from repeating it.
             DiagnosticBuilderState::AlreadyEmittedOrDuringCancellation => {}
@@ -237,10 +189,10 @@ impl EmissionGuarantee for () {
     }
 
     fn make_diagnostic_builder(
-        handler: &Handler,
+        dcx: &DiagCtxt,
         msg: impl Into<DiagnosticMessage>,
     ) -> DiagnosticBuilder<'_, Self> {
-        DiagnosticBuilder::new(handler, Level::Warning(None), msg)
+        DiagnosticBuilder::new(dcx, Level::Warning(None), msg)
     }
 }
 
@@ -249,35 +201,13 @@ impl EmissionGuarantee for () {
 #[derive(Copy, Clone)]
 pub struct Noted;
 
-impl<'a> DiagnosticBuilder<'a, Noted> {
-    /// Convenience function for internal use, clients should use one of the
-    /// `struct_*` methods on [`Handler`].
-    pub(crate) fn new_note(handler: &'a Handler, message: impl Into<DiagnosticMessage>) -> Self {
-        let diagnostic = Diagnostic::new_with_code(Level::Note, None, message);
-        Self::new_diagnostic_note(handler, diagnostic)
-    }
-
-    /// Creates a new `DiagnosticBuilder` with an already constructed
-    /// diagnostic.
-    pub(crate) fn new_diagnostic_note(handler: &'a Handler, diagnostic: Diagnostic) -> Self {
-        debug!("Created new diagnostic");
-        Self {
-            inner: DiagnosticBuilderInner {
-                state: DiagnosticBuilderState::Emittable(handler),
-                diagnostic: Box::new(diagnostic),
-            },
-            _marker: PhantomData,
-        }
-    }
-}
-
 impl EmissionGuarantee for Noted {
     fn diagnostic_builder_emit_producing_guarantee(db: &mut DiagnosticBuilder<'_, Self>) -> Self {
         match db.inner.state {
-            // First `.emit()` call, the `&Handler` is still available.
-            DiagnosticBuilderState::Emittable(handler) => {
+            // First `.emit()` call, the `&DiagCtxt` is still available.
+            DiagnosticBuilderState::Emittable(dcx) => {
                 db.inner.state = DiagnosticBuilderState::AlreadyEmittedOrDuringCancellation;
-                handler.emit_diagnostic(&mut db.inner.diagnostic);
+                dcx.emit_diagnostic_without_consuming(&mut db.inner.diagnostic);
             }
             // `.emit()` was previously called, disallowed from repeating it.
             DiagnosticBuilderState::AlreadyEmittedOrDuringCancellation => {}
@@ -287,10 +217,10 @@ impl EmissionGuarantee for Noted {
     }
 
     fn make_diagnostic_builder(
-        handler: &Handler,
+        dcx: &DiagCtxt,
         msg: impl Into<DiagnosticMessage>,
     ) -> DiagnosticBuilder<'_, Self> {
-        DiagnosticBuilder::new_note(handler, msg)
+        DiagnosticBuilder::new(dcx, Level::Note, msg)
     }
 }
 
@@ -299,37 +229,14 @@ impl EmissionGuarantee for Noted {
 #[derive(Copy, Clone)]
 pub struct Bug;
 
-impl<'a> DiagnosticBuilder<'a, Bug> {
-    /// Convenience function for internal use, clients should use one of the
-    /// `struct_*` methods on [`Handler`].
-    #[track_caller]
-    pub(crate) fn new_bug(handler: &'a Handler, message: impl Into<DiagnosticMessage>) -> Self {
-        let diagnostic = Diagnostic::new_with_code(Level::Bug, None, message);
-        Self::new_diagnostic_bug(handler, diagnostic)
-    }
-
-    /// Creates a new `DiagnosticBuilder` with an already constructed
-    /// diagnostic.
-    pub(crate) fn new_diagnostic_bug(handler: &'a Handler, diagnostic: Diagnostic) -> Self {
-        debug!("Created new diagnostic bug");
-        Self {
-            inner: DiagnosticBuilderInner {
-                state: DiagnosticBuilderState::Emittable(handler),
-                diagnostic: Box::new(diagnostic),
-            },
-            _marker: PhantomData,
-        }
-    }
-}
-
 impl EmissionGuarantee for Bug {
     fn diagnostic_builder_emit_producing_guarantee(db: &mut DiagnosticBuilder<'_, Self>) -> Self {
         match db.inner.state {
-            // First `.emit()` call, the `&Handler` is still available.
-            DiagnosticBuilderState::Emittable(handler) => {
+            // First `.emit()` call, the `&DiagCtxt` is still available.
+            DiagnosticBuilderState::Emittable(dcx) => {
                 db.inner.state = DiagnosticBuilderState::AlreadyEmittedOrDuringCancellation;
 
-                handler.emit_diagnostic(&mut db.inner.diagnostic);
+                dcx.emit_diagnostic_without_consuming(&mut db.inner.diagnostic);
             }
             // `.emit()` was previously called, disallowed from repeating it.
             DiagnosticBuilderState::AlreadyEmittedOrDuringCancellation => {}
@@ -339,44 +246,21 @@ impl EmissionGuarantee for Bug {
     }
 
     fn make_diagnostic_builder(
-        handler: &Handler,
+        dcx: &DiagCtxt,
         msg: impl Into<DiagnosticMessage>,
     ) -> DiagnosticBuilder<'_, Self> {
-        DiagnosticBuilder::new_bug(handler, msg)
-    }
-}
-
-impl<'a> DiagnosticBuilder<'a, !> {
-    /// Convenience function for internal use, clients should use one of the
-    /// `struct_*` methods on [`Handler`].
-    #[track_caller]
-    pub(crate) fn new_fatal(handler: &'a Handler, message: impl Into<DiagnosticMessage>) -> Self {
-        let diagnostic = Diagnostic::new_with_code(Level::Fatal, None, message);
-        Self::new_diagnostic_fatal(handler, diagnostic)
-    }
-
-    /// Creates a new `DiagnosticBuilder` with an already constructed
-    /// diagnostic.
-    pub(crate) fn new_diagnostic_fatal(handler: &'a Handler, diagnostic: Diagnostic) -> Self {
-        debug!("Created new diagnostic");
-        Self {
-            inner: DiagnosticBuilderInner {
-                state: DiagnosticBuilderState::Emittable(handler),
-                diagnostic: Box::new(diagnostic),
-            },
-            _marker: PhantomData,
-        }
+        DiagnosticBuilder::new(dcx, Level::Bug, msg)
     }
 }
 
 impl EmissionGuarantee for ! {
     fn diagnostic_builder_emit_producing_guarantee(db: &mut DiagnosticBuilder<'_, Self>) -> Self {
         match db.inner.state {
-            // First `.emit()` call, the `&Handler` is still available.
-            DiagnosticBuilderState::Emittable(handler) => {
+            // First `.emit()` call, the `&DiagCtxt` is still available.
+            DiagnosticBuilderState::Emittable(dcx) => {
                 db.inner.state = DiagnosticBuilderState::AlreadyEmittedOrDuringCancellation;
 
-                handler.emit_diagnostic(&mut db.inner.diagnostic);
+                dcx.emit_diagnostic_without_consuming(&mut db.inner.diagnostic);
             }
             // `.emit()` was previously called, disallowed from repeating it.
             DiagnosticBuilderState::AlreadyEmittedOrDuringCancellation => {}
@@ -386,50 +270,21 @@ impl EmissionGuarantee for ! {
     }
 
     fn make_diagnostic_builder(
-        handler: &Handler,
+        dcx: &DiagCtxt,
         msg: impl Into<DiagnosticMessage>,
     ) -> DiagnosticBuilder<'_, Self> {
-        DiagnosticBuilder::new_fatal(handler, msg)
-    }
-}
-
-impl<'a> DiagnosticBuilder<'a, rustc_span::fatal_error::FatalError> {
-    /// Convenience function for internal use, clients should use one of the
-    /// `struct_*` methods on [`Handler`].
-    #[track_caller]
-    pub(crate) fn new_almost_fatal(
-        handler: &'a Handler,
-        message: impl Into<DiagnosticMessage>,
-    ) -> Self {
-        let diagnostic = Diagnostic::new_with_code(Level::Fatal, None, message);
-        Self::new_diagnostic_almost_fatal(handler, diagnostic)
-    }
-
-    /// Creates a new `DiagnosticBuilder` with an already constructed
-    /// diagnostic.
-    pub(crate) fn new_diagnostic_almost_fatal(
-        handler: &'a Handler,
-        diagnostic: Diagnostic,
-    ) -> Self {
-        debug!("Created new diagnostic");
-        Self {
-            inner: DiagnosticBuilderInner {
-                state: DiagnosticBuilderState::Emittable(handler),
-                diagnostic: Box::new(diagnostic),
-            },
-            _marker: PhantomData,
-        }
+        DiagnosticBuilder::new(dcx, Level::Fatal, msg)
     }
 }
 
 impl EmissionGuarantee for rustc_span::fatal_error::FatalError {
     fn diagnostic_builder_emit_producing_guarantee(db: &mut DiagnosticBuilder<'_, Self>) -> Self {
         match db.inner.state {
-            // First `.emit()` call, the `&Handler` is still available.
-            DiagnosticBuilderState::Emittable(handler) => {
+            // First `.emit()` call, the `&DiagCtxt` is still available.
+            DiagnosticBuilderState::Emittable(dcx) => {
                 db.inner.state = DiagnosticBuilderState::AlreadyEmittedOrDuringCancellation;
 
-                handler.emit_diagnostic(&mut db.inner.diagnostic);
+                dcx.emit_diagnostic_without_consuming(&mut db.inner.diagnostic);
             }
             // `.emit()` was previously called, disallowed from repeating it.
             DiagnosticBuilderState::AlreadyEmittedOrDuringCancellation => {}
@@ -439,10 +294,10 @@ impl EmissionGuarantee for rustc_span::fatal_error::FatalError {
     }
 
     fn make_diagnostic_builder(
-        handler: &Handler,
+        dcx: &DiagCtxt,
         msg: impl Into<DiagnosticMessage>,
     ) -> DiagnosticBuilder<'_, Self> {
-        DiagnosticBuilder::new_almost_fatal(handler, msg)
+        DiagnosticBuilder::new(dcx, Level::Fatal, msg)
     }
 }
 
@@ -484,7 +339,35 @@ impl<G: EmissionGuarantee> DerefMut for DiagnosticBuilder<'_, G> {
 }
 
 impl<'a, G: EmissionGuarantee> DiagnosticBuilder<'a, G> {
-    /// Emit the diagnostic.
+    /// Convenience function for internal use, clients should use one of the
+    /// `struct_*` methods on [`DiagCtxt`].
+    #[track_caller]
+    pub(crate) fn new<M: Into<DiagnosticMessage>>(
+        dcx: &'a DiagCtxt,
+        level: Level,
+        message: M,
+    ) -> Self {
+        let diagnostic = Diagnostic::new(level, message);
+        Self::new_diagnostic(dcx, diagnostic)
+    }
+
+    /// Creates a new `DiagnosticBuilder` with an already constructed
+    /// diagnostic.
+    #[track_caller]
+    pub(crate) fn new_diagnostic(dcx: &'a DiagCtxt, diagnostic: Diagnostic) -> Self {
+        debug!("Created new diagnostic");
+        Self {
+            inner: DiagnosticBuilderInner {
+                state: DiagnosticBuilderState::Emittable(dcx),
+                diagnostic: Box::new(diagnostic),
+            },
+            _marker: PhantomData,
+        }
+    }
+
+    /// Emit the diagnostic. Does not consume `self`, which may be surprising,
+    /// but there are various places that rely on continuing to use `self`
+    /// after calling `emit`.
     #[track_caller]
     pub fn emit(&mut self) -> G {
         G::diagnostic_builder_emit_producing_guarantee(self)
@@ -515,7 +398,7 @@ impl<'a, G: EmissionGuarantee> DiagnosticBuilder<'a, G> {
 
     /// Stashes diagnostic for possible later improvement in a different,
     /// later stage of the compiler. The diagnostic can be accessed with
-    /// the provided `span` and `key` through [`Handler::steal_diagnostic()`].
+    /// the provided `span` and `key` through [`DiagCtxt::steal_diagnostic()`].
     ///
     /// As with `buffer`, this is unless the handler has disabled such buffering.
     pub fn stash(self, span: Span, key: StashKey) {
@@ -526,18 +409,18 @@ impl<'a, G: EmissionGuarantee> DiagnosticBuilder<'a, G> {
 
     /// Converts the builder to a `Diagnostic` for later emission,
     /// unless handler has disabled such buffering, or `.emit()` was called.
-    pub fn into_diagnostic(mut self) -> Option<(Diagnostic, &'a Handler)> {
-        let handler = match self.inner.state {
-            // No `.emit()` calls, the `&Handler` is still available.
-            DiagnosticBuilderState::Emittable(handler) => handler,
+    pub fn into_diagnostic(mut self) -> Option<(Diagnostic, &'a DiagCtxt)> {
+        let dcx = match self.inner.state {
+            // No `.emit()` calls, the `&DiagCtxt` is still available.
+            DiagnosticBuilderState::Emittable(dcx) => dcx,
             // `.emit()` was previously called, nothing we can do.
             DiagnosticBuilderState::AlreadyEmittedOrDuringCancellation => {
                 return None;
             }
         };
 
-        if handler.inner.lock().flags.dont_buffer_diagnostics
-            || handler.inner.lock().flags.treat_err_as_bug.is_some()
+        if dcx.inner.lock().flags.dont_buffer_diagnostics
+            || dcx.inner.lock().flags.treat_err_as_bug.is_some()
         {
             self.emit();
             return None;
@@ -554,13 +437,13 @@ impl<'a, G: EmissionGuarantee> DiagnosticBuilder<'a, G> {
         // actually emitted.
         debug!("buffer: diagnostic={:?}", diagnostic);
 
-        Some((diagnostic, handler))
+        Some((diagnostic, dcx))
     }
 
-    /// Retrieves the [`Handler`] if available
-    pub fn handler(&self) -> Option<&Handler> {
+    /// Retrieves the [`DiagCtxt`] if available
+    pub fn dcx(&self) -> Option<&DiagCtxt> {
         match self.inner.state {
-            DiagnosticBuilderState::Emittable(handler) => Some(handler),
+            DiagnosticBuilderState::Emittable(dcx) => Some(dcx),
             DiagnosticBuilderState::AlreadyEmittedOrDuringCancellation => None,
         }
     }
@@ -634,12 +517,6 @@ impl<'a, G: EmissionGuarantee> DiagnosticBuilder<'a, G> {
         found_extra: &dyn fmt::Display,
     ) -> &mut Self);
 
-    forward!(pub fn note_unsuccessful_coercion(
-        &mut self,
-        expected: DiagnosticStyledString,
-        found: DiagnosticStyledString,
-    ) -> &mut Self);
-
     forward!(pub fn note(&mut self, msg: impl Into<SubdiagnosticMessage>) -> &mut Self);
     forward!(pub fn note_once(&mut self, msg: impl Into<SubdiagnosticMessage>) -> &mut Self);
     forward!(pub fn span_note(
@@ -668,7 +545,6 @@ impl<'a, G: EmissionGuarantee> DiagnosticBuilder<'a, G> {
     forward!(pub fn set_is_lint(&mut self,) -> &mut Self);
 
     forward!(pub fn disable_suggestions(&mut self,) -> &mut Self);
-    forward!(pub fn clear_suggestions(&mut self,) -> &mut Self);
 
     forward!(pub fn multipart_suggestion(
         &mut self,
@@ -764,15 +640,15 @@ impl Drop for DiagnosticBuilderInner<'_> {
     fn drop(&mut self) {
         match self.state {
             // No `.emit()` or `.cancel()` calls.
-            DiagnosticBuilderState::Emittable(handler) => {
+            DiagnosticBuilderState::Emittable(dcx) => {
                 if !panicking() {
-                    handler.emit_diagnostic(&mut Diagnostic::new(
+                    dcx.emit_diagnostic(Diagnostic::new(
                         Level::Bug,
                         DiagnosticMessage::from(
                             "the following error was constructed but not emitted",
                         ),
                     ));
-                    handler.emit_diagnostic(&mut self.diagnostic);
+                    dcx.emit_diagnostic_without_consuming(&mut self.diagnostic);
                     panic!("error was constructed but not emitted");
                 }
             }
diff --git a/compiler/rustc_errors/src/diagnostic_impls.rs b/compiler/rustc_errors/src/diagnostic_impls.rs
index 4f77f09b2..3e4b3ee75 100644
--- a/compiler/rustc_errors/src/diagnostic_impls.rs
+++ b/compiler/rustc_errors/src/diagnostic_impls.rs
@@ -1,6 +1,6 @@
 use crate::diagnostic::DiagnosticLocation;
 use crate::{fluent_generated as fluent, AddToDiagnostic};
-use crate::{DiagnosticArgValue, DiagnosticBuilder, Handler, IntoDiagnostic, IntoDiagnosticArg};
+use crate::{DiagCtxt, DiagnosticArgValue, DiagnosticBuilder, IntoDiagnostic, IntoDiagnosticArg};
 use rustc_ast as ast;
 use rustc_ast_pretty::pprust;
 use rustc_hir as hir;
@@ -246,18 +246,18 @@ impl<Id> IntoDiagnosticArg for hir::def::Res<Id> {
 }
 
 impl IntoDiagnostic<'_, !> for TargetDataLayoutErrors<'_> {
-    fn into_diagnostic(self, handler: &Handler) -> DiagnosticBuilder<'_, !> {
+    fn into_diagnostic(self, dcx: &DiagCtxt) -> DiagnosticBuilder<'_, !> {
         let mut diag;
         match self {
             TargetDataLayoutErrors::InvalidAddressSpace { addr_space, err, cause } => {
-                diag = handler.struct_fatal(fluent::errors_target_invalid_address_space);
+                diag = dcx.struct_fatal(fluent::errors_target_invalid_address_space);
                 diag.set_arg("addr_space", addr_space);
                 diag.set_arg("cause", cause);
                 diag.set_arg("err", err);
                 diag
             }
             TargetDataLayoutErrors::InvalidBits { kind, bit, cause, err } => {
-                diag = handler.struct_fatal(fluent::errors_target_invalid_bits);
+                diag = dcx.struct_fatal(fluent::errors_target_invalid_bits);
                 diag.set_arg("kind", kind);
                 diag.set_arg("bit", bit);
                 diag.set_arg("cause", cause);
@@ -265,31 +265,31 @@ impl IntoDiagnostic<'_, !> for TargetDataLayoutErrors<'_> {
                 diag
             }
             TargetDataLayoutErrors::MissingAlignment { cause } => {
-                diag = handler.struct_fatal(fluent::errors_target_missing_alignment);
+                diag = dcx.struct_fatal(fluent::errors_target_missing_alignment);
                 diag.set_arg("cause", cause);
                 diag
             }
             TargetDataLayoutErrors::InvalidAlignment { cause, err } => {
-                diag = handler.struct_fatal(fluent::errors_target_invalid_alignment);
+                diag = dcx.struct_fatal(fluent::errors_target_invalid_alignment);
                 diag.set_arg("cause", cause);
                 diag.set_arg("err_kind", err.diag_ident());
                 diag.set_arg("align", err.align());
                 diag
             }
             TargetDataLayoutErrors::InconsistentTargetArchitecture { dl, target } => {
-                diag = handler.struct_fatal(fluent::errors_target_inconsistent_architecture);
+                diag = dcx.struct_fatal(fluent::errors_target_inconsistent_architecture);
                 diag.set_arg("dl", dl);
                 diag.set_arg("target", target);
                 diag
             }
             TargetDataLayoutErrors::InconsistentTargetPointerWidth { pointer_size, target } => {
-                diag = handler.struct_fatal(fluent::errors_target_inconsistent_pointer_width);
+                diag = dcx.struct_fatal(fluent::errors_target_inconsistent_pointer_width);
                 diag.set_arg("pointer_size", pointer_size);
                 diag.set_arg("target", target);
                 diag
             }
             TargetDataLayoutErrors::InvalidBitsSize { err } => {
-                diag = handler.struct_fatal(fluent::errors_target_invalid_bits_size);
+                diag = dcx.struct_fatal(fluent::errors_target_invalid_bits_size);
                 diag.set_arg("err", err);
                 diag
             }
@@ -378,3 +378,9 @@ pub struct IndicateAnonymousLifetime {
     pub count: usize,
     pub suggestion: String,
 }
+
+impl IntoDiagnosticArg for type_ir::ClosureKind {
+    fn into_diagnostic_arg(self) -> DiagnosticArgValue<'static> {
+        DiagnosticArgValue::Str(self.as_str().into())
+    }
+}
diff --git a/compiler/rustc_errors/src/emitter.rs b/compiler/rustc_errors/src/emitter.rs
index 68dba8602..3fb993c36 100644
--- a/compiler/rustc_errors/src/emitter.rs
+++ b/compiler/rustc_errors/src/emitter.rs
@@ -16,8 +16,8 @@ use crate::snippet::{
 use crate::styled_buffer::StyledBuffer;
 use crate::translation::{to_fluent_args, Translate};
 use crate::{
-    diagnostic::DiagnosticLocation, CodeSuggestion, Diagnostic, DiagnosticId, DiagnosticMessage,
-    FluentBundle, Handler, LazyFallbackBundle, Level, MultiSpan, SubDiagnostic,
+    diagnostic::DiagnosticLocation, CodeSuggestion, DiagCtxt, Diagnostic, DiagnosticId,
+    DiagnosticMessage, FluentBundle, LazyFallbackBundle, Level, MultiSpan, SubDiagnostic,
     SubstitutionHighlight, SuggestionStyle, TerminalUrl,
 };
 use rustc_lint_defs::pluralize;
@@ -553,10 +553,10 @@ impl Emitter for EmitterWriter {
 }
 
 /// An emitter that does nothing when emitting a non-fatal diagnostic.
-/// Fatal diagnostics are forwarded to `fatal_handler` to avoid silent
+/// Fatal diagnostics are forwarded to `fatal_dcx` to avoid silent
 /// failures of rustc, as witnessed e.g. in issue #89358.
 pub struct SilentEmitter {
-    pub fatal_handler: Handler,
+    pub fatal_dcx: DiagCtxt,
     pub fatal_note: Option<String>,
 }
 
@@ -581,7 +581,7 @@ impl Emitter for SilentEmitter {
             if let Some(ref note) = self.fatal_note {
                 d.note(note.clone());
             }
-            self.fatal_handler.emit_diagnostic(&mut d);
+            self.fatal_dcx.emit_diagnostic(d);
         }
     }
 }
@@ -1297,7 +1297,7 @@ impl EmitterWriter {
                     buffer.append(line_number, line, style_or_override(*style, override_style));
                 }
             } else {
-                buffer.append(line_number, &text, style_or_override(*style, override_style));
+                buffer.append(line_number, text, style_or_override(*style, override_style));
             }
         }
     }
@@ -1931,7 +1931,7 @@ impl EmitterWriter {
                 self.draw_code_line(
                     &mut buffer,
                     &mut row_num,
-                    &highlight_parts,
+                    highlight_parts,
                     line_pos + line_start,
                     line,
                     show_code_change,
@@ -2338,7 +2338,7 @@ impl FileWithAnnotatedLines {
         let mut output = vec![];
         let mut multiline_annotations = vec![];
 
-        if let Some(ref sm) = emitter.source_map() {
+        if let Some(sm) = emitter.source_map() {
             for SpanLabel { span, is_primary, label } in msp.span_labels() {
                 // If we don't have a useful span, pick the primary span if that exists.
                 // Worst case we'll just print an error at the top of the main file.
@@ -2362,7 +2362,7 @@ impl FileWithAnnotatedLines {
 
                 let label = label.as_ref().map(|m| {
                     normalize_whitespace(
-                        &emitter.translate_message(m, &args).map_err(Report::new).unwrap(),
+                        &emitter.translate_message(m, args).map_err(Report::new).unwrap(),
                     )
                 });
 
@@ -2674,6 +2674,11 @@ fn from_stderr(color: ColorConfig) -> Destination {
     }
 }
 
+/// On Windows, BRIGHT_BLUE is hard to read on black. Use cyan instead.
+///
+/// See #36178.
+const BRIGHT_BLUE: Color = if cfg!(windows) { Color::Cyan } else { Color::Blue };
+
 impl Style {
     fn color_spec(&self, lvl: Level) -> ColorSpec {
         let mut spec = ColorSpec::new();
@@ -2688,11 +2693,7 @@ impl Style {
             Style::LineNumber => {
                 spec.set_bold(true);
                 spec.set_intense(true);
-                if cfg!(windows) {
-                    spec.set_fg(Some(Color::Cyan));
-                } else {
-                    spec.set_fg(Some(Color::Blue));
-                }
+                spec.set_fg(Some(BRIGHT_BLUE));
             }
             Style::Quotation => {}
             Style::MainHeaderMsg => {
@@ -2707,11 +2708,7 @@ impl Style {
             }
             Style::UnderlineSecondary | Style::LabelSecondary => {
                 spec.set_bold(true).set_intense(true);
-                if cfg!(windows) {
-                    spec.set_fg(Some(Color::Cyan));
-                } else {
-                    spec.set_fg(Some(Color::Blue));
-                }
+                spec.set_fg(Some(BRIGHT_BLUE));
             }
             Style::HeaderMsg | Style::NoStyle => {}
             Style::Level(lvl) => {
@@ -2719,7 +2716,7 @@ impl Style {
                 spec.set_bold(true);
             }
             Style::Highlight => {
-                spec.set_bold(true);
+                spec.set_bold(true).set_fg(Some(Color::Magenta));
             }
         }
         spec
diff --git a/compiler/rustc_errors/src/json.rs b/compiler/rustc_errors/src/json.rs
index 0cb75c71b..aa3749334 100644
--- a/compiler/rustc_errors/src/json.rs
+++ b/compiler/rustc_errors/src/json.rs
@@ -145,6 +145,25 @@ impl JsonEmitter {
     pub fn ignored_directories_in_source_blocks(self, value: Vec<String>) -> Self {
         Self { ignored_directories_in_source_blocks: value, ..self }
     }
+
+    fn emit(&mut self, val: EmitTyped<'_>) -> io::Result<()> {
+        if self.pretty {
+            serde_json::to_writer_pretty(&mut *self.dst, &val)?
+        } else {
+            serde_json::to_writer(&mut *self.dst, &val)?
+        };
+        self.dst.write_all(b"\n")?;
+        self.dst.flush()
+    }
+}
+
+#[derive(Serialize)]
+#[serde(tag = "$message_type", rename_all = "snake_case")]
+enum EmitTyped<'a> {
+    Diagnostic(Diagnostic),
+    Artifact(ArtifactNotification<'a>),
+    FutureIncompat(FutureIncompatReport<'a>),
+    UnusedExtern(UnusedExterns<'a, 'a, 'a>),
 }
 
 impl Translate for JsonEmitter {
@@ -160,12 +179,7 @@ impl Translate for JsonEmitter {
 impl Emitter for JsonEmitter {
     fn emit_diagnostic(&mut self, diag: &crate::Diagnostic) {
         let data = Diagnostic::from_errors_diagnostic(diag, self);
-        let result = if self.pretty {
-            writeln!(&mut self.dst, "{}", serde_json::to_string_pretty(&data).unwrap())
-        } else {
-            writeln!(&mut self.dst, "{}", serde_json::to_string(&data).unwrap())
-        }
-        .and_then(|_| self.dst.flush());
+        let result = self.emit(EmitTyped::Diagnostic(data));
         if let Err(e) = result {
             panic!("failed to print diagnostics: {e:?}");
         }
@@ -173,34 +187,28 @@ impl Emitter for JsonEmitter {
 
     fn emit_artifact_notification(&mut self, path: &Path, artifact_type: &str) {
         let data = ArtifactNotification { artifact: path, emit: artifact_type };
-        let result = if self.pretty {
-            writeln!(&mut self.dst, "{}", serde_json::to_string_pretty(&data).unwrap())
-        } else {
-            writeln!(&mut self.dst, "{}", serde_json::to_string(&data).unwrap())
-        }
-        .and_then(|_| self.dst.flush());
+        let result = self.emit(EmitTyped::Artifact(data));
         if let Err(e) = result {
             panic!("failed to print notification: {e:?}");
         }
     }
 
     fn emit_future_breakage_report(&mut self, diags: Vec<crate::Diagnostic>) {
-        let data: Vec<FutureBreakageItem> = diags
+        let data: Vec<FutureBreakageItem<'_>> = diags
             .into_iter()
             .map(|mut diag| {
                 if diag.level == crate::Level::Allow {
                     diag.level = crate::Level::Warning(None);
                 }
-                FutureBreakageItem { diagnostic: Diagnostic::from_errors_diagnostic(&diag, self) }
+                FutureBreakageItem {
+                    diagnostic: EmitTyped::Diagnostic(Diagnostic::from_errors_diagnostic(
+                        &diag, self,
+                    )),
+                }
             })
             .collect();
         let report = FutureIncompatReport { future_incompat_report: data };
-        let result = if self.pretty {
-            writeln!(&mut self.dst, "{}", serde_json::to_string_pretty(&report).unwrap())
-        } else {
-            writeln!(&mut self.dst, "{}", serde_json::to_string(&report).unwrap())
-        }
-        .and_then(|_| self.dst.flush());
+        let result = self.emit(EmitTyped::FutureIncompat(report));
         if let Err(e) = result {
             panic!("failed to print future breakage report: {e:?}");
         }
@@ -209,12 +217,7 @@ impl Emitter for JsonEmitter {
     fn emit_unused_externs(&mut self, lint_level: rustc_lint_defs::Level, unused_externs: &[&str]) {
         let lint_level = lint_level.as_str();
         let data = UnusedExterns { lint_level, unused_extern_names: unused_externs };
-        let result = if self.pretty {
-            writeln!(&mut self.dst, "{}", serde_json::to_string_pretty(&data).unwrap())
-        } else {
-            writeln!(&mut self.dst, "{}", serde_json::to_string(&data).unwrap())
-        }
-        .and_then(|_| self.dst.flush());
+        let result = self.emit(EmitTyped::UnusedExtern(data));
         if let Err(e) = result {
             panic!("failed to print unused externs: {e:?}");
         }
@@ -313,13 +316,15 @@ struct ArtifactNotification<'a> {
 }
 
 #[derive(Serialize)]
-struct FutureBreakageItem {
-    diagnostic: Diagnostic,
+struct FutureBreakageItem<'a> {
+    // Always EmitTyped::Diagnostic, but we want to make sure it gets serialized
+    // with "$message_type".
+    diagnostic: EmitTyped<'a>,
 }
 
 #[derive(Serialize)]
-struct FutureIncompatReport {
-    future_incompat_report: Vec<FutureBreakageItem>,
+struct FutureIncompatReport<'a> {
+    future_incompat_report: Vec<FutureBreakageItem<'a>>,
 }
 
 // NOTE: Keep this in sync with the equivalent structs in rustdoc's
diff --git a/compiler/rustc_errors/src/json/tests.rs b/compiler/rustc_errors/src/json/tests.rs
index 1f9a2981e..303de0a93 100644
--- a/compiler/rustc_errors/src/json/tests.rs
+++ b/compiler/rustc_errors/src/json/tests.rs
@@ -1,11 +1,8 @@
 use super::*;
 
-use crate::json::JsonEmitter;
-use rustc_span::source_map::{FilePathMapping, SourceMap};
-
-use crate::emitter::{ColorConfig, HumanReadableErrorType};
-use crate::{Handler, TerminalUrl};
-use rustc_span::{BytePos, Span};
+use crate::emitter::ColorConfig;
+use crate::DiagCtxt;
+use rustc_span::BytePos;
 
 use std::str;
 
@@ -64,8 +61,8 @@ fn test_positions(code: &str, span: (u32, u32), expected_output: SpanTestData) {
         );
 
         let span = Span::with_root_ctxt(BytePos(span.0), BytePos(span.1));
-        let handler = Handler::with_emitter(Box::new(je));
-        handler.span_err(span, "foo");
+        let dcx = DiagCtxt::with_emitter(Box::new(je));
+        dcx.span_err(span, "foo");
 
         let bytes = output.lock().unwrap();
         let actual_output = str::from_utf8(&bytes).unwrap();
diff --git a/compiler/rustc_errors/src/lib.rs b/compiler/rustc_errors/src/lib.rs
index dd462cc64..959e26fec 100644
--- a/compiler/rustc_errors/src/lib.rs
+++ b/compiler/rustc_errors/src/lib.rs
@@ -3,14 +3,13 @@
 //! This module contains the code for creating and emitting diagnostics.
 
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
+#![doc(rust_logo)]
+#![feature(rustdoc_internals)]
 #![feature(array_windows)]
 #![feature(extract_if)]
 #![feature(if_let_guard)]
 #![feature(let_chains)]
 #![feature(never_type)]
-#![feature(result_option_inspect)]
 #![feature(rustc_attrs)]
 #![feature(yeet_expr)]
 #![feature(try_blocks)]
@@ -42,7 +41,6 @@ pub use rustc_error_messages::{
     fallback_fluent_bundle, fluent_bundle, DelayDm, DiagnosticMessage, FluentBundle,
     LanguageIdentifier, LazyFallbackBundle, MultiSpan, SpanLabel, SubdiagnosticMessage,
 };
-use rustc_fluent_macro::fluent_messages;
 pub use rustc_lint_defs::{pluralize, Applicability};
 use rustc_span::source_map::SourceMap;
 pub use rustc_span::ErrorGuaranteed;
@@ -83,7 +81,7 @@ pub use snippet::Style;
 pub type PErr<'a> = DiagnosticBuilder<'a, ErrorGuaranteed>;
 pub type PResult<'a, T> = Result<T, PErr<'a>>;
 
-fluent_messages! { "../messages.ftl" }
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
 
 // `PResult` is used a lot. Make sure it doesn't unintentionally get bigger.
 // (See also the comment on `DiagnosticBuilderInner`'s `diagnostic` field.)
@@ -414,15 +412,15 @@ use std::backtrace::{Backtrace, BacktraceStatus};
 /// A handler deals with errors and other compiler output.
 /// Certain errors (fatal, bug, unimpl) may cause immediate exit,
 /// others log errors for later reporting.
-pub struct Handler {
-    inner: Lock<HandlerInner>,
+pub struct DiagCtxt {
+    inner: Lock<DiagCtxtInner>,
 }
 
 /// This inner struct exists to keep it all behind a single lock;
 /// this is done to prevent possible deadlocks in a multi-threaded compiler,
 /// as well as inconsistent state observation.
-struct HandlerInner {
-    flags: HandlerFlags,
+struct DiagCtxtInner {
+    flags: DiagCtxtFlags,
     /// The number of lint errors that have been emitted.
     lint_err_count: usize,
     /// The number of errors that have been emitted, including duplicates.
@@ -433,10 +431,10 @@ struct HandlerInner {
     warn_count: usize,
     deduplicated_err_count: usize,
     emitter: Box<DynEmitter>,
-    delayed_span_bugs: Vec<DelayedDiagnostic>,
-    delayed_good_path_bugs: Vec<DelayedDiagnostic>,
+    span_delayed_bugs: Vec<DelayedDiagnostic>,
+    good_path_delayed_bugs: Vec<DelayedDiagnostic>,
     /// This flag indicates that an expected diagnostic was emitted and suppressed.
-    /// This is used for the `delayed_good_path_bugs` check.
+    /// This is used for the `good_path_delayed_bugs` check.
     suppressed_expected_diag: bool,
 
     /// This set contains the `DiagnosticId` of all emitted diagnostics to avoid
@@ -520,7 +518,7 @@ pub static TRACK_DIAGNOSTICS: AtomicRef<fn(&mut Diagnostic, &mut dyn FnMut(&mut
     AtomicRef::new(&(default_track_diagnostic as _));
 
 #[derive(Copy, Clone, Default)]
-pub struct HandlerFlags {
+pub struct DiagCtxtFlags {
     /// If false, warning-level lints are suppressed.
     /// (rustc: see `--allow warnings` and `--cap-lints`)
     pub can_emit_warnings: bool,
@@ -530,7 +528,7 @@ pub struct HandlerFlags {
     /// If true, immediately emit diagnostics that would otherwise be buffered.
     /// (rustc: see `-Z dont-buffer-diagnostics` and `-Z treat-err-as-bug`)
     pub dont_buffer_diagnostics: bool,
-    /// If true, immediately print bugs registered with `delay_span_bug`.
+    /// If true, immediately print bugs registered with `span_delayed_bug`.
     /// (rustc: see `-Z report-delayed-bugs`)
     pub report_delayed_bugs: bool,
     /// Show macro backtraces.
@@ -542,25 +540,26 @@ pub struct HandlerFlags {
     pub track_diagnostics: bool,
 }
 
-impl Drop for HandlerInner {
+impl Drop for DiagCtxtInner {
     fn drop(&mut self) {
         self.emit_stashed_diagnostics();
 
         if !self.has_errors() {
-            let bugs = std::mem::replace(&mut self.delayed_span_bugs, Vec::new());
-            self.flush_delayed(bugs, "no errors encountered even though `delay_span_bug` issued");
+            let bugs = std::mem::replace(&mut self.span_delayed_bugs, Vec::new());
+            self.flush_delayed(bugs, "no errors encountered even though `span_delayed_bug` issued");
         }
 
-        // FIXME(eddyb) this explains what `delayed_good_path_bugs` are!
-        // They're `delayed_span_bugs` but for "require some diagnostic happened"
+        // FIXME(eddyb) this explains what `good_path_delayed_bugs` are!
+        // They're `span_delayed_bugs` but for "require some diagnostic happened"
         // instead of "require some error happened". Sadly that isn't ideal, as
         // lints can be `#[allow]`'d, potentially leading to this triggering.
         // Also, "good path" should be replaced with a better naming.
-        if !self.has_any_message() && !self.suppressed_expected_diag && !std::thread::panicking() {
-            let bugs = std::mem::replace(&mut self.delayed_good_path_bugs, Vec::new());
+        let has_any_message = self.err_count > 0 || self.lint_err_count > 0 || self.warn_count > 0;
+        if !has_any_message && !self.suppressed_expected_diag && !std::thread::panicking() {
+            let bugs = std::mem::replace(&mut self.good_path_delayed_bugs, Vec::new());
             self.flush_delayed(
                 bugs,
-                "no warnings or errors encountered even though `delayed_good_path_bugs` issued",
+                "no warnings or errors encountered even though `good_path_delayed_bugs` issued",
             );
         }
 
@@ -573,7 +572,7 @@ impl Drop for HandlerInner {
     }
 }
 
-impl Handler {
+impl DiagCtxt {
     pub fn with_tty_emitter(
         sm: Option<Lrc<SourceMap>>,
         fallback_bundle: LazyFallbackBundle,
@@ -586,12 +585,7 @@ impl Handler {
         self
     }
 
-    pub fn treat_err_as_bug(mut self, treat_err_as_bug: NonZeroUsize) -> Self {
-        self.inner.get_mut().flags.treat_err_as_bug = Some(treat_err_as_bug);
-        self
-    }
-
-    pub fn with_flags(mut self, flags: HandlerFlags) -> Self {
+    pub fn with_flags(mut self, flags: DiagCtxtFlags) -> Self {
         self.inner.get_mut().flags = flags;
         self
     }
@@ -603,16 +597,16 @@ impl Handler {
 
     pub fn with_emitter(emitter: Box<DynEmitter>) -> Self {
         Self {
-            inner: Lock::new(HandlerInner {
-                flags: HandlerFlags { can_emit_warnings: true, ..Default::default() },
+            inner: Lock::new(DiagCtxtInner {
+                flags: DiagCtxtFlags { can_emit_warnings: true, ..Default::default() },
                 lint_err_count: 0,
                 err_count: 0,
                 warn_count: 0,
                 deduplicated_err_count: 0,
                 deduplicated_warn_count: 0,
                 emitter,
-                delayed_span_bugs: Vec::new(),
-                delayed_good_path_bugs: Vec::new(),
+                span_delayed_bugs: Vec::new(),
+                good_path_delayed_bugs: Vec::new(),
                 suppressed_expected_diag: false,
                 taught_diagnostics: Default::default(),
                 emitted_diagnostic_codes: Default::default(),
@@ -650,7 +644,7 @@ impl Handler {
     // This is here to not allow mutation of flags;
     // as of this writing it's only used in tests in librustc_middle.
     pub fn can_emit_warnings(&self) -> bool {
-        self.inner.lock().flags.can_emit_warnings
+        self.inner.borrow_mut().flags.can_emit_warnings
     }
 
     /// Resets the diagnostic error count as well as the cached emitted diagnostics.
@@ -666,8 +660,8 @@ impl Handler {
         inner.deduplicated_warn_count = 0;
 
         // actually free the underlying memory (which `clear` would not do)
-        inner.delayed_span_bugs = Default::default();
-        inner.delayed_good_path_bugs = Default::default();
+        inner.span_delayed_bugs = Default::default();
+        inner.good_path_delayed_bugs = Default::default();
         inner.taught_diagnostics = Default::default();
         inner.emitted_diagnostic_codes = Default::default();
         inner.emitted_diagnostics = Default::default();
@@ -678,15 +672,45 @@ impl Handler {
     /// Retrieve a stashed diagnostic with `steal_diagnostic`.
     pub fn stash_diagnostic(&self, span: Span, key: StashKey, diag: Diagnostic) {
         let mut inner = self.inner.borrow_mut();
-        inner.stash((span.with_parent(None), key), diag);
+
+        let key = (span.with_parent(None), key);
+
+        if diag.is_error() {
+            if matches!(diag.level, Level::Error { lint: true }) {
+                inner.lint_err_count += 1;
+            } else {
+                inner.err_count += 1;
+            }
+        } else {
+            // Warnings are only automatically flushed if they're forced.
+            if diag.is_force_warn() {
+                inner.warn_count += 1;
+            }
+        }
+
+        // FIXME(Centril, #69537): Consider reintroducing panic on overwriting a stashed diagnostic
+        // if/when we have a more robust macro-friendly replacement for `(span, key)` as a key.
+        // See the PR for a discussion.
+        inner.stashed_diagnostics.insert(key, diag);
     }
 
     /// Steal a previously stashed diagnostic with the given `Span` and [`StashKey`] as the key.
     pub fn steal_diagnostic(&self, span: Span, key: StashKey) -> Option<DiagnosticBuilder<'_, ()>> {
         let mut inner = self.inner.borrow_mut();
-        inner
-            .steal((span.with_parent(None), key))
-            .map(|diag| DiagnosticBuilder::new_diagnostic(self, diag))
+        let key = (span.with_parent(None), key);
+        let diag = inner.stashed_diagnostics.remove(&key)?;
+        if diag.is_error() {
+            if matches!(diag.level, Level::Error { lint: true }) {
+                inner.lint_err_count -= 1;
+            } else {
+                inner.err_count -= 1;
+            }
+        } else {
+            if diag.is_force_warn() {
+                inner.warn_count -= 1;
+            }
+        }
+        Some(DiagnosticBuilder::new_diagnostic(self, diag))
     }
 
     pub fn has_stashed_diagnostic(&self, span: Span, key: StashKey) -> bool {
@@ -698,7 +722,12 @@ impl Handler {
         self.inner.borrow_mut().emit_stashed_diagnostics()
     }
 
-    /// Construct a builder with the `msg` at the level appropriate for the specific `EmissionGuarantee`.
+    /// Construct a builder with the `msg` at the level appropriate for the
+    /// specific `EmissionGuarantee`.
+    ///
+    /// Note: this is necessary for `derive(Diagnostic)`, but shouldn't be used
+    /// outside of that. Instead use `struct_err`, `struct_warn`, etc., which
+    /// make the diagnostic kind clearer.
     #[rustc_lint_diagnostics]
     #[track_caller]
     pub fn struct_diagnostic<G: EmissionGuarantee>(
@@ -850,7 +879,7 @@ impl Handler {
         &self,
         msg: impl Into<DiagnosticMessage>,
     ) -> DiagnosticBuilder<'_, ErrorGuaranteed> {
-        DiagnosticBuilder::new_guaranteeing_error(self, msg)
+        DiagnosticBuilder::new(self, Level::Error { lint: false }, msg)
     }
 
     /// This should only be used by `rustc_middle::lint::struct_lint_level`. Do not use it for hard errors.
@@ -913,11 +942,21 @@ impl Handler {
         result
     }
 
-    /// Construct a builder at the `Error` level with the `msg`.
+    /// Construct a builder at the `Fatal` level with the `msg`.
     #[rustc_lint_diagnostics]
     #[track_caller]
     pub fn struct_fatal(&self, msg: impl Into<DiagnosticMessage>) -> DiagnosticBuilder<'_, !> {
-        DiagnosticBuilder::new_fatal(self, msg)
+        DiagnosticBuilder::new(self, Level::Fatal, msg)
+    }
+
+    /// Construct a builder at the `Fatal` level with the `msg`, that doesn't abort.
+    #[rustc_lint_diagnostics]
+    #[track_caller]
+    pub fn struct_almost_fatal(
+        &self,
+        msg: impl Into<DiagnosticMessage>,
+    ) -> DiagnosticBuilder<'_, FatalError> {
+        DiagnosticBuilder::new(self, Level::Fatal, msg)
     }
 
     /// Construct a builder at the `Help` level with the `msg`.
@@ -929,18 +968,14 @@ impl Handler {
     /// Construct a builder at the `Note` level with the `msg`.
     #[rustc_lint_diagnostics]
     #[track_caller]
-    pub fn struct_note_without_error(
-        &self,
-        msg: impl Into<DiagnosticMessage>,
-    ) -> DiagnosticBuilder<'_, ()> {
+    pub fn struct_note(&self, msg: impl Into<DiagnosticMessage>) -> DiagnosticBuilder<'_, ()> {
         DiagnosticBuilder::new(self, Level::Note, msg)
     }
 
     #[rustc_lint_diagnostics]
     #[track_caller]
     pub fn span_fatal(&self, span: impl Into<MultiSpan>, msg: impl Into<DiagnosticMessage>) -> ! {
-        self.emit_diag_at_span(Diagnostic::new(Fatal, msg), span);
-        FatalError.raise()
+        self.struct_span_fatal(span, msg).emit()
     }
 
     #[rustc_lint_diagnostics]
@@ -951,8 +986,7 @@ impl Handler {
         msg: impl Into<DiagnosticMessage>,
         code: DiagnosticId,
     ) -> ! {
-        self.emit_diag_at_span(Diagnostic::new_with_code(Fatal, Some(code), msg), span);
-        FatalError.raise()
+        self.struct_span_fatal_with_code(span, msg, code).emit()
     }
 
     #[rustc_lint_diagnostics]
@@ -962,7 +996,7 @@ impl Handler {
         span: impl Into<MultiSpan>,
         msg: impl Into<DiagnosticMessage>,
     ) -> ErrorGuaranteed {
-        self.emit_diag_at_span(Diagnostic::new(Error { lint: false }, msg), span).unwrap()
+        self.struct_span_err(span, msg).emit()
     }
 
     #[rustc_lint_diagnostics]
@@ -972,17 +1006,14 @@ impl Handler {
         span: impl Into<MultiSpan>,
         msg: impl Into<DiagnosticMessage>,
         code: DiagnosticId,
-    ) {
-        self.emit_diag_at_span(
-            Diagnostic::new_with_code(Error { lint: false }, Some(code), msg),
-            span,
-        );
+    ) -> ErrorGuaranteed {
+        self.struct_span_err_with_code(span, msg, code).emit()
     }
 
     #[rustc_lint_diagnostics]
     #[track_caller]
     pub fn span_warn(&self, span: impl Into<MultiSpan>, msg: impl Into<DiagnosticMessage>) {
-        self.emit_diag_at_span(Diagnostic::new(Warning(None), msg), span);
+        self.struct_span_warn(span, msg).emit()
     }
 
     #[rustc_lint_diagnostics]
@@ -993,49 +1024,70 @@ impl Handler {
         msg: impl Into<DiagnosticMessage>,
         code: DiagnosticId,
     ) {
-        self.emit_diag_at_span(Diagnostic::new_with_code(Warning(None), Some(code), msg), span);
+        self.struct_span_warn_with_code(span, msg, code).emit()
     }
 
-    pub fn span_bug(&self, span: impl Into<MultiSpan>, msg: impl Into<String>) -> ! {
+    pub fn span_bug(&self, span: impl Into<MultiSpan>, msg: impl Into<DiagnosticMessage>) -> ! {
         self.inner.borrow_mut().span_bug(span, msg)
     }
 
-    /// For documentation on this, see `Session::delay_span_bug`.
+    /// For documentation on this, see `Session::span_delayed_bug`.
+    ///
+    /// Note: this function used to be called `delay_span_bug`. It was renamed
+    /// to match similar functions like `span_bug`, `span_err`, etc.
     #[track_caller]
-    pub fn delay_span_bug(
+    pub fn span_delayed_bug(
         &self,
-        span: impl Into<MultiSpan>,
-        msg: impl Into<String>,
+        sp: impl Into<MultiSpan>,
+        msg: impl Into<DiagnosticMessage>,
     ) -> ErrorGuaranteed {
-        self.inner.borrow_mut().delay_span_bug(span, msg)
+        let mut inner = self.inner.borrow_mut();
+
+        // This is technically `self.treat_err_as_bug()` but `span_delayed_bug` is called before
+        // incrementing `err_count` by one, so we need to +1 the comparing.
+        // FIXME: Would be nice to increment err_count in a more coherent way.
+        if inner.flags.treat_err_as_bug.is_some_and(|c| {
+            inner.err_count + inner.lint_err_count + inner.delayed_bug_count() + 1 >= c.get()
+        }) {
+            // FIXME: don't abort here if report_delayed_bugs is off
+            inner.span_bug(sp, msg);
+        }
+        let mut diagnostic = Diagnostic::new(Level::DelayedBug, msg);
+        diagnostic.set_span(sp);
+        inner.emit_diagnostic(diagnostic).unwrap()
     }
 
-    // FIXME(eddyb) note the comment inside `impl Drop for HandlerInner`, that's
+    // FIXME(eddyb) note the comment inside `impl Drop for DiagCtxtInner`, that's
     // where the explanation of what "good path" is (also, it should be renamed).
-    pub fn delay_good_path_bug(&self, msg: impl Into<DiagnosticMessage>) {
-        self.inner.borrow_mut().delay_good_path_bug(msg)
+    pub fn good_path_delayed_bug(&self, msg: impl Into<DiagnosticMessage>) {
+        let mut inner = self.inner.borrow_mut();
+
+        let mut diagnostic = Diagnostic::new(Level::DelayedBug, msg);
+        if inner.flags.report_delayed_bugs {
+            inner.emit_diagnostic_without_consuming(&mut diagnostic);
+        }
+        let backtrace = std::backtrace::Backtrace::capture();
+        inner.good_path_delayed_bugs.push(DelayedDiagnostic::with_backtrace(diagnostic, backtrace));
     }
 
     #[track_caller]
     pub fn span_bug_no_panic(&self, span: impl Into<MultiSpan>, msg: impl Into<DiagnosticMessage>) {
-        self.emit_diag_at_span(Diagnostic::new(Bug, msg), span);
+        let mut diag = Diagnostic::new(Bug, msg);
+        diag.set_span(span);
+        self.emit_diagnostic(diag);
     }
 
     #[track_caller]
     #[rustc_lint_diagnostics]
-    pub fn span_note_without_error(
-        &self,
-        span: impl Into<MultiSpan>,
-        msg: impl Into<DiagnosticMessage>,
-    ) {
-        self.emit_diag_at_span(Diagnostic::new(Note, msg), span);
+    pub fn span_note(&self, span: impl Into<MultiSpan>, msg: impl Into<DiagnosticMessage>) {
+        self.struct_span_note(span, msg).emit()
     }
 
     #[track_caller]
     #[rustc_lint_diagnostics]
-    pub fn span_note_diag(
+    pub fn struct_span_note(
         &self,
-        span: Span,
+        span: impl Into<MultiSpan>,
         msg: impl Into<DiagnosticMessage>,
     ) -> DiagnosticBuilder<'_, ()> {
         let mut db = DiagnosticBuilder::new(self, Note, msg);
@@ -1043,35 +1095,34 @@ impl Handler {
         db
     }
 
-    // NOTE: intentionally doesn't raise an error so rustc_codegen_ssa only reports fatal errors in the main thread
     #[rustc_lint_diagnostics]
-    pub fn fatal(&self, msg: impl Into<DiagnosticMessage>) -> FatalError {
-        self.inner.borrow_mut().fatal(msg)
+    pub fn fatal(&self, msg: impl Into<DiagnosticMessage>) -> ! {
+        self.struct_fatal(msg).emit()
     }
 
     #[rustc_lint_diagnostics]
     pub fn err(&self, msg: impl Into<DiagnosticMessage>) -> ErrorGuaranteed {
-        self.inner.borrow_mut().err(msg)
+        self.struct_err(msg).emit()
     }
 
     #[rustc_lint_diagnostics]
     pub fn warn(&self, msg: impl Into<DiagnosticMessage>) {
-        let mut db = DiagnosticBuilder::new(self, Warning(None), msg);
-        db.emit();
+        self.struct_warn(msg).emit()
     }
 
     #[rustc_lint_diagnostics]
-    pub fn note_without_error(&self, msg: impl Into<DiagnosticMessage>) {
-        DiagnosticBuilder::new(self, Note, msg).emit();
+    pub fn note(&self, msg: impl Into<DiagnosticMessage>) {
+        self.struct_note(msg).emit()
     }
 
     pub fn bug(&self, msg: impl Into<DiagnosticMessage>) -> ! {
-        self.inner.borrow_mut().bug(msg)
+        DiagnosticBuilder::<diagnostic_builder::Bug>::new(self, Bug, msg).emit();
+        panic::panic_any(ExplicitBug);
     }
 
     #[inline]
     pub fn err_count(&self) -> usize {
-        self.inner.borrow().err_count()
+        self.inner.borrow().err_count
     }
 
     pub fn has_errors(&self) -> Option<ErrorGuaranteed> {
@@ -1082,26 +1133,103 @@ impl Handler {
     }
 
     pub fn has_errors_or_lint_errors(&self) -> Option<ErrorGuaranteed> {
-        self.inner.borrow().has_errors_or_lint_errors().then(|| {
+        let inner = self.inner.borrow();
+        let has_errors_or_lint_errors = inner.has_errors() || inner.lint_err_count > 0;
+        has_errors_or_lint_errors.then(|| {
             #[allow(deprecated)]
             ErrorGuaranteed::unchecked_claim_error_was_emitted()
         })
     }
-    pub fn has_errors_or_delayed_span_bugs(&self) -> Option<ErrorGuaranteed> {
-        self.inner.borrow().has_errors_or_delayed_span_bugs().then(|| {
+
+    pub fn has_errors_or_span_delayed_bugs(&self) -> Option<ErrorGuaranteed> {
+        let inner = self.inner.borrow();
+        let has_errors_or_span_delayed_bugs =
+            inner.has_errors() || !inner.span_delayed_bugs.is_empty();
+        has_errors_or_span_delayed_bugs.then(|| {
             #[allow(deprecated)]
             ErrorGuaranteed::unchecked_claim_error_was_emitted()
         })
     }
+
     pub fn is_compilation_going_to_fail(&self) -> Option<ErrorGuaranteed> {
-        self.inner.borrow().is_compilation_going_to_fail().then(|| {
+        let inner = self.inner.borrow();
+        let will_fail =
+            inner.has_errors() || inner.lint_err_count > 0 || !inner.span_delayed_bugs.is_empty();
+        will_fail.then(|| {
             #[allow(deprecated)]
             ErrorGuaranteed::unchecked_claim_error_was_emitted()
         })
     }
 
     pub fn print_error_count(&self, registry: &Registry) {
-        self.inner.borrow_mut().print_error_count(registry)
+        let mut inner = self.inner.borrow_mut();
+
+        inner.emit_stashed_diagnostics();
+
+        let warnings = match inner.deduplicated_warn_count {
+            0 => Cow::from(""),
+            1 => Cow::from("1 warning emitted"),
+            count => Cow::from(format!("{count} warnings emitted")),
+        };
+        let errors = match inner.deduplicated_err_count {
+            0 => Cow::from(""),
+            1 => Cow::from("aborting due to 1 previous error"),
+            count => Cow::from(format!("aborting due to {count} previous errors")),
+        };
+        if inner.treat_err_as_bug() {
+            return;
+        }
+
+        match (errors.len(), warnings.len()) {
+            (0, 0) => return,
+            (0, _) => inner.emitter.emit_diagnostic(&Diagnostic::new(
+                Level::Warning(None),
+                DiagnosticMessage::Str(warnings),
+            )),
+            (_, 0) => {
+                inner.emit_diagnostic(Diagnostic::new(Fatal, errors));
+            }
+            (_, _) => {
+                inner.emit_diagnostic(Diagnostic::new(Fatal, format!("{errors}; {warnings}")));
+            }
+        }
+
+        let can_show_explain = inner.emitter.should_show_explain();
+        let are_there_diagnostics = !inner.emitted_diagnostic_codes.is_empty();
+        if can_show_explain && are_there_diagnostics {
+            let mut error_codes = inner
+                .emitted_diagnostic_codes
+                .iter()
+                .filter_map(|x| match &x {
+                    DiagnosticId::Error(s) if registry.try_find_description(s).is_ok() => {
+                        Some(s.clone())
+                    }
+                    _ => None,
+                })
+                .collect::<Vec<_>>();
+            if !error_codes.is_empty() {
+                error_codes.sort();
+                if error_codes.len() > 1 {
+                    let limit = if error_codes.len() > 9 { 9 } else { error_codes.len() };
+                    inner.failure_note(format!(
+                        "Some errors have detailed explanations: {}{}",
+                        error_codes[..limit].join(", "),
+                        if error_codes.len() > 9 { "..." } else { "." }
+                    ));
+                    inner.failure_note(format!(
+                        "For more information about an error, try \
+                         `rustc --explain {}`.",
+                        &error_codes[0]
+                    ));
+                } else {
+                    inner.failure_note(format!(
+                        "For more information about this error, try \
+                         `rustc --explain {}`.",
+                        &error_codes[0]
+                    ));
+                }
+            }
+        }
     }
 
     pub fn take_future_breakage_diagnostics(&self) -> Vec<Diagnostic> {
@@ -1109,7 +1237,11 @@ impl Handler {
     }
 
     pub fn abort_if_errors(&self) {
-        self.inner.borrow_mut().abort_if_errors()
+        let mut inner = self.inner.borrow_mut();
+        inner.emit_stashed_diagnostics();
+        if inner.has_errors() {
+            FatalError.raise();
+        }
     }
 
     /// `true` if we haven't taught a diagnostic with this code already.
@@ -1118,15 +1250,24 @@ impl Handler {
     /// Used to suppress emitting the same error multiple times with extended explanation when
     /// calling `-Zteach`.
     pub fn must_teach(&self, code: &DiagnosticId) -> bool {
-        self.inner.borrow_mut().must_teach(code)
+        self.inner.borrow_mut().taught_diagnostics.insert(code.clone())
     }
 
     pub fn force_print_diagnostic(&self, db: Diagnostic) {
-        self.inner.borrow_mut().force_print_diagnostic(db)
+        self.inner.borrow_mut().emitter.emit_diagnostic(&db);
+    }
+
+    pub fn emit_diagnostic(&self, mut diagnostic: Diagnostic) -> Option<ErrorGuaranteed> {
+        self.emit_diagnostic_without_consuming(&mut diagnostic)
     }
 
-    pub fn emit_diagnostic(&self, diagnostic: &mut Diagnostic) -> Option<ErrorGuaranteed> {
-        self.inner.borrow_mut().emit_diagnostic(diagnostic)
+    // It's unfortunate this exists. `emit_diagnostic` is preferred, because it
+    // consumes the diagnostic, thus ensuring it is emitted just once.
+    pub(crate) fn emit_diagnostic_without_consuming(
+        &self,
+        diagnostic: &mut Diagnostic,
+    ) -> Option<ErrorGuaranteed> {
+        self.inner.borrow_mut().emit_diagnostic_without_consuming(diagnostic)
     }
 
     pub fn emit_err<'a>(&'a self, err: impl IntoDiagnostic<'a>) -> ErrorGuaranteed {
@@ -1201,17 +1342,8 @@ impl Handler {
         note.into_diagnostic(self)
     }
 
-    fn emit_diag_at_span(
-        &self,
-        mut diag: Diagnostic,
-        sp: impl Into<MultiSpan>,
-    ) -> Option<ErrorGuaranteed> {
-        let mut inner = self.inner.borrow_mut();
-        inner.emit_diagnostic(diag.set_span(sp))
-    }
-
     pub fn emit_artifact_notification(&self, path: &Path, artifact_type: &str) {
-        self.inner.borrow_mut().emit_artifact_notification(path, artifact_type)
+        self.inner.borrow_mut().emitter.emit_artifact_notification(path, artifact_type);
     }
 
     pub fn emit_future_breakage_report(&self, diags: Vec<Diagnostic>) {
@@ -1230,7 +1362,7 @@ impl Handler {
             inner.bump_err_count();
         }
 
-        inner.emit_unused_externs(lint_level, unused_externs)
+        inner.emitter.emit_unused_externs(lint_level, unused_externs)
     }
 
     pub fn update_unstable_expectation_id(
@@ -1249,7 +1381,7 @@ impl Handler {
                 // Here the diagnostic is given back to `emit_diagnostic` where it was first
                 // intercepted. Now it should be processed as usual, since the unstable expectation
                 // id is now stable.
-                inner.emit_diagnostic(&mut diag);
+                inner.emit_diagnostic(diag);
             }
         }
 
@@ -1264,38 +1396,34 @@ impl Handler {
     }
 
     /// This methods steals all [`LintExpectationId`]s that are stored inside
-    /// [`HandlerInner`] and indicate that the linked expectation has been fulfilled.
+    /// [`DiagCtxtInner`] and indicate that the linked expectation has been fulfilled.
     #[must_use]
     pub fn steal_fulfilled_expectation_ids(&self) -> FxHashSet<LintExpectationId> {
         assert!(
             self.inner.borrow().unstable_expect_diagnostics.is_empty(),
-            "`HandlerInner::unstable_expect_diagnostics` should be empty at this point",
+            "`DiagCtxtInner::unstable_expect_diagnostics` should be empty at this point",
         );
         std::mem::take(&mut self.inner.borrow_mut().fulfilled_expectations)
     }
 
     pub fn flush_delayed(&self) {
-        let mut inner = self.inner.lock();
-        let bugs = std::mem::replace(&mut inner.delayed_span_bugs, Vec::new());
-        inner.flush_delayed(bugs, "no errors encountered even though `delay_span_bug` issued");
+        let mut inner = self.inner.borrow_mut();
+        let bugs = std::mem::replace(&mut inner.span_delayed_bugs, Vec::new());
+        inner.flush_delayed(bugs, "no errors encountered even though `span_delayed_bug` issued");
     }
 }
 
-impl HandlerInner {
-    fn must_teach(&mut self, code: &DiagnosticId) -> bool {
-        self.taught_diagnostics.insert(code.clone())
-    }
-
-    fn force_print_diagnostic(&mut self, db: Diagnostic) {
-        self.emitter.emit_diagnostic(&db);
-    }
-
+// Note: we prefer implementing operations on `DiagCtxt`, rather than
+// `DiagCtxtInner`, whenever possible. This minimizes functions where
+// `DiagCtxt::foo()` just borrows `inner` and forwards a call to
+// `HanderInner::foo`.
+impl DiagCtxtInner {
     /// Emit all stashed diagnostics.
     fn emit_stashed_diagnostics(&mut self) -> Option<ErrorGuaranteed> {
         let has_errors = self.has_errors();
         let diags = self.stashed_diagnostics.drain(..).map(|x| x.1).collect::<Vec<_>>();
         let mut reported = None;
-        for mut diag in diags {
+        for diag in diags {
             // Decrement the count tracking the stash; emitting will increment it.
             if diag.is_error() {
                 if matches!(diag.level, Level::Error { lint: true }) {
@@ -1315,14 +1443,25 @@ impl HandlerInner {
                     }
                 }
             }
-            let reported_this = self.emit_diagnostic(&mut diag);
+            let reported_this = self.emit_diagnostic(diag);
             reported = reported.or(reported_this);
         }
         reported
     }
 
-    // FIXME(eddyb) this should ideally take `diagnostic` by value.
-    fn emit_diagnostic(&mut self, diagnostic: &mut Diagnostic) -> Option<ErrorGuaranteed> {
+    fn emit_diagnostic(&mut self, mut diagnostic: Diagnostic) -> Option<ErrorGuaranteed> {
+        self.emit_diagnostic_without_consuming(&mut diagnostic)
+    }
+
+    fn emit_diagnostic_without_consuming(
+        &mut self,
+        diagnostic: &mut Diagnostic,
+    ) -> Option<ErrorGuaranteed> {
+        if matches!(diagnostic.level, Level::Error { .. } | Level::Fatal) && self.treat_err_as_bug()
+        {
+            diagnostic.level = Level::Bug;
+        }
+
         // The `LintExpectationId` can be stable or unstable depending on when it was created.
         // Diagnostics created before the definition of `HirId`s are unstable and can not yet
         // be stored. Instead, they are buffered until the `LintExpectationId` is replaced by
@@ -1334,11 +1473,11 @@ impl HandlerInner {
 
         if diagnostic.level == Level::DelayedBug {
             // FIXME(eddyb) this should check for `has_errors` and stop pushing
-            // once *any* errors were emitted (and truncate `delayed_span_bugs`
+            // once *any* errors were emitted (and truncate `span_delayed_bugs`
             // when an error is first emitted, also), but maybe there's a case
             // in which that's not sound? otherwise this is really inefficient.
             let backtrace = std::backtrace::Backtrace::capture();
-            self.delayed_span_bugs
+            self.span_delayed_bugs
                 .push(DelayedDiagnostic::with_backtrace(diagnostic.clone(), backtrace));
 
             if !self.flags.report_delayed_bugs {
@@ -1438,227 +1577,30 @@ impl HandlerInner {
         guaranteed
     }
 
-    fn emit_artifact_notification(&mut self, path: &Path, artifact_type: &str) {
-        self.emitter.emit_artifact_notification(path, artifact_type);
-    }
-
-    fn emit_unused_externs(&mut self, lint_level: rustc_lint_defs::Level, unused_externs: &[&str]) {
-        self.emitter.emit_unused_externs(lint_level, unused_externs);
-    }
-
     fn treat_err_as_bug(&self) -> bool {
         self.flags.treat_err_as_bug.is_some_and(|c| {
-            self.err_count() + self.lint_err_count + self.delayed_bug_count() >= c.get()
+            self.err_count + self.lint_err_count + self.delayed_bug_count() >= c.get()
         })
     }
 
     fn delayed_bug_count(&self) -> usize {
-        self.delayed_span_bugs.len() + self.delayed_good_path_bugs.len()
-    }
-
-    fn print_error_count(&mut self, registry: &Registry) {
-        self.emit_stashed_diagnostics();
-
-        let warnings = match self.deduplicated_warn_count {
-            0 => Cow::from(""),
-            1 => Cow::from("1 warning emitted"),
-            count => Cow::from(format!("{count} warnings emitted")),
-        };
-        let errors = match self.deduplicated_err_count {
-            0 => Cow::from(""),
-            1 => Cow::from("aborting due to previous error"),
-            count => Cow::from(format!("aborting due to {count} previous errors")),
-        };
-        if self.treat_err_as_bug() {
-            return;
-        }
-
-        match (errors.len(), warnings.len()) {
-            (0, 0) => return,
-            (0, _) => self.emitter.emit_diagnostic(&Diagnostic::new(
-                Level::Warning(None),
-                DiagnosticMessage::Str(warnings),
-            )),
-            (_, 0) => {
-                let _ = self.fatal(errors);
-            }
-            (_, _) => {
-                let _ = self.fatal(format!("{errors}; {warnings}"));
-            }
-        }
-
-        let can_show_explain = self.emitter.should_show_explain();
-        let are_there_diagnostics = !self.emitted_diagnostic_codes.is_empty();
-        if can_show_explain && are_there_diagnostics {
-            let mut error_codes = self
-                .emitted_diagnostic_codes
-                .iter()
-                .filter_map(|x| match &x {
-                    DiagnosticId::Error(s) if registry.try_find_description(s).is_ok() => {
-                        Some(s.clone())
-                    }
-                    _ => None,
-                })
-                .collect::<Vec<_>>();
-            if !error_codes.is_empty() {
-                error_codes.sort();
-                if error_codes.len() > 1 {
-                    let limit = if error_codes.len() > 9 { 9 } else { error_codes.len() };
-                    self.failure(format!(
-                        "Some errors have detailed explanations: {}{}",
-                        error_codes[..limit].join(", "),
-                        if error_codes.len() > 9 { "..." } else { "." }
-                    ));
-                    self.failure(format!(
-                        "For more information about an error, try \
-                         `rustc --explain {}`.",
-                        &error_codes[0]
-                    ));
-                } else {
-                    self.failure(format!(
-                        "For more information about this error, try \
-                         `rustc --explain {}`.",
-                        &error_codes[0]
-                    ));
-                }
-            }
-        }
-    }
-
-    fn stash(&mut self, key: (Span, StashKey), diagnostic: Diagnostic) {
-        // Track the diagnostic for counts, but don't panic-if-treat-err-as-bug
-        // yet; that happens when we actually emit the diagnostic.
-        if diagnostic.is_error() {
-            if matches!(diagnostic.level, Level::Error { lint: true }) {
-                self.lint_err_count += 1;
-            } else {
-                self.err_count += 1;
-            }
-        } else {
-            // Warnings are only automatically flushed if they're forced.
-            if diagnostic.is_force_warn() {
-                self.warn_count += 1;
-            }
-        }
-
-        // FIXME(Centril, #69537): Consider reintroducing panic on overwriting a stashed diagnostic
-        // if/when we have a more robust macro-friendly replacement for `(span, key)` as a key.
-        // See the PR for a discussion.
-        self.stashed_diagnostics.insert(key, diagnostic);
-    }
-
-    fn steal(&mut self, key: (Span, StashKey)) -> Option<Diagnostic> {
-        let diagnostic = self.stashed_diagnostics.remove(&key)?;
-        if diagnostic.is_error() {
-            if matches!(diagnostic.level, Level::Error { lint: true }) {
-                self.lint_err_count -= 1;
-            } else {
-                self.err_count -= 1;
-            }
-        } else {
-            if diagnostic.is_force_warn() {
-                self.warn_count -= 1;
-            }
-        }
-        Some(diagnostic)
-    }
-
-    #[inline]
-    fn err_count(&self) -> usize {
-        self.err_count
+        self.span_delayed_bugs.len() + self.good_path_delayed_bugs.len()
     }
 
     fn has_errors(&self) -> bool {
-        self.err_count() > 0
-    }
-    fn has_errors_or_lint_errors(&self) -> bool {
-        self.has_errors() || self.lint_err_count > 0
-    }
-    fn has_errors_or_delayed_span_bugs(&self) -> bool {
-        self.has_errors() || !self.delayed_span_bugs.is_empty()
-    }
-    fn has_any_message(&self) -> bool {
-        self.err_count() > 0 || self.lint_err_count > 0 || self.warn_count > 0
-    }
-
-    fn is_compilation_going_to_fail(&self) -> bool {
-        self.has_errors() || self.lint_err_count > 0 || !self.delayed_span_bugs.is_empty()
-    }
-
-    fn abort_if_errors(&mut self) {
-        self.emit_stashed_diagnostics();
-
-        if self.has_errors() {
-            FatalError.raise();
-        }
+        self.err_count > 0
     }
 
     #[track_caller]
-    fn span_bug(&mut self, sp: impl Into<MultiSpan>, msg: impl Into<String>) -> ! {
-        self.emit_diag_at_span(Diagnostic::new(Bug, msg.into()), sp);
+    fn span_bug(&mut self, sp: impl Into<MultiSpan>, msg: impl Into<DiagnosticMessage>) -> ! {
+        let mut diag = Diagnostic::new(Bug, msg);
+        diag.set_span(sp);
+        self.emit_diagnostic(diag);
         panic::panic_any(ExplicitBug);
     }
 
-    fn emit_diag_at_span(&mut self, mut diag: Diagnostic, sp: impl Into<MultiSpan>) {
-        self.emit_diagnostic(diag.set_span(sp));
-    }
-
-    /// For documentation on this, see `Session::delay_span_bug`.
-    #[track_caller]
-    fn delay_span_bug(
-        &mut self,
-        sp: impl Into<MultiSpan>,
-        msg: impl Into<String>,
-    ) -> ErrorGuaranteed {
-        // This is technically `self.treat_err_as_bug()` but `delay_span_bug` is called before
-        // incrementing `err_count` by one, so we need to +1 the comparing.
-        // FIXME: Would be nice to increment err_count in a more coherent way.
-        if self.flags.treat_err_as_bug.is_some_and(|c| {
-            self.err_count() + self.lint_err_count + self.delayed_bug_count() + 1 >= c.get()
-        }) {
-            // FIXME: don't abort here if report_delayed_bugs is off
-            self.span_bug(sp, msg.into());
-        }
-        let mut diagnostic = Diagnostic::new(Level::DelayedBug, msg.into());
-        diagnostic.set_span(sp.into());
-        self.emit_diagnostic(&mut diagnostic).unwrap()
-    }
-
-    // FIXME(eddyb) note the comment inside `impl Drop for HandlerInner`, that's
-    // where the explanation of what "good path" is (also, it should be renamed).
-    fn delay_good_path_bug(&mut self, msg: impl Into<DiagnosticMessage>) {
-        let mut diagnostic = Diagnostic::new(Level::DelayedBug, msg);
-        if self.flags.report_delayed_bugs {
-            self.emit_diagnostic(&mut diagnostic);
-        }
-        let backtrace = std::backtrace::Backtrace::capture();
-        self.delayed_good_path_bugs.push(DelayedDiagnostic::with_backtrace(diagnostic, backtrace));
-    }
-
-    fn failure(&mut self, msg: impl Into<DiagnosticMessage>) {
-        self.emit_diagnostic(&mut Diagnostic::new(FailureNote, msg));
-    }
-
-    fn fatal(&mut self, msg: impl Into<DiagnosticMessage>) -> FatalError {
-        self.emit(Fatal, msg);
-        FatalError
-    }
-
-    fn err(&mut self, msg: impl Into<DiagnosticMessage>) -> ErrorGuaranteed {
-        self.emit(Error { lint: false }, msg)
-    }
-
-    /// Emit an error; level should be `Error` or `Fatal`.
-    fn emit(&mut self, level: Level, msg: impl Into<DiagnosticMessage>) -> ErrorGuaranteed {
-        if self.treat_err_as_bug() {
-            self.bug(msg);
-        }
-        self.emit_diagnostic(&mut Diagnostic::new(level, msg)).unwrap()
-    }
-
-    fn bug(&mut self, msg: impl Into<DiagnosticMessage>) -> ! {
-        self.emit_diagnostic(&mut Diagnostic::new(Bug, msg));
-        panic::panic_any(ExplicitBug);
+    fn failure_note(&mut self, msg: impl Into<DiagnosticMessage>) {
+        self.emit_diagnostic(Diagnostic::new(FailureNote, msg));
     }
 
     fn flush_delayed(
@@ -1690,7 +1632,7 @@ impl HandlerInner {
             if no_bugs {
                 // Put the overall explanation before the `DelayedBug`s, to
                 // frame them better (e.g. separate warnings from them).
-                self.emit_diagnostic(&mut Diagnostic::new(Bug, explanation));
+                self.emit_diagnostic(Diagnostic::new(Bug, explanation));
                 no_bugs = false;
             }
 
@@ -1705,7 +1647,7 @@ impl HandlerInner {
             }
             bug.level = Level::Bug;
 
-            self.emit_diagnostic(&mut bug);
+            self.emit_diagnostic(bug);
         }
 
         // Panic with `DelayedBugPanic` to avoid "unexpected panic" messages.
@@ -1731,7 +1673,7 @@ impl HandlerInner {
     fn panic_if_treat_err_as_bug(&self) {
         if self.treat_err_as_bug() {
             match (
-                self.err_count() + self.lint_err_count,
+                self.err_count + self.lint_err_count,
                 self.delayed_bug_count(),
                 self.flags.treat_err_as_bug.map(|c| c.get()).unwrap(),
             ) {
diff --git a/compiler/rustc_errors/src/markdown/parse.rs b/compiler/rustc_errors/src/markdown/parse.rs
index d3a08da62..67e4963fd 100644
--- a/compiler/rustc_errors/src/markdown/parse.rs
+++ b/compiler/rustc_errors/src/markdown/parse.rs
@@ -329,7 +329,7 @@ fn parse_with_end_pat<'a>(
     end_sep: &[u8],
     ignore_esc: bool,
 ) -> Option<(&'a [u8], &'a [u8])> {
-    // Find positions that start with the end seperator
+    // Find positions that start with the end separator
     for idx in (0..buf.len()).filter(|idx| buf[*idx..].starts_with(end_sep)) {
         if !ignore_esc && idx > 0 && buf[idx - 1] == b'\\' {
             continue;
diff --git a/compiler/rustc_errors/src/markdown/tests/term.rs b/compiler/rustc_errors/src/markdown/tests/term.rs
index 6f68fb25a..a0d956bf0 100644
--- a/compiler/rustc_errors/src/markdown/tests/term.rs
+++ b/compiler/rustc_errors/src/markdown/tests/term.rs
@@ -3,7 +3,6 @@ use std::path::PathBuf;
 use termcolor::{BufferWriter, ColorChoice};
 
 use super::*;
-use crate::markdown::MdStream;
 
 const INPUT: &str = include_str!("input.md");
 const OUTPUT_PATH: &[&str] = &[env!("CARGO_MANIFEST_DIR"), "src","markdown","tests","output.stdout"];
diff --git a/compiler/rustc_expand/messages.ftl b/compiler/rustc_expand/messages.ftl
index 8b9382962..475dd348e 100644
--- a/compiler/rustc_expand/messages.ftl
+++ b/compiler/rustc_expand/messages.ftl
@@ -35,9 +35,6 @@ expand_explain_doc_comment_outer =
 expand_expr_repeat_no_syntax_vars =
     attempted to repeat an expression containing no syntax variables matched as repeating at this depth
 
-expand_feature_included_in_edition =
-    the feature `{$feature}` is included in the Rust {$edition} edition
-
 expand_feature_not_allowed =
     the feature `{$name}` is not in the list of allowed features
 
@@ -71,6 +68,8 @@ expand_macro_const_stability =
     .label = invalid const stability attribute
     .label2 = const stability attribute affects this macro
 
+expand_macro_expands_to_match_arm = macros cannot expand to match arms
+
 expand_malformed_feature_attribute =
     malformed `feature` attribute input
     .expected = expected just one word
diff --git a/compiler/rustc_expand/src/base.rs b/compiler/rustc_expand/src/base.rs
index c4d2a374f..b63609c48 100644
--- a/compiler/rustc_expand/src/base.rs
+++ b/compiler/rustc_expand/src/base.rs
@@ -21,7 +21,7 @@ use rustc_errors::{
 use rustc_feature::Features;
 use rustc_lint_defs::builtin::PROC_MACRO_BACK_COMPAT;
 use rustc_lint_defs::{BufferedEarlyLint, BuiltinLintDiagnostics, RegisteredTools};
-use rustc_parse::{self, parser, MACRO_ARGUMENTS};
+use rustc_parse::{parser, MACRO_ARGUMENTS};
 use rustc_session::errors::report_lit_error;
 use rustc_session::{parse::ParseSess, Limit, Session};
 use rustc_span::def_id::{CrateNum, DefId, LocalDefId};
@@ -777,7 +777,7 @@ impl SyntaxExtension {
         attrs: &[ast::Attribute],
     ) -> SyntaxExtension {
         let allow_internal_unstable =
-            attr::allow_internal_unstable(sess, &attrs).collect::<Vec<Symbol>>();
+            attr::allow_internal_unstable(sess, attrs).collect::<Vec<Symbol>>();
 
         let allow_internal_unsafe = attr::contains_name(attrs, sym::allow_internal_unsafe);
         let local_inner_macros = attr::find_by_name(attrs, sym::macro_export)
@@ -790,15 +790,15 @@ impl SyntaxExtension {
             .map(|attr| {
                 // Override `helper_attrs` passed above if it's a built-in macro,
                 // marking `proc_macro_derive` macros as built-in is not a realistic use case.
-                parse_macro_name_and_helper_attrs(sess.diagnostic(), attr, "built-in").map_or_else(
+                parse_macro_name_and_helper_attrs(sess.dcx(), attr, "built-in").map_or_else(
                     || (Some(name), Vec::new()),
                     |(name, helper_attrs)| (Some(name), helper_attrs),
                 )
             })
             .unwrap_or_else(|| (None, helper_attrs));
-        let stability = attr::find_stability(&sess, attrs, span);
-        let const_stability = attr::find_const_stability(&sess, attrs, span);
-        let body_stability = attr::find_body_stability(&sess, attrs);
+        let stability = attr::find_stability(sess, attrs, span);
+        let const_stability = attr::find_const_stability(sess, attrs, span);
+        let body_stability = attr::find_body_stability(sess, attrs);
         if let Some((_, sp)) = const_stability {
             sess.emit_err(errors::MacroConstStability {
                 span: sp,
@@ -818,7 +818,7 @@ impl SyntaxExtension {
             allow_internal_unstable: (!allow_internal_unstable.is_empty())
                 .then(|| allow_internal_unstable.into()),
             stability: stability.map(|(s, _)| s),
-            deprecation: attr::find_deprecation(&sess, features, attrs).map(|(d, _)| d),
+            deprecation: attr::find_deprecation(sess, features, attrs).map(|(d, _)| d),
             helper_attrs,
             edition,
             builtin_name,
@@ -1119,7 +1119,7 @@ impl<'a> ExtCtxt<'a> {
         sp: S,
         msg: impl Into<DiagnosticMessage>,
     ) -> DiagnosticBuilder<'a, ErrorGuaranteed> {
-        self.sess.parse_sess.span_diagnostic.struct_span_err(sp, msg)
+        self.sess.dcx().struct_span_err(sp, msg)
     }
 
     #[track_caller]
@@ -1143,15 +1143,10 @@ impl<'a> ExtCtxt<'a> {
     #[rustc_lint_diagnostics]
     #[track_caller]
     pub fn span_err<S: Into<MultiSpan>>(&self, sp: S, msg: impl Into<DiagnosticMessage>) {
-        self.sess.parse_sess.span_diagnostic.span_err(sp, msg);
+        self.sess.dcx().span_err(sp, msg);
     }
-    #[rustc_lint_diagnostics]
-    #[track_caller]
-    pub fn span_warn<S: Into<MultiSpan>>(&self, sp: S, msg: impl Into<DiagnosticMessage>) {
-        self.sess.parse_sess.span_diagnostic.span_warn(sp, msg);
-    }
-    pub fn span_bug<S: Into<MultiSpan>>(&self, sp: S, msg: impl Into<String>) -> ! {
-        self.sess.parse_sess.span_diagnostic.span_bug(sp, msg);
+    pub fn span_bug<S: Into<MultiSpan>>(&self, sp: S, msg: impl Into<DiagnosticMessage>) -> ! {
+        self.sess.dcx().span_bug(sp, msg);
     }
     pub fn trace_macros_diag(&mut self) {
         for (span, notes) in self.expansions.iter() {
@@ -1165,7 +1160,7 @@ impl<'a> ExtCtxt<'a> {
         self.expansions.clear();
     }
     pub fn bug(&self, msg: &'static str) -> ! {
-        self.sess.parse_sess.span_diagnostic.bug(msg);
+        self.sess.dcx().bug(msg);
     }
     pub fn trace_macros(&self) -> bool {
         self.ecfg.trace_mac
@@ -1213,7 +1208,7 @@ pub fn resolve_path(
                     span,
                     path: parse_sess.source_map().filename_for_diagnostics(&other).to_string(),
                 }
-                .into_diagnostic(&parse_sess.span_diagnostic));
+                .into_diagnostic(&parse_sess.dcx));
             }
         };
         result.pop();
@@ -1286,9 +1281,8 @@ pub fn expr_to_string(
 
 /// Non-fatally assert that `tts` is empty. Note that this function
 /// returns even when `tts` is non-empty, macros that *need* to stop
-/// compilation should call
-/// `cx.parse_sess.span_diagnostic.abort_if_errors()` (this should be
-/// done as rarely as possible).
+/// compilation should call `cx.diagnostic().abort_if_errors()`
+/// (this should be done as rarely as possible).
 pub fn check_zero_tts(cx: &ExtCtxt<'_>, span: Span, tts: TokenStream, name: &str) {
     if !tts.is_empty() {
         cx.emit_err(errors::TakesNoArguments { span, name });
@@ -1356,7 +1350,7 @@ pub fn get_exprs_from_tts(cx: &mut ExtCtxt<'_>, tts: TokenStream) -> Option<Vec<
 }
 
 pub fn parse_macro_name_and_helper_attrs(
-    diag: &rustc_errors::Handler,
+    dcx: &rustc_errors::DiagCtxt,
     attr: &Attribute,
     macro_type: &str,
 ) -> Option<(Symbol, Vec<Symbol>)> {
@@ -1365,23 +1359,23 @@ pub fn parse_macro_name_and_helper_attrs(
     // `#[proc_macro_derive(Foo, attributes(A, ..))]`
     let list = attr.meta_item_list()?;
     if list.len() != 1 && list.len() != 2 {
-        diag.emit_err(errors::AttrNoArguments { span: attr.span });
+        dcx.emit_err(errors::AttrNoArguments { span: attr.span });
         return None;
     }
     let Some(trait_attr) = list[0].meta_item() else {
-        diag.emit_err(errors::NotAMetaItem { span: list[0].span() });
+        dcx.emit_err(errors::NotAMetaItem { span: list[0].span() });
         return None;
     };
     let trait_ident = match trait_attr.ident() {
         Some(trait_ident) if trait_attr.is_word() => trait_ident,
         _ => {
-            diag.emit_err(errors::OnlyOneWord { span: trait_attr.span });
+            dcx.emit_err(errors::OnlyOneWord { span: trait_attr.span });
             return None;
         }
     };
 
     if !trait_ident.name.can_be_raw() {
-        diag.emit_err(errors::CannotBeNameOfMacro {
+        dcx.emit_err(errors::CannotBeNameOfMacro {
             span: trait_attr.span,
             trait_ident,
             macro_type,
@@ -1391,29 +1385,29 @@ pub fn parse_macro_name_and_helper_attrs(
     let attributes_attr = list.get(1);
     let proc_attrs: Vec<_> = if let Some(attr) = attributes_attr {
         if !attr.has_name(sym::attributes) {
-            diag.emit_err(errors::ArgumentNotAttributes { span: attr.span() });
+            dcx.emit_err(errors::ArgumentNotAttributes { span: attr.span() });
         }
         attr.meta_item_list()
             .unwrap_or_else(|| {
-                diag.emit_err(errors::AttributesWrongForm { span: attr.span() });
+                dcx.emit_err(errors::AttributesWrongForm { span: attr.span() });
                 &[]
             })
             .iter()
             .filter_map(|attr| {
                 let Some(attr) = attr.meta_item() else {
-                    diag.emit_err(errors::AttributeMetaItem { span: attr.span() });
+                    dcx.emit_err(errors::AttributeMetaItem { span: attr.span() });
                     return None;
                 };
 
                 let ident = match attr.ident() {
                     Some(ident) if attr.is_word() => ident,
                     _ => {
-                        diag.emit_err(errors::AttributeSingleWord { span: attr.span });
+                        dcx.emit_err(errors::AttributeSingleWord { span: attr.span });
                         return None;
                     }
                 };
                 if !ident.name.can_be_raw() {
-                    diag.emit_err(errors::HelperAttributeNameInvalid {
+                    dcx.emit_err(errors::HelperAttributeNameInvalid {
                         span: attr.span,
                         name: ident,
                     });
@@ -1464,7 +1458,7 @@ fn pretty_printing_compatibility_hack(item: &Item, sess: &ParseSess) -> bool {
 
                             if crate_matches {
                                 sess.buffer_lint_with_diagnostic(
-                                        &PROC_MACRO_BACK_COMPAT,
+                                        PROC_MACRO_BACK_COMPAT,
                                         item.ident.span,
                                         ast::CRATE_NODE_ID,
                                         "using an old version of `rental`",
diff --git a/compiler/rustc_expand/src/build.rs b/compiler/rustc_expand/src/build.rs
index 7de469944..86f555fa0 100644
--- a/compiler/rustc_expand/src/build.rs
+++ b/compiler/rustc_expand/src/build.rs
@@ -4,7 +4,7 @@ use rustc_ast::{self as ast, AttrVec, BlockCheckMode, Expr, LocalKind, PatKind,
 use rustc_ast::{attr, token, util::literal};
 use rustc_span::source_map::Spanned;
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
-use rustc_span::Span;
+use rustc_span::{Span, DUMMY_SP};
 use thin_vec::{thin_vec, ThinVec};
 
 impl<'a> ExtCtxt<'a> {
@@ -135,7 +135,7 @@ impl<'a> ExtCtxt<'a> {
         ast::GenericBound::Trait(
             self.poly_trait_ref(path.span, path),
             if is_const {
-                ast::TraitBoundModifier::MaybeConst
+                ast::TraitBoundModifier::MaybeConst(DUMMY_SP)
             } else {
                 ast::TraitBoundModifier::None
             },
@@ -505,7 +505,7 @@ impl<'a> ExtCtxt<'a> {
             attrs: AttrVec::new(),
             pat,
             guard: None,
-            body: expr,
+            body: Some(expr),
             span,
             id: ast::DUMMY_NODE_ID,
             is_placeholder: false,
@@ -547,7 +547,7 @@ impl<'a> ExtCtxt<'a> {
                 binder: ast::ClosureBinder::NotPresent,
                 capture_clause: ast::CaptureBy::Ref,
                 constness: ast::Const::No,
-                asyncness: ast::Async::No,
+                coroutine_kind: None,
                 movability: ast::Movability::Movable,
                 fn_decl,
                 body,
diff --git a/compiler/rustc_expand/src/config.rs b/compiler/rustc_expand/src/config.rs
index bef487659..0b56dbb2c 100644
--- a/compiler/rustc_expand/src/config.rs
+++ b/compiler/rustc_expand/src/config.rs
@@ -1,25 +1,22 @@
 //! Conditional compilation stripping.
 
 use crate::errors::{
-    FeatureIncludedInEdition, FeatureNotAllowed, FeatureRemoved, FeatureRemovedReason, InvalidCfg,
-    MalformedFeatureAttribute, MalformedFeatureAttributeHelp, RemoveExprNotSupported,
+    FeatureNotAllowed, FeatureRemoved, FeatureRemovedReason, InvalidCfg, MalformedFeatureAttribute,
+    MalformedFeatureAttributeHelp, RemoveExprNotSupported,
 };
 use rustc_ast::ptr::P;
 use rustc_ast::token::{Delimiter, Token, TokenKind};
-use rustc_ast::tokenstream::{AttrTokenStream, AttrTokenTree};
-use rustc_ast::tokenstream::{DelimSpan, Spacing};
+use rustc_ast::tokenstream::{AttrTokenStream, AttrTokenTree, DelimSpacing, DelimSpan, Spacing};
 use rustc_ast::tokenstream::{LazyAttrTokenStream, TokenTree};
 use rustc_ast::NodeId;
 use rustc_ast::{self as ast, AttrStyle, Attribute, HasAttrs, HasTokens, MetaItem};
 use rustc_attr as attr;
 use rustc_data_structures::flat_map_in_place::FlatMapInPlace;
-use rustc_data_structures::fx::FxHashSet;
 use rustc_feature::Features;
 use rustc_feature::{ACCEPTED_FEATURES, REMOVED_FEATURES, UNSTABLE_FEATURES};
 use rustc_parse::validate_attr;
 use rustc_session::parse::feature_err;
 use rustc_session::Session;
-use rustc_span::edition::ALL_EDITIONS;
 use rustc_span::symbol::{sym, Symbol};
 use rustc_span::Span;
 use thin_vec::ThinVec;
@@ -48,40 +45,6 @@ pub fn features(sess: &Session, krate_attrs: &[Attribute], crate_name: Symbol) -
 
     let mut features = Features::default();
 
-    // The edition from `--edition`.
-    let crate_edition = sess.edition();
-
-    // The maximum of (a) the edition from `--edition` and (b) any edition
-    // umbrella feature-gates declared in the code.
-    // - E.g. if `crate_edition` is 2015 but `rust_2018_preview` is present,
-    //   `feature_edition` is 2018
-    let mut features_edition = crate_edition;
-    for attr in krate_attrs {
-        for mi in feature_list(attr) {
-            if mi.is_word() {
-                let name = mi.name_or_empty();
-                let edition = ALL_EDITIONS.iter().find(|e| name == e.feature_name()).copied();
-                if let Some(edition) = edition
-                    && edition > features_edition
-                {
-                    features_edition = edition;
-                }
-            }
-        }
-    }
-
-    // Enable edition-dependent features based on `features_edition`.
-    // - E.g. enable `test_2018_feature` if `features_edition` is 2018 or higher
-    let mut edition_enabled_features = FxHashSet::default();
-    for f in UNSTABLE_FEATURES {
-        if let Some(edition) = f.feature.edition && edition <= features_edition {
-            // FIXME(Manishearth) there is currently no way to set lib features by
-            // edition.
-            edition_enabled_features.insert(f.feature.name);
-            (f.set_enabled)(&mut features);
-        }
-    }
-
     // Process all features declared in the code.
     for attr in krate_attrs {
         for mi in feature_list(attr) {
@@ -106,38 +69,6 @@ pub fn features(sess: &Session, krate_attrs: &[Attribute], crate_name: Symbol) -
                 }
             };
 
-            // If the declared feature is an edition umbrella feature-gate,
-            // warn if it was redundant w.r.t. `crate_edition`.
-            // - E.g. warn if `rust_2018_preview` is declared when
-            //   `crate_edition` is 2018
-            // - E.g. don't warn if `rust_2018_preview` is declared when
-            //   `crate_edition` is 2015.
-            if let Some(&edition) = ALL_EDITIONS.iter().find(|e| name == e.feature_name()) {
-                if edition <= crate_edition {
-                    sess.emit_warning(FeatureIncludedInEdition {
-                        span: mi.span(),
-                        feature: name,
-                        edition,
-                    });
-                }
-                features.set_declared_lang_feature(name, mi.span(), None);
-                continue;
-            }
-
-            // If the declared feature is edition-dependent and was already
-            // enabled due to `feature_edition`, give a warning.
-            // - E.g. warn if `test_2018_feature` is declared when
-            //   `feature_edition` is 2018 or higher.
-            if edition_enabled_features.contains(&name) {
-                sess.emit_warning(FeatureIncludedInEdition {
-                    span: mi.span(),
-                    feature: name,
-                    edition: features_edition,
-                });
-                features.set_declared_lang_feature(name, mi.span(), None);
-                continue;
-            }
-
             // If the declared feature has been removed, issue an error.
             if let Some(f) = REMOVED_FEATURES.iter().find(|f| name == f.feature.name) {
                 sess.emit_err(FeatureRemoved {
@@ -240,7 +171,7 @@ impl<'a> StripUnconfigured<'a> {
             stream.0.iter().all(|tree| match tree {
                 AttrTokenTree::Attributes(_) => false,
                 AttrTokenTree::Token(..) => true,
-                AttrTokenTree::Delimited(_, _, inner) => can_skip(inner),
+                AttrTokenTree::Delimited(.., inner) => can_skip(inner),
             })
         }
 
@@ -251,8 +182,7 @@ impl<'a> StripUnconfigured<'a> {
         let trees: Vec<_> = stream
             .0
             .iter()
-            .flat_map(|tree| {
-                match tree.clone() {
+            .flat_map(|tree| match tree.clone() {
                 AttrTokenTree::Attributes(mut data) => {
                     data.attrs.flat_map_in_place(|attr| self.process_cfg_attr(&attr));
 
@@ -265,9 +195,9 @@ impl<'a> StripUnconfigured<'a> {
                         None.into_iter()
                     }
                 }
-                AttrTokenTree::Delimited(sp, delim, mut inner) => {
+                AttrTokenTree::Delimited(sp, spacing, delim, mut inner) => {
                     inner = self.configure_tokens(&inner);
-                    Some(AttrTokenTree::Delimited(sp, delim, inner)).into_iter()
+                    Some(AttrTokenTree::Delimited(sp, spacing, delim, inner)).into_iter()
                 }
                 AttrTokenTree::Token(ref token, _)
                     if let TokenKind::Interpolated(nt) = &token.kind =>
@@ -277,7 +207,6 @@ impl<'a> StripUnconfigured<'a> {
                 AttrTokenTree::Token(token, spacing) => {
                     Some(AttrTokenTree::Token(token, spacing)).into_iter()
                 }
-            }
             })
             .collect();
         AttrTokenStream::new(trees)
@@ -372,27 +301,32 @@ impl<'a> StripUnconfigured<'a> {
         };
         let pound_span = pound_token.span;
 
-        let mut trees = vec![AttrTokenTree::Token(pound_token, Spacing::Alone)];
-        if attr.style == AttrStyle::Inner {
-            // For inner attributes, we do the same thing for the `!` in `#![some_attr]`
-            let TokenTree::Token(bang_token @ Token { kind: TokenKind::Not, .. }, _) =
-                orig_trees.next().unwrap().clone()
-            else {
-                panic!("Bad tokens for attribute {attr:?}");
-            };
-            trees.push(AttrTokenTree::Token(bang_token, Spacing::Alone));
-        }
         // We don't really have a good span to use for the synthesized `[]`
         // in `#[attr]`, so just use the span of the `#` token.
         let bracket_group = AttrTokenTree::Delimited(
             DelimSpan::from_single(pound_span),
+            DelimSpacing::new(Spacing::JointHidden, Spacing::Alone),
             Delimiter::Bracket,
             item.tokens
                 .as_ref()
                 .unwrap_or_else(|| panic!("Missing tokens for {item:?}"))
                 .to_attr_token_stream(),
         );
-        trees.push(bracket_group);
+        let trees = if attr.style == AttrStyle::Inner {
+            // For inner attributes, we do the same thing for the `!` in `#![some_attr]`
+            let TokenTree::Token(bang_token @ Token { kind: TokenKind::Not, .. }, _) =
+                orig_trees.next().unwrap().clone()
+            else {
+                panic!("Bad tokens for attribute {attr:?}");
+            };
+            vec![
+                AttrTokenTree::Token(pound_token, Spacing::Joint),
+                AttrTokenTree::Token(bang_token, Spacing::JointHidden),
+                bracket_group,
+            ]
+        } else {
+            vec![AttrTokenTree::Token(pound_token, Spacing::JointHidden), bracket_group]
+        };
         let tokens = Some(LazyAttrTokenStream::new(AttrTokenStream::new(trees)));
         let attr = attr::mk_attr_from_item(
             &self.sess.parse_sess.attr_id_generator,
@@ -434,9 +368,9 @@ impl<'a> StripUnconfigured<'a> {
             }
         };
         (
-            parse_cfg(&meta_item, &self.sess).map_or(true, |meta_item| {
+            parse_cfg(&meta_item, self.sess).map_or(true, |meta_item| {
                 attr::cfg_matches(
-                    &meta_item,
+                    meta_item,
                     &self.sess.parse_sess,
                     self.lint_node_id,
                     self.features,
diff --git a/compiler/rustc_expand/src/errors.rs b/compiler/rustc_expand/src/errors.rs
index d86632c47..2b43fae68 100644
--- a/compiler/rustc_expand/src/errors.rs
+++ b/compiler/rustc_expand/src/errors.rs
@@ -1,7 +1,6 @@
 use rustc_ast::ast;
 use rustc_macros::Diagnostic;
 use rustc_session::Limit;
-use rustc_span::edition::Edition;
 use rustc_span::symbol::{Ident, MacroRulesNormalizedIdent};
 use rustc_span::{Span, Symbol};
 use std::borrow::Cow;
@@ -169,15 +168,6 @@ pub(crate) struct TakesNoArguments<'a> {
 }
 
 #[derive(Diagnostic)]
-#[diag(expand_feature_included_in_edition, code = "E0705")]
-pub(crate) struct FeatureIncludedInEdition {
-    #[primary_span]
-    pub span: Span,
-    pub feature: Symbol,
-    pub edition: Edition,
-}
-
-#[derive(Diagnostic)]
 #[diag(expand_feature_removed, code = "E0557")]
 pub(crate) struct FeatureRemoved<'a> {
     #[primary_span]
@@ -304,6 +294,8 @@ pub(crate) struct IncompleteParse<'a> {
     pub label_span: Span,
     pub macro_path: &'a ast::Path,
     pub kind_name: &'a str,
+    #[note(expand_macro_expands_to_match_arm)]
+    pub expands_to_match_arm: Option<()>,
 
     #[suggestion(
         expand_suggestion_add_semi,
diff --git a/compiler/rustc_expand/src/expand.rs b/compiler/rustc_expand/src/expand.rs
index f87f4aba2..89caf8aa2 100644
--- a/compiler/rustc_expand/src/expand.rs
+++ b/compiler/rustc_expand/src/expand.rs
@@ -41,6 +41,7 @@ use std::path::PathBuf;
 use std::rc::Rc;
 use std::{iter, mem};
 
+#[cfg(bootstrap)]
 macro_rules! ast_fragments {
     (
         $($Kind:ident($AstTy:ty) {
@@ -165,6 +166,131 @@ macro_rules! ast_fragments {
     }
 }
 
+#[cfg(not(bootstrap))]
+macro_rules! ast_fragments {
+    (
+        $($Kind:ident($AstTy:ty) {
+            $kind_name:expr;
+            $(one fn $mut_visit_ast:ident; fn $visit_ast:ident;)?
+            $(many fn $flat_map_ast_elt:ident; fn $visit_ast_elt:ident($($args:tt)*);)?
+            fn $make_ast:ident;
+        })*
+    ) => {
+        /// A fragment of AST that can be produced by a single macro expansion.
+        /// Can also serve as an input and intermediate result for macro expansion operations.
+        pub enum AstFragment {
+            OptExpr(Option<P<ast::Expr>>),
+            MethodReceiverExpr(P<ast::Expr>),
+            $($Kind($AstTy),)*
+        }
+
+        /// "Discriminant" of an AST fragment.
+        #[derive(Copy, Clone, PartialEq, Eq)]
+        pub enum AstFragmentKind {
+            OptExpr,
+            MethodReceiverExpr,
+            $($Kind,)*
+        }
+
+        impl AstFragmentKind {
+            pub fn name(self) -> &'static str {
+                match self {
+                    AstFragmentKind::OptExpr => "expression",
+                    AstFragmentKind::MethodReceiverExpr => "expression",
+                    $(AstFragmentKind::$Kind => $kind_name,)*
+                }
+            }
+
+            fn make_from<'a>(self, result: Box<dyn MacResult + 'a>) -> Option<AstFragment> {
+                match self {
+                    AstFragmentKind::OptExpr =>
+                        result.make_expr().map(Some).map(AstFragment::OptExpr),
+                    AstFragmentKind::MethodReceiverExpr =>
+                        result.make_expr().map(AstFragment::MethodReceiverExpr),
+                    $(AstFragmentKind::$Kind => result.$make_ast().map(AstFragment::$Kind),)*
+                }
+            }
+        }
+
+        impl AstFragment {
+            pub fn add_placeholders(&mut self, placeholders: &[NodeId]) {
+                if placeholders.is_empty() {
+                    return;
+                }
+                match self {
+                    $($(AstFragment::$Kind(ast) => ast.extend(placeholders.iter().flat_map(|id| {
+                        ${ignore($flat_map_ast_elt)}
+                        placeholder(AstFragmentKind::$Kind, *id, None).$make_ast()
+                    })),)?)*
+                    _ => panic!("unexpected AST fragment kind")
+                }
+            }
+
+            pub fn make_opt_expr(self) -> Option<P<ast::Expr>> {
+                match self {
+                    AstFragment::OptExpr(expr) => expr,
+                    _ => panic!("AstFragment::make_* called on the wrong kind of fragment"),
+                }
+            }
+
+            pub fn make_method_receiver_expr(self) -> P<ast::Expr> {
+                match self {
+                    AstFragment::MethodReceiverExpr(expr) => expr,
+                    _ => panic!("AstFragment::make_* called on the wrong kind of fragment"),
+                }
+            }
+
+            $(pub fn $make_ast(self) -> $AstTy {
+                match self {
+                    AstFragment::$Kind(ast) => ast,
+                    _ => panic!("AstFragment::make_* called on the wrong kind of fragment"),
+                }
+            })*
+
+            fn make_ast<T: InvocationCollectorNode>(self) -> T::OutputTy {
+                T::fragment_to_output(self)
+            }
+
+            pub fn mut_visit_with<F: MutVisitor>(&mut self, vis: &mut F) {
+                match self {
+                    AstFragment::OptExpr(opt_expr) => {
+                        visit_clobber(opt_expr, |opt_expr| {
+                            if let Some(expr) = opt_expr {
+                                vis.filter_map_expr(expr)
+                            } else {
+                                None
+                            }
+                        });
+                    }
+                    AstFragment::MethodReceiverExpr(expr) => vis.visit_method_receiver_expr(expr),
+                    $($(AstFragment::$Kind(ast) => vis.$mut_visit_ast(ast),)?)*
+                    $($(AstFragment::$Kind(ast) =>
+                        ast.flat_map_in_place(|ast| vis.$flat_map_ast_elt(ast)),)?)*
+                }
+            }
+
+            pub fn visit_with<'a, V: Visitor<'a>>(&'a self, visitor: &mut V) {
+                match self {
+                    AstFragment::OptExpr(Some(expr)) => visitor.visit_expr(expr),
+                    AstFragment::OptExpr(None) => {}
+                    AstFragment::MethodReceiverExpr(expr) => visitor.visit_method_receiver_expr(expr),
+                    $($(AstFragment::$Kind(ast) => visitor.$visit_ast(ast),)?)*
+                    $($(AstFragment::$Kind(ast) => for ast_elt in &ast[..] {
+                        visitor.$visit_ast_elt(ast_elt, $($args)*);
+                    })?)*
+                }
+            }
+        }
+
+        impl<'a> MacResult for crate::mbe::macro_rules::ParserAnyMacro<'a> {
+            $(fn $make_ast(self: Box<crate::mbe::macro_rules::ParserAnyMacro<'a>>)
+                           -> Option<$AstTy> {
+                Some(self.make(AstFragmentKind::$Kind).$make_ast())
+            })*
+        }
+    }
+}
+
 ast_fragments! {
     Expr(P<ast::Expr>) { "expression"; one fn visit_expr; fn visit_expr; fn make_expr; }
     Pat(P<ast::Pat>) { "pattern"; one fn visit_pat; fn visit_pat; fn make_pat; }
@@ -435,7 +561,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                 invocations = mem::take(&mut undetermined_invocations);
                 force = !mem::replace(&mut progress, false);
                 if force && self.monotonic {
-                    self.cx.sess.delay_span_bug(
+                    self.cx.sess.span_delayed_bug(
                         invocations.last().unwrap().0.span(),
                         "expansion entered force mode without producing any errors",
                     );
@@ -955,12 +1081,15 @@ pub fn ensure_complete_parse<'a>(
             _ => None,
         };
 
+        let expands_to_match_arm = kind_name == "pattern" && parser.token == token::FatArrow;
+
         parser.sess.emit_err(IncompleteParse {
             span: def_site_span,
             token,
             label_span: span,
             macro_path,
             kind_name,
+            expands_to_match_arm: expands_to_match_arm.then_some(()),
             add_semicolon,
         });
     }
@@ -1096,7 +1225,7 @@ impl InvocationCollectorNode for P<ast::Item> {
             ModKind::Loaded(_, inline, _) => {
                 // Inline `mod foo { ... }`, but we still need to push directories.
                 let (dir_path, dir_ownership) = mod_dir_path(
-                    &ecx.sess,
+                    ecx.sess,
                     ident,
                     &attrs,
                     &ecx.current_expansion.module,
@@ -1111,7 +1240,7 @@ impl InvocationCollectorNode for P<ast::Item> {
                 let old_attrs_len = attrs.len();
                 let ParsedExternalMod { items, spans, file_path, dir_path, dir_ownership } =
                     parse_external_mod(
-                        &ecx.sess,
+                        ecx.sess,
                         ident,
                         span,
                         &ecx.current_expansion.module,
@@ -1168,14 +1297,14 @@ impl InvocationCollectorNode for P<ast::Item> {
                     ast::UseTreeKind::Simple(_) => idents.push(ut.ident()),
                     ast::UseTreeKind::Nested(nested) => {
                         for (ut, _) in nested {
-                            collect_use_tree_leaves(&ut, idents);
+                            collect_use_tree_leaves(ut, idents);
                         }
                     }
                 }
             }
 
             let mut idents = Vec::new();
-            collect_use_tree_leaves(&ut, &mut idents);
+            collect_use_tree_leaves(ut, &mut idents);
             return idents;
         }
 
@@ -1531,7 +1660,7 @@ impl InvocationCollectorNode for AstNodeWrapper<P<ast::Expr>, OptExprTag> {
         }
     }
     fn pre_flat_map_node_collect_attr(cfg: &StripUnconfigured<'_>, attr: &ast::Attribute) {
-        cfg.maybe_emit_expr_attr_err(&attr);
+        cfg.maybe_emit_expr_attr_err(attr);
     }
 }
 
@@ -1580,7 +1709,7 @@ struct InvocationCollector<'a, 'b> {
 impl<'a, 'b> InvocationCollector<'a, 'b> {
     fn cfg(&self) -> StripUnconfigured<'_> {
         StripUnconfigured {
-            sess: &self.cx.sess,
+            sess: self.cx.sess,
             features: Some(self.cx.ecfg.features),
             config_tokens: false,
             lint_node_id: self.cx.current_expansion.lint_node_id,
@@ -1693,7 +1822,7 @@ impl<'a, 'b> InvocationCollector<'a, 'b> {
 
             if attr.is_doc_comment() {
                 self.cx.sess.parse_sess.buffer_lint_with_diagnostic(
-                    &UNUSED_DOC_COMMENTS,
+                    UNUSED_DOC_COMMENTS,
                     current_span,
                     self.cx.current_expansion.lint_node_id,
                     "unused doc comment",
@@ -1705,7 +1834,7 @@ impl<'a, 'b> InvocationCollector<'a, 'b> {
                 // eagerly evaluated.
                 if attr_name != sym::cfg && attr_name != sym::cfg_attr {
                     self.cx.sess.parse_sess.buffer_lint_with_diagnostic(
-                        &UNUSED_ATTRIBUTES,
+                        UNUSED_ATTRIBUTES,
                         attr.span,
                         self.cx.current_expansion.lint_node_id,
                         format!("unused attribute `{attr_name}`"),
diff --git a/compiler/rustc_expand/src/lib.rs b/compiler/rustc_expand/src/lib.rs
index 5a774164a..bed667048 100644
--- a/compiler/rustc_expand/src/lib.rs
+++ b/compiler/rustc_expand/src/lib.rs
@@ -1,5 +1,5 @@
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
+#![doc(rust_logo)]
+#![feature(rustdoc_internals)]
 #![feature(array_windows)]
 #![feature(associated_type_bounds)]
 #![feature(associated_type_defaults)]
@@ -23,9 +23,6 @@ extern crate tracing;
 
 extern crate proc_macro as pm;
 
-use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
-use rustc_fluent_macro::fluent_messages;
-
 mod placeholders;
 mod proc_macro_server;
 
@@ -67,4 +64,4 @@ mod mut_visit {
     mod tests;
 }
 
-fluent_messages! { "../messages.ftl" }
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
diff --git a/compiler/rustc_expand/src/mbe.rs b/compiler/rustc_expand/src/mbe.rs
index a43b2a001..ca4a1f327 100644
--- a/compiler/rustc_expand/src/mbe.rs
+++ b/compiler/rustc_expand/src/mbe.rs
@@ -13,7 +13,7 @@ pub(crate) mod transcribe;
 
 use metavar_expr::MetaVarExpr;
 use rustc_ast::token::{Delimiter, NonterminalKind, Token, TokenKind};
-use rustc_ast::tokenstream::DelimSpan;
+use rustc_ast::tokenstream::{DelimSpacing, DelimSpan};
 use rustc_span::symbol::Ident;
 use rustc_span::Span;
 
@@ -68,7 +68,7 @@ pub(crate) enum KleeneOp {
 enum TokenTree {
     Token(Token),
     /// A delimited sequence, e.g. `($e:expr)` (RHS) or `{ $e }` (LHS).
-    Delimited(DelimSpan, Delimited),
+    Delimited(DelimSpan, DelimSpacing, Delimited),
     /// A kleene-style repetition sequence, e.g. `$($e:expr)*` (RHS) or `$($e),*` (LHS).
     Sequence(DelimSpan, SequenceRepetition),
     /// e.g., `$var`.
@@ -99,7 +99,7 @@ impl TokenTree {
             TokenTree::Token(Token { span, .. })
             | TokenTree::MetaVar(span, _)
             | TokenTree::MetaVarDecl(span, _, _) => span,
-            TokenTree::Delimited(span, _)
+            TokenTree::Delimited(span, ..)
             | TokenTree::MetaVarExpr(span, _)
             | TokenTree::Sequence(span, _) => span.entire(),
         }
diff --git a/compiler/rustc_expand/src/mbe/diagnostics.rs b/compiler/rustc_expand/src/mbe/diagnostics.rs
index e06037564..8f80e6e29 100644
--- a/compiler/rustc_expand/src/mbe/diagnostics.rs
+++ b/compiler/rustc_expand/src/mbe/diagnostics.rs
@@ -34,7 +34,7 @@ pub(super) fn failed_to_match_macro<'cx>(
     if try_success_result.is_ok() {
         // Nonterminal parser recovery might turn failed matches into successful ones,
         // but for that it must have emitted an error already
-        tracker.cx.sess.delay_span_bug(sp, "Macro matching returned a success on the second try");
+        tracker.cx.sess.span_delayed_bug(sp, "Macro matching returned a success on the second try");
     }
 
     if let Some(result) = tracker.result {
@@ -67,6 +67,12 @@ pub(super) fn failed_to_match_macro<'cx>(
         && (matches!(expected_token.kind, TokenKind::Interpolated(_))
             || matches!(token.kind, TokenKind::Interpolated(_)))
     {
+        if let TokenKind::Interpolated(node) = &expected_token.kind {
+            err.span_label(node.1, "");
+        }
+        if let TokenKind::Interpolated(node) = &token.kind {
+            err.span_label(node.1, "");
+        }
         err.note("captured metavariables except for `:tt`, `:ident` and `:lifetime` cannot be compared to other tokens");
         err.note("see <https://doc.rust-lang.org/nightly/reference/macros-by-example.html#forwarding-a-matched-fragment> for more information");
 
@@ -145,7 +151,7 @@ impl<'a, 'cx, 'matcher> Tracker<'matcher> for CollectTrackerAndEmitter<'a, 'cx,
             Success(_) => {
                 // Nonterminal parser recovery might turn failed matches into successful ones,
                 // but for that it must have emitted an error already
-                self.cx.sess.delay_span_bug(
+                self.cx.sess.span_delayed_bug(
                     self.root_span,
                     "should not collect detailed info for successful macro match",
                 );
diff --git a/compiler/rustc_expand/src/mbe/macro_check.rs b/compiler/rustc_expand/src/mbe/macro_check.rs
index 95f5bb2d2..e66cfbe6f 100644
--- a/compiler/rustc_expand/src/mbe/macro_check.rs
+++ b/compiler/rustc_expand/src/mbe/macro_check.rs
@@ -205,7 +205,7 @@ pub(super) fn check_meta_variables(
     rhses: &[TokenTree],
 ) -> bool {
     if lhses.len() != rhses.len() {
-        sess.span_diagnostic.span_bug(span, "length mismatch between LHSes and RHSes")
+        sess.dcx.span_bug(span, "length mismatch between LHSes and RHSes")
     }
     let mut valid = true;
     for (lhs, rhs) in iter::zip(lhses, rhses) {
@@ -244,7 +244,7 @@ fn check_binders(
         // MetaVar(fragment) and not as MetaVarDecl(y, fragment).
         TokenTree::MetaVar(span, name) => {
             if macros.is_empty() {
-                sess.span_diagnostic.span_bug(span, "unexpected MetaVar in lhs");
+                sess.dcx.span_bug(span, "unexpected MetaVar in lhs");
             }
             let name = MacroRulesNormalizedIdent::new(name);
             // There are 3 possibilities:
@@ -275,14 +275,13 @@ fn check_binders(
                 );
             }
             if !macros.is_empty() {
-                sess.span_diagnostic.span_bug(span, "unexpected MetaVarDecl in nested lhs");
+                sess.dcx.span_bug(span, "unexpected MetaVarDecl in nested lhs");
             }
             let name = MacroRulesNormalizedIdent::new(name);
             if let Some(prev_info) = get_binder_info(macros, binders, name) {
                 // Duplicate binders at the top-level macro definition are errors. The lint is only
                 // for nested macro definitions.
-                sess.span_diagnostic
-                    .emit_err(errors::DuplicateMatcherBinding { span, prev: prev_info.span });
+                sess.dcx.emit_err(errors::DuplicateMatcherBinding { span, prev: prev_info.span });
                 *valid = false;
             } else {
                 binders.insert(name, BinderInfo { span, ops: ops.into() });
@@ -290,7 +289,7 @@ fn check_binders(
         }
         // `MetaVarExpr` can not appear in the LHS of a macro arm
         TokenTree::MetaVarExpr(..) => {}
-        TokenTree::Delimited(_, ref del) => {
+        TokenTree::Delimited(.., ref del) => {
             for tt in &del.tts {
                 check_binders(sess, node_id, tt, macros, binders, ops, valid);
             }
@@ -341,7 +340,7 @@ fn check_occurrences(
     match *rhs {
         TokenTree::Token(..) => {}
         TokenTree::MetaVarDecl(span, _name, _kind) => {
-            sess.span_diagnostic.span_bug(span, "unexpected MetaVarDecl in rhs")
+            sess.dcx.span_bug(span, "unexpected MetaVarDecl in rhs")
         }
         TokenTree::MetaVar(span, name) => {
             let name = MacroRulesNormalizedIdent::new(name);
@@ -353,7 +352,7 @@ fn check_occurrences(
             };
             check_ops_is_prefix(sess, node_id, macros, binders, ops, dl.entire(), name);
         }
-        TokenTree::Delimited(_, ref del) => {
+        TokenTree::Delimited(.., ref del) => {
             check_nested_occurrences(sess, node_id, &del.tts, macros, binders, ops, valid);
         }
         TokenTree::Sequence(_, ref seq) => {
@@ -435,8 +434,8 @@ fn check_nested_occurrences(
                 // We check that the meta-variable is correctly used.
                 check_occurrences(sess, node_id, tt, macros, binders, ops, valid);
             }
-            (NestedMacroState::MacroRulesNotName, TokenTree::Delimited(_, del))
-            | (NestedMacroState::MacroName, TokenTree::Delimited(_, del))
+            (NestedMacroState::MacroRulesNotName, TokenTree::Delimited(.., del))
+            | (NestedMacroState::MacroName, TokenTree::Delimited(.., del))
                 if del.delim == Delimiter::Brace =>
             {
                 let macro_rules = state == NestedMacroState::MacroRulesNotName;
@@ -466,7 +465,7 @@ fn check_nested_occurrences(
                 // We check that the meta-variable is correctly used.
                 check_occurrences(sess, node_id, tt, macros, binders, ops, valid);
             }
-            (NestedMacroState::MacroName, TokenTree::Delimited(_, del))
+            (NestedMacroState::MacroName, TokenTree::Delimited(.., del))
                 if del.delim == Delimiter::Parenthesis =>
             {
                 state = NestedMacroState::MacroNameParen;
@@ -481,7 +480,7 @@ fn check_nested_occurrences(
                     valid,
                 );
             }
-            (NestedMacroState::MacroNameParen, TokenTree::Delimited(_, del))
+            (NestedMacroState::MacroNameParen, TokenTree::Delimited(.., del))
                 if del.delim == Delimiter::Brace =>
             {
                 state = NestedMacroState::Empty;
@@ -650,6 +649,6 @@ fn buffer_lint(
 ) {
     // Macros loaded from other crates have dummy node ids.
     if node_id != DUMMY_NODE_ID {
-        sess.buffer_lint(&META_VARIABLE_MISUSE, span, node_id, message);
+        sess.buffer_lint(META_VARIABLE_MISUSE, span, node_id, message);
     }
 }
diff --git a/compiler/rustc_expand/src/mbe/macro_parser.rs b/compiler/rustc_expand/src/mbe/macro_parser.rs
index 7e85beaad..b248a1fe3 100644
--- a/compiler/rustc_expand/src/mbe/macro_parser.rs
+++ b/compiler/rustc_expand/src/mbe/macro_parser.rs
@@ -184,7 +184,7 @@ pub(super) fn compute_locs(matcher: &[TokenTree]) -> Vec<MatcherLoc> {
                 TokenTree::Token(token) => {
                     locs.push(MatcherLoc::Token { token: token.clone() });
                 }
-                TokenTree::Delimited(span, delimited) => {
+                TokenTree::Delimited(span, _, delimited) => {
                     let open_token = Token::new(token::OpenDelim(delimited.delim), span.open);
                     let close_token = Token::new(token::CloseDelim(delimited.delim), span.close);
 
@@ -335,7 +335,7 @@ pub(super) fn count_metavar_decls(matcher: &[TokenTree]) -> usize {
         .map(|tt| match tt {
             TokenTree::MetaVarDecl(..) => 1,
             TokenTree::Sequence(_, seq) => seq.num_captures,
-            TokenTree::Delimited(_, delim) => count_metavar_decls(&delim.tts),
+            TokenTree::Delimited(.., delim) => count_metavar_decls(&delim.tts),
             TokenTree::Token(..) => 0,
             TokenTree::MetaVar(..) | TokenTree::MetaVarExpr(..) => unreachable!(),
         })
@@ -397,7 +397,7 @@ pub(crate) enum NamedMatch {
     MatchedTokenTree(rustc_ast::tokenstream::TokenTree),
 
     // A metavar match of any type other than `tt`.
-    MatchedNonterminal(Lrc<Nonterminal>),
+    MatchedNonterminal(Lrc<(Nonterminal, rustc_span::Span)>),
 }
 
 /// Performs a token equality check, ignoring syntax context (that is, an unhygienic comparison)
@@ -483,7 +483,7 @@ impl TtParser {
                     if matches!(t, Token { kind: DocComment(..), .. }) {
                         mp.idx += 1;
                         self.cur_mps.push(mp);
-                    } else if token_name_eq(&t, token) {
+                    } else if token_name_eq(t, token) {
                         mp.idx += 1;
                         self.next_mps.push(mp);
                     }
@@ -692,7 +692,7 @@ impl TtParser {
                             Ok(nt) => nt,
                         };
                         let m = match nt {
-                            ParseNtResult::Nt(nt) => MatchedNonterminal(Lrc::new(nt)),
+                            ParseNtResult::Nt(nt) => MatchedNonterminal(Lrc::new((nt, span))),
                             ParseNtResult::Tt(tt) => MatchedTokenTree(tt),
                         };
                         mp.push_match(next_metavar, seq_depth, m);
diff --git a/compiler/rustc_expand/src/mbe/macro_rules.rs b/compiler/rustc_expand/src/mbe/macro_rules.rs
index ebdd3cb54..44f10e7d3 100644
--- a/compiler/rustc_expand/src/mbe/macro_rules.rs
+++ b/compiler/rustc_expand/src/mbe/macro_rules.rs
@@ -207,13 +207,13 @@ fn expand_macro<'cx>(
     match try_success_result {
         Ok((i, named_matches)) => {
             let (rhs, rhs_span): (&mbe::Delimited, DelimSpan) = match &rhses[i] {
-                mbe::TokenTree::Delimited(span, delimited) => (&delimited, *span),
+                mbe::TokenTree::Delimited(span, _, delimited) => (&delimited, *span),
                 _ => cx.span_bug(sp, "malformed macro rhs"),
             };
             let arm_span = rhses[i].span();
 
             // rhs has holes ( `$id` and `$(...)` that need filled)
-            let mut tts = match transcribe(cx, &named_matches, &rhs, rhs_span, transparency) {
+            let mut tts = match transcribe(cx, &named_matches, rhs, rhs_span, transparency) {
                 Ok(tts) => tts,
                 Err(mut err) => {
                     err.emit();
@@ -236,6 +236,13 @@ fn expand_macro<'cx>(
                             target_sp.open = source_sp.open.with_ctxt(ctxt);
                             target_sp.close = source_sp.close.with_ctxt(ctxt);
                         }
+                        (
+                            TokenTree::Delimited(target_sp, ..),
+                            mbe::TokenTree::MetaVar(source_sp, ..),
+                        ) => {
+                            target_sp.open = source_sp.with_ctxt(ctxt);
+                            target_sp.close = source_sp.with_ctxt(ctxt).shrink_to_hi();
+                        }
                         _ => {
                             let sp = rhs_tt.span().with_ctxt(ctxt);
                             tt.set_span(sp);
@@ -395,7 +402,7 @@ pub fn compile_declarative_macro(
     };
     let dummy_syn_ext = || (mk_syn_ext(Box::new(macro_rules_dummy_expander)), Vec::new());
 
-    let diag = &sess.parse_sess.span_diagnostic;
+    let dcx = &sess.parse_sess.dcx;
     let lhs_nm = Ident::new(sym::lhs, def.span);
     let rhs_nm = Ident::new(sym::rhs, def.span);
     let tt_spec = Some(NonterminalKind::TT);
@@ -475,17 +482,14 @@ pub fn compile_declarative_macro(
 
                 let s = parse_failure_msg(&token);
                 let sp = token.span.substitute_dummy(def.span);
-                let mut err = sess.parse_sess.span_diagnostic.struct_span_err(sp, s);
+                let mut err = sess.dcx().struct_span_err(sp, s);
                 err.span_label(sp, msg);
                 annotate_doc_comment(&mut err, sess.source_map(), sp);
                 err.emit();
                 return dummy_syn_ext();
             }
             Error(sp, msg) => {
-                sess.parse_sess
-                    .span_diagnostic
-                    .struct_span_err(sp.substitute_dummy(def.span), msg)
-                    .emit();
+                sess.dcx().struct_span_err(sp.substitute_dummy(def.span), msg).emit();
                 return dummy_syn_ext();
             }
             ErrorReported(_) => {
@@ -511,13 +515,13 @@ pub fn compile_declarative_macro(
                     )
                     .pop()
                     .unwrap();
-                    valid &= check_lhs_nt_follows(&sess.parse_sess, &def, &tt);
+                    valid &= check_lhs_nt_follows(&sess.parse_sess, def, &tt);
                     return tt;
                 }
-                sess.parse_sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs")
+                sess.dcx().span_bug(def.span, "wrong-structured lhs")
             })
             .collect::<Vec<mbe::TokenTree>>(),
-        _ => sess.parse_sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs"),
+        _ => sess.dcx().span_bug(def.span, "wrong-structured lhs"),
     };
 
     let rhses = match &argument_map[&MacroRulesNormalizedIdent::new(rhs_nm)] {
@@ -536,10 +540,10 @@ pub fn compile_declarative_macro(
                     .pop()
                     .unwrap();
                 }
-                sess.parse_sess.span_diagnostic.span_bug(def.span, "wrong-structured rhs")
+                sess.dcx().span_bug(def.span, "wrong-structured rhs")
             })
             .collect::<Vec<mbe::TokenTree>>(),
-        _ => sess.parse_sess.span_diagnostic.span_bug(def.span, "wrong-structured rhs"),
+        _ => sess.dcx().span_bug(def.span, "wrong-structured rhs"),
     };
 
     for rhs in &rhses {
@@ -556,10 +560,10 @@ pub fn compile_declarative_macro(
     let (transparency, transparency_error) = attr::find_transparency(&def.attrs, macro_rules);
     match transparency_error {
         Some(TransparencyError::UnknownTransparency(value, span)) => {
-            diag.span_err(span, format!("unknown macro transparency: `{value}`"));
+            dcx.span_err(span, format!("unknown macro transparency: `{value}`"));
         }
         Some(TransparencyError::MultipleTransparencyAttrs(old_span, new_span)) => {
-            diag.span_err(vec![old_span, new_span], "multiple macro transparency attributes");
+            dcx.span_err(vec![old_span, new_span], "multiple macro transparency attributes");
         }
         None => {}
     }
@@ -592,10 +596,10 @@ pub fn compile_declarative_macro(
             .map(|lhs| {
                 // Ignore the delimiters around the matcher.
                 match lhs {
-                    mbe::TokenTree::Delimited(_, delimited) => {
+                    mbe::TokenTree::Delimited(.., delimited) => {
                         mbe::macro_parser::compute_locs(&delimited.tts)
                     }
-                    _ => sess.parse_sess.span_diagnostic.span_bug(def.span, "malformed macro lhs"),
+                    _ => sess.dcx().span_bug(def.span, "malformed macro lhs"),
                 }
             })
             .collect()
@@ -618,11 +622,11 @@ pub fn compile_declarative_macro(
 fn check_lhs_nt_follows(sess: &ParseSess, def: &ast::Item, lhs: &mbe::TokenTree) -> bool {
     // lhs is going to be like TokenTree::Delimited(...), where the
     // entire lhs is those tts. Or, it can be a "bare sequence", not wrapped in parens.
-    if let mbe::TokenTree::Delimited(_, delimited) = lhs {
+    if let mbe::TokenTree::Delimited(.., delimited) = lhs {
         check_matcher(sess, def, &delimited.tts)
     } else {
         let msg = "invalid macro matcher; matchers must be contained in balanced delimiters";
-        sess.span_diagnostic.span_err(lhs.span(), msg);
+        sess.dcx.span_err(lhs.span(), msg);
         false
     }
     // we don't abort on errors on rejection, the driver will do that for us
@@ -648,10 +652,7 @@ fn is_empty_token_tree(sess: &ParseSess, seq: &mbe::SequenceRepetition) -> bool
                         iter.next();
                     }
                     let span = t.span.to(now.span);
-                    sess.span_diagnostic.span_note_without_error(
-                        span,
-                        "doc comments are ignored in matcher position",
-                    );
+                    sess.dcx.span_note(span, "doc comments are ignored in matcher position");
                 }
                 mbe::TokenTree::Sequence(_, sub_seq)
                     if (sub_seq.kleene.op == mbe::KleeneOp::ZeroOrMore
@@ -673,7 +674,7 @@ fn check_lhs_no_empty_seq(sess: &ParseSess, tts: &[mbe::TokenTree]) -> bool {
             | TokenTree::MetaVar(..)
             | TokenTree::MetaVarDecl(..)
             | TokenTree::MetaVarExpr(..) => (),
-            TokenTree::Delimited(_, del) => {
+            TokenTree::Delimited(.., del) => {
                 if !check_lhs_no_empty_seq(sess, &del.tts) {
                     return false;
                 }
@@ -681,7 +682,7 @@ fn check_lhs_no_empty_seq(sess: &ParseSess, tts: &[mbe::TokenTree]) -> bool {
             TokenTree::Sequence(span, seq) => {
                 if is_empty_token_tree(sess, seq) {
                     let sp = span.entire();
-                    sess.span_diagnostic.span_err(sp, "repetition matches empty token tree");
+                    sess.dcx.span_err(sp, "repetition matches empty token tree");
                     return false;
                 }
                 if !check_lhs_no_empty_seq(sess, &seq.tts) {
@@ -698,7 +699,7 @@ fn check_rhs(sess: &ParseSess, rhs: &mbe::TokenTree) -> bool {
     match *rhs {
         mbe::TokenTree::Delimited(..) => return true,
         _ => {
-            sess.span_diagnostic.span_err(rhs.span(), "macro rhs must be delimited");
+            sess.dcx.span_err(rhs.span(), "macro rhs must be delimited");
         }
     }
     false
@@ -707,21 +708,21 @@ fn check_rhs(sess: &ParseSess, rhs: &mbe::TokenTree) -> bool {
 fn check_matcher(sess: &ParseSess, def: &ast::Item, matcher: &[mbe::TokenTree]) -> bool {
     let first_sets = FirstSets::new(matcher);
     let empty_suffix = TokenSet::empty();
-    let err = sess.span_diagnostic.err_count();
+    let err = sess.dcx.err_count();
     check_matcher_core(sess, def, &first_sets, matcher, &empty_suffix);
-    err == sess.span_diagnostic.err_count()
+    err == sess.dcx.err_count()
 }
 
 fn has_compile_error_macro(rhs: &mbe::TokenTree) -> bool {
     match rhs {
-        mbe::TokenTree::Delimited(_sp, d) => {
+        mbe::TokenTree::Delimited(.., d) => {
             let has_compile_error = d.tts.array_windows::<3>().any(|[ident, bang, args]| {
                 if let mbe::TokenTree::Token(ident) = ident
                     && let TokenKind::Ident(ident, _) = ident.kind
                     && ident == sym::compile_error
                     && let mbe::TokenTree::Token(bang) = bang
                     && let TokenKind::Not = bang.kind
-                    && let mbe::TokenTree::Delimited(_, del) = args
+                    && let mbe::TokenTree::Delimited(.., del) = args
                     && del.delim != Delimiter::Invisible
                 {
                     true
@@ -778,7 +779,7 @@ impl<'tt> FirstSets<'tt> {
                     | TokenTree::MetaVarExpr(..) => {
                         first.replace_with(TtHandle::TtRef(tt));
                     }
-                    TokenTree::Delimited(span, delimited) => {
+                    TokenTree::Delimited(span, _, delimited) => {
                         build_recur(sets, &delimited.tts);
                         first.replace_with(TtHandle::from_token_kind(
                             token::OpenDelim(delimited.delim),
@@ -847,7 +848,7 @@ impl<'tt> FirstSets<'tt> {
                     first.add_one(TtHandle::TtRef(tt));
                     return first;
                 }
-                TokenTree::Delimited(span, delimited) => {
+                TokenTree::Delimited(span, _, delimited) => {
                     first.add_one(TtHandle::from_token_kind(
                         token::OpenDelim(delimited.delim),
                         span.open,
@@ -927,7 +928,7 @@ impl<'tt> TtHandle<'tt> {
     fn get(&'tt self) -> &'tt mbe::TokenTree {
         match self {
             TtHandle::TtRef(tt) => tt,
-            TtHandle::Token(token_tt) => &token_tt,
+            TtHandle::Token(token_tt) => token_tt,
         }
     }
 }
@@ -1092,7 +1093,7 @@ fn check_matcher_core<'tt>(
                     suffix_first = build_suffix_first();
                 }
             }
-            TokenTree::Delimited(span, d) => {
+            TokenTree::Delimited(span, _, d) => {
                 let my_suffix = TokenSet::singleton(TtHandle::from_token_kind(
                     token::CloseDelim(d.delim),
                     span.close,
@@ -1170,7 +1171,7 @@ fn check_matcher_core<'tt>(
                             Some(NonterminalKind::PatParam { inferred: false }),
                         ));
                         sess.buffer_lint_with_diagnostic(
-                            &RUST_2021_INCOMPATIBLE_OR_PATTERNS,
+                            RUST_2021_INCOMPATIBLE_OR_PATTERNS,
                             span,
                             ast::CRATE_NODE_ID,
                             "the meaning of the `pat` fragment specifier is changing in Rust 2021, which may affect this macro",
@@ -1188,7 +1189,7 @@ fn check_matcher_core<'tt>(
                             };
 
                             let sp = next_token.span();
-                            let mut err = sess.span_diagnostic.struct_span_err(
+                            let mut err = sess.dcx.struct_span_err(
                                 sp,
                                 format!(
                                     "`${name}:{frag}` {may_be} followed by `{next}`, which \
@@ -1407,7 +1408,7 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
 
 fn quoted_tt_to_string(tt: &mbe::TokenTree) -> String {
     match tt {
-        mbe::TokenTree::Token(token) => pprust::token_to_string(&token).into(),
+        mbe::TokenTree::Token(token) => pprust::token_to_string(token).into(),
         mbe::TokenTree::MetaVar(_, name) => format!("${name}"),
         mbe::TokenTree::MetaVarDecl(_, name, Some(kind)) => format!("${name}:{kind}"),
         mbe::TokenTree::MetaVarDecl(_, name, None) => format!("${name}:"),
diff --git a/compiler/rustc_expand/src/mbe/metavar_expr.rs b/compiler/rustc_expand/src/mbe/metavar_expr.rs
index 7cb279a98..e3dc73d0d 100644
--- a/compiler/rustc_expand/src/mbe/metavar_expr.rs
+++ b/compiler/rustc_expand/src/mbe/metavar_expr.rs
@@ -10,9 +10,8 @@ use rustc_span::Span;
 /// A meta-variable expression, for expansions based on properties of meta-variables.
 #[derive(Debug, Clone, PartialEq, Encodable, Decodable)]
 pub(crate) enum MetaVarExpr {
-    /// The number of repetitions of an identifier, optionally limited to a number
-    /// of outer-most repetition depths. If the depth limit is `None` then the depth is unlimited.
-    Count(Ident, Option<usize>),
+    /// The number of repetitions of an identifier.
+    Count(Ident, usize),
 
     /// Ignore a meta-variable for repetition without expansion.
     Ignore(Ident),
@@ -35,20 +34,23 @@ impl MetaVarExpr {
     ) -> PResult<'sess, MetaVarExpr> {
         let mut tts = input.trees();
         let ident = parse_ident(&mut tts, sess, outer_span)?;
-        let Some(TokenTree::Delimited(_, Delimiter::Parenthesis, args)) = tts.next() else {
+        let Some(TokenTree::Delimited(.., Delimiter::Parenthesis, args)) = tts.next() else {
             let msg = "meta-variable expression parameter must be wrapped in parentheses";
-            return Err(sess.span_diagnostic.struct_span_err(ident.span, msg));
+            return Err(sess.dcx.struct_span_err(ident.span, msg));
         };
         check_trailing_token(&mut tts, sess)?;
         let mut iter = args.trees();
         let rslt = match ident.as_str() {
             "count" => parse_count(&mut iter, sess, ident.span)?,
-            "ignore" => MetaVarExpr::Ignore(parse_ident(&mut iter, sess, ident.span)?),
+            "ignore" => {
+                eat_dollar(&mut iter, sess, ident.span)?;
+                MetaVarExpr::Ignore(parse_ident(&mut iter, sess, ident.span)?)
+            }
             "index" => MetaVarExpr::Index(parse_depth(&mut iter, sess, ident.span)?),
             "length" => MetaVarExpr::Length(parse_depth(&mut iter, sess, ident.span)?),
             _ => {
                 let err_msg = "unrecognized meta-variable expression";
-                let mut err = sess.span_diagnostic.struct_span_err(ident.span, err_msg);
+                let mut err = sess.dcx.struct_span_err(ident.span, err_msg);
                 err.span_suggestion(
                     ident.span,
                     "supported expressions are count, ignore, index and length",
@@ -77,7 +79,7 @@ fn check_trailing_token<'sess>(
 ) -> PResult<'sess, ()> {
     if let Some(tt) = iter.next() {
         let mut diag = sess
-            .span_diagnostic
+            .dcx
             .struct_span_err(tt.span(), format!("unexpected token: {}", pprust::tt_to_string(tt)));
         diag.span_note(tt.span(), "meta-variable expression must not have trailing tokens");
         Err(diag)
@@ -92,17 +94,18 @@ fn parse_count<'sess>(
     sess: &'sess ParseSess,
     span: Span,
 ) -> PResult<'sess, MetaVarExpr> {
+    eat_dollar(iter, sess, span)?;
     let ident = parse_ident(iter, sess, span)?;
     let depth = if try_eat_comma(iter) {
         if iter.look_ahead(0).is_none() {
-            return Err(sess.span_diagnostic.struct_span_err(
+            return Err(sess.dcx.struct_span_err(
                 span,
                 "`count` followed by a comma must have an associated index indicating its depth",
             ));
         }
-        Some(parse_depth(iter, sess, span)?)
+        parse_depth(iter, sess, span)?
     } else {
-        None
+        0
     };
     Ok(MetaVarExpr::Count(ident, depth))
 }
@@ -116,7 +119,7 @@ fn parse_depth<'sess>(
     let Some(tt) = iter.next() else { return Ok(0) };
     let TokenTree::Token(token::Token { kind: token::TokenKind::Literal(lit), .. }, _) = tt else {
         return Err(sess
-            .span_diagnostic
+            .dcx
             .struct_span_err(span, "meta-variable expression depth must be a literal"));
     };
     if let Ok(lit_kind) = LitKind::from_token_lit(*lit)
@@ -126,7 +129,7 @@ fn parse_depth<'sess>(
         Ok(n_usize)
     } else {
         let msg = "only unsuffixes integer literals are supported in meta-variable expressions";
-        Err(sess.span_diagnostic.struct_span_err(span, msg))
+        Err(sess.dcx.struct_span_err(span, msg))
     }
 }
 
@@ -143,9 +146,8 @@ fn parse_ident<'sess>(
             return Ok(elem);
         }
         let token_str = pprust::token_to_string(token);
-        let mut err = sess
-            .span_diagnostic
-            .struct_span_err(span, format!("expected identifier, found `{}`", &token_str));
+        let mut err =
+            sess.dcx.struct_span_err(span, format!("expected identifier, found `{}`", &token_str));
         err.span_suggestion(
             token.span,
             format!("try removing `{}`", &token_str),
@@ -154,7 +156,7 @@ fn parse_ident<'sess>(
         );
         return Err(err);
     }
-    Err(sess.span_diagnostic.struct_span_err(span, "expected identifier"))
+    Err(sess.dcx.struct_span_err(span, "expected identifier"))
 }
 
 /// Tries to move the iterator forward returning `true` if there is a comma. If not, then the
@@ -166,3 +168,20 @@ fn try_eat_comma(iter: &mut RefTokenTreeCursor<'_>) -> bool {
     }
     false
 }
+
+/// Expects that the next item is a dollar sign.
+fn eat_dollar<'sess>(
+    iter: &mut RefTokenTreeCursor<'_>,
+    sess: &'sess ParseSess,
+    span: Span,
+) -> PResult<'sess, ()> {
+    if let Some(TokenTree::Token(token::Token { kind: token::Dollar, .. }, _)) = iter.look_ahead(0)
+    {
+        let _ = iter.next();
+        return Ok(());
+    }
+    Err(sess.dcx.struct_span_err(
+        span,
+        "meta-variables within meta-variable expressions must be referenced using a dollar sign",
+    ))
+}
diff --git a/compiler/rustc_expand/src/mbe/quoted.rs b/compiler/rustc_expand/src/mbe/quoted.rs
index 6546199f5..445be01bc 100644
--- a/compiler/rustc_expand/src/mbe/quoted.rs
+++ b/compiler/rustc_expand/src/mbe/quoted.rs
@@ -84,7 +84,7 @@ pub(super) fn parse(
                                                     "invalid fragment specifier `{}`",
                                                     frag.name
                                                 );
-                                                sess.span_diagnostic
+                                                sess.dcx
                                                     .struct_span_err(span, msg)
                                                     .help(VALID_FRAGMENT_NAMES_MSG)
                                                     .emit();
@@ -116,7 +116,7 @@ pub(super) fn parse(
 fn maybe_emit_macro_metavar_expr_feature(features: &Features, sess: &ParseSess, span: Span) {
     if !features.macro_metavar_expr {
         let msg = "meta-variable expressions are unstable";
-        feature_err(&sess, sym::macro_metavar_expr, span, msg).emit();
+        feature_err(sess, sym::macro_metavar_expr, span, msg).emit();
     }
 }
 
@@ -151,7 +151,7 @@ fn parse_tree<'a>(
             // during parsing.
             let mut next = outer_trees.next();
             let mut trees: Box<dyn Iterator<Item = &tokenstream::TokenTree>>;
-            if let Some(tokenstream::TokenTree::Delimited(_, Delimiter::Invisible, tts)) = next {
+            if let Some(tokenstream::TokenTree::Delimited(.., Delimiter::Invisible, tts)) = next {
                 trees = Box::new(tts.trees());
                 next = trees.next();
             } else {
@@ -160,7 +160,7 @@ fn parse_tree<'a>(
 
             match next {
                 // `tree` is followed by a delimited set of token trees.
-                Some(&tokenstream::TokenTree::Delimited(delim_span, delim, ref tts)) => {
+                Some(&tokenstream::TokenTree::Delimited(delim_span, _, delim, ref tts)) => {
                     if parsing_patterns {
                         if delim != Delimiter::Parenthesis {
                             span_dollar_dollar_or_metavar_in_the_lhs_err(
@@ -174,7 +174,7 @@ fn parse_tree<'a>(
                                 // The delimiter is `{`. This indicates the beginning
                                 // of a meta-variable expression (e.g. `${count(ident)}`).
                                 // Try to parse the meta-variable expression.
-                                match MetaVarExpr::parse(&tts, delim_span.entire(), sess) {
+                                match MetaVarExpr::parse(tts, delim_span.entire(), sess) {
                                     Err(mut err) => {
                                         err.emit();
                                         // Returns early the same read `$` to avoid spanning
@@ -195,7 +195,7 @@ fn parse_tree<'a>(
                             _ => {
                                 let tok = pprust::token_kind_to_string(&token::OpenDelim(delim));
                                 let msg = format!("expected `(` or `{{`, found `{tok}`");
-                                sess.span_diagnostic.span_err(delim_span.entire(), msg);
+                                sess.dcx.span_err(delim_span.entire(), msg);
                             }
                         }
                     }
@@ -242,11 +242,9 @@ fn parse_tree<'a>(
 
                 // `tree` is followed by some other token. This is an error.
                 Some(tokenstream::TokenTree::Token(token, _)) => {
-                    let msg = format!(
-                        "expected identifier, found `{}`",
-                        pprust::token_to_string(&token),
-                    );
-                    sess.span_diagnostic.span_err(token.span, msg);
+                    let msg =
+                        format!("expected identifier, found `{}`", pprust::token_to_string(token),);
+                    sess.dcx.span_err(token.span, msg);
                     TokenTree::MetaVar(token.span, Ident::empty())
                 }
 
@@ -260,8 +258,9 @@ fn parse_tree<'a>(
 
         // `tree` is the beginning of a delimited set of tokens (e.g., `(` or `{`). We need to
         // descend into the delimited set and further parse it.
-        &tokenstream::TokenTree::Delimited(span, delim, ref tts) => TokenTree::Delimited(
+        &tokenstream::TokenTree::Delimited(span, spacing, delim, ref tts) => TokenTree::Delimited(
             span,
+            spacing,
             Delimited {
                 delim,
                 tts: parse(tts, parsing_patterns, sess, node_id, features, edition),
@@ -291,7 +290,7 @@ fn parse_kleene_op<'a>(
     span: Span,
 ) -> Result<Result<(KleeneOp, Span), Token>, Span> {
     match input.next() {
-        Some(tokenstream::TokenTree::Token(token, _)) => match kleene_op(&token) {
+        Some(tokenstream::TokenTree::Token(token, _)) => match kleene_op(token) {
             Some(op) => Ok(Ok((op, token.span))),
             None => Ok(Err(token.clone())),
         },
@@ -326,7 +325,7 @@ fn parse_sep_and_kleene_op<'a>(
             // #2 is the `?` Kleene op, which does not take a separator (error)
             Ok(Ok((KleeneOp::ZeroOrOne, span))) => {
                 // Error!
-                sess.span_diagnostic.span_err(
+                sess.dcx.span_err(
                     token.span,
                     "the `?` macro repetition operator does not take a separator",
                 );
@@ -347,7 +346,7 @@ fn parse_sep_and_kleene_op<'a>(
     };
 
     // If we ever get to this point, we have experienced an "unexpected token" error
-    sess.span_diagnostic.span_err(span, "expected one of: `*`, `+`, or `?`");
+    sess.dcx.span_err(span, "expected one of: `*`, `+`, or `?`");
 
     // Return a dummy
     (None, KleeneToken::new(KleeneOp::ZeroOrMore, span))
@@ -357,9 +356,8 @@ fn parse_sep_and_kleene_op<'a>(
 //
 // For example, `macro_rules! foo { ( ${length()} ) => {} }`
 fn span_dollar_dollar_or_metavar_in_the_lhs_err(sess: &ParseSess, token: &Token) {
-    sess.span_diagnostic
-        .span_err(token.span, format!("unexpected token: {}", pprust::token_to_string(token)));
-    sess.span_diagnostic.span_note_without_error(
+    sess.dcx.span_err(token.span, format!("unexpected token: {}", pprust::token_to_string(token)));
+    sess.dcx.span_note(
         token.span,
         "`$$` and meta-variable expressions are not allowed inside macro parameter definitions",
     );
diff --git a/compiler/rustc_expand/src/mbe/transcribe.rs b/compiler/rustc_expand/src/mbe/transcribe.rs
index 15e7ab3fe..80fd82e03 100644
--- a/compiler/rustc_expand/src/mbe/transcribe.rs
+++ b/compiler/rustc_expand/src/mbe/transcribe.rs
@@ -7,7 +7,7 @@ use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, MatchedTokenTree,
 use crate::mbe::{self, MetaVarExpr};
 use rustc_ast::mut_visit::{self, MutVisitor};
 use rustc_ast::token::{self, Delimiter, Token, TokenKind};
-use rustc_ast::tokenstream::{DelimSpan, Spacing, TokenStream, TokenTree};
+use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_errors::{pluralize, PResult};
 use rustc_errors::{DiagnosticBuilder, ErrorGuaranteed};
@@ -31,14 +31,24 @@ impl MutVisitor for Marker {
 
 /// An iterator over the token trees in a delimited token tree (`{ ... }`) or a sequence (`$(...)`).
 enum Frame<'a> {
-    Delimited { tts: &'a [mbe::TokenTree], idx: usize, delim: Delimiter, span: DelimSpan },
-    Sequence { tts: &'a [mbe::TokenTree], idx: usize, sep: Option<Token> },
+    Delimited {
+        tts: &'a [mbe::TokenTree],
+        idx: usize,
+        delim: Delimiter,
+        span: DelimSpan,
+        spacing: DelimSpacing,
+    },
+    Sequence {
+        tts: &'a [mbe::TokenTree],
+        idx: usize,
+        sep: Option<Token>,
+    },
 }
 
 impl<'a> Frame<'a> {
     /// Construct a new frame around the delimited set of tokens.
-    fn new(src: &'a mbe::Delimited, span: DelimSpan) -> Frame<'a> {
-        Frame::Delimited { tts: &src.tts, idx: 0, delim: src.delim, span }
+    fn new(src: &'a mbe::Delimited, span: DelimSpan, spacing: DelimSpacing) -> Frame<'a> {
+        Frame::Delimited { tts: &src.tts, idx: 0, delim: src.delim, span, spacing }
     }
 }
 
@@ -89,8 +99,10 @@ pub(super) fn transcribe<'a>(
     }
 
     // We descend into the RHS (`src`), expanding things as we go. This stack contains the things
-    // we have yet to expand/are still expanding. We start the stack off with the whole RHS.
-    let mut stack: SmallVec<[Frame<'_>; 1]> = smallvec![Frame::new(&src, src_span)];
+    // we have yet to expand/are still expanding. We start the stack off with the whole RHS. The
+    // choice of spacing values doesn't matter.
+    let mut stack: SmallVec<[Frame<'_>; 1]> =
+        smallvec![Frame::new(src, src_span, DelimSpacing::new(Spacing::Alone, Spacing::Alone))];
 
     // As we descend in the RHS, we will need to be able to match nested sequences of matchers.
     // `repeats` keeps track of where we are in matching at each level, with the last element being
@@ -144,14 +156,19 @@ pub(super) fn transcribe<'a>(
                 // We are done processing a Delimited. If this is the top-level delimited, we are
                 // done. Otherwise, we unwind the result_stack to append what we have produced to
                 // any previous results.
-                Frame::Delimited { delim, span, .. } => {
+                Frame::Delimited { delim, span, mut spacing, .. } => {
+                    // Hack to force-insert a space after `]` in certain case.
+                    // See discussion of the `hex-literal` crate in #114571.
+                    if delim == Delimiter::Bracket {
+                        spacing.close = Spacing::Alone;
+                    }
                     if result_stack.is_empty() {
                         // No results left to compute! We are back at the top-level.
                         return Ok(TokenStream::new(result));
                     }
 
                     // Step back into the parent Delimited.
-                    let tree = TokenTree::Delimited(span, delim, TokenStream::new(result));
+                    let tree = TokenTree::Delimited(span, spacing, delim, TokenStream::new(result));
                     result = result_stack.pop().unwrap();
                     result.push(tree);
                 }
@@ -166,7 +183,7 @@ pub(super) fn transcribe<'a>(
             // and the matches in `interp` have the same shape. Otherwise, either the caller or the
             // macro writer has made a mistake.
             seq @ mbe::TokenTree::Sequence(_, delimited) => {
-                match lockstep_iter_size(&seq, interp, &repeats) {
+                match lockstep_iter_size(seq, interp, &repeats) {
                     LockstepIterSize::Unconstrained => {
                         return Err(cx.create_err(NoSyntaxVarsExprRepeat { span: seq.span() }));
                     }
@@ -240,7 +257,7 @@ pub(super) fn transcribe<'a>(
                     // with modified syntax context. (I believe this supports nested macros).
                     marker.visit_span(&mut sp);
                     marker.visit_ident(&mut original_ident);
-                    result.push(TokenTree::token_alone(token::Dollar, sp));
+                    result.push(TokenTree::token_joint_hidden(token::Dollar, sp));
                     result.push(TokenTree::Token(
                         Token::from_ast_ident(original_ident),
                         Spacing::Alone,
@@ -250,7 +267,7 @@ pub(super) fn transcribe<'a>(
 
             // Replace meta-variable expressions with the result of their expansion.
             mbe::TokenTree::MetaVarExpr(sp, expr) => {
-                transcribe_metavar_expr(cx, expr, interp, &mut marker, &repeats, &mut result, &sp)?;
+                transcribe_metavar_expr(cx, expr, interp, &mut marker, &repeats, &mut result, sp)?;
             }
 
             // If we are entering a new delimiter, we push its contents to the `stack` to be
@@ -258,13 +275,14 @@ pub(super) fn transcribe<'a>(
             // We will produce all of the results of the inside of the `Delimited` and then we will
             // jump back out of the Delimited, pop the result_stack and add the new results back to
             // the previous results (from outside the Delimited).
-            mbe::TokenTree::Delimited(mut span, delimited) => {
+            mbe::TokenTree::Delimited(mut span, spacing, delimited) => {
                 mut_visit::visit_delim_span(&mut span, &mut marker);
                 stack.push(Frame::Delimited {
                     tts: &delimited.tts,
                     delim: delimited.delim,
                     idx: 0,
                     span,
+                    spacing: *spacing,
                 });
                 result_stack.push(mem::take(&mut result));
             }
@@ -374,7 +392,7 @@ fn lockstep_iter_size(
 ) -> LockstepIterSize {
     use mbe::TokenTree;
     match tree {
-        TokenTree::Delimited(_, delimited) => {
+        TokenTree::Delimited(.., delimited) => {
             delimited.tts.iter().fold(LockstepIterSize::Unconstrained, |size, tt| {
                 size.with(lockstep_iter_size(tt, interpolations, repeats))
             })
@@ -422,7 +440,7 @@ fn lockstep_iter_size(
 ///   declared inside a single repetition and the index `1` implies two nested repetitions.
 fn count_repetitions<'a>(
     cx: &ExtCtxt<'a>,
-    depth_opt: Option<usize>,
+    depth_user: usize,
     mut matched: &NamedMatch,
     repeats: &[(usize, usize)],
     sp: &DelimSpan,
@@ -431,37 +449,45 @@ fn count_repetitions<'a>(
     // (or at the top-level of `matched` if no depth is given).
     fn count<'a>(
         cx: &ExtCtxt<'a>,
-        declared_lhs_depth: usize,
-        depth_opt: Option<usize>,
+        depth_curr: usize,
+        depth_max: usize,
         matched: &NamedMatch,
         sp: &DelimSpan,
     ) -> PResult<'a, usize> {
         match matched {
-            MatchedTokenTree(_) | MatchedNonterminal(_) => {
-                if declared_lhs_depth == 0 {
-                    return Err(cx.create_err(CountRepetitionMisplaced { span: sp.entire() }));
-                }
-                match depth_opt {
-                    None => Ok(1),
-                    Some(_) => Err(out_of_bounds_err(cx, declared_lhs_depth, sp.entire(), "count")),
-                }
-            }
+            MatchedTokenTree(_) | MatchedNonterminal(_) => Ok(1),
             MatchedSeq(named_matches) => {
-                let new_declared_lhs_depth = declared_lhs_depth + 1;
-                match depth_opt {
-                    None => named_matches
-                        .iter()
-                        .map(|elem| count(cx, new_declared_lhs_depth, None, elem, sp))
-                        .sum(),
-                    Some(0) => Ok(named_matches.len()),
-                    Some(depth) => named_matches
+                if depth_curr == depth_max {
+                    Ok(named_matches.len())
+                } else {
+                    named_matches
                         .iter()
-                        .map(|elem| count(cx, new_declared_lhs_depth, Some(depth - 1), elem, sp))
-                        .sum(),
+                        .map(|elem| count(cx, depth_curr + 1, depth_max, elem, sp))
+                        .sum()
                 }
             }
         }
     }
+
+    /// Maximum depth
+    fn depth(counter: usize, matched: &NamedMatch) -> usize {
+        match matched {
+            MatchedTokenTree(_) | MatchedNonterminal(_) => counter,
+            MatchedSeq(named_matches) => {
+                let rslt = counter + 1;
+                if let Some(elem) = named_matches.first() { depth(rslt, elem) } else { rslt }
+            }
+        }
+    }
+
+    let depth_max = depth(0, matched)
+        .checked_sub(1)
+        .and_then(|el| el.checked_sub(repeats.len()))
+        .unwrap_or_default();
+    if depth_user > depth_max {
+        return Err(out_of_bounds_err(cx, depth_max + 1, sp.entire(), "count"));
+    }
+
     // `repeats` records all of the nested levels at which we are currently
     // matching meta-variables. The meta-var-expr `count($x)` only counts
     // matches that occur in this "subtree" of the `NamedMatch` where we
@@ -473,7 +499,12 @@ fn count_repetitions<'a>(
             matched = &ads[idx];
         }
     }
-    count(cx, 0, depth_opt, matched, sp)
+
+    if let MatchedTokenTree(_) | MatchedNonterminal(_) = matched {
+        return Err(cx.create_err(CountRepetitionMisplaced { span: sp.entire() }));
+    }
+
+    count(cx, depth_user, depth_max, matched, sp)
 }
 
 /// Returns a `NamedMatch` item declared on the LHS given an arbitrary [Ident]
@@ -505,7 +536,7 @@ fn out_of_bounds_err<'a>(
         )
     } else {
         format!(
-            "depth parameter on meta-variable expression `{ty}` \
+            "depth parameter of meta-variable expression `{ty}` \
              must be less than {max}"
         )
     };
@@ -527,9 +558,9 @@ fn transcribe_metavar_expr<'a>(
         span
     };
     match *expr {
-        MetaVarExpr::Count(original_ident, depth_opt) => {
+        MetaVarExpr::Count(original_ident, depth) => {
             let matched = matched_from_ident(cx, original_ident, interp)?;
-            let count = count_repetitions(cx, depth_opt, matched, &repeats, sp)?;
+            let count = count_repetitions(cx, depth, matched, repeats, sp)?;
             let tt = TokenTree::token_alone(
                 TokenKind::lit(token::Integer, sym::integer(count), None),
                 visited_span(),
diff --git a/compiler/rustc_expand/src/module.rs b/compiler/rustc_expand/src/module.rs
index df6bdc695..a0dec89d6 100644
--- a/compiler/rustc_expand/src/module.rs
+++ b/compiler/rustc_expand/src/module.rs
@@ -57,7 +57,7 @@ pub(crate) fn parse_external_mod(
     // We bail on the first error, but that error does not cause a fatal error... (1)
     let result: Result<_, ModError<'_>> = try {
         // Extract the file path and the new ownership.
-        let mp = mod_file_path(sess, ident, &attrs, &module.dir_path, dir_ownership)?;
+        let mp = mod_file_path(sess, ident, attrs, &module.dir_path, dir_ownership)?;
         dir_ownership = mp.dir_ownership;
 
         // Ensure file paths are acyclic.
@@ -119,7 +119,7 @@ pub(crate) fn mod_dir_path(
         Inline::No => {
             // FIXME: This is a subset of `parse_external_mod` without actual parsing,
             // check whether the logic for unloaded, loaded and inline modules can be unified.
-            let file_path = mod_file_path(sess, ident, &attrs, &module.dir_path, dir_ownership)
+            let file_path = mod_file_path(sess, ident, attrs, &module.dir_path, dir_ownership)
                 .map(|mp| {
                     dir_ownership = mp.dir_ownership;
                     mp.file_path
diff --git a/compiler/rustc_expand/src/parse/tests.rs b/compiler/rustc_expand/src/parse/tests.rs
index bdc20882a..7a888250c 100644
--- a/compiler/rustc_expand/src/parse/tests.rs
+++ b/compiler/rustc_expand/src/parse/tests.rs
@@ -4,7 +4,7 @@ use crate::tests::{
 
 use rustc_ast::ptr::P;
 use rustc_ast::token::{self, Delimiter, Token};
-use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree};
+use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree};
 use rustc_ast::visit;
 use rustc_ast::{self as ast, PatKind};
 use rustc_ast_pretty::pprust::item_to_string;
@@ -77,14 +77,14 @@ fn string_to_tts_macro() {
                 TokenTree::Token(Token { kind: token::Ident(name_macro_rules, false), .. }, _),
                 TokenTree::Token(Token { kind: token::Not, .. }, _),
                 TokenTree::Token(Token { kind: token::Ident(name_zip, false), .. }, _),
-                TokenTree::Delimited(_, macro_delim, macro_tts),
+                TokenTree::Delimited(.., macro_delim, macro_tts),
             ] if name_macro_rules == &kw::MacroRules && name_zip.as_str() == "zip" => {
                 let tts = &macro_tts.trees().collect::<Vec<_>>();
                 match &tts[..] {
                     [
-                        TokenTree::Delimited(_, first_delim, first_tts),
+                        TokenTree::Delimited(.., first_delim, first_tts),
                         TokenTree::Token(Token { kind: token::FatArrow, .. }, _),
-                        TokenTree::Delimited(_, second_delim, second_tts),
+                        TokenTree::Delimited(.., second_delim, second_tts),
                     ] if macro_delim == &Delimiter::Parenthesis => {
                         let tts = &first_tts.trees().collect::<Vec<_>>();
                         match &tts[..] {
@@ -116,27 +116,36 @@ fn string_to_tts_macro() {
 #[test]
 fn string_to_tts_1() {
     create_default_session_globals_then(|| {
-        let tts = string_to_stream("fn a (b : i32) { b; }".to_string());
+        let tts = string_to_stream("fn a(b: i32) { b; }".to_string());
 
         let expected = TokenStream::new(vec![
             TokenTree::token_alone(token::Ident(kw::Fn, false), sp(0, 2)),
-            TokenTree::token_alone(token::Ident(Symbol::intern("a"), false), sp(3, 4)),
+            TokenTree::token_joint_hidden(token::Ident(Symbol::intern("a"), false), sp(3, 4)),
             TokenTree::Delimited(
-                DelimSpan::from_pair(sp(5, 6), sp(13, 14)),
+                DelimSpan::from_pair(sp(4, 5), sp(11, 12)),
+                // `JointHidden` because the `(` is followed immediately by
+                // `b`, `Alone` because the `)` is followed by whitespace.
+                DelimSpacing::new(Spacing::JointHidden, Spacing::Alone),
                 Delimiter::Parenthesis,
                 TokenStream::new(vec![
-                    TokenTree::token_alone(token::Ident(Symbol::intern("b"), false), sp(6, 7)),
-                    TokenTree::token_alone(token::Colon, sp(8, 9)),
-                    TokenTree::token_alone(token::Ident(sym::i32, false), sp(10, 13)),
+                    TokenTree::token_joint(token::Ident(Symbol::intern("b"), false), sp(5, 6)),
+                    TokenTree::token_alone(token::Colon, sp(6, 7)),
+                    // `JointHidden` because the `i32` is immediately followed by the `)`.
+                    TokenTree::token_joint_hidden(token::Ident(sym::i32, false), sp(8, 11)),
                 ])
                 .into(),
             ),
             TokenTree::Delimited(
-                DelimSpan::from_pair(sp(15, 16), sp(20, 21)),
+                DelimSpan::from_pair(sp(13, 14), sp(18, 19)),
+                // First `Alone` because the `{` is followed by whitespace,
+                // second `Alone` because the `}` is followed immediately by
+                // EOF.
+                DelimSpacing::new(Spacing::Alone, Spacing::Alone),
                 Delimiter::Brace,
                 TokenStream::new(vec![
-                    TokenTree::token_joint(token::Ident(Symbol::intern("b"), false), sp(17, 18)),
-                    TokenTree::token_alone(token::Semi, sp(18, 19)),
+                    TokenTree::token_joint(token::Ident(Symbol::intern("b"), false), sp(15, 16)),
+                    // `Alone` because the `;` is followed by whitespace.
+                    TokenTree::token_alone(token::Semi, sp(16, 17)),
                 ])
                 .into(),
             ),
diff --git a/compiler/rustc_expand/src/placeholders.rs b/compiler/rustc_expand/src/placeholders.rs
index 1292a8552..2c4187031 100644
--- a/compiler/rustc_expand/src/placeholders.rs
+++ b/compiler/rustc_expand/src/placeholders.rs
@@ -119,7 +119,7 @@ pub fn placeholder(
         }]),
         AstFragmentKind::Arms => AstFragment::Arms(smallvec![ast::Arm {
             attrs: Default::default(),
-            body: expr_placeholder(),
+            body: Some(expr_placeholder()),
             guard: None,
             id,
             pat: pat(),
@@ -174,7 +174,7 @@ pub fn placeholder(
         }]),
         AstFragmentKind::Variants => AstFragment::Variants(smallvec![ast::Variant {
             attrs: Default::default(),
-            data: ast::VariantData::Struct(Default::default(), false),
+            data: ast::VariantData::Struct { fields: Default::default(), recovered: false },
             disr_expr: None,
             id,
             ident,
diff --git a/compiler/rustc_expand/src/proc_macro.rs b/compiler/rustc_expand/src/proc_macro.rs
index c617cd76e..429bfa614 100644
--- a/compiler/rustc_expand/src/proc_macro.rs
+++ b/compiler/rustc_expand/src/proc_macro.rs
@@ -126,7 +126,7 @@ impl MultiItemModifier for DeriveProcMacro {
                 Annotatable::Stmt(stmt) => token::NtStmt(stmt),
                 _ => unreachable!(),
             };
-            TokenStream::token_alone(token::Interpolated(Lrc::new(nt)), DUMMY_SP)
+            TokenStream::token_alone(token::Interpolated(Lrc::new((nt, span))), DUMMY_SP)
         } else {
             item.to_tokens()
         };
@@ -156,7 +156,7 @@ impl MultiItemModifier for DeriveProcMacro {
             }
         };
 
-        let error_count_before = ecx.sess.parse_sess.span_diagnostic.err_count();
+        let error_count_before = ecx.sess.dcx().err_count();
         let mut parser =
             rustc_parse::stream_to_parser(&ecx.sess.parse_sess, stream, Some("proc-macro derive"));
         let mut items = vec![];
@@ -179,7 +179,7 @@ impl MultiItemModifier for DeriveProcMacro {
         }
 
         // fail if there have been errors emitted
-        if ecx.sess.parse_sess.span_diagnostic.err_count() > error_count_before {
+        if ecx.sess.dcx().err_count() > error_count_before {
             ecx.sess.emit_err(errors::ProcMacroDeriveTokens { span });
         }
 
diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs
index aa4c7a531..c412b064c 100644
--- a/compiler/rustc_expand/src/proc_macro_server.rs
+++ b/compiler/rustc_expand/src/proc_macro_server.rs
@@ -5,7 +5,7 @@ use pm::bridge::{
 use pm::{Delimiter, Level};
 use rustc_ast as ast;
 use rustc_ast::token;
-use rustc_ast::tokenstream::{self, Spacing::*, TokenStream};
+use rustc_ast::tokenstream::{self, DelimSpacing, Spacing, TokenStream};
 use rustc_ast::util::literal::escape_byte_str_symbol;
 use rustc_ast_pretty::pprust;
 use rustc_data_structures::fx::FxHashMap;
@@ -98,7 +98,7 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
 
         while let Some(tree) = cursor.next() {
             let (Token { kind, span }, joint) = match tree.clone() {
-                tokenstream::TokenTree::Delimited(span, delim, tts) => {
+                tokenstream::TokenTree::Delimited(span, _, delim, tts) => {
                     let delimiter = pm::Delimiter::from_internal(delim);
                     trees.push(TokenTree::Group(Group {
                         delimiter,
@@ -111,7 +111,22 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
                     }));
                     continue;
                 }
-                tokenstream::TokenTree::Token(token, spacing) => (token, spacing == Joint),
+                tokenstream::TokenTree::Token(token, spacing) => {
+                    // Do not be tempted to check here that the `spacing`
+                    // values are "correct" w.r.t. the token stream (e.g. that
+                    // `Spacing::Joint` is actually followed by a `Punct` token
+                    // tree). Because the problem in #76399 was introduced that
+                    // way.
+                    //
+                    // This is where the `Hidden` in `JointHidden` applies,
+                    // because the jointness is effectively hidden from proc
+                    // macros.
+                    let joint = match spacing {
+                        Spacing::Alone | Spacing::JointHidden => false,
+                        Spacing::Joint => true,
+                    };
+                    (token, joint)
+                }
             };
 
             // Split the operator into one or more `Punct`s, one per character.
@@ -133,7 +148,8 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
                     } else {
                         span
                     };
-                    TokenTree::Punct(Punct { ch, joint: if is_final { joint } else { true }, span })
+                    let joint = if is_final { joint } else { true };
+                    TokenTree::Punct(Punct { ch, joint, span })
                 }));
             };
 
@@ -226,18 +242,23 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
                     }));
                 }
 
-                Interpolated(nt) if let NtIdent(ident, is_raw) = *nt => trees
-                    .push(TokenTree::Ident(Ident { sym: ident.name, is_raw, span: ident.span })),
+                Interpolated(ref nt) if let NtIdent(ident, is_raw) = &nt.0 => {
+                    trees.push(TokenTree::Ident(Ident {
+                        sym: ident.name,
+                        is_raw: *is_raw,
+                        span: ident.span,
+                    }))
+                }
 
                 Interpolated(nt) => {
-                    let stream = TokenStream::from_nonterminal_ast(&nt);
+                    let stream = TokenStream::from_nonterminal_ast(&nt.0);
                     // A hack used to pass AST fragments to attribute and derive
                     // macros as a single nonterminal token instead of a token
                     // stream. Such token needs to be "unwrapped" and not
                     // represented as a delimited group.
                     // FIXME: It needs to be removed, but there are some
                     // compatibility issues (see #73345).
-                    if crate::base::nt_pretty_printing_compatibility_hack(&nt, rustc.sess()) {
+                    if crate::base::nt_pretty_printing_compatibility_hack(&nt.0, rustc.sess()) {
                         trees.extend(Self::from_internal((stream, rustc)));
                     } else {
                         trees.push(TokenTree::Group(Group {
@@ -263,6 +284,11 @@ impl ToInternal<SmallVec<[tokenstream::TokenTree; 2]>>
     fn to_internal(self) -> SmallVec<[tokenstream::TokenTree; 2]> {
         use rustc_ast::token::*;
 
+        // The code below is conservative, using `token_alone`/`Spacing::Alone`
+        // in most places. When the resulting code is pretty-printed by
+        // `print_tts` it ends up with spaces between most tokens, which is
+        // safe but ugly. It's hard in general to do better when working at the
+        // token level.
         let (tree, rustc) = self;
         match tree {
             TokenTree::Punct(Punct { ch, joint, span }) => {
@@ -291,6 +317,11 @@ impl ToInternal<SmallVec<[tokenstream::TokenTree; 2]>>
                     b'\'' => SingleQuote,
                     _ => unreachable!(),
                 };
+                // We never produce `token::Spacing::JointHidden` here, which
+                // means the pretty-printing of code produced by proc macros is
+                // ugly, with lots of whitespace between tokens. This is
+                // unavoidable because `proc_macro::Spacing` only applies to
+                // `Punct` token trees.
                 smallvec![if joint {
                     tokenstream::TokenTree::token_joint(kind, span)
                 } else {
@@ -300,6 +331,7 @@ impl ToInternal<SmallVec<[tokenstream::TokenTree; 2]>>
             TokenTree::Group(Group { delimiter, stream, span: DelimSpan { open, close, .. } }) => {
                 smallvec![tokenstream::TokenTree::Delimited(
                     tokenstream::DelimSpan { open, close },
+                    DelimSpacing::new(Spacing::Alone, Spacing::Alone),
                     delimiter.to_internal(),
                     stream.unwrap_or_default(),
                 )]
@@ -317,7 +349,7 @@ impl ToInternal<SmallVec<[tokenstream::TokenTree; 2]>>
                 let minus = BinOp(BinOpToken::Minus);
                 let symbol = Symbol::intern(&symbol.as_str()[1..]);
                 let integer = TokenKind::lit(token::Integer, symbol, suffix);
-                let a = tokenstream::TokenTree::token_alone(minus, span);
+                let a = tokenstream::TokenTree::token_joint_hidden(minus, span);
                 let b = tokenstream::TokenTree::token_alone(integer, span);
                 smallvec![a, b]
             }
@@ -330,7 +362,7 @@ impl ToInternal<SmallVec<[tokenstream::TokenTree; 2]>>
                 let minus = BinOp(BinOpToken::Minus);
                 let symbol = Symbol::intern(&symbol.as_str()[1..]);
                 let float = TokenKind::lit(token::Float, symbol, suffix);
-                let a = tokenstream::TokenTree::token_alone(minus, span);
+                let a = tokenstream::TokenTree::token_joint_hidden(minus, span);
                 let b = tokenstream::TokenTree::token_alone(float, span);
                 smallvec![a, b]
             }
@@ -394,6 +426,10 @@ impl server::Types for Rustc<'_, '_> {
 }
 
 impl server::FreeFunctions for Rustc<'_, '_> {
+    fn injected_env_var(&mut self, var: &str) -> Option<String> {
+        self.ecx.sess.opts.logical_env.get(var).cloned()
+    }
+
     fn track_env_var(&mut self, var: &str, value: Option<&str>) {
         self.sess()
             .env_depinfo
@@ -470,7 +506,7 @@ impl server::FreeFunctions for Rustc<'_, '_> {
                 None,
             );
         }
-        self.sess().span_diagnostic.emit_diagnostic(&mut diag);
+        self.sess().dcx.emit_diagnostic(diag);
     }
 }
 
@@ -541,7 +577,10 @@ impl server::TokenStream for Rustc<'_, '_> {
                         Ok(Self::TokenStream::from_iter([
                             // FIXME: The span of the `-` token is lost when
                             // parsing, so we cannot faithfully recover it here.
-                            tokenstream::TokenTree::token_alone(token::BinOp(token::Minus), e.span),
+                            tokenstream::TokenTree::token_joint_hidden(
+                                token::BinOp(token::Minus),
+                                e.span,
+                            ),
                             tokenstream::TokenTree::token_alone(token::Literal(*token_lit), e.span),
                         ]))
                     }
@@ -779,6 +818,6 @@ impl server::Server for Rustc<'_, '_> {
     }
 
     fn with_symbol_string(symbol: &Self::Symbol, f: impl FnOnce(&str)) {
-        f(&symbol.as_str())
+        f(symbol.as_str())
     }
 }
diff --git a/compiler/rustc_expand/src/tests.rs b/compiler/rustc_expand/src/tests.rs
index 9f52669e1..0b8598418 100644
--- a/compiler/rustc_expand/src/tests.rs
+++ b/compiler/rustc_expand/src/tests.rs
@@ -8,7 +8,7 @@ use rustc_span::{BytePos, Span};
 
 use rustc_data_structures::sync::Lrc;
 use rustc_errors::emitter::EmitterWriter;
-use rustc_errors::{Handler, MultiSpan, PResult};
+use rustc_errors::{DiagCtxt, MultiSpan, PResult};
 use termcolor::WriteColor;
 
 use std::io;
@@ -23,7 +23,7 @@ fn string_to_parser(ps: &ParseSess, source_str: String) -> Parser<'_> {
     new_parser_from_source_str(ps, PathBuf::from("bogofile").into(), source_str)
 }
 
-fn create_test_handler() -> (Handler, Lrc<SourceMap>, Arc<Mutex<Vec<u8>>>) {
+fn create_test_handler() -> (DiagCtxt, Lrc<SourceMap>, Arc<Mutex<Vec<u8>>>) {
     let output = Arc::new(Mutex::new(Vec::new()));
     let source_map = Lrc::new(SourceMap::new(FilePathMapping::empty()));
     let fallback_bundle = rustc_errors::fallback_fluent_bundle(
@@ -33,8 +33,8 @@ fn create_test_handler() -> (Handler, Lrc<SourceMap>, Arc<Mutex<Vec<u8>>>) {
     let emitter = EmitterWriter::new(Box::new(Shared { data: output.clone() }), fallback_bundle)
         .sm(Some(source_map.clone()))
         .diagnostic_width(Some(140));
-    let handler = Handler::with_emitter(Box::new(emitter));
-    (handler, source_map, output)
+    let dcx = DiagCtxt::with_emitter(Box::new(emitter));
+    (dcx, source_map, output)
 }
 
 /// Returns the result of parsing the given string via the given callback.
@@ -46,7 +46,7 @@ where
 {
     let mut p = string_to_parser(&ps, s);
     let x = f(&mut p).unwrap();
-    p.sess.span_diagnostic.abort_if_errors();
+    p.sess.dcx.abort_if_errors();
     x
 }
 
@@ -57,7 +57,7 @@ where
     F: for<'a> FnOnce(&mut Parser<'a>) -> PResult<'a, T>,
 {
     let (handler, source_map, output) = create_test_handler();
-    let ps = ParseSess::with_span_handler(handler, source_map);
+    let ps = ParseSess::with_dcx(handler, source_map);
     let mut p = string_to_parser(&ps, source_str.to_string());
     let result = f(&mut p);
     assert!(result.is_ok());
@@ -135,7 +135,7 @@ pub(crate) fn matches_codepattern(a: &str, b: &str) -> bool {
 
 /// Advances the given peekable `Iterator` until it reaches a non-whitespace character.
 fn scan_for_non_ws_or_end<I: Iterator<Item = char>>(iter: &mut Peekable<I>) {
-    while iter.peek().copied().map(rustc_lexer::is_whitespace) == Some(true) {
+    while iter.peek().copied().is_some_and(rustc_lexer::is_whitespace) {
         iter.next();
     }
 }
diff --git a/compiler/rustc_feature/src/accepted.rs b/compiler/rustc_feature/src/accepted.rs
index 577657ff7..be04fa743 100644
--- a/compiler/rustc_feature/src/accepted.rs
+++ b/compiler/rustc_feature/src/accepted.rs
@@ -5,7 +5,7 @@ use rustc_span::symbol::sym;
 
 macro_rules! declare_features {
     ($(
-        $(#[doc = $doc:tt])* (accepted, $feature:ident, $ver:expr, $issue:expr, None),
+        $(#[doc = $doc:tt])* (accepted, $feature:ident, $ver:expr, $issue:expr),
     )+) => {
         /// Formerly unstable features that have now been accepted (stabilized).
         pub const ACCEPTED_FEATURES: &[Feature] = &[
@@ -13,7 +13,6 @@ macro_rules! declare_features {
                 name: sym::$feature,
                 since: $ver,
                 issue: to_nonzero($issue),
-                edition: None,
             }),+
         ];
     }
@@ -27,10 +26,10 @@ declare_features! (
 
     /// A temporary feature gate used to enable parser extensions needed
     /// to bootstrap fix for #5723.
-    (accepted, issue_5723_bootstrap, "1.0.0", None, None),
+    (accepted, issue_5723_bootstrap, "1.0.0", None),
     /// These are used to test this portion of the compiler,
     /// they don't actually mean anything.
-    (accepted, test_accepted_feature, "1.0.0", None, None),
+    (accepted, test_accepted_feature, "1.0.0", None),
     // !!!!    !!!!    !!!!    !!!!   !!!!    !!!!    !!!!    !!!!    !!!!    !!!!    !!!!
     // Features are listed in alphabetical order. Tidy will fail if you don't keep it this way.
     // !!!!    !!!!    !!!!    !!!!   !!!!    !!!!    !!!!    !!!!    !!!!    !!!!    !!!!
@@ -44,333 +43,333 @@ declare_features! (
     // -------------------------------------------------------------------------
 
     /// Allows `#[target_feature(...)]` on aarch64 platforms
-    (accepted, aarch64_target_feature, "1.61.0", Some(44839), None),
+    (accepted, aarch64_target_feature, "1.61.0", Some(44839)),
     /// Allows using the `efiapi` ABI.
-    (accepted, abi_efiapi, "1.68.0", Some(65815), None),
+    (accepted, abi_efiapi, "1.68.0", Some(65815)),
     /// Allows the sysV64 ABI to be specified on all platforms
     /// instead of just the platforms on which it is the C ABI.
-    (accepted, abi_sysv64, "1.24.0", Some(36167), None),
+    (accepted, abi_sysv64, "1.24.0", Some(36167)),
     /// Allows using the `thiscall` ABI.
-    (accepted, abi_thiscall, "1.73.0", None, None),
+    (accepted, abi_thiscall, "1.73.0", None),
     /// Allows using ADX intrinsics from `core::arch::{x86, x86_64}`.
-    (accepted, adx_target_feature, "1.61.0", Some(44839), None),
+    (accepted, adx_target_feature, "1.61.0", Some(44839)),
     /// Allows explicit discriminants on non-unit enum variants.
-    (accepted, arbitrary_enum_discriminant, "1.66.0", Some(60553), None),
+    (accepted, arbitrary_enum_discriminant, "1.66.0", Some(60553)),
     /// Allows using `sym` operands in inline assembly.
-    (accepted, asm_sym, "1.66.0", Some(93333), None),
+    (accepted, asm_sym, "1.66.0", Some(93333)),
     /// Allows the definition of associated constants in `trait` or `impl` blocks.
-    (accepted, associated_consts, "1.20.0", Some(29646), None),
+    (accepted, associated_consts, "1.20.0", Some(29646)),
     /// Allows using associated `type`s in `trait`s.
-    (accepted, associated_types, "1.0.0", None, None),
+    (accepted, associated_types, "1.0.0", None),
     /// Allows free and inherent `async fn`s, `async` blocks, and `<expr>.await` expressions.
-    (accepted, async_await, "1.39.0", Some(50547), None),
+    (accepted, async_await, "1.39.0", Some(50547)),
     /// Allows async functions to be declared, implemented, and used in traits.
-    (accepted, async_fn_in_trait, "1.75.0", Some(91611), None),
+    (accepted, async_fn_in_trait, "1.75.0", Some(91611)),
     /// Allows all literals in attribute lists and values of key-value pairs.
-    (accepted, attr_literals, "1.30.0", Some(34981), None),
+    (accepted, attr_literals, "1.30.0", Some(34981)),
     /// Allows overloading augmented assignment operations like `a += b`.
-    (accepted, augmented_assignments, "1.8.0", Some(28235), None),
+    (accepted, augmented_assignments, "1.8.0", Some(28235)),
     /// Allows mixing bind-by-move in patterns and references to those identifiers in guards.
-    (accepted, bind_by_move_pattern_guards, "1.39.0", Some(15287), None),
+    (accepted, bind_by_move_pattern_guards, "1.39.0", Some(15287)),
     /// Allows bindings in the subpattern of a binding pattern.
     /// For example, you can write `x @ Some(y)`.
-    (accepted, bindings_after_at, "1.56.0", Some(65490), None),
+    (accepted, bindings_after_at, "1.56.0", Some(65490)),
     /// Allows empty structs and enum variants with braces.
-    (accepted, braced_empty_structs, "1.8.0", Some(29720), None),
+    (accepted, braced_empty_structs, "1.8.0", Some(29720)),
     /// Allows `#[cfg_attr(predicate, multiple, attributes, here)]`.
-    (accepted, cfg_attr_multi, "1.33.0", Some(54881), None),
+    (accepted, cfg_attr_multi, "1.33.0", Some(54881)),
     /// Allows the use of `#[cfg(doctest)]`, set when rustdoc is collecting doctests.
-    (accepted, cfg_doctest, "1.40.0", Some(62210), None),
+    (accepted, cfg_doctest, "1.40.0", Some(62210)),
     /// Enables `#[cfg(panic = "...")]` config key.
-    (accepted, cfg_panic, "1.60.0", Some(77443), None),
+    (accepted, cfg_panic, "1.60.0", Some(77443)),
     /// Allows `cfg(target_feature = "...")`.
-    (accepted, cfg_target_feature, "1.27.0", Some(29717), None),
+    (accepted, cfg_target_feature, "1.27.0", Some(29717)),
     /// Allows `cfg(target_vendor = "...")`.
-    (accepted, cfg_target_vendor, "1.33.0", Some(29718), None),
+    (accepted, cfg_target_vendor, "1.33.0", Some(29718)),
     /// Allows implementing `Clone` for closures where possible (RFC 2132).
-    (accepted, clone_closures, "1.26.0", Some(44490), None),
+    (accepted, clone_closures, "1.26.0", Some(44490)),
     /// Allows coercing non capturing closures to function pointers.
-    (accepted, closure_to_fn_coercion, "1.19.0", Some(39817), None),
+    (accepted, closure_to_fn_coercion, "1.19.0", Some(39817)),
     /// Allows using the CMPXCHG16B target feature.
-    (accepted, cmpxchg16b_target_feature, "1.69.0", Some(44839), None),
+    (accepted, cmpxchg16b_target_feature, "1.69.0", Some(44839)),
     /// Allows usage of the `compile_error!` macro.
-    (accepted, compile_error, "1.20.0", Some(40872), None),
+    (accepted, compile_error, "1.20.0", Some(40872)),
     /// Allows `impl Trait` in function return types.
-    (accepted, conservative_impl_trait, "1.26.0", Some(34511), None),
+    (accepted, conservative_impl_trait, "1.26.0", Some(34511)),
     /// Allows calling constructor functions in `const fn`.
-    (accepted, const_constructor, "1.40.0", Some(61456), None),
+    (accepted, const_constructor, "1.40.0", Some(61456)),
     /// Allows using and casting function pointers in a `const fn`.
-    (accepted, const_fn_fn_ptr_basics, "1.61.0", Some(57563), None),
+    (accepted, const_fn_fn_ptr_basics, "1.61.0", Some(57563)),
     /// Allows trait bounds in `const fn`.
-    (accepted, const_fn_trait_bound, "1.61.0", Some(93706), None),
+    (accepted, const_fn_trait_bound, "1.61.0", Some(93706)),
     /// Allows calling `transmute` in const fn
-    (accepted, const_fn_transmute, "1.56.0", Some(53605), None),
+    (accepted, const_fn_transmute, "1.56.0", Some(53605)),
     /// Allows accessing fields of unions inside `const` functions.
-    (accepted, const_fn_union, "1.56.0", Some(51909), None),
+    (accepted, const_fn_union, "1.56.0", Some(51909)),
     /// Allows unsizing coercions in `const fn`.
-    (accepted, const_fn_unsize, "1.54.0", Some(64992), None),
+    (accepted, const_fn_unsize, "1.54.0", Some(64992)),
     /// Allows const generics to have default values (e.g. `struct Foo<const N: usize = 3>(...);`).
-    (accepted, const_generics_defaults, "1.59.0", Some(44580), None),
+    (accepted, const_generics_defaults, "1.59.0", Some(44580)),
     /// Allows the use of `if` and `match` in constants.
-    (accepted, const_if_match, "1.46.0", Some(49146), None),
+    (accepted, const_if_match, "1.46.0", Some(49146)),
     /// Allows argument and return position `impl Trait` in a `const fn`.
-    (accepted, const_impl_trait, "1.61.0", Some(77463), None),
+    (accepted, const_impl_trait, "1.61.0", Some(77463)),
     /// Allows indexing into constant arrays.
-    (accepted, const_indexing, "1.26.0", Some(29947), None),
+    (accepted, const_indexing, "1.26.0", Some(29947)),
     /// Allows let bindings, assignments and destructuring in `const` functions and constants.
     /// As long as control flow is not implemented in const eval, `&&` and `||` may not be used
     /// at the same time as let bindings.
-    (accepted, const_let, "1.33.0", Some(48821), None),
+    (accepted, const_let, "1.33.0", Some(48821)),
     /// Allows the use of `loop` and `while` in constants.
-    (accepted, const_loop, "1.46.0", Some(52000), None),
+    (accepted, const_loop, "1.46.0", Some(52000)),
     /// Allows panicking during const eval (producing compile-time errors).
-    (accepted, const_panic, "1.57.0", Some(51999), None),
+    (accepted, const_panic, "1.57.0", Some(51999)),
     /// Allows dereferencing raw pointers during const eval.
-    (accepted, const_raw_ptr_deref, "1.58.0", Some(51911), None),
+    (accepted, const_raw_ptr_deref, "1.58.0", Some(51911)),
     /// Allows implementing `Copy` for closures where possible (RFC 2132).
-    (accepted, copy_closures, "1.26.0", Some(44490), None),
+    (accepted, copy_closures, "1.26.0", Some(44490)),
     /// Allows `crate` in paths.
-    (accepted, crate_in_paths, "1.30.0", Some(45477), None),
+    (accepted, crate_in_paths, "1.30.0", Some(45477)),
     /// Allows using `#[debugger_visualizer]` attribute.
-    (accepted, debugger_visualizer, "1.71.0", Some(95939), None),
+    (accepted, debugger_visualizer, "1.71.0", Some(95939)),
     /// Allows rustc to inject a default alloc_error_handler
-    (accepted, default_alloc_error_handler, "1.68.0", Some(66741), None),
+    (accepted, default_alloc_error_handler, "1.68.0", Some(66741)),
     /// Allows using assigning a default type to type parameters in algebraic data type definitions.
-    (accepted, default_type_params, "1.0.0", None, None),
+    (accepted, default_type_params, "1.0.0", None),
     /// Allows `#[deprecated]` attribute.
-    (accepted, deprecated, "1.9.0", Some(29935), None),
+    (accepted, deprecated, "1.9.0", Some(29935)),
     /// Allows `#[derive(Default)]` and `#[default]` on enums.
-    (accepted, derive_default_enum, "1.62.0", Some(86985), None),
+    (accepted, derive_default_enum, "1.62.0", Some(86985)),
     /// Allows the use of destructuring assignments.
-    (accepted, destructuring_assignment, "1.59.0", Some(71126), None),
+    (accepted, destructuring_assignment, "1.59.0", Some(71126)),
     /// Allows `#[doc(alias = "...")]`.
-    (accepted, doc_alias, "1.48.0", Some(50146), None),
+    (accepted, doc_alias, "1.48.0", Some(50146)),
     /// Allows `..` in tuple (struct) patterns.
-    (accepted, dotdot_in_tuple_patterns, "1.14.0", Some(33627), None),
+    (accepted, dotdot_in_tuple_patterns, "1.14.0", Some(33627)),
     /// Allows `..=` in patterns (RFC 1192).
-    (accepted, dotdoteq_in_patterns, "1.26.0", Some(28237), None),
+    (accepted, dotdoteq_in_patterns, "1.26.0", Some(28237)),
     /// Allows `Drop` types in constants (RFC 1440).
-    (accepted, drop_types_in_const, "1.22.0", Some(33156), None),
+    (accepted, drop_types_in_const, "1.22.0", Some(33156)),
     /// Allows using `dyn Trait` as a syntax for trait objects.
-    (accepted, dyn_trait, "1.27.0", Some(44662), None),
+    (accepted, dyn_trait, "1.27.0", Some(44662)),
     /// Allows integer match exhaustiveness checking (RFC 2591).
-    (accepted, exhaustive_integer_patterns, "1.33.0", Some(50907), None),
+    (accepted, exhaustive_integer_patterns, "1.33.0", Some(50907)),
     /// Allows explicit generic arguments specification with `impl Trait` present.
-    (accepted, explicit_generic_args_with_impl_trait, "1.63.0", Some(83701), None),
+    (accepted, explicit_generic_args_with_impl_trait, "1.63.0", Some(83701)),
     /// Allows arbitrary expressions in key-value attributes at parse time.
-    (accepted, extended_key_value_attributes, "1.54.0", Some(78835), None),
+    (accepted, extended_key_value_attributes, "1.54.0", Some(78835)),
     /// Allows resolving absolute paths as paths from other crates.
-    (accepted, extern_absolute_paths, "1.30.0", Some(44660), None),
+    (accepted, extern_absolute_paths, "1.30.0", Some(44660)),
     /// Allows `extern crate foo as bar;`. This puts `bar` into extern prelude.
-    (accepted, extern_crate_item_prelude, "1.31.0", Some(55599), None),
+    (accepted, extern_crate_item_prelude, "1.31.0", Some(55599)),
     /// Allows `extern crate self as foo;`.
     /// This puts local crate root into extern prelude under name `foo`.
-    (accepted, extern_crate_self, "1.34.0", Some(56409), None),
+    (accepted, extern_crate_self, "1.34.0", Some(56409)),
     /// Allows access to crate names passed via `--extern` through prelude.
-    (accepted, extern_prelude, "1.30.0", Some(44660), None),
+    (accepted, extern_prelude, "1.30.0", Some(44660)),
     /// Allows using F16C intrinsics from `core::arch::{x86, x86_64}`.
-    (accepted, f16c_target_feature, "1.68.0", Some(44839), None),
+    (accepted, f16c_target_feature, "1.68.0", Some(44839)),
     /// Allows field shorthands (`x` meaning `x: x`) in struct literal expressions.
-    (accepted, field_init_shorthand, "1.17.0", Some(37340), None),
+    (accepted, field_init_shorthand, "1.17.0", Some(37340)),
     /// Allows `#[must_use]` on functions, and introduces must-use operators (RFC 1940).
-    (accepted, fn_must_use, "1.27.0", Some(43302), None),
+    (accepted, fn_must_use, "1.27.0", Some(43302)),
     /// Allows capturing variables in scope using format_args!
-    (accepted, format_args_capture, "1.58.0", Some(67984), None),
+    (accepted, format_args_capture, "1.58.0", Some(67984)),
     /// Allows associated types to be generic, e.g., `type Foo<T>;` (RFC 1598).
-    (accepted, generic_associated_types, "1.65.0", Some(44265), None),
+    (accepted, generic_associated_types, "1.65.0", Some(44265)),
     /// Allows attributes on lifetime/type formal parameters in generics (RFC 1327).
-    (accepted, generic_param_attrs, "1.27.0", Some(48848), None),
+    (accepted, generic_param_attrs, "1.27.0", Some(48848)),
     /// Allows the `#[global_allocator]` attribute.
-    (accepted, global_allocator, "1.28.0", Some(27389), None),
+    (accepted, global_allocator, "1.28.0", Some(27389)),
     // FIXME: explain `globs`.
-    (accepted, globs, "1.0.0", None, None),
+    (accepted, globs, "1.0.0", None),
     /// Allows using `..=X` as a pattern.
-    (accepted, half_open_range_patterns, "1.66.0", Some(67264), None),
+    (accepted, half_open_range_patterns, "1.66.0", Some(67264)),
     /// Allows using the `u128` and `i128` types.
-    (accepted, i128_type, "1.26.0", Some(35118), None),
+    (accepted, i128_type, "1.26.0", Some(35118)),
     /// Allows the use of `if let` expressions.
-    (accepted, if_let, "1.0.0", None, None),
+    (accepted, if_let, "1.0.0", None),
     /// Allows top level or-patterns (`p | q`) in `if let` and `while let`.
-    (accepted, if_while_or_patterns, "1.33.0", Some(48215), None),
+    (accepted, if_while_or_patterns, "1.33.0", Some(48215)),
     /// Allows lifetime elision in `impl` headers. For example:
     /// + `impl<I:Iterator> Iterator for &mut Iterator`
     /// + `impl Debug for Foo<'_>`
-    (accepted, impl_header_lifetime_elision, "1.31.0", Some(15872), None),
+    (accepted, impl_header_lifetime_elision, "1.31.0", Some(15872)),
     /// Allows referencing `Self` and projections in impl-trait.
-    (accepted, impl_trait_projections, "1.74.0", Some(103532), None),
+    (accepted, impl_trait_projections, "1.74.0", Some(103532)),
     /// Allows using `a..=b` and `..=b` as inclusive range syntaxes.
-    (accepted, inclusive_range_syntax, "1.26.0", Some(28237), None),
+    (accepted, inclusive_range_syntax, "1.26.0", Some(28237)),
     /// Allows inferring outlives requirements (RFC 2093).
-    (accepted, infer_outlives_requirements, "1.30.0", Some(44493), None),
+    (accepted, infer_outlives_requirements, "1.30.0", Some(44493)),
     /// Allows irrefutable patterns in `if let` and `while let` statements (RFC 2086).
-    (accepted, irrefutable_let_patterns, "1.33.0", Some(44495), None),
+    (accepted, irrefutable_let_patterns, "1.33.0", Some(44495)),
     /// Allows `#[instruction_set(_)]` attribute.
-    (accepted, isa_attribute, "1.67.0", Some(74727), None),
+    (accepted, isa_attribute, "1.67.0", Some(74727)),
     /// Allows some increased flexibility in the name resolution rules,
     /// especially around globs and shadowing (RFC 1560).
-    (accepted, item_like_imports, "1.15.0", Some(35120), None),
+    (accepted, item_like_imports, "1.15.0", Some(35120)),
     /// Allows `'a: { break 'a; }`.
-    (accepted, label_break_value, "1.65.0", Some(48594), None),
+    (accepted, label_break_value, "1.65.0", Some(48594)),
     /// Allows `let...else` statements.
-    (accepted, let_else, "1.65.0", Some(87335), None),
+    (accepted, let_else, "1.65.0", Some(87335)),
     /// Allows `break {expr}` with a value inside `loop`s.
-    (accepted, loop_break_value, "1.19.0", Some(37339), None),
+    (accepted, loop_break_value, "1.19.0", Some(37339)),
     /// Allows use of `?` as the Kleene "at most one" operator in macros.
-    (accepted, macro_at_most_once_rep, "1.32.0", Some(48075), None),
+    (accepted, macro_at_most_once_rep, "1.32.0", Some(48075)),
     /// Allows macro attributes to observe output of `#[derive]`.
-    (accepted, macro_attributes_in_derive_output, "1.57.0", Some(81119), None),
+    (accepted, macro_attributes_in_derive_output, "1.57.0", Some(81119)),
     /// Allows use of the `:lifetime` macro fragment specifier.
-    (accepted, macro_lifetime_matcher, "1.27.0", Some(34303), None),
+    (accepted, macro_lifetime_matcher, "1.27.0", Some(34303)),
     /// Allows use of the `:literal` macro fragment specifier (RFC 1576).
-    (accepted, macro_literal_matcher, "1.32.0", Some(35625), None),
+    (accepted, macro_literal_matcher, "1.32.0", Some(35625)),
     /// Allows `macro_rules!` items.
-    (accepted, macro_rules, "1.0.0", None, None),
+    (accepted, macro_rules, "1.0.0", None),
     /// Allows use of the `:vis` macro fragment specifier
-    (accepted, macro_vis_matcher, "1.30.0", Some(41022), None),
+    (accepted, macro_vis_matcher, "1.30.0", Some(41022)),
     /// Allows macro invocations in `extern {}` blocks.
-    (accepted, macros_in_extern, "1.40.0", Some(49476), None),
+    (accepted, macros_in_extern, "1.40.0", Some(49476)),
     /// Allows '|' at beginning of match arms (RFC 1925).
-    (accepted, match_beginning_vert, "1.25.0", Some(44101), None),
+    (accepted, match_beginning_vert, "1.25.0", Some(44101)),
     /// Allows default match binding modes (RFC 2005).
-    (accepted, match_default_bindings, "1.26.0", Some(42640), None),
+    (accepted, match_default_bindings, "1.26.0", Some(42640)),
     /// Allows `impl Trait` with multiple unrelated lifetimes.
-    (accepted, member_constraints, "1.54.0", Some(61997), None),
+    (accepted, member_constraints, "1.54.0", Some(61997)),
     /// Allows the definition of `const fn` functions.
-    (accepted, min_const_fn, "1.31.0", Some(53555), None),
+    (accepted, min_const_fn, "1.31.0", Some(53555)),
     /// The smallest useful subset of const generics.
-    (accepted, min_const_generics, "1.51.0", Some(74878), None),
+    (accepted, min_const_generics, "1.51.0", Some(74878)),
     /// Allows calling `const unsafe fn` inside `unsafe` blocks in `const fn` functions.
-    (accepted, min_const_unsafe_fn, "1.33.0", Some(55607), None),
+    (accepted, min_const_unsafe_fn, "1.33.0", Some(55607)),
     /// Allows using `Self` and associated types in struct expressions and patterns.
-    (accepted, more_struct_aliases, "1.16.0", Some(37544), None),
+    (accepted, more_struct_aliases, "1.16.0", Some(37544)),
     /// Allows using the MOVBE target feature.
-    (accepted, movbe_target_feature, "1.70.0", Some(44839), None),
+    (accepted, movbe_target_feature, "1.70.0", Some(44839)),
     /// Allows patterns with concurrent by-move and by-ref bindings.
     /// For example, you can write `Foo(a, ref b)` where `a` is by-move and `b` is by-ref.
-    (accepted, move_ref_pattern, "1.49.0", Some(68354), None),
+    (accepted, move_ref_pattern, "1.49.0", Some(68354)),
     /// Allows specifying modifiers in the link attribute: `#[link(modifiers = "...")]`
-    (accepted, native_link_modifiers, "1.61.0", Some(81490), None),
+    (accepted, native_link_modifiers, "1.61.0", Some(81490)),
     /// Allows specifying the bundle link modifier
-    (accepted, native_link_modifiers_bundle, "1.63.0", Some(81490), None),
+    (accepted, native_link_modifiers_bundle, "1.63.0", Some(81490)),
     /// Allows specifying the verbatim link modifier
-    (accepted, native_link_modifiers_verbatim, "1.67.0", Some(81490), None),
+    (accepted, native_link_modifiers_verbatim, "1.67.0", Some(81490)),
     /// Allows specifying the whole-archive link modifier
-    (accepted, native_link_modifiers_whole_archive, "1.61.0", Some(81490), None),
+    (accepted, native_link_modifiers_whole_archive, "1.61.0", Some(81490)),
     /// Allows using non lexical lifetimes (RFC 2094).
-    (accepted, nll, "1.63.0", Some(43234), None),
+    (accepted, nll, "1.63.0", Some(43234)),
     /// Allows using `#![no_std]`.
-    (accepted, no_std, "1.6.0", None, None),
+    (accepted, no_std, "1.6.0", None),
     /// Allows defining identifiers beyond ASCII.
-    (accepted, non_ascii_idents, "1.53.0", Some(55467), None),
+    (accepted, non_ascii_idents, "1.53.0", Some(55467)),
     /// Allows future-proofing enums/structs with the `#[non_exhaustive]` attribute (RFC 2008).
-    (accepted, non_exhaustive, "1.40.0", Some(44109), None),
+    (accepted, non_exhaustive, "1.40.0", Some(44109)),
     /// Allows `foo.rs` as an alternative to `foo/mod.rs`.
-    (accepted, non_modrs_mods, "1.30.0", Some(44660), None),
+    (accepted, non_modrs_mods, "1.30.0", Some(44660)),
     /// Allows the use of or-patterns (e.g., `0 | 1`).
-    (accepted, or_patterns, "1.53.0", Some(54883), None),
+    (accepted, or_patterns, "1.53.0", Some(54883)),
     /// Allows using `+bundle,+whole-archive` link modifiers with native libs.
-    (accepted, packed_bundled_libs, "1.74.0", Some(108081), None),
+    (accepted, packed_bundled_libs, "1.74.0", Some(108081)),
     /// Allows annotating functions conforming to `fn(&PanicInfo) -> !` with `#[panic_handler]`.
     /// This defines the behavior of panics.
-    (accepted, panic_handler, "1.30.0", Some(44489), None),
+    (accepted, panic_handler, "1.30.0", Some(44489)),
     /// Allows attributes in formal function parameters.
-    (accepted, param_attrs, "1.39.0", Some(60406), None),
+    (accepted, param_attrs, "1.39.0", Some(60406)),
     /// Allows parentheses in patterns.
-    (accepted, pattern_parentheses, "1.31.0", Some(51087), None),
+    (accepted, pattern_parentheses, "1.31.0", Some(51087)),
     /// Allows procedural macros in `proc-macro` crates.
-    (accepted, proc_macro, "1.29.0", Some(38356), None),
+    (accepted, proc_macro, "1.29.0", Some(38356)),
     /// Allows multi-segment paths in attributes and derives.
-    (accepted, proc_macro_path_invoc, "1.30.0", Some(38356), None),
+    (accepted, proc_macro_path_invoc, "1.30.0", Some(38356)),
     /// Allows `pub(restricted)` visibilities (RFC 1422).
-    (accepted, pub_restricted, "1.18.0", Some(32409), None),
+    (accepted, pub_restricted, "1.18.0", Some(32409)),
     /// Allows use of the postfix `?` operator in expressions.
-    (accepted, question_mark, "1.13.0", Some(31436), None),
+    (accepted, question_mark, "1.13.0", Some(31436)),
     /// Allows the use of raw-dylibs (RFC 2627).
-    (accepted, raw_dylib, "1.71.0", Some(58713), None),
+    (accepted, raw_dylib, "1.71.0", Some(58713)),
     /// Allows keywords to be escaped for use as identifiers.
-    (accepted, raw_identifiers, "1.30.0", Some(48589), None),
+    (accepted, raw_identifiers, "1.30.0", Some(48589)),
     /// Allows relaxing the coherence rules such that
     /// `impl<T> ForeignTrait<LocalType> for ForeignType<T>` is permitted.
-    (accepted, re_rebalance_coherence, "1.41.0", Some(55437), None),
+    (accepted, re_rebalance_coherence, "1.41.0", Some(55437)),
     /// Allows numeric fields in struct expressions and patterns.
-    (accepted, relaxed_adts, "1.19.0", Some(35626), None),
+    (accepted, relaxed_adts, "1.19.0", Some(35626)),
     /// Lessens the requirements for structs to implement `Unsize`.
-    (accepted, relaxed_struct_unsize, "1.58.0", Some(81793), None),
+    (accepted, relaxed_struct_unsize, "1.58.0", Some(81793)),
     /// Allows `repr(align(16))` struct attribute (RFC 1358).
-    (accepted, repr_align, "1.25.0", Some(33626), None),
+    (accepted, repr_align, "1.25.0", Some(33626)),
     /// Allows using `#[repr(align(X))]` on enums with equivalent semantics
     /// to wrapping an enum in a wrapper struct with `#[repr(align(X))]`.
-    (accepted, repr_align_enum, "1.37.0", Some(57996), None),
+    (accepted, repr_align_enum, "1.37.0", Some(57996)),
     /// Allows `#[repr(packed(N))]` attribute on structs.
-    (accepted, repr_packed, "1.33.0", Some(33158), None),
+    (accepted, repr_packed, "1.33.0", Some(33158)),
     /// Allows `#[repr(transparent)]` attribute on newtype structs.
-    (accepted, repr_transparent, "1.28.0", Some(43036), None),
+    (accepted, repr_transparent, "1.28.0", Some(43036)),
     /// Allows return-position `impl Trait` in traits.
-    (accepted, return_position_impl_trait_in_trait, "1.75.0", Some(91611), None),
+    (accepted, return_position_impl_trait_in_trait, "1.75.0", Some(91611)),
     /// Allows code like `let x: &'static u32 = &42` to work (RFC 1414).
-    (accepted, rvalue_static_promotion, "1.21.0", Some(38865), None),
+    (accepted, rvalue_static_promotion, "1.21.0", Some(38865)),
     /// Allows `Self` in type definitions (RFC 2300).
-    (accepted, self_in_typedefs, "1.32.0", Some(49303), None),
+    (accepted, self_in_typedefs, "1.32.0", Some(49303)),
     /// Allows `Self` struct constructor (RFC 2302).
-    (accepted, self_struct_ctor, "1.32.0", Some(51994), None),
+    (accepted, self_struct_ctor, "1.32.0", Some(51994)),
     /// Allows using subslice patterns, `[a, .., b]` and `[a, xs @ .., b]`.
-    (accepted, slice_patterns, "1.42.0", Some(62254), None),
+    (accepted, slice_patterns, "1.42.0", Some(62254)),
     /// Allows use of `&foo[a..b]` as a slicing syntax.
-    (accepted, slicing_syntax, "1.0.0", None, None),
+    (accepted, slicing_syntax, "1.0.0", None),
     /// Allows elision of `'static` lifetimes in `static`s and `const`s.
-    (accepted, static_in_const, "1.17.0", Some(35897), None),
+    (accepted, static_in_const, "1.17.0", Some(35897)),
     /// Allows the definition recursive static items.
-    (accepted, static_recursion, "1.17.0", Some(29719), None),
+    (accepted, static_recursion, "1.17.0", Some(29719)),
     /// Allows attributes on struct literal fields.
-    (accepted, struct_field_attributes, "1.20.0", Some(38814), None),
+    (accepted, struct_field_attributes, "1.20.0", Some(38814)),
     /// Allows struct variants `Foo { baz: u8, .. }` in enums (RFC 418).
-    (accepted, struct_variant, "1.0.0", None, None),
+    (accepted, struct_variant, "1.0.0", None),
     /// Allows `#[target_feature(...)]`.
-    (accepted, target_feature, "1.27.0", None, None),
+    (accepted, target_feature, "1.27.0", None),
     /// Allows `fn main()` with return types which implements `Termination` (RFC 1937).
-    (accepted, termination_trait, "1.26.0", Some(43301), None),
+    (accepted, termination_trait, "1.26.0", Some(43301)),
     /// Allows `#[test]` functions where the return type implements `Termination` (RFC 1937).
-    (accepted, termination_trait_test, "1.27.0", Some(48854), None),
+    (accepted, termination_trait_test, "1.27.0", Some(48854)),
     /// Allows attributes scoped to tools.
-    (accepted, tool_attributes, "1.30.0", Some(44690), None),
+    (accepted, tool_attributes, "1.30.0", Some(44690)),
     /// Allows scoped lints.
-    (accepted, tool_lints, "1.31.0", Some(44690), None),
+    (accepted, tool_lints, "1.31.0", Some(44690)),
     /// Allows `#[track_caller]` to be used which provides
     /// accurate caller location reporting during panic (RFC 2091).
-    (accepted, track_caller, "1.46.0", Some(47809), None),
+    (accepted, track_caller, "1.46.0", Some(47809)),
     /// Allows #[repr(transparent)] on univariant enums (RFC 2645).
-    (accepted, transparent_enums, "1.42.0", Some(60405), None),
+    (accepted, transparent_enums, "1.42.0", Some(60405)),
     /// Allows indexing tuples.
-    (accepted, tuple_indexing, "1.0.0", None, None),
+    (accepted, tuple_indexing, "1.0.0", None),
     /// Allows paths to enum variants on type aliases including `Self`.
-    (accepted, type_alias_enum_variants, "1.37.0", Some(49683), None),
+    (accepted, type_alias_enum_variants, "1.37.0", Some(49683)),
     /// Allows macros to appear in the type position.
-    (accepted, type_macros, "1.13.0", Some(27245), None),
+    (accepted, type_macros, "1.13.0", Some(27245)),
     /// Allows `const _: TYPE = VALUE`.
-    (accepted, underscore_const_names, "1.37.0", Some(54912), None),
+    (accepted, underscore_const_names, "1.37.0", Some(54912)),
     /// Allows `use path as _;` and `extern crate c as _;`.
-    (accepted, underscore_imports, "1.33.0", Some(48216), None),
+    (accepted, underscore_imports, "1.33.0", Some(48216)),
     /// Allows `'_` placeholder lifetimes.
-    (accepted, underscore_lifetimes, "1.26.0", Some(44524), None),
+    (accepted, underscore_lifetimes, "1.26.0", Some(44524)),
     /// Allows `use x::y;` to search `x` in the current scope.
-    (accepted, uniform_paths, "1.32.0", Some(53130), None),
+    (accepted, uniform_paths, "1.32.0", Some(53130)),
     /// Allows `impl Trait` in function arguments.
-    (accepted, universal_impl_trait, "1.26.0", Some(34511), None),
+    (accepted, universal_impl_trait, "1.26.0", Some(34511)),
     /// Allows arbitrary delimited token streams in non-macro attributes.
-    (accepted, unrestricted_attribute_tokens, "1.34.0", Some(55208), None),
+    (accepted, unrestricted_attribute_tokens, "1.34.0", Some(55208)),
     /// The `unsafe_op_in_unsafe_fn` lint (allowed by default): no longer treat an unsafe function as an unsafe block.
-    (accepted, unsafe_block_in_unsafe_fn, "1.52.0", Some(71668), None),
+    (accepted, unsafe_block_in_unsafe_fn, "1.52.0", Some(71668)),
     /// Allows importing and reexporting macros with `use`,
     /// enables macro modularization in general.
-    (accepted, use_extern_macros, "1.30.0", Some(35896), None),
+    (accepted, use_extern_macros, "1.30.0", Some(35896)),
     /// Allows nested groups in `use` items (RFC 2128).
-    (accepted, use_nested_groups, "1.25.0", Some(44494), None),
+    (accepted, use_nested_groups, "1.25.0", Some(44494)),
     /// Allows `#[used]` to preserve symbols (see llvm.compiler.used).
-    (accepted, used, "1.30.0", Some(40289), None),
+    (accepted, used, "1.30.0", Some(40289)),
     /// Allows the use of `while let` expressions.
-    (accepted, while_let, "1.0.0", None, None),
+    (accepted, while_let, "1.0.0", None),
     /// Allows `#![windows_subsystem]`.
-    (accepted, windows_subsystem, "1.18.0", Some(37499), None),
+    (accepted, windows_subsystem, "1.18.0", Some(37499)),
     // !!!!    !!!!    !!!!    !!!!   !!!!    !!!!    !!!!    !!!!    !!!!    !!!!    !!!!
     // Features are listed in alphabetical order. Tidy will fail if you don't keep it this way.
     // !!!!    !!!!    !!!!    !!!!   !!!!    !!!!    !!!!    !!!!    !!!!    !!!!    !!!!
diff --git a/compiler/rustc_feature/src/builtin_attrs.rs b/compiler/rustc_feature/src/builtin_attrs.rs
index 214de3ca4..5523543cd 100644
--- a/compiler/rustc_feature/src/builtin_attrs.rs
+++ b/compiler/rustc_feature/src/builtin_attrs.rs
@@ -719,12 +719,6 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
         and it is only intended to be used in `alloc`."
     ),
 
-    rustc_attr!(
-        rustc_host, AttributeType::Normal, template!(Word), ErrorFollowing,
-        "#[rustc_host] annotates const generic parameters as the `host` effect param, \
-        and it is only intended for internal use and as a desugaring."
-    ),
-
     BuiltinAttribute {
         name: sym::rustc_diagnostic_item,
         // FIXME: This can be `true` once we always use `tcx.is_diagnostic_item`.
@@ -813,7 +807,7 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
     rustc_attr!(TEST, rustc_regions, Normal, template!(Word), WarnFollowing),
     rustc_attr!(
         TEST, rustc_error, Normal,
-        template!(Word, List: "delay_span_bug_from_inside_query"), WarnFollowingWordOnly
+        template!(Word, List: "span_delayed_bug_from_inside_query"), WarnFollowingWordOnly
     ),
     rustc_attr!(TEST, rustc_dump_user_args, Normal, template!(Word), WarnFollowing),
     rustc_attr!(TEST, rustc_evaluate_where_clauses, Normal, template!(Word), WarnFollowing),
diff --git a/compiler/rustc_feature/src/lib.rs b/compiler/rustc_feature/src/lib.rs
index 070234df9..cd1d9b13d 100644
--- a/compiler/rustc_feature/src/lib.rs
+++ b/compiler/rustc_feature/src/lib.rs
@@ -11,9 +11,9 @@
 //! even if it is stabilized or removed, *do not remove it*. Instead, move the
 //! symbol to the `accepted` or `removed` modules respectively.
 
-#![cfg_attr(not(bootstrap), allow(internal_features))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
+#![allow(internal_features)]
+#![feature(rustdoc_internals)]
+#![doc(rust_logo)]
 #![feature(lazy_cell)]
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
@@ -26,7 +26,7 @@ mod unstable;
 #[cfg(test)]
 mod tests;
 
-use rustc_span::{edition::Edition, symbol::Symbol};
+use rustc_span::symbol::Symbol;
 use std::num::NonZeroU32;
 
 #[derive(Debug, Clone)]
@@ -34,7 +34,6 @@ pub struct Feature {
     pub name: Symbol,
     pub since: &'static str,
     issue: Option<NonZeroU32>,
-    pub edition: Option<Edition>,
 }
 
 #[derive(Copy, Clone, Debug)]
diff --git a/compiler/rustc_feature/src/removed.rs b/compiler/rustc_feature/src/removed.rs
index 4385e745b..67ee53d8a 100644
--- a/compiler/rustc_feature/src/removed.rs
+++ b/compiler/rustc_feature/src/removed.rs
@@ -10,7 +10,7 @@ pub struct RemovedFeature {
 
 macro_rules! declare_features {
     ($(
-        $(#[doc = $doc:tt])* (removed, $feature:ident, $ver:expr, $issue:expr, None, $reason:expr),
+        $(#[doc = $doc:tt])* (removed, $feature:ident, $ver:expr, $issue:expr, $reason:expr),
     )+) => {
         /// Formerly unstable features that have now been removed.
         pub const REMOVED_FEATURES: &[RemovedFeature] = &[
@@ -19,7 +19,6 @@ macro_rules! declare_features {
                     name: sym::$feature,
                     since: $ver,
                     issue: to_nonzero($issue),
-                    edition: None,
                 },
                 reason: $reason
             }),+
@@ -33,173 +32,178 @@ declare_features! (
     // feature-group-start: removed features
     // -------------------------------------------------------------------------
 
-    (removed, advanced_slice_patterns, "1.0.0", Some(62254), None,
+    (removed, advanced_slice_patterns, "1.0.0", Some(62254),
      Some("merged into `#![feature(slice_patterns)]`")),
-    (removed, allocator, "1.0.0", None, None, None),
+    (removed, allocator, "1.0.0", None, None),
     /// Allows a test to fail without failing the whole suite.
-    (removed, allow_fail, "1.19.0", Some(46488), None, Some("removed due to no clear use cases")),
-    (removed, await_macro, "1.38.0", Some(50547), None,
+    (removed, allow_fail, "1.19.0", Some(46488), Some("removed due to no clear use cases")),
+    (removed, await_macro, "1.38.0", Some(50547),
      Some("subsumed by `.await` syntax")),
     /// Allows using the `box $expr` syntax.
-    (removed, box_syntax, "1.70.0", Some(49733), None, Some("replaced with `#[rustc_box]`")),
+    (removed, box_syntax, "1.70.0", Some(49733), Some("replaced with `#[rustc_box]`")),
     /// Allows capturing disjoint fields in a closure/coroutine (RFC 2229).
-    (removed, capture_disjoint_fields, "1.49.0", Some(53488), None, Some("stabilized in Rust 2021")),
+    (removed, capture_disjoint_fields, "1.49.0", Some(53488), Some("stabilized in Rust 2021")),
     /// Allows comparing raw pointers during const eval.
-    (removed, const_compare_raw_pointers, "1.46.0", Some(53020), None,
+    (removed, const_compare_raw_pointers, "1.46.0", Some(53020),
      Some("cannot be allowed in const eval in any meaningful way")),
     /// Allows limiting the evaluation steps of const expressions
-    (removed, const_eval_limit, "1.43.0", Some(67217), None, Some("removed the limit entirely")),
+    (removed, const_eval_limit, "1.43.0", Some(67217), Some("removed the limit entirely")),
     /// Allows non-trivial generic constants which have to be manually propagated upwards.
-    (removed, const_evaluatable_checked, "1.48.0", Some(76560), None, Some("renamed to `generic_const_exprs`")),
+    (removed, const_evaluatable_checked, "1.48.0", Some(76560), Some("renamed to `generic_const_exprs`")),
     /// Allows the definition of `const` functions with some advanced features.
-    (removed, const_fn, "1.54.0", Some(57563), None,
+    (removed, const_fn, "1.54.0", Some(57563),
      Some("split into finer-grained feature gates")),
     /// Allows const generic types (e.g. `struct Foo<const N: usize>(...);`).
-    (removed, const_generics, "1.34.0", Some(44580), None,
+    (removed, const_generics, "1.34.0", Some(44580),
      Some("removed in favor of `#![feature(adt_const_params)]` and `#![feature(generic_const_exprs)]`")),
     /// Allows `[x; N]` where `x` is a constant (RFC 2203).
-    (removed, const_in_array_repeat_expressions,  "1.37.0", Some(49147), None,
+    (removed, const_in_array_repeat_expressions,  "1.37.0", Some(49147),
      Some("removed due to causing promotable bugs")),
     /// Allows casting raw pointers to `usize` during const eval.
-    (removed, const_raw_ptr_to_usize_cast, "1.55.0", Some(51910), None,
+    (removed, const_raw_ptr_to_usize_cast, "1.55.0", Some(51910),
      Some("at compile-time, pointers do not have an integer value, so these casts cannot be properly supported")),
     /// Allows `T: ?const Trait` syntax in bounds.
-    (removed, const_trait_bound_opt_out, "1.42.0", Some(67794), None,
+    (removed, const_trait_bound_opt_out, "1.42.0", Some(67794),
      Some("Removed in favor of `~const` bound in #![feature(const_trait_impl)]")),
     /// Allows using `crate` as visibility modifier, synonymous with `pub(crate)`.
-    (removed, crate_visibility_modifier, "1.63.0", Some(53120), None, Some("removed in favor of `pub(crate)`")),
+    (removed, crate_visibility_modifier, "1.63.0", Some(53120), Some("removed in favor of `pub(crate)`")),
     /// Allows using custom attributes (RFC 572).
-    (removed, custom_attribute, "1.0.0", Some(29642), None,
+    (removed, custom_attribute, "1.0.0", Some(29642),
      Some("removed in favor of `#![register_tool]` and `#![register_attr]`")),
     /// Allows the use of `#[derive(Anything)]` as sugar for `#[derive_Anything]`.
-    (removed, custom_derive, "1.32.0", Some(29644), None,
+    (removed, custom_derive, "1.32.0", Some(29644),
      Some("subsumed by `#[proc_macro_derive]`")),
     /// Allows using `#[doc(keyword = "...")]`.
-    (removed, doc_keyword, "1.28.0", Some(51315), None,
+    (removed, doc_keyword, "1.28.0", Some(51315),
      Some("merged into `#![feature(rustdoc_internals)]`")),
     /// Allows using `doc(primitive)` without a future-incompat warning.
-    (removed, doc_primitive, "1.56.0", Some(88070), None,
+    (removed, doc_primitive, "1.56.0", Some(88070),
      Some("merged into `#![feature(rustdoc_internals)]`")),
     /// Allows `#[doc(spotlight)]`.
     /// The attribute was renamed to `#[doc(notable_trait)]`
     /// and the feature to `doc_notable_trait`.
-    (removed, doc_spotlight, "1.22.0", Some(45040), None,
+    (removed, doc_spotlight, "1.22.0", Some(45040),
      Some("renamed to `doc_notable_trait`")),
     /// Allows using `#[unsafe_destructor_blind_to_params]` (RFC 1238).
-    (removed, dropck_parametricity, "1.38.0", Some(28498), None, None),
+    (removed, dropck_parametricity, "1.38.0", Some(28498), None),
     /// Allows defining `existential type`s.
-    (removed, existential_type, "1.38.0", Some(63063), None,
+    (removed, existential_type, "1.38.0", Some(63063),
      Some("removed in favor of `#![feature(type_alias_impl_trait)]`")),
     /// Paths of the form: `extern::foo::bar`
-    (removed, extern_in_paths, "1.33.0", Some(55600), None,
+    (removed, extern_in_paths, "1.33.0", Some(55600),
      Some("subsumed by `::foo::bar` paths")),
     /// Allows `#[doc(include = "some-file")]`.
-    (removed, external_doc, "1.54.0", Some(44732), None,
+    (removed, external_doc, "1.54.0", Some(44732),
      Some("use #[doc = include_str!(\"filename\")] instead, which handles macro invocations")),
     /// Allows generators to be cloned.
-    (removed, generator_clone, "1.65.0", Some(95360), None, Some("renamed to `coroutine_clone`")),
+    (removed, generator_clone, "1.65.0", Some(95360), Some("renamed to `coroutine_clone`")),
     /// Allows defining generators.
-    (removed, generators, "1.21.0", Some(43122), None, Some("renamed to `coroutines`")),
+    (removed, generators, "1.21.0", Some(43122), Some("renamed to `coroutines`")),
     /// Allows `impl Trait` in bindings (`let`, `const`, `static`).
-    (removed, impl_trait_in_bindings, "1.55.0", Some(63065), None,
+    (removed, impl_trait_in_bindings, "1.55.0", Some(63065),
      Some("the implementation was not maintainable, the feature may get reintroduced once the current refactorings are done")),
-    (removed, import_shadowing, "1.0.0", None, None, None),
+    (removed, import_shadowing, "1.0.0", None, None),
     /// Allows in-band quantification of lifetime bindings (e.g., `fn foo(x: &'a u8) -> &'a u8`).
-    (removed, in_band_lifetimes, "1.23.0", Some(44524), None,
+    (removed, in_band_lifetimes, "1.23.0", Some(44524),
      Some("removed due to unsolved ergonomic questions and added lifetime resolution complexity")),
     /// Allows inferring `'static` outlives requirements (RFC 2093).
-    (removed, infer_static_outlives_requirements, "1.63.0", Some(54185), None,
+    (removed, infer_static_outlives_requirements, "1.63.0", Some(54185),
      Some("removed as it caused some confusion and discussion was inactive for years")),
     /// Lazily evaluate constants. This allows constants to depend on type parameters.
-    (removed, lazy_normalization_consts, "1.46.0", Some(72219), None, Some("superseded by `generic_const_exprs`")),
+    (removed, lazy_normalization_consts, "1.46.0", Some(72219), Some("superseded by `generic_const_exprs`")),
     /// Allows using the `#[link_args]` attribute.
-    (removed, link_args, "1.53.0", Some(29596), None,
+    (removed, link_args, "1.53.0", Some(29596),
      Some("removed in favor of using `-C link-arg=ARG` on command line, \
            which is available from cargo build scripts with `cargo:rustc-link-arg` now")),
-    (removed, macro_reexport, "1.0.0", Some(29638), None,
+    (removed, macro_reexport, "1.0.0", Some(29638),
      Some("subsumed by `pub use`")),
     /// Allows using `#[main]` to replace the entrypoint `#[lang = "start"]` calls.
-    (removed, main, "1.53.0", Some(29634), None, None),
-    (removed, managed_boxes, "1.0.0", None, None, None),
+    (removed, main, "1.53.0", Some(29634), None),
+    (removed, managed_boxes, "1.0.0", None, None),
     /// Allows the use of type alias impl trait in function return positions
-    (removed, min_type_alias_impl_trait, "1.56.0", Some(63063), None,
+    (removed, min_type_alias_impl_trait, "1.56.0", Some(63063),
      Some("removed in favor of full type_alias_impl_trait")),
-    (removed, needs_allocator, "1.4.0", Some(27389), None,
+    (removed, needs_allocator, "1.4.0", Some(27389),
      Some("subsumed by `#![feature(allocator_internals)]`")),
     /// Allows use of unary negate on unsigned integers, e.g., -e for e: u8
-    (removed, negate_unsigned, "1.0.0", Some(29645), None, None),
+    (removed, negate_unsigned, "1.0.0", Some(29645), None),
     /// Allows `#[no_coverage]` on functions.
     /// The feature was renamed to `coverage_attribute` and the attribute to `#[coverage(on|off)]`
-    (removed, no_coverage, "1.74.0", Some(84605), None, Some("renamed to `coverage_attribute`")),
+    (removed, no_coverage, "1.74.0", Some(84605), Some("renamed to `coverage_attribute`")),
     /// Allows `#[no_debug]`.
-    (removed, no_debug, "1.43.0", Some(29721), None, Some("removed due to lack of demand")),
+    (removed, no_debug, "1.43.0", Some(29721), Some("removed due to lack of demand")),
     /// Note: this feature was previously recorded in a separate
     /// `STABLE_REMOVED` list because it, uniquely, was once stable but was
     /// then removed. But there was no utility storing it separately, so now
     /// it's in this list.
-    (removed, no_stack_check, "1.0.0", None, None, None),
+    (removed, no_stack_check, "1.0.0", None, None),
     /// Allows using `#[on_unimplemented(..)]` on traits.
     /// (Moved to `rustc_attrs`.)
-    (removed, on_unimplemented, "1.40.0", None, None, None),
+    (removed, on_unimplemented, "1.40.0", None, None),
     /// A way to temporarily opt out of opt-in copy. This will *never* be accepted.
-    (removed, opt_out_copy, "1.0.0", None, None, None),
+    (removed, opt_out_copy, "1.0.0", None, None),
     /// Allows features specific to OIBIT (now called auto traits).
     /// Renamed to `auto_traits`.
-    (removed, optin_builtin_traits, "1.0.0", Some(13231), None,
+    (removed, optin_builtin_traits, "1.0.0", Some(13231),
      Some("renamed to `auto_traits`")),
     /// Allows overlapping impls of marker traits.
-    (removed, overlapping_marker_traits, "1.42.0", Some(29864), None,
+    (removed, overlapping_marker_traits, "1.42.0", Some(29864),
      Some("removed in favor of `#![feature(marker_trait_attr)]`")),
-    (removed, panic_implementation, "1.28.0", Some(44489), None,
+    (removed, panic_implementation, "1.28.0", Some(44489),
      Some("subsumed by `#[panic_handler]`")),
     /// Allows using `#![plugin(myplugin)]`.
-    (removed, plugin, "1.75.0", Some(29597), None,
+    (removed, plugin, "1.75.0", Some(29597),
      Some("plugins are no longer supported")),
     /// Allows using `#[plugin_registrar]` on functions.
-    (removed, plugin_registrar, "1.54.0", Some(29597), None,
+    (removed, plugin_registrar, "1.54.0", Some(29597),
      Some("plugins are no longer supported")),
-    (removed, proc_macro_expr, "1.27.0", Some(54727), None,
+    /// Allows exhaustive integer pattern matching with `usize::MAX`/`isize::MIN`/`isize::MAX`.
+    (removed, precise_pointer_size_matching, "1.32.0", Some(56354),
+     Some("removed in favor of half-open ranges")),
+    (removed, proc_macro_expr, "1.27.0", Some(54727),
      Some("subsumed by `#![feature(proc_macro_hygiene)]`")),
-    (removed, proc_macro_gen, "1.27.0", Some(54727), None,
+    (removed, proc_macro_gen, "1.27.0", Some(54727),
      Some("subsumed by `#![feature(proc_macro_hygiene)]`")),
-    (removed, proc_macro_mod, "1.27.0", Some(54727), None,
+    (removed, proc_macro_mod, "1.27.0", Some(54727),
      Some("subsumed by `#![feature(proc_macro_hygiene)]`")),
-    (removed, proc_macro_non_items, "1.27.0", Some(54727), None,
+    (removed, proc_macro_non_items, "1.27.0", Some(54727),
      Some("subsumed by `#![feature(proc_macro_hygiene)]`")),
-    (removed, pub_macro_rules, "1.53.0", Some(78855), None,
+    (removed, pub_macro_rules, "1.53.0", Some(78855),
      Some("removed due to being incomplete, in particular it does not work across crates")),
-    (removed, pushpop_unsafe, "1.2.0", None, None, None),
-    (removed, quad_precision_float, "1.0.0", None, None, None),
-    (removed, quote, "1.33.0", Some(29601), None, None),
-    (removed, reflect, "1.0.0", Some(27749), None, None),
+    (removed, pushpop_unsafe, "1.2.0", None, None),
+    (removed, quad_precision_float, "1.0.0", None, None),
+    (removed, quote, "1.33.0", Some(29601), None),
+    (removed, reflect, "1.0.0", Some(27749), None),
     /// Allows using the `#[register_attr]` attribute.
-    (removed, register_attr, "1.65.0", Some(66080), None,
+    (removed, register_attr, "1.65.0", Some(66080),
      Some("removed in favor of `#![register_tool]`")),
+    (removed, rust_2018_preview, "1.76.0", None,
+     Some("2018 Edition preview is no longer relevant")),
     /// Allows using the macros:
     /// + `__diagnostic_used`
     /// + `__register_diagnostic`
     /// +`__build_diagnostic_array`
-    (removed, rustc_diagnostic_macros, "1.38.0", None, None, None),
+    (removed, rustc_diagnostic_macros, "1.38.0", None, None),
     /// Allows identifying crates that contain sanitizer runtimes.
-    (removed, sanitizer_runtime, "1.17.0", None, None, None),
-    (removed, simd, "1.0.0", Some(27731), None,
+    (removed, sanitizer_runtime, "1.17.0", None, None),
+    (removed, simd, "1.0.0", Some(27731),
      Some("removed in favor of `#[repr(simd)]`")),
     /// Allows `#[link(kind = "static-nobundle", ...)]`.
-    (removed, static_nobundle, "1.16.0", Some(37403), None,
+    (removed, static_nobundle, "1.16.0", Some(37403),
      Some(r#"subsumed by `#[link(kind = "static", modifiers = "-bundle", ...)]`"#)),
-    (removed, struct_inherit, "1.0.0", None, None, None),
-    (removed, test_removed_feature, "1.0.0", None, None, None),
+    (removed, struct_inherit, "1.0.0", None, None),
+    (removed, test_removed_feature, "1.0.0", None, None),
     /// Allows using items which are missing stability attributes
-    (removed, unmarked_api, "1.0.0", None, None, None),
-    (removed, unsafe_no_drop_flag, "1.0.0", None, None, None),
+    (removed, unmarked_api, "1.0.0", None, None),
+    (removed, unsafe_no_drop_flag, "1.0.0", None, None),
     /// Allows `union` fields that don't implement `Copy` as long as they don't have any drop glue.
-    (removed, untagged_unions, "1.13.0", Some(55149), None,
+    (removed, untagged_unions, "1.13.0", Some(55149),
      Some("unions with `Copy` and `ManuallyDrop` fields are stable; there is no intent to stabilize more")),
     /// Allows `#[unwind(..)]`.
     ///
     /// Permits specifying whether a function should permit unwinding or abort on unwind.
-    (removed, unwind_attributes, "1.56.0", Some(58760), None, Some("use the C-unwind ABI instead")),
-    (removed, visible_private_types, "1.0.0", None, None, None),
+    (removed, unwind_attributes, "1.56.0", Some(58760), Some("use the C-unwind ABI instead")),
+    (removed, visible_private_types, "1.0.0", None, None),
     // !!!!    !!!!    !!!!    !!!!   !!!!    !!!!    !!!!    !!!!    !!!!    !!!!    !!!!
     // Features are listed in alphabetical order. Tidy will fail if you don't keep it this way.
     // !!!!    !!!!    !!!!    !!!!   !!!!    !!!!    !!!!    !!!!    !!!!    !!!!    !!!!
diff --git a/compiler/rustc_feature/src/unstable.rs b/compiler/rustc_feature/src/unstable.rs
index b11b190bd..eadb19f6f 100644
--- a/compiler/rustc_feature/src/unstable.rs
+++ b/compiler/rustc_feature/src/unstable.rs
@@ -3,7 +3,6 @@
 use super::{to_nonzero, Feature};
 
 use rustc_data_structures::fx::FxHashSet;
-use rustc_span::edition::Edition;
 use rustc_span::symbol::{sym, Symbol};
 use rustc_span::Span;
 
@@ -33,7 +32,7 @@ macro_rules! status_to_enum {
 
 macro_rules! declare_features {
     ($(
-        $(#[doc = $doc:tt])* ($status:ident, $feature:ident, $ver:expr, $issue:expr, $edition:expr),
+        $(#[doc = $doc:tt])* ($status:ident, $feature:ident, $ver:expr, $issue:expr),
     )+) => {
         /// Unstable language features that are being implemented or being
         /// considered for acceptance (stabilization) or removal.
@@ -43,19 +42,22 @@ macro_rules! declare_features {
                     name: sym::$feature,
                     since: $ver,
                     issue: to_nonzero($issue),
-                    edition: $edition,
                 },
                 // Sets this feature's corresponding bool within `features`.
                 set_enabled: |features| features.$feature = true,
             }),+
         ];
 
+        const NUM_FEATURES: usize = UNSTABLE_FEATURES.len();
+
         /// A set of features to be used by later passes.
         #[derive(Clone, Default, Debug)]
         pub struct Features {
             /// `#![feature]` attrs for language features, for error reporting.
+            /// "declared" here means that the feature is actually enabled in the current crate.
             pub declared_lang_features: Vec<(Symbol, Span, Option<Symbol>)>,
             /// `#![feature]` attrs for non-language (library) features.
+            /// "declared" here means that the feature is actually enabled in the current crate.
             pub declared_lib_features: Vec<(Symbol, Span)>,
             /// `declared_lang_features` + `declared_lib_features`.
             pub declared_features: FxHashSet<Symbol>,
@@ -82,8 +84,14 @@ macro_rules! declare_features {
                 self.declared_features.insert(symbol);
             }
 
-            pub fn walk_feature_fields(&self, mut f: impl FnMut(&str, bool)) {
-                $(f(stringify!($feature), self.$feature);)+
+            /// This is intended for hashing the set of active features.
+            ///
+            /// The expectation is that this produces much smaller code than other alternatives.
+            ///
+            /// Note that the total feature count is pretty small, so this is not a huge array.
+            #[inline]
+            pub fn all_features(&self) -> [u8; NUM_FEATURES] {
+                [$(self.$feature as u8),+]
             }
 
             /// Is the given feature explicitly declared, i.e. named in a
@@ -92,8 +100,7 @@ macro_rules! declare_features {
                 self.declared_features.contains(&feature)
             }
 
-            /// Is the given feature active, i.e. declared or automatically
-            /// enabled due to the edition?
+            /// Is the given feature active (enabled by the user)?
             ///
             /// Panics if the symbol doesn't correspond to a declared feature.
             pub fn active(&self, feature: Symbol) -> bool {
@@ -125,9 +132,18 @@ macro_rules! declare_features {
                     $(
                         sym::$feature => status_to_enum!($status) == FeatureStatus::Internal,
                     )*
-                    // Accepted/removed features aren't in this file but are never internal
-                    // (a removed feature might have been internal, but that's now irrelevant).
-                    _ if self.declared_features.contains(&feature) => false,
+                    _ if self.declared_features.contains(&feature) => {
+                        // This could be accepted/removed, or a libs feature.
+                        // Accepted/removed features aren't in this file but are never internal
+                        // (a removed feature might have been internal, but that's now irrelevant).
+                        // Libs features are internal if they end in `_internal` or `_internals`.
+                        // As a special exception we also consider `core_intrinsics` internal;
+                        // renaming that age-old feature is just not worth the hassle.
+                        // We just always test the name; it's not a big deal if we accidentally hit
+                        // an accepted/removed lang feature that way.
+                        let name = feature.as_str();
+                        name == "core_intrinsics" || name.ends_with("_internal") || name.ends_with("_internals")
+                    }
                     _ => panic!("`{}` was not listed in `declare_features`", feature),
                 }
             }
@@ -147,7 +163,7 @@ macro_rules! declare_features {
 // was set.
 //
 // Note that the features are grouped into internal/user-facing and then
-// sorted by version inside those groups. This is enforced with tidy.
+// sorted alphabetically inside those groups. This is enforced with tidy.
 //
 // N.B., `tools/tidy/src/features.rs` parses this information directly out of the
 // source, so take care when modifying it.
@@ -160,59 +176,56 @@ declare_features! (
     // no-tracking-issue-start
 
     /// Allows using the `unadjusted` ABI; perma-unstable.
-    (unstable, abi_unadjusted, "1.16.0", None, None),
+    (internal, abi_unadjusted, "1.16.0", None),
     /// Allows using the `vectorcall` ABI.
-    (unstable, abi_vectorcall, "1.7.0", None, None),
+    (unstable, abi_vectorcall, "1.7.0", None),
     /// Allows using `#![needs_allocator]`, an implementation detail of `#[global_allocator]`.
-    (internal, allocator_internals, "1.20.0", None, None),
+    (internal, allocator_internals, "1.20.0", None),
     /// Allows using `#[allow_internal_unsafe]`. This is an
     /// attribute on `macro_rules!` and can't use the attribute handling
     /// below (it has to be checked before expansion possibly makes
     /// macros disappear).
-    (internal, allow_internal_unsafe, "1.0.0", None, None),
+    (internal, allow_internal_unsafe, "1.0.0", None),
     /// Allows using `#[allow_internal_unstable]`. This is an
     /// attribute on `macro_rules!` and can't use the attribute handling
     /// below (it has to be checked before expansion possibly makes
     /// macros disappear).
-    (internal, allow_internal_unstable, "1.0.0", None, None),
+    (internal, allow_internal_unstable, "1.0.0", None),
     /// Allows using anonymous lifetimes in argument-position impl-trait.
-    (unstable, anonymous_lifetime_in_impl_trait, "1.63.0", None, None),
+    (unstable, anonymous_lifetime_in_impl_trait, "1.63.0", None),
     /// Allows identifying the `compiler_builtins` crate.
-    (internal, compiler_builtins, "1.13.0", None, None),
+    (internal, compiler_builtins, "1.13.0", None),
     /// Allows writing custom MIR
-    (internal, custom_mir, "1.65.0", None, None),
+    (internal, custom_mir, "1.65.0", None),
     /// Outputs useful `assert!` messages
-    (unstable, generic_assert, "1.63.0", None, None),
+    (unstable, generic_assert, "1.63.0", None),
     /// Allows using the `rust-intrinsic`'s "ABI".
-    (internal, intrinsics, "1.0.0", None, None),
+    (internal, intrinsics, "1.0.0", None),
     /// Allows using `#[lang = ".."]` attribute for linking items to special compiler logic.
-    (internal, lang_items, "1.0.0", None, None),
+    (internal, lang_items, "1.0.0", None),
+    /// Changes `impl Trait` to capture all lifetimes in scope.
+    (unstable, lifetime_capture_rules_2024, "1.76.0", None),
     /// Allows `#[link(..., cfg(..))]`; perma-unstable per #37406
-    (unstable, link_cfg, "1.14.0", None, None),
+    (unstable, link_cfg, "1.14.0", None),
     /// Allows the `multiple_supertrait_upcastable` lint.
-    (unstable, multiple_supertrait_upcastable, "1.69.0", None, None),
+    (unstable, multiple_supertrait_upcastable, "1.69.0", None),
     /// Allow negative trait bounds. This is an internal-only feature for testing the trait solver!
-    (incomplete, negative_bounds, "1.71.0", None, None),
+    (incomplete, negative_bounds, "1.71.0", None),
     /// Allows using `#[omit_gdb_pretty_printer_section]`.
-    (internal, omit_gdb_pretty_printer_section, "1.5.0", None, None),
+    (internal, omit_gdb_pretty_printer_section, "1.5.0", None),
     /// Allows using `#[prelude_import]` on glob `use` items.
-    (internal, prelude_import, "1.2.0", None, None),
+    (internal, prelude_import, "1.2.0", None),
     /// Used to identify crates that contain the profiler runtime.
-    (internal, profiler_runtime, "1.18.0", None, None),
+    (internal, profiler_runtime, "1.18.0", None),
     /// Allows using `rustc_*` attributes (RFC 572).
-    (internal, rustc_attrs, "1.0.0", None, None),
+    (internal, rustc_attrs, "1.0.0", None),
     /// Allows using the `#[stable]` and `#[unstable]` attributes.
-    (internal, staged_api, "1.0.0", None, None),
-    /// Added for testing E0705; perma-unstable.
-    (internal, test_2018_feature, "1.31.0", None, Some(Edition::Edition2018)),
+    (internal, staged_api, "1.0.0", None),
     /// Added for testing unstable lints; perma-unstable.
-    (internal, test_unstable_lint, "1.60.0", None, None),
-    /// Allows non-`unsafe` —and thus, unsound— access to `Pin` constructions.
-    /// Marked `internal` since perma-unstable and unsound.
-    (internal, unsafe_pin_internals, "1.60.0", None, None),
+    (internal, test_unstable_lint, "1.60.0", None),
     /// Use for stable + negative coherence and strict coherence depending on trait's
     /// rustc_strict_coherence value.
-    (unstable, with_negative_coherence, "1.60.0", None, None),
+    (unstable, with_negative_coherence, "1.60.0", None),
     // !!!!    !!!!    !!!!    !!!!   !!!!    !!!!    !!!!    !!!!    !!!!    !!!!    !!!!
     // Features are listed in alphabetical order. Tidy will fail if you don't keep it this way.
     // !!!!    !!!!    !!!!    !!!!   !!!!    !!!!    !!!!    !!!!    !!!!    !!!!    !!!!
@@ -228,44 +241,44 @@ declare_features! (
 
     /// Allows features specific to auto traits.
     /// Renamed from `optin_builtin_traits`.
-    (unstable, auto_traits, "1.50.0", Some(13231), None),
+    (unstable, auto_traits, "1.50.0", Some(13231)),
     /// Allows using `box` in patterns (RFC 469).
-    (unstable, box_patterns, "1.0.0", Some(29641), None),
+    (unstable, box_patterns, "1.0.0", Some(29641)),
     /// Allows `#[doc(notable_trait)]`.
     /// Renamed from `doc_spotlight`.
-    (unstable, doc_notable_trait, "1.52.0", Some(45040), None),
+    (unstable, doc_notable_trait, "1.52.0", Some(45040)),
     /// Allows using the `may_dangle` attribute (RFC 1327).
-    (unstable, dropck_eyepatch, "1.10.0", Some(34761), None),
+    (unstable, dropck_eyepatch, "1.10.0", Some(34761)),
     /// Allows using the `#[fundamental]` attribute.
-    (unstable, fundamental, "1.0.0", Some(29635), None),
+    (unstable, fundamental, "1.0.0", Some(29635)),
     /// Allows using `#[link_name="llvm.*"]`.
-    (internal, link_llvm_intrinsics, "1.0.0", Some(29602), None),
+    (internal, link_llvm_intrinsics, "1.0.0", Some(29602)),
     /// Allows using the `#[linkage = ".."]` attribute.
-    (unstable, linkage, "1.0.0", Some(29603), None),
+    (unstable, linkage, "1.0.0", Some(29603)),
     /// Allows declaring with `#![needs_panic_runtime]` that a panic runtime is needed.
-    (internal, needs_panic_runtime, "1.10.0", Some(32837), None),
+    (internal, needs_panic_runtime, "1.10.0", Some(32837)),
     /// Allows using the `#![panic_runtime]` attribute.
-    (internal, panic_runtime, "1.10.0", Some(32837), None),
+    (internal, panic_runtime, "1.10.0", Some(32837)),
     /// Allows `extern "platform-intrinsic" { ... }`.
-    (internal, platform_intrinsics, "1.4.0", Some(27731), None),
+    (internal, platform_intrinsics, "1.4.0", Some(27731)),
     /// Allows using `#[rustc_allow_const_fn_unstable]`.
     /// This is an attribute on `const fn` for the same
     /// purpose as `#[allow_internal_unstable]`.
-    (internal, rustc_allow_const_fn_unstable, "1.49.0", Some(69399), None),
+    (internal, rustc_allow_const_fn_unstable, "1.49.0", Some(69399)),
     /// Allows using compiler's own crates.
-    (unstable, rustc_private, "1.0.0", Some(27812), None),
+    (unstable, rustc_private, "1.0.0", Some(27812)),
     /// Allows using internal rustdoc features like `doc(keyword)`.
-    (internal, rustdoc_internals, "1.58.0", Some(90418), None),
+    (internal, rustdoc_internals, "1.58.0", Some(90418)),
     /// Allows using the `rustdoc::missing_doc_code_examples` lint
-    (unstable, rustdoc_missing_doc_code_examples, "1.31.0", Some(101730), None),
+    (unstable, rustdoc_missing_doc_code_examples, "1.31.0", Some(101730)),
     /// Allows using `#[start]` on a function indicating that it is the program entrypoint.
-    (unstable, start, "1.0.0", Some(29633), None),
+    (unstable, start, "1.0.0", Some(29633)),
     /// Allows using `#[structural_match]` which indicates that a type is structurally matchable.
     /// FIXME: Subsumed by trait `StructuralPartialEq`, cannot move to removed until a library
     /// feature with the same name exists.
-    (unstable, structural_match, "1.8.0", Some(31434), None),
+    (unstable, structural_match, "1.8.0", Some(31434)),
     /// Allows using the `rust-call` ABI.
-    (unstable, unboxed_closures, "1.0.0", Some(29625), None),
+    (unstable, unboxed_closures, "1.0.0", Some(29625)),
     // !!!!    !!!!    !!!!    !!!!   !!!!    !!!!    !!!!    !!!!    !!!!    !!!!    !!!!
     // Features are listed in alphabetical order. Tidy will fail if you don't keep it this way.
     // !!!!    !!!!    !!!!    !!!!   !!!!    !!!!    !!!!    !!!!    !!!!    !!!!    !!!!
@@ -281,21 +294,21 @@ declare_features! (
     // FIXME: Document these and merge with the list below.
 
     // Unstable `#[target_feature]` directives.
-    (unstable, aarch64_ver_target_feature, "1.27.0", Some(44839), None),
-    (unstable, arm_target_feature, "1.27.0", Some(44839), None),
-    (unstable, avx512_target_feature, "1.27.0", Some(44839), None),
-    (unstable, bpf_target_feature, "1.54.0", Some(44839), None),
-    (unstable, csky_target_feature, "1.73.0", Some(44839), None),
-    (unstable, ermsb_target_feature, "1.49.0", Some(44839), None),
-    (unstable, hexagon_target_feature, "1.27.0", Some(44839), None),
-    (unstable, loongarch_target_feature, "1.73.0", Some(44839), None),
-    (unstable, mips_target_feature, "1.27.0", Some(44839), None),
-    (unstable, powerpc_target_feature, "1.27.0", Some(44839), None),
-    (unstable, riscv_target_feature, "1.45.0", Some(44839), None),
-    (unstable, rtm_target_feature, "1.35.0", Some(44839), None),
-    (unstable, sse4a_target_feature, "1.27.0", Some(44839), None),
-    (unstable, tbm_target_feature, "1.27.0", Some(44839), None),
-    (unstable, wasm_target_feature, "1.30.0", Some(44839), None),
+    (unstable, aarch64_ver_target_feature, "1.27.0", Some(44839)),
+    (unstable, arm_target_feature, "1.27.0", Some(44839)),
+    (unstable, avx512_target_feature, "1.27.0", Some(44839)),
+    (unstable, bpf_target_feature, "1.54.0", Some(44839)),
+    (unstable, csky_target_feature, "1.73.0", Some(44839)),
+    (unstable, ermsb_target_feature, "1.49.0", Some(44839)),
+    (unstable, hexagon_target_feature, "1.27.0", Some(44839)),
+    (unstable, loongarch_target_feature, "1.73.0", Some(44839)),
+    (unstable, mips_target_feature, "1.27.0", Some(44839)),
+    (unstable, powerpc_target_feature, "1.27.0", Some(44839)),
+    (unstable, riscv_target_feature, "1.45.0", Some(44839)),
+    (unstable, rtm_target_feature, "1.35.0", Some(44839)),
+    (unstable, sse4a_target_feature, "1.27.0", Some(44839)),
+    (unstable, tbm_target_feature, "1.27.0", Some(44839)),
+    (unstable, wasm_target_feature, "1.30.0", Some(44839)),
     // !!!!    !!!!    !!!!    !!!!   !!!!    !!!!    !!!!    !!!!    !!!!    !!!!    !!!!
     // Features are listed in alphabetical order. Tidy will fail if you don't keep it this way.
     // !!!!    !!!!    !!!!    !!!!   !!!!    !!!!    !!!!    !!!!    !!!!    !!!!    !!!!
@@ -309,295 +322,300 @@ declare_features! (
     // -------------------------------------------------------------------------
 
     /// Allows using the `amdgpu-kernel` ABI.
-    (unstable, abi_amdgpu_kernel, "1.29.0", Some(51575), None),
+    (unstable, abi_amdgpu_kernel, "1.29.0", Some(51575)),
     /// Allows `extern "avr-interrupt" fn()` and `extern "avr-non-blocking-interrupt" fn()`.
-    (unstable, abi_avr_interrupt, "1.45.0", Some(69664), None),
+    (unstable, abi_avr_interrupt, "1.45.0", Some(69664)),
     /// Allows `extern "C-cmse-nonsecure-call" fn()`.
-    (unstable, abi_c_cmse_nonsecure_call, "1.51.0", Some(81391), None),
+    (unstable, abi_c_cmse_nonsecure_call, "1.51.0", Some(81391)),
     /// Allows `extern "msp430-interrupt" fn()`.
-    (unstable, abi_msp430_interrupt, "1.16.0", Some(38487), None),
+    (unstable, abi_msp430_interrupt, "1.16.0", Some(38487)),
     /// Allows `extern "ptx-*" fn()`.
-    (unstable, abi_ptx, "1.15.0", Some(38788), None),
+    (unstable, abi_ptx, "1.15.0", Some(38788)),
     /// Allows `extern "riscv-interrupt-m" fn()` and `extern "riscv-interrupt-s" fn()`.
-    (unstable, abi_riscv_interrupt, "1.73.0", Some(111889), None),
+    (unstable, abi_riscv_interrupt, "1.73.0", Some(111889)),
     /// Allows `extern "x86-interrupt" fn()`.
-    (unstable, abi_x86_interrupt, "1.17.0", Some(40180), None),
+    (unstable, abi_x86_interrupt, "1.17.0", Some(40180)),
     /// Allows additional const parameter types, such as `&'static str` or user defined types
-    (incomplete, adt_const_params, "1.56.0", Some(95174), None),
+    (incomplete, adt_const_params, "1.56.0", Some(95174)),
     /// Allows defining an `#[alloc_error_handler]`.
-    (unstable, alloc_error_handler, "1.29.0", Some(51540), None),
+    (unstable, alloc_error_handler, "1.29.0", Some(51540)),
     /// Allows trait methods with arbitrary self types.
-    (unstable, arbitrary_self_types, "1.23.0", Some(44874), None),
+    (unstable, arbitrary_self_types, "1.23.0", Some(44874)),
     /// Allows using `const` operands in inline assembly.
-    (unstable, asm_const, "1.58.0", Some(93332), None),
+    (unstable, asm_const, "1.58.0", Some(93332)),
     /// Enables experimental inline assembly support for additional architectures.
-    (unstable, asm_experimental_arch, "1.58.0", Some(93335), None),
+    (unstable, asm_experimental_arch, "1.58.0", Some(93335)),
     /// Allows the `may_unwind` option in inline assembly.
-    (unstable, asm_unwind, "1.58.0", Some(93334), None),
+    (unstable, asm_unwind, "1.58.0", Some(93334)),
     /// Allows users to enforce equality of associated constants `TraitImpl<AssocConst=3>`.
-    (unstable, associated_const_equality, "1.58.0", Some(92827), None),
+    (unstable, associated_const_equality, "1.58.0", Some(92827)),
     /// Allows the user of associated type bounds.
-    (unstable, associated_type_bounds, "1.34.0", Some(52662), None),
+    (unstable, associated_type_bounds, "1.34.0", Some(52662)),
     /// Allows associated type defaults.
-    (unstable, associated_type_defaults, "1.2.0", Some(29661), None),
+    (unstable, associated_type_defaults, "1.2.0", Some(29661)),
     /// Allows `async || body` closures.
-    (unstable, async_closure, "1.37.0", Some(62290), None),
+    (unstable, async_closure, "1.37.0", Some(62290)),
     /// Allows `#[track_caller]` on async functions.
-    (unstable, async_fn_track_caller, "1.73.0", Some(110011), None),
+    (unstable, async_fn_track_caller, "1.73.0", Some(110011)),
     /// Allows builtin # foo() syntax
-    (unstable, builtin_syntax, "1.71.0", Some(110680), None),
+    (unstable, builtin_syntax, "1.71.0", Some(110680)),
     /// Allows `c"foo"` literals.
-    (unstable, c_str_literals, "1.71.0", Some(105723), None),
+    (unstable, c_str_literals, "1.71.0", Some(105723)),
     /// Treat `extern "C"` function as nounwind.
-    (unstable, c_unwind, "1.52.0", Some(74990), None),
+    (unstable, c_unwind, "1.52.0", Some(74990)),
     /// Allows using C-variadics.
-    (unstable, c_variadic, "1.34.0", Some(44930), None),
+    (unstable, c_variadic, "1.34.0", Some(44930)),
     /// Allows the use of `#[cfg(overflow_checks)` to check if integer overflow behaviour.
-    (unstable, cfg_overflow_checks, "1.71.0", Some(111466), None),
+    (unstable, cfg_overflow_checks, "1.71.0", Some(111466)),
     /// Provides the relocation model information as cfg entry
-    (unstable, cfg_relocation_model, "1.73.0", Some(114929), None),
+    (unstable, cfg_relocation_model, "1.73.0", Some(114929)),
     /// Allows the use of `#[cfg(sanitize = "option")]`; set when -Zsanitizer is used.
-    (unstable, cfg_sanitize, "1.41.0", Some(39699), None),
+    (unstable, cfg_sanitize, "1.41.0", Some(39699)),
     /// Allows `cfg(target_abi = "...")`.
-    (unstable, cfg_target_abi, "1.55.0", Some(80970), None),
+    (unstable, cfg_target_abi, "1.55.0", Some(80970)),
     /// Allows `cfg(target(abi = "..."))`.
-    (unstable, cfg_target_compact, "1.63.0", Some(96901), None),
+    (unstable, cfg_target_compact, "1.63.0", Some(96901)),
     /// Allows `cfg(target_has_atomic_load_store = "...")`.
-    (unstable, cfg_target_has_atomic, "1.60.0", Some(94039), None),
+    (unstable, cfg_target_has_atomic, "1.60.0", Some(94039)),
     /// Allows `cfg(target_has_atomic_equal_alignment = "...")`.
-    (unstable, cfg_target_has_atomic_equal_alignment, "1.60.0", Some(93822), None),
+    (unstable, cfg_target_has_atomic_equal_alignment, "1.60.0", Some(93822)),
     /// Allows `cfg(target_thread_local)`.
-    (unstable, cfg_target_thread_local, "1.7.0", Some(29594), None),
+    (unstable, cfg_target_thread_local, "1.7.0", Some(29594)),
     /// Allow conditional compilation depending on rust version
-    (unstable, cfg_version, "1.45.0", Some(64796), None),
+    (unstable, cfg_version, "1.45.0", Some(64796)),
     /// Allows to use the `#[cfi_encoding = ""]` attribute.
-    (unstable, cfi_encoding, "1.71.0", Some(89653), None),
+    (unstable, cfi_encoding, "1.71.0", Some(89653)),
     /// Allows `for<...>` on closures and coroutines.
-    (unstable, closure_lifetime_binder, "1.64.0", Some(97362), None),
+    (unstable, closure_lifetime_binder, "1.64.0", Some(97362)),
     /// Allows `#[track_caller]` on closures and coroutines.
-    (unstable, closure_track_caller, "1.57.0", Some(87417), None),
+    (unstable, closure_track_caller, "1.57.0", Some(87417)),
     /// Allows to use the `#[cmse_nonsecure_entry]` attribute.
-    (unstable, cmse_nonsecure_entry, "1.48.0", Some(75835), None),
+    (unstable, cmse_nonsecure_entry, "1.48.0", Some(75835)),
     /// Allows use of the `#[collapse_debuginfo]` attribute.
-    (unstable, collapse_debuginfo, "1.65.0", Some(100758), None),
+    (unstable, collapse_debuginfo, "1.65.0", Some(100758)),
     /// Allows `async {}` expressions in const contexts.
-    (unstable, const_async_blocks, "1.53.0", Some(85368), None),
+    (unstable, const_async_blocks, "1.53.0", Some(85368)),
     /// Allows `const || {}` closures in const contexts.
-    (incomplete, const_closures, "1.68.0", Some(106003), None),
+    (incomplete, const_closures, "1.68.0", Some(106003)),
     /// Allows the definition of `const extern fn` and `const unsafe extern fn`.
-    (unstable, const_extern_fn, "1.40.0", Some(64926), None),
+    (unstable, const_extern_fn, "1.40.0", Some(64926)),
     /// Allows basic arithmetic on floating point types in a `const fn`.
-    (unstable, const_fn_floating_point_arithmetic, "1.48.0", Some(57241), None),
+    (unstable, const_fn_floating_point_arithmetic, "1.48.0", Some(57241)),
     /// Allows `for _ in _` loops in const contexts.
-    (unstable, const_for, "1.56.0", Some(87575), None),
+    (unstable, const_for, "1.56.0", Some(87575)),
     /// Allows using `&mut` in constant functions.
-    (unstable, const_mut_refs, "1.41.0", Some(57349), None),
+    (unstable, const_mut_refs, "1.41.0", Some(57349)),
     /// Be more precise when looking for live drops in a const context.
-    (unstable, const_precise_live_drops, "1.46.0", Some(73255), None),
+    (unstable, const_precise_live_drops, "1.46.0", Some(73255)),
     /// Allows references to types with interior mutability within constants
-    (unstable, const_refs_to_cell, "1.51.0", Some(80384), None),
+    (unstable, const_refs_to_cell, "1.51.0", Some(80384)),
     /// Allows `impl const Trait for T` syntax.
-    (unstable, const_trait_impl, "1.42.0", Some(67792), None),
+    (unstable, const_trait_impl, "1.42.0", Some(67792)),
     /// Allows the `?` operator in const contexts.
-    (unstable, const_try, "1.56.0", Some(74935), None),
+    (unstable, const_try, "1.56.0", Some(74935)),
     /// Allows coroutines to be cloned.
-    (unstable, coroutine_clone, "1.65.0", Some(95360), None),
+    (unstable, coroutine_clone, "1.65.0", Some(95360)),
     /// Allows defining coroutines.
-    (unstable, coroutines, "1.21.0", Some(43122), None),
+    (unstable, coroutines, "1.21.0", Some(43122)),
     /// Allows function attribute `#[coverage(on/off)]`, to control coverage
     /// instrumentation of that function.
-    (unstable, coverage_attribute, "1.74.0", Some(84605), None),
+    (unstable, coverage_attribute, "1.74.0", Some(84605)),
     /// Allows users to provide classes for fenced code block using `class:classname`.
-    (unstable, custom_code_classes_in_docs, "1.74.0", Some(79483), None),
+    (unstable, custom_code_classes_in_docs, "1.74.0", Some(79483)),
     /// Allows non-builtin attributes in inner attribute position.
-    (unstable, custom_inner_attributes, "1.30.0", Some(54726), None),
+    (unstable, custom_inner_attributes, "1.30.0", Some(54726)),
     /// Allows custom test frameworks with `#![test_runner]` and `#[test_case]`.
-    (unstable, custom_test_frameworks, "1.30.0", Some(50297), None),
+    (unstable, custom_test_frameworks, "1.30.0", Some(50297)),
     /// Allows declarative macros 2.0 (`macro`).
-    (unstable, decl_macro, "1.17.0", Some(39412), None),
+    (unstable, decl_macro, "1.17.0", Some(39412)),
     /// Allows default type parameters to influence type inference.
-    (unstable, default_type_parameter_fallback, "1.3.0", Some(27336), None),
+    (unstable, default_type_parameter_fallback, "1.3.0", Some(27336)),
     /// Allows using `#[deprecated_safe]` to deprecate the safeness of a function or trait
-    (unstable, deprecated_safe, "1.61.0", Some(94978), None),
+    (unstable, deprecated_safe, "1.61.0", Some(94978)),
     /// Allows having using `suggestion` in the `#[deprecated]` attribute.
-    (unstable, deprecated_suggestion, "1.61.0", Some(94785), None),
+    (unstable, deprecated_suggestion, "1.61.0", Some(94785)),
     /// Allows using the `#[diagnostic]` attribute tool namespace
-    (unstable, diagnostic_namespace, "1.73.0", Some(111996), None),
+    (unstable, diagnostic_namespace, "1.73.0", Some(111996)),
     /// Controls errors in trait implementations.
-    (unstable, do_not_recommend, "1.67.0", Some(51992), None),
+    (unstable, do_not_recommend, "1.67.0", Some(51992)),
     /// Tells rustdoc to automatically generate `#[doc(cfg(...))]`.
-    (unstable, doc_auto_cfg, "1.58.0", Some(43781), None),
+    (unstable, doc_auto_cfg, "1.58.0", Some(43781)),
     /// Allows `#[doc(cfg(...))]`.
-    (unstable, doc_cfg, "1.21.0", Some(43781), None),
+    (unstable, doc_cfg, "1.21.0", Some(43781)),
     /// Allows `#[doc(cfg_hide(...))]`.
-    (unstable, doc_cfg_hide, "1.57.0", Some(43781), None),
+    (unstable, doc_cfg_hide, "1.57.0", Some(43781)),
     /// Allows `#[doc(masked)]`.
-    (unstable, doc_masked, "1.21.0", Some(44027), None),
+    (unstable, doc_masked, "1.21.0", Some(44027)),
     /// Allows `dyn* Trait` objects.
-    (incomplete, dyn_star, "1.65.0", Some(102425), None),
+    (incomplete, dyn_star, "1.65.0", Some(102425)),
     // Uses generic effect parameters for ~const bounds
-    (unstable, effects, "1.72.0", Some(102090), None),
+    (unstable, effects, "1.72.0", Some(102090)),
     /// Allows `X..Y` patterns.
-    (unstable, exclusive_range_pattern, "1.11.0", Some(37854), None),
+    (unstable, exclusive_range_pattern, "1.11.0", Some(37854)),
     /// Allows exhaustive pattern matching on types that contain uninhabited types.
-    (unstable, exhaustive_patterns, "1.13.0", Some(51085), None),
+    (unstable, exhaustive_patterns, "1.13.0", Some(51085)),
     /// Allows explicit tail calls via `become` expression.
-    (incomplete, explicit_tail_calls, "1.72.0", Some(112788), None),
+    (incomplete, explicit_tail_calls, "1.72.0", Some(112788)),
     /// Allows using `efiapi`, `sysv64` and `win64` as calling convention
     /// for functions with varargs.
-    (unstable, extended_varargs_abi_support, "1.65.0", Some(100189), None),
+    (unstable, extended_varargs_abi_support, "1.65.0", Some(100189)),
     /// Allows defining `extern type`s.
-    (unstable, extern_types, "1.23.0", Some(43467), None),
+    (unstable, extern_types, "1.23.0", Some(43467)),
     /// Allows the use of `#[ffi_const]` on foreign functions.
-    (unstable, ffi_const, "1.45.0", Some(58328), None),
+    (unstable, ffi_const, "1.45.0", Some(58328)),
     /// Allows the use of `#[ffi_pure]` on foreign functions.
-    (unstable, ffi_pure, "1.45.0", Some(58329), None),
+    (unstable, ffi_pure, "1.45.0", Some(58329)),
     /// Allows using `#[ffi_returns_twice]` on foreign functions.
-    (unstable, ffi_returns_twice, "1.34.0", Some(58314), None),
+    (unstable, ffi_returns_twice, "1.34.0", Some(58314)),
     /// Allows using `#[repr(align(...))]` on function items
-    (unstable, fn_align, "1.53.0", Some(82232), None),
+    (unstable, fn_align, "1.53.0", Some(82232)),
+    /// Support delegating implementation of functions to other already implemented functions.
+    (incomplete, fn_delegation, "1.76.0", Some(118212)),
     /// Allows defining gen blocks and `gen fn`.
-    (unstable, gen_blocks, "1.75.0", Some(117078), None),
+    (unstable, gen_blocks, "1.75.0", Some(117078)),
     /// Infer generic args for both consts and types.
-    (unstable, generic_arg_infer, "1.55.0", Some(85077), None),
+    (unstable, generic_arg_infer, "1.55.0", Some(85077)),
     /// An extension to the `generic_associated_types` feature, allowing incomplete features.
-    (incomplete, generic_associated_types_extended, "1.61.0", Some(95451), None),
+    (incomplete, generic_associated_types_extended, "1.61.0", Some(95451)),
     /// Allows non-trivial generic constants which have to have wfness manually propagated to callers
-    (incomplete, generic_const_exprs, "1.56.0", Some(76560), None),
+    (incomplete, generic_const_exprs, "1.56.0", Some(76560)),
     /// Allows generic parameters and where-clauses on free & associated const items.
-    (incomplete, generic_const_items, "1.73.0", Some(113521), None),
+    (incomplete, generic_const_items, "1.73.0", Some(113521)),
     /// Allows using `..=X` as a patterns in slices.
-    (unstable, half_open_range_patterns_in_slices, "1.66.0", Some(67264), None),
+    (unstable, half_open_range_patterns_in_slices, "1.66.0", Some(67264)),
     /// Allows `if let` guard in match arms.
-    (unstable, if_let_guard, "1.47.0", Some(51114), None),
+    (unstable, if_let_guard, "1.47.0", Some(51114)),
     /// Allows `impl Trait` to be used inside associated types (RFC 2515).
-    (unstable, impl_trait_in_assoc_type, "1.70.0", Some(63063), None),
+    (unstable, impl_trait_in_assoc_type, "1.70.0", Some(63063)),
     /// Allows `impl Trait` as output type in `Fn` traits in return position of functions.
-    (unstable, impl_trait_in_fn_trait_return, "1.64.0", Some(99697), None),
+    (unstable, impl_trait_in_fn_trait_return, "1.64.0", Some(99697)),
     /// Allows using imported `main` function
-    (unstable, imported_main, "1.53.0", Some(28937), None),
+    (unstable, imported_main, "1.53.0", Some(28937)),
     /// Allows associated types in inherent impls.
-    (incomplete, inherent_associated_types, "1.52.0", Some(8995), None),
+    (incomplete, inherent_associated_types, "1.52.0", Some(8995)),
     /// Allow anonymous constants from an inline `const` block
-    (unstable, inline_const, "1.49.0", Some(76001), None),
+    (unstable, inline_const, "1.49.0", Some(76001)),
     /// Allow anonymous constants from an inline `const` block in pattern position
-    (incomplete, inline_const_pat, "1.58.0", Some(76001), None),
+    (incomplete, inline_const_pat, "1.58.0", Some(76001)),
     /// Allows using `pointer` and `reference` in intra-doc links
-    (unstable, intra_doc_pointers, "1.51.0", Some(80896), None),
+    (unstable, intra_doc_pointers, "1.51.0", Some(80896)),
     // Allows setting the threshold for the `large_assignments` lint.
-    (unstable, large_assignments, "1.52.0", Some(83518), None),
+    (unstable, large_assignments, "1.52.0", Some(83518)),
     /// Allow to have type alias types for inter-crate use.
-    (incomplete, lazy_type_alias, "1.72.0", Some(112792), None),
+    (incomplete, lazy_type_alias, "1.72.0", Some(112792)),
     /// Allows `if/while p && let q = r && ...` chains.
-    (unstable, let_chains, "1.37.0", Some(53667), None),
+    (unstable, let_chains, "1.37.0", Some(53667)),
+    /// Allows using `#[link(kind = "link-arg", name = "...")]`
+    /// to pass custom arguments to the linker.
+    (unstable, link_arg_attribute, "1.76.0", Some(99427)),
     /// Allows using `reason` in lint attributes and the `#[expect(lint)]` lint check.
-    (unstable, lint_reasons, "1.31.0", Some(54503), None),
+    (unstable, lint_reasons, "1.31.0", Some(54503)),
     /// Give access to additional metadata about declarative macro meta-variables.
-    (unstable, macro_metavar_expr, "1.61.0", Some(83527), None),
+    (unstable, macro_metavar_expr, "1.61.0", Some(83527)),
     /// Allows `#[marker]` on certain traits allowing overlapping implementations.
-    (unstable, marker_trait_attr, "1.30.0", Some(29864), None),
+    (unstable, marker_trait_attr, "1.30.0", Some(29864)),
     /// A minimal, sound subset of specialization intended to be used by the
     /// standard library until the soundness issues with specialization
     /// are fixed.
-    (unstable, min_specialization, "1.7.0", Some(31844), None),
+    (unstable, min_specialization, "1.7.0", Some(31844)),
     /// Allows qualified paths in struct expressions, struct patterns and tuple struct patterns.
-    (unstable, more_qualified_paths, "1.54.0", Some(86935), None),
+    (unstable, more_qualified_paths, "1.54.0", Some(86935)),
     /// Allows the `#[must_not_suspend]` attribute.
-    (unstable, must_not_suspend, "1.57.0", Some(83310), None),
+    (unstable, must_not_suspend, "1.57.0", Some(83310)),
     /// Allows using `#[naked]` on functions.
-    (unstable, naked_functions, "1.9.0", Some(32408), None),
+    (unstable, naked_functions, "1.9.0", Some(32408)),
     /// Allows specifying the as-needed link modifier
-    (unstable, native_link_modifiers_as_needed, "1.53.0", Some(81490), None),
+    (unstable, native_link_modifiers_as_needed, "1.53.0", Some(81490)),
     /// Allow negative trait implementations.
-    (unstable, negative_impls, "1.44.0", Some(68318), None),
+    (unstable, negative_impls, "1.44.0", Some(68318)),
+    /// Allows the `!` pattern.
+    (incomplete, never_patterns, "1.76.0", Some(118155)),
     /// Allows the `!` type. Does not imply 'exhaustive_patterns' (below) any more.
-    (unstable, never_type, "1.13.0", Some(35121), None),
+    (unstable, never_type, "1.13.0", Some(35121)),
     /// Allows diverging expressions to fall back to `!` rather than `()`.
-    (unstable, never_type_fallback, "1.41.0", Some(65992), None),
+    (unstable, never_type_fallback, "1.41.0", Some(65992)),
     /// Allows `#![no_core]`.
-    (unstable, no_core, "1.3.0", Some(29639), None),
+    (unstable, no_core, "1.3.0", Some(29639)),
     /// Allows the use of `no_sanitize` attribute.
-    (unstable, no_sanitize, "1.42.0", Some(39699), None),
+    (unstable, no_sanitize, "1.42.0", Some(39699)),
     /// Allows using the `non_exhaustive_omitted_patterns` lint.
-    (unstable, non_exhaustive_omitted_patterns_lint, "1.57.0", Some(89554), None),
+    (unstable, non_exhaustive_omitted_patterns_lint, "1.57.0", Some(89554)),
     /// Allows `for<T>` binders in where-clauses
-    (incomplete, non_lifetime_binders, "1.69.0", Some(108185), None),
+    (incomplete, non_lifetime_binders, "1.69.0", Some(108185)),
     /// Allows making `dyn Trait` well-formed even if `Trait` is not object safe.
     /// In that case, `dyn Trait: Trait` does not hold. Moreover, coercions and
     /// casts in safe Rust to `dyn Trait` for such a `Trait` is also forbidden.
-    (unstable, object_safe_for_dispatch, "1.40.0", Some(43561), None),
+    (unstable, object_safe_for_dispatch, "1.40.0", Some(43561)),
     /// Allows using enums in offset_of!
-    (unstable, offset_of_enum, "1.75.0", Some(106655), None),
+    (unstable, offset_of_enum, "1.75.0", Some(106655)),
     /// Allows using `#[optimize(X)]`.
-    (unstable, optimize_attribute, "1.34.0", Some(54882), None),
-    /// Allows exhaustive integer pattern matching on `usize` and `isize`.
-    (unstable, precise_pointer_size_matching, "1.32.0", Some(56354), None),
+    (unstable, optimize_attribute, "1.34.0", Some(54882)),
     /// Allows macro attributes on expressions, statements and non-inline modules.
-    (unstable, proc_macro_hygiene, "1.30.0", Some(54727), None),
+    (unstable, proc_macro_hygiene, "1.30.0", Some(54727)),
     /// Allows `&raw const $place_expr` and `&raw mut $place_expr` expressions.
-    (unstable, raw_ref_op, "1.41.0", Some(64490), None),
+    (unstable, raw_ref_op, "1.41.0", Some(64490)),
     /// Allows using the `#[register_tool]` attribute.
-    (unstable, register_tool, "1.41.0", Some(66079), None),
+    (unstable, register_tool, "1.41.0", Some(66079)),
     /// Allows the `#[repr(i128)]` attribute for enums.
-    (incomplete, repr128, "1.16.0", Some(56071), None),
+    (incomplete, repr128, "1.16.0", Some(56071)),
     /// Allows `repr(simd)` and importing the various simd intrinsics.
-    (unstable, repr_simd, "1.4.0", Some(27731), None),
+    (unstable, repr_simd, "1.4.0", Some(27731)),
     /// Allows bounding the return type of AFIT/RPITIT.
-    (incomplete, return_type_notation, "1.70.0", Some(109417), None),
+    (incomplete, return_type_notation, "1.70.0", Some(109417)),
     /// Allows `extern "rust-cold"`.
-    (unstable, rust_cold_cc, "1.63.0", Some(97544), None),
+    (unstable, rust_cold_cc, "1.63.0", Some(97544)),
     /// Allows the use of SIMD types in functions declared in `extern` blocks.
-    (unstable, simd_ffi, "1.0.0", Some(27731), None),
+    (unstable, simd_ffi, "1.0.0", Some(27731)),
     /// Allows specialization of implementations (RFC 1210).
-    (incomplete, specialization, "1.7.0", Some(31844), None),
+    (incomplete, specialization, "1.7.0", Some(31844)),
     /// Allows attributes on expressions and non-item statements.
-    (unstable, stmt_expr_attributes, "1.6.0", Some(15701), None),
+    (unstable, stmt_expr_attributes, "1.6.0", Some(15701)),
     /// Allows lints part of the strict provenance effort.
-    (unstable, strict_provenance, "1.61.0", Some(95228), None),
+    (unstable, strict_provenance, "1.61.0", Some(95228)),
     /// Allows string patterns to dereference values to match them.
-    (unstable, string_deref_patterns, "1.67.0", Some(87121), None),
+    (unstable, string_deref_patterns, "1.67.0", Some(87121)),
     /// Allows the use of `#[target_feature]` on safe functions.
-    (unstable, target_feature_11, "1.45.0", Some(69098), None),
+    (unstable, target_feature_11, "1.45.0", Some(69098)),
     /// Allows using `#[thread_local]` on `static` items.
-    (unstable, thread_local, "1.0.0", Some(29594), None),
+    (unstable, thread_local, "1.0.0", Some(29594)),
     /// Allows defining `trait X = A + B;` alias items.
-    (unstable, trait_alias, "1.24.0", Some(41517), None),
+    (unstable, trait_alias, "1.24.0", Some(41517)),
     /// Allows dyn upcasting trait objects via supertraits.
     /// Dyn upcasting is casting, e.g., `dyn Foo -> dyn Bar` where `Foo: Bar`.
-    (unstable, trait_upcasting, "1.56.0", Some(65991), None),
+    (unstable, trait_upcasting, "1.56.0", Some(65991)),
     /// Allows for transmuting between arrays with sizes that contain generic consts.
-    (unstable, transmute_generic_consts, "1.70.0", Some(109929), None),
+    (unstable, transmute_generic_consts, "1.70.0", Some(109929)),
     /// Allows #[repr(transparent)] on unions (RFC 2645).
-    (unstable, transparent_unions, "1.37.0", Some(60405), None),
+    (unstable, transparent_unions, "1.37.0", Some(60405)),
     /// Allows inconsistent bounds in where clauses.
-    (unstable, trivial_bounds, "1.28.0", Some(48214), None),
+    (unstable, trivial_bounds, "1.28.0", Some(48214)),
     /// Allows using `try {...}` expressions.
-    (unstable, try_blocks, "1.29.0", Some(31436), None),
+    (unstable, try_blocks, "1.29.0", Some(31436)),
     /// Allows `impl Trait` to be used inside type aliases (RFC 2515).
-    (unstable, type_alias_impl_trait, "1.38.0", Some(63063), None),
+    (unstable, type_alias_impl_trait, "1.38.0", Some(63063)),
     /// Allows the use of type ascription in expressions.
-    (unstable, type_ascription, "1.6.0", Some(23416), None),
+    (unstable, type_ascription, "1.6.0", Some(23416)),
     /// Allows creation of instances of a struct by moving fields that have
     /// not changed from prior instances of the same struct (RFC #2528)
-    (unstable, type_changing_struct_update, "1.58.0", Some(86555), None),
+    (unstable, type_changing_struct_update, "1.58.0", Some(86555)),
     /// Allows using type privacy lints (`private_interfaces`, `private_bounds`, `unnameable_types`).
-    (unstable, type_privacy_lints, "1.72.0", Some(48054), None),
+    (unstable, type_privacy_lints, "1.72.0", Some(48054)),
     /// Enables rustc to generate code that instructs libstd to NOT ignore SIGPIPE.
-    (unstable, unix_sigpipe, "1.65.0", Some(97889), None),
+    (unstable, unix_sigpipe, "1.65.0", Some(97889)),
     /// Allows unnamed fields of struct and union type
-    (incomplete, unnamed_fields, "1.74.0", Some(49804), None),
+    (incomplete, unnamed_fields, "1.74.0", Some(49804)),
     /// Allows unsized fn parameters.
-    (unstable, unsized_fn_params, "1.49.0", Some(48055), None),
+    (unstable, unsized_fn_params, "1.49.0", Some(48055)),
     /// Allows unsized rvalues at arguments and parameters.
-    (incomplete, unsized_locals, "1.30.0", Some(48055), None),
+    (incomplete, unsized_locals, "1.30.0", Some(48055)),
     /// Allows unsized tuple coercion.
-    (unstable, unsized_tuple_coercion, "1.20.0", Some(42877), None),
+    (unstable, unsized_tuple_coercion, "1.20.0", Some(42877)),
     /// Allows using the `#[used(linker)]` (or `#[used(compiler)]`) attribute.
-    (unstable, used_with_arg, "1.60.0", Some(93798), None),
+    (unstable, used_with_arg, "1.60.0", Some(93798)),
     /// Allows `extern "wasm" fn`
-    (unstable, wasm_abi, "1.53.0", Some(83788), None),
+    (unstable, wasm_abi, "1.53.0", Some(83788)),
     /// Allows `do yeet` expressions
-    (unstable, yeet_expr, "1.62.0", Some(96373), None),
+    (unstable, yeet_expr, "1.62.0", Some(96373)),
     // !!!!    !!!!    !!!!    !!!!   !!!!    !!!!    !!!!    !!!!    !!!!    !!!!    !!!!
     // Features are listed in alphabetical order. Tidy will fail if you don't keep it this way.
     // !!!!    !!!!    !!!!    !!!!   !!!!    !!!!    !!!!    !!!!    !!!!    !!!!    !!!!
diff --git a/compiler/rustc_fluent_macro/src/fluent.rs b/compiler/rustc_fluent_macro/src/fluent.rs
index 7479e4ef2..3b1b63455 100644
--- a/compiler/rustc_fluent_macro/src/fluent.rs
+++ b/compiler/rustc_fluent_macro/src/fluent.rs
@@ -57,20 +57,20 @@ fn finish(body: TokenStream, resource: TokenStream) -> proc_macro::TokenStream {
             /// identifiers for different subdiagnostic kinds.
             pub mod _subdiag {
                 /// Default for `#[help]`
-                pub const help: crate::SubdiagnosticMessage =
-                    crate::SubdiagnosticMessage::FluentAttr(std::borrow::Cow::Borrowed("help"));
+                pub const help: rustc_errors::SubdiagnosticMessage =
+                    rustc_errors::SubdiagnosticMessage::FluentAttr(std::borrow::Cow::Borrowed("help"));
                 /// Default for `#[note]`
-                pub const note: crate::SubdiagnosticMessage =
-                    crate::SubdiagnosticMessage::FluentAttr(std::borrow::Cow::Borrowed("note"));
+                pub const note: rustc_errors::SubdiagnosticMessage =
+                    rustc_errors::SubdiagnosticMessage::FluentAttr(std::borrow::Cow::Borrowed("note"));
                 /// Default for `#[warn]`
-                pub const warn: crate::SubdiagnosticMessage =
-                    crate::SubdiagnosticMessage::FluentAttr(std::borrow::Cow::Borrowed("warn"));
+                pub const warn: rustc_errors::SubdiagnosticMessage =
+                    rustc_errors::SubdiagnosticMessage::FluentAttr(std::borrow::Cow::Borrowed("warn"));
                 /// Default for `#[label]`
-                pub const label: crate::SubdiagnosticMessage =
-                    crate::SubdiagnosticMessage::FluentAttr(std::borrow::Cow::Borrowed("label"));
+                pub const label: rustc_errors::SubdiagnosticMessage =
+                    rustc_errors::SubdiagnosticMessage::FluentAttr(std::borrow::Cow::Borrowed("label"));
                 /// Default for `#[suggestion]`
-                pub const suggestion: crate::SubdiagnosticMessage =
-                    crate::SubdiagnosticMessage::FluentAttr(std::borrow::Cow::Borrowed("suggestion"));
+                pub const suggestion: rustc_errors::SubdiagnosticMessage =
+                    rustc_errors::SubdiagnosticMessage::FluentAttr(std::borrow::Cow::Borrowed("suggestion"));
             }
         }
     }
@@ -248,11 +248,8 @@ pub(crate) fn fluent_messages(input: proc_macro::TokenStream) -> proc_macro::Tok
                 format!("Constant referring to Fluent message `{name}` from `{crate_name}`");
             constants.extend(quote! {
                 #[doc = #docstr]
-                pub const #snake_name: crate::DiagnosticMessage =
-                    crate::DiagnosticMessage::FluentIdentifier(
-                        std::borrow::Cow::Borrowed(#name),
-                        None
-                    );
+                pub const #snake_name: rustc_errors::DiagnosticMessage =
+                    rustc_errors::DiagnosticMessage::FluentIdentifier(std::borrow::Cow::Borrowed(#name), None);
             });
 
             for Attribute { id: Identifier { name: attr_name }, .. } in attributes {
@@ -279,10 +276,8 @@ pub(crate) fn fluent_messages(input: proc_macro::TokenStream) -> proc_macro::Tok
                 );
                 constants.extend(quote! {
                     #[doc = #msg]
-                    pub const #snake_name: crate::SubdiagnosticMessage =
-                        crate::SubdiagnosticMessage::FluentAttr(
-                            std::borrow::Cow::Borrowed(#attr_name)
-                        );
+                    pub const #snake_name: rustc_errors::SubdiagnosticMessage =
+                        rustc_errors::SubdiagnosticMessage::FluentAttr(std::borrow::Cow::Borrowed(#attr_name));
                 });
             }
 
diff --git a/compiler/rustc_fluent_macro/src/lib.rs b/compiler/rustc_fluent_macro/src/lib.rs
index 191fb787f..fc65d1eb8 100644
--- a/compiler/rustc_fluent_macro/src/lib.rs
+++ b/compiler/rustc_fluent_macro/src/lib.rs
@@ -1,7 +1,7 @@
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
-#![cfg_attr(not(bootstrap), allow(internal_features))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
+#![doc(rust_logo)]
+#![allow(internal_features)]
+#![feature(rustdoc_internals)]
 #![feature(proc_macro_diagnostic)]
 #![feature(proc_macro_span)]
 #![deny(rustc::untranslatable_diagnostic)]
@@ -61,6 +61,10 @@ mod fluent;
 /// );
 /// err.emit();
 /// ```
+///
+/// Note: any crate using this macro must also have a dependency on
+/// `rustc_errors`, because the generated code refers to things from that
+/// crate.
 #[proc_macro]
 pub fn fluent_messages(input: TokenStream) -> TokenStream {
     fluent::fluent_messages(input)
diff --git a/compiler/rustc_graphviz/src/lib.rs b/compiler/rustc_graphviz/src/lib.rs
index 9cb279e3e..eba3215d9 100644
--- a/compiler/rustc_graphviz/src/lib.rs
+++ b/compiler/rustc_graphviz/src/lib.rs
@@ -273,9 +273,9 @@
     html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/",
     test(attr(allow(unused_variables), deny(warnings)))
 )]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
-#![cfg_attr(not(bootstrap), allow(internal_features))]
+#![feature(rustdoc_internals)]
+#![doc(rust_logo)]
+#![allow(internal_features)]
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
 
@@ -522,33 +522,6 @@ impl<'a> LabelText<'a> {
             HtmlStr(ref s) => format!("<{s}>"),
         }
     }
-
-    /// Decomposes content into string suitable for making EscStr that
-    /// yields same content as self. The result obeys the law
-    /// render(`lt`) == render(`EscStr(lt.pre_escaped_content())`) for
-    /// all `lt: LabelText`.
-    fn pre_escaped_content(self) -> Cow<'a, str> {
-        match self {
-            EscStr(s) => s,
-            LabelStr(s) => {
-                if s.contains('\\') {
-                    s.escape_default().to_string().into()
-                } else {
-                    s
-                }
-            }
-            HtmlStr(s) => s,
-        }
-    }
-
-    /// Puts `suffix` on a line below this label, with a blank line separator.
-    pub fn suffix_line(self, suffix: LabelText<'_>) -> LabelText<'static> {
-        let mut prefix = self.pre_escaped_content().into_owned();
-        let suffix = suffix.pre_escaped_content();
-        prefix.push_str(r"\n\n");
-        prefix.push_str(&suffix);
-        EscStr(prefix.into())
-    }
 }
 
 pub type Nodes<'a, N> = Cow<'a, [N]>;
diff --git a/compiler/rustc_hir/Cargo.toml b/compiler/rustc_hir/Cargo.toml
index a72c4d0f1..ac24c47d0 100644
--- a/compiler/rustc_hir/Cargo.toml
+++ b/compiler/rustc_hir/Cargo.toml
@@ -9,7 +9,6 @@ odht = { version = "0.3.1", features = ["nightly"] }
 rustc_arena = { path = "../rustc_arena" }
 rustc_ast = { path = "../rustc_ast" }
 rustc_data_structures = { path = "../rustc_data_structures" }
-rustc_error_messages = { path = "../rustc_error_messages" }
 rustc_index = { path = "../rustc_index" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_serialize = { path = "../rustc_serialize" }
diff --git a/compiler/rustc_hir/src/arena.rs b/compiler/rustc_hir/src/arena.rs
index e213623e0..f1f624269 100644
--- a/compiler/rustc_hir/src/arena.rs
+++ b/compiler/rustc_hir/src/arena.rs
@@ -5,7 +5,6 @@ macro_rules! arena_types {
     ($macro:path) => (
         $macro!([
             // HIR types
-            [] hir_krate: rustc_hir::Crate<'tcx>,
             [] asm_template: rustc_ast::InlineAsmTemplatePiece,
             [] attribute: rustc_ast::Attribute,
             [] owner_info: rustc_hir::OwnerInfo<'tcx>,
diff --git a/compiler/rustc_hir/src/def.rs b/compiler/rustc_hir/src/def.rs
index ed1dc751f..81ec7ddb6 100644
--- a/compiler/rustc_hir/src/def.rs
+++ b/compiler/rustc_hir/src/def.rs
@@ -1,9 +1,10 @@
+use crate::definitions::DefPathData;
 use crate::hir;
 
 use rustc_ast as ast;
 use rustc_ast::NodeId;
-use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::stable_hasher::ToStableHashKey;
+use rustc_data_structures::unord::UnordMap;
 use rustc_macros::HashStable_Generic;
 use rustc_span::def_id::{DefId, LocalDefId};
 use rustc_span::hygiene::MacroKind;
@@ -13,8 +14,7 @@ use std::array::IntoIter;
 use std::fmt::Debug;
 
 /// Encodes if a `DefKind::Ctor` is the constructor of an enum variant or a struct.
-#[derive(Clone, Copy, PartialEq, Eq, Encodable, Decodable, Hash, Debug)]
-#[derive(HashStable_Generic)]
+#[derive(Clone, Copy, PartialEq, Eq, Encodable, Decodable, Hash, Debug, HashStable_Generic)]
 pub enum CtorOf {
     /// This `DefKind::Ctor` is a synthesized constructor of a tuple or unit struct.
     Struct,
@@ -23,8 +23,7 @@ pub enum CtorOf {
 }
 
 /// What kind of constructor something is.
-#[derive(Clone, Copy, PartialEq, Eq, Encodable, Decodable, Hash, Debug)]
-#[derive(HashStable_Generic)]
+#[derive(Clone, Copy, PartialEq, Eq, Encodable, Decodable, Hash, Debug, HashStable_Generic)]
 pub enum CtorKind {
     /// Constructor function automatically created by a tuple struct/variant.
     Fn,
@@ -33,8 +32,7 @@ pub enum CtorKind {
 }
 
 /// An attribute that is not a macro; e.g., `#[inline]` or `#[rustfmt::skip]`.
-#[derive(Clone, Copy, PartialEq, Eq, Encodable, Decodable, Hash, Debug)]
-#[derive(HashStable_Generic)]
+#[derive(Clone, Copy, PartialEq, Eq, Encodable, Decodable, Hash, Debug, HashStable_Generic)]
 pub enum NonMacroAttrKind {
     /// Single-segment attribute defined by the language (`#[inline]`)
     Builtin(Symbol),
@@ -48,8 +46,8 @@ pub enum NonMacroAttrKind {
 }
 
 /// What kind of definition something is; e.g., `mod` vs `struct`.
-#[derive(Clone, Copy, PartialEq, Eq, Encodable, Decodable, Hash, Debug)]
-#[derive(HashStable_Generic)]
+/// `enum DefPathData` may need to be updated if a new variant is added here.
+#[derive(Clone, Copy, PartialEq, Eq, Encodable, Decodable, Hash, Debug, HashStable_Generic)]
 pub enum DefKind {
     // Type namespace
     Mod,
@@ -118,7 +116,6 @@ pub enum DefKind {
         of_trait: bool,
     },
     Closure,
-    Coroutine,
 }
 
 impl DefKind {
@@ -161,7 +158,6 @@ impl DefKind {
             DefKind::Field => "field",
             DefKind::Impl { .. } => "implementation",
             DefKind::Closure => "closure",
-            DefKind::Coroutine => "coroutine",
             DefKind::ExternCrate => "extern crate",
             DefKind::GlobalAsm => "global assembly block",
         }
@@ -220,7 +216,6 @@ impl DefKind {
             | DefKind::LifetimeParam
             | DefKind::ExternCrate
             | DefKind::Closure
-            | DefKind::Coroutine
             | DefKind::Use
             | DefKind::ForeignMod
             | DefKind::GlobalAsm
@@ -228,9 +223,44 @@ impl DefKind {
         }
     }
 
+    pub fn def_path_data(self, name: Symbol) -> DefPathData {
+        match self {
+            DefKind::Mod
+            | DefKind::Struct
+            | DefKind::Union
+            | DefKind::Enum
+            | DefKind::Variant
+            | DefKind::Trait
+            | DefKind::TyAlias
+            | DefKind::ForeignTy
+            | DefKind::TraitAlias
+            | DefKind::AssocTy
+            | DefKind::TyParam
+            | DefKind::ExternCrate => DefPathData::TypeNs(name),
+            DefKind::Fn
+            | DefKind::Const
+            | DefKind::ConstParam
+            | DefKind::Static(..)
+            | DefKind::AssocFn
+            | DefKind::AssocConst
+            | DefKind::Field => DefPathData::ValueNs(name),
+            DefKind::Macro(..) => DefPathData::MacroNs(name),
+            DefKind::LifetimeParam => DefPathData::LifetimeNs(name),
+            DefKind::Ctor(..) => DefPathData::Ctor,
+            DefKind::Use => DefPathData::Use,
+            DefKind::ForeignMod => DefPathData::ForeignMod,
+            DefKind::AnonConst => DefPathData::AnonConst,
+            DefKind::InlineConst => DefPathData::AnonConst,
+            DefKind::OpaqueTy => DefPathData::OpaqueTy,
+            DefKind::GlobalAsm => DefPathData::GlobalAsm,
+            DefKind::Impl { .. } => DefPathData::Impl,
+            DefKind::Closure => DefPathData::Closure,
+        }
+    }
+
     #[inline]
     pub fn is_fn_like(self) -> bool {
-        matches!(self, DefKind::Fn | DefKind::AssocFn | DefKind::Closure | DefKind::Coroutine)
+        matches!(self, DefKind::Fn | DefKind::AssocFn | DefKind::Closure)
     }
 
     /// Whether `query get_codegen_attrs` should be used with this definition.
@@ -240,7 +270,6 @@ impl DefKind {
             | DefKind::AssocFn
             | DefKind::Ctor(..)
             | DefKind::Closure
-            | DefKind::Coroutine
             | DefKind::Static(_) => true,
             DefKind::Mod
             | DefKind::Struct
@@ -299,8 +328,7 @@ impl DefKind {
 /// - the call to `str_to_string` will resolve to [`Res::Def`], with the [`DefId`]
 ///   pointing to the definition of `str_to_string` in the current crate.
 //
-#[derive(Clone, Copy, PartialEq, Eq, Encodable, Decodable, Hash, Debug)]
-#[derive(HashStable_Generic)]
+#[derive(Clone, Copy, PartialEq, Eq, Encodable, Decodable, Hash, Debug, HashStable_Generic)]
 pub enum Res<Id = hir::HirId> {
     /// Definition having a unique ID (`DefId`), corresponds to something defined in user code.
     ///
@@ -575,7 +603,7 @@ impl CtorKind {
         match *vdata {
             ast::VariantData::Tuple(_, node_id) => Some((CtorKind::Fn, node_id)),
             ast::VariantData::Unit(node_id) => Some((CtorKind::Const, node_id)),
-            ast::VariantData::Struct(..) => None,
+            ast::VariantData::Struct { .. } => None,
         }
     }
 }
@@ -591,6 +619,8 @@ impl NonMacroAttrKind {
         }
     }
 
+    // Currently trivial, but exists in case a new kind is added in the future whose name starts
+    // with a vowel.
     pub fn article(self) -> &'static str {
         "a"
     }
@@ -776,4 +806,4 @@ pub enum LifetimeRes {
     ElidedAnchor { start: NodeId, end: NodeId },
 }
 
-pub type DocLinkResMap = FxHashMap<(Symbol, Namespace), Option<Res<NodeId>>>;
+pub type DocLinkResMap = UnordMap<(Symbol, Namespace), Option<Res<NodeId>>>;
diff --git a/compiler/rustc_hir/src/definitions.rs b/compiler/rustc_hir/src/definitions.rs
index 168b336e3..2ab9a6ef3 100644
--- a/compiler/rustc_hir/src/definitions.rs
+++ b/compiler/rustc_hir/src/definitions.rs
@@ -8,8 +8,8 @@ pub use crate::def_id::DefPathHash;
 use crate::def_id::{CrateNum, DefIndex, LocalDefId, StableCrateId, CRATE_DEF_INDEX, LOCAL_CRATE};
 use crate::def_path_hash_map::DefPathHashMap;
 
-use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::stable_hasher::{Hash64, StableHasher};
+use rustc_data_structures::unord::UnordMap;
 use rustc_index::IndexVec;
 use rustc_span::symbol::{kw, sym, Symbol};
 
@@ -95,7 +95,7 @@ impl DefPathTable {
 #[derive(Debug)]
 pub struct Definitions {
     table: DefPathTable,
-    next_disambiguator: FxHashMap<(LocalDefId, DefPathData), u32>,
+    next_disambiguator: UnordMap<(LocalDefId, DefPathData), u32>,
 
     /// The [StableCrateId] of the local crate.
     stable_crate_id: StableCrateId,
@@ -246,6 +246,7 @@ impl DefPath {
     }
 }
 
+/// New variants should only be added in synchronization with `enum DefKind`.
 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, Encodable, Decodable)]
 pub enum DefPathData {
     // Root: these should only be used for the root nodes, because
@@ -271,7 +272,7 @@ pub enum DefPathData {
     /// Something in the lifetime namespace.
     LifetimeNs(Symbol),
     /// A closure expression.
-    ClosureExpr,
+    Closure,
 
     // Subportions of items:
     /// Implicit constructor for a unit or tuple-like struct or enum variant.
@@ -280,9 +281,7 @@ pub enum DefPathData {
     AnonConst,
     /// An existential `impl Trait` type node.
     /// Argument position `impl Trait` have a `TypeNs` with their pretty-printed name.
-    ImplTrait,
-    /// `impl Trait` generated associated type node.
-    ImplTraitAssocTy,
+    OpaqueTy,
 }
 
 impl Definitions {
@@ -403,16 +402,17 @@ impl DefPathData {
     pub fn get_opt_name(&self) -> Option<Symbol> {
         use self::DefPathData::*;
         match *self {
+            TypeNs(name) if name == kw::Empty => None,
             TypeNs(name) | ValueNs(name) | MacroNs(name) | LifetimeNs(name) => Some(name),
-
-            Impl | ForeignMod | CrateRoot | Use | GlobalAsm | ClosureExpr | Ctor | AnonConst
-            | ImplTrait | ImplTraitAssocTy => None,
+            Impl | ForeignMod | CrateRoot | Use | GlobalAsm | Closure | Ctor | AnonConst
+            | OpaqueTy => None,
         }
     }
 
     pub fn name(&self) -> DefPathDataName {
         use self::DefPathData::*;
         match *self {
+            TypeNs(name) if name == kw::Empty => DefPathDataName::Anon { namespace: sym::opaque },
             TypeNs(name) | ValueNs(name) | MacroNs(name) | LifetimeNs(name) => {
                 DefPathDataName::Named(name)
             }
@@ -422,10 +422,10 @@ impl DefPathData {
             ForeignMod => DefPathDataName::Anon { namespace: kw::Extern },
             Use => DefPathDataName::Anon { namespace: kw::Use },
             GlobalAsm => DefPathDataName::Anon { namespace: sym::global_asm },
-            ClosureExpr => DefPathDataName::Anon { namespace: sym::closure },
+            Closure => DefPathDataName::Anon { namespace: sym::closure },
             Ctor => DefPathDataName::Anon { namespace: sym::constructor },
             AnonConst => DefPathDataName::Anon { namespace: sym::constant },
-            ImplTrait | ImplTraitAssocTy => DefPathDataName::Anon { namespace: sym::opaque },
+            OpaqueTy => DefPathDataName::Anon { namespace: sym::opaque },
         }
     }
 }
diff --git a/compiler/rustc_hir/src/diagnostic_items.rs b/compiler/rustc_hir/src/diagnostic_items.rs
index 243014b00..d4d09f9a4 100644
--- a/compiler/rustc_hir/src/diagnostic_items.rs
+++ b/compiler/rustc_hir/src/diagnostic_items.rs
@@ -1,12 +1,13 @@
 use crate::def_id::DefId;
-use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::fx::FxIndexMap;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
+use rustc_span::def_id::DefIdMap;
 use rustc_span::Symbol;
 
 #[derive(Debug, Default)]
 pub struct DiagnosticItems {
-    pub id_to_name: FxHashMap<DefId, Symbol>,
-    pub name_to_id: FxHashMap<Symbol, DefId>,
+    pub id_to_name: DefIdMap<Symbol>,
+    pub name_to_id: FxIndexMap<Symbol, DefId>,
 }
 
 impl<CTX: crate::HashStableContext> HashStable<CTX> for DiagnosticItems {
diff --git a/compiler/rustc_hir/src/hir.rs b/compiler/rustc_hir/src/hir.rs
index c4e44a6a4..3179fd736 100644
--- a/compiler/rustc_hir/src/hir.rs
+++ b/compiler/rustc_hir/src/hir.rs
@@ -1,19 +1,17 @@
 use crate::def::{CtorKind, DefKind, Res};
-use crate::def_id::DefId;
-pub(crate) use crate::hir_id::{HirId, ItemLocalId, OwnerId};
+use crate::def_id::{DefId, LocalDefIdMap};
+pub(crate) use crate::hir_id::{HirId, ItemLocalId, ItemLocalMap, OwnerId};
 use crate::intravisit::FnKind;
 use crate::LangItem;
 
 use rustc_ast as ast;
 use rustc_ast::util::parser::ExprPrecedence;
 use rustc_ast::{Attribute, FloatTy, IntTy, Label, LitKind, TraitObjectSyntax, UintTy};
-pub use rustc_ast::{BindingAnnotation, BorrowKind, ByRef, ImplPolarity, IsAuto};
-pub use rustc_ast::{CaptureBy, Movability, Mutability};
+pub use rustc_ast::{BinOp, BinOpKind, BindingAnnotation, BorrowKind, ByRef, CaptureBy};
+pub use rustc_ast::{ImplPolarity, IsAuto, Movability, Mutability, UnOp};
 use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
 use rustc_data_structures::fingerprint::Fingerprint;
-use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sorted_map::SortedMap;
-use rustc_error_messages::MultiSpan;
 use rustc_index::IndexVec;
 use rustc_macros::HashStable_Generic;
 use rustc_span::hygiene::MacroKind;
@@ -76,13 +74,6 @@ impl ParamName {
             ParamName::Fresh | ParamName::Error => Ident::with_dummy_span(kw::UnderscoreLifetime),
         }
     }
-
-    pub fn normalize_to_macros_2_0(&self) -> ParamName {
-        match *self {
-            ParamName::Plain(ident) => ParamName::Plain(ident.normalize_to_macros_2_0()),
-            param_name => param_name,
-        }
-    }
 }
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq, HashStable_Generic)]
@@ -116,7 +107,7 @@ pub enum LifetimeName {
 }
 
 impl LifetimeName {
-    pub fn is_elided(&self) -> bool {
+    fn is_elided(&self) -> bool {
         match self {
             LifetimeName::ImplicitObjectLifetimeDefault | LifetimeName::Infer => true,
 
@@ -289,10 +280,6 @@ impl GenericArg<'_> {
         }
     }
 
-    pub fn is_synthetic(&self) -> bool {
-        matches!(self, GenericArg::Lifetime(lifetime) if lifetime.ident == Ident::empty())
-    }
-
     pub fn descr(&self) -> &'static str {
         match self {
             GenericArg::Lifetime(_) => "lifetime",
@@ -368,11 +355,6 @@ impl<'hir> GenericArgs<'hir> {
         panic!("GenericArgs::inputs: not a `Fn(T) -> U`");
     }
 
-    #[inline]
-    pub fn has_type_params(&self) -> bool {
-        self.args.iter().any(|arg| matches!(arg, GenericArg::Type(_)))
-    }
-
     pub fn has_err(&self) -> bool {
         self.args.iter().any(|arg| match arg {
             GenericArg::Type(ty) => matches!(ty.kind, TyKind::Err(_)),
@@ -384,11 +366,6 @@ impl<'hir> GenericArgs<'hir> {
     }
 
     #[inline]
-    pub fn num_type_params(&self) -> usize {
-        self.args.iter().filter(|arg| matches!(arg, GenericArg::Type(_))).count()
-    }
-
-    #[inline]
     pub fn num_lifetime_params(&self) -> usize {
         self.args.iter().filter(|arg| matches!(arg, GenericArg::Lifetime(_))).count()
     }
@@ -457,8 +434,6 @@ pub enum TraitBoundModifier {
 #[derive(Clone, Copy, Debug, HashStable_Generic)]
 pub enum GenericBound<'hir> {
     Trait(PolyTraitRef<'hir>, TraitBoundModifier),
-    // FIXME(davidtwco): Introduce `PolyTraitRef::LangItem`
-    LangItemTrait(LangItem, Span, HirId, &'hir GenericArgs<'hir>),
     Outlives(&'hir Lifetime),
 }
 
@@ -473,7 +448,6 @@ impl GenericBound<'_> {
     pub fn span(&self) -> Span {
         match self {
             GenericBound::Trait(t, ..) => t.span,
-            GenericBound::LangItemTrait(_, span, ..) => *span,
             GenericBound::Outlives(l) => l.ident.span,
         }
     }
@@ -509,6 +483,7 @@ pub enum GenericParamKind<'hir> {
         ty: &'hir Ty<'hir>,
         /// Optional default value for the const generic param
         default: Option<AnonConst>,
+        is_host_effect: bool,
     },
 }
 
@@ -589,14 +564,6 @@ impl<'hir> Generics<'hir> {
         self.params.iter().find(|&param| name == param.name.ident().name)
     }
 
-    pub fn spans(&self) -> MultiSpan {
-        if self.params.is_empty() {
-            self.span.into()
-        } else {
-            self.params.iter().map(|p| p.span).collect::<Vec<Span>>().into()
-        }
-    }
-
     /// If there are generic parameters, return where to introduce a new one.
     pub fn span_for_lifetime_suggestion(&self) -> Option<Span> {
         if let Some(first) = self.params.first()
@@ -679,7 +646,7 @@ impl<'hir> Generics<'hir> {
         )
     }
 
-    pub fn span_for_predicate_removal(&self, pos: usize) -> Span {
+    fn span_for_predicate_removal(&self, pos: usize) -> Span {
         let predicate = &self.predicates[pos];
         let span = predicate.span();
 
@@ -812,7 +779,7 @@ pub struct WhereRegionPredicate<'hir> {
 
 impl<'hir> WhereRegionPredicate<'hir> {
     /// Returns `true` if `param_def_id` matches the `lifetime` of this predicate.
-    pub fn is_param_bound(&self, param_def_id: LocalDefId) -> bool {
+    fn is_param_bound(&self, param_def_id: LocalDefId) -> bool {
         self.lifetime.res == LifetimeName::Param(param_def_id)
     }
 }
@@ -869,7 +836,7 @@ pub struct OwnerNodes<'tcx> {
 }
 
 impl<'tcx> OwnerNodes<'tcx> {
-    pub fn node(&self) -> OwnerNode<'tcx> {
+    fn node(&self) -> OwnerNode<'tcx> {
         use rustc_index::Idx;
         let node = self.nodes[ItemLocalId::new(0)].as_ref().unwrap().node;
         let node = node.as_owner().unwrap(); // Indexing must ensure it is an OwnerNode.
@@ -906,12 +873,12 @@ pub struct OwnerInfo<'hir> {
     /// Contents of the HIR.
     pub nodes: OwnerNodes<'hir>,
     /// Map from each nested owner to its parent's local id.
-    pub parenting: FxHashMap<LocalDefId, ItemLocalId>,
+    pub parenting: LocalDefIdMap<ItemLocalId>,
     /// Collected attributes of the HIR nodes.
     pub attrs: AttributeMap<'hir>,
     /// Map indicating what traits are in scope for places where this
     /// is relevant; generated by resolve.
-    pub trait_map: FxHashMap<ItemLocalId, Box<[TraitCandidate]>>,
+    pub trait_map: ItemLocalMap<Box<[TraitCandidate]>>,
 }
 
 impl<'tcx> OwnerInfo<'tcx> {
@@ -1032,7 +999,7 @@ impl<'hir> Pat<'hir> {
 
         use PatKind::*;
         match self.kind {
-            Wild | Lit(_) | Range(..) | Binding(.., None) | Path(_) => true,
+            Wild | Never | Lit(_) | Range(..) | Binding(.., None) | Path(_) => true,
             Box(s) | Ref(s, _) | Binding(.., Some(s)) => s.walk_short_(it),
             Struct(_, fields, _) => fields.iter().all(|field| field.pat.walk_short_(it)),
             TupleStruct(_, s, _) | Tuple(s, _) | Or(s) => s.iter().all(|p| p.walk_short_(it)),
@@ -1059,7 +1026,7 @@ impl<'hir> Pat<'hir> {
 
         use PatKind::*;
         match self.kind {
-            Wild | Lit(_) | Range(..) | Binding(.., None) | Path(_) => {}
+            Wild | Never | Lit(_) | Range(..) | Binding(.., None) | Path(_) => {}
             Box(s) | Ref(s, _) | Binding(.., Some(s)) => s.walk_(it),
             Struct(_, fields, _) => fields.iter().for_each(|field| field.pat.walk_(it)),
             TupleStruct(_, s, _) | Tuple(s, _) | Or(s) => s.iter().for_each(|p| p.walk_(it)),
@@ -1085,6 +1052,23 @@ impl<'hir> Pat<'hir> {
             true
         })
     }
+
+    /// Whether this a never pattern.
+    pub fn is_never_pattern(&self) -> bool {
+        let mut is_never_pattern = false;
+        self.walk(|pat| match &pat.kind {
+            PatKind::Never => {
+                is_never_pattern = true;
+                false
+            }
+            PatKind::Or(s) => {
+                is_never_pattern = s.iter().all(|p| p.is_never_pattern());
+                false
+            }
+            _ => true,
+        });
+        is_never_pattern
+    }
 }
 
 /// A single field in a struct pattern.
@@ -1172,6 +1156,9 @@ pub enum PatKind<'hir> {
     /// Invariant: `pats.len() >= 2`.
     Or(&'hir [Pat<'hir>]),
 
+    /// A never pattern `!`.
+    Never,
+
     /// A path pattern for a unit struct/variant or a (maybe-associated) constant.
     Path(QPath<'hir>),
 
@@ -1204,159 +1191,6 @@ pub enum PatKind<'hir> {
     Slice(&'hir [Pat<'hir>], Option<&'hir Pat<'hir>>, &'hir [Pat<'hir>]),
 }
 
-#[derive(Copy, Clone, PartialEq, Debug, HashStable_Generic)]
-pub enum BinOpKind {
-    /// The `+` operator (addition).
-    Add,
-    /// The `-` operator (subtraction).
-    Sub,
-    /// The `*` operator (multiplication).
-    Mul,
-    /// The `/` operator (division).
-    Div,
-    /// The `%` operator (modulus).
-    Rem,
-    /// The `&&` operator (logical and).
-    And,
-    /// The `||` operator (logical or).
-    Or,
-    /// The `^` operator (bitwise xor).
-    BitXor,
-    /// The `&` operator (bitwise and).
-    BitAnd,
-    /// The `|` operator (bitwise or).
-    BitOr,
-    /// The `<<` operator (shift left).
-    Shl,
-    /// The `>>` operator (shift right).
-    Shr,
-    /// The `==` operator (equality).
-    Eq,
-    /// The `<` operator (less than).
-    Lt,
-    /// The `<=` operator (less than or equal to).
-    Le,
-    /// The `!=` operator (not equal to).
-    Ne,
-    /// The `>=` operator (greater than or equal to).
-    Ge,
-    /// The `>` operator (greater than).
-    Gt,
-}
-
-impl BinOpKind {
-    pub fn as_str(self) -> &'static str {
-        match self {
-            BinOpKind::Add => "+",
-            BinOpKind::Sub => "-",
-            BinOpKind::Mul => "*",
-            BinOpKind::Div => "/",
-            BinOpKind::Rem => "%",
-            BinOpKind::And => "&&",
-            BinOpKind::Or => "||",
-            BinOpKind::BitXor => "^",
-            BinOpKind::BitAnd => "&",
-            BinOpKind::BitOr => "|",
-            BinOpKind::Shl => "<<",
-            BinOpKind::Shr => ">>",
-            BinOpKind::Eq => "==",
-            BinOpKind::Lt => "<",
-            BinOpKind::Le => "<=",
-            BinOpKind::Ne => "!=",
-            BinOpKind::Ge => ">=",
-            BinOpKind::Gt => ">",
-        }
-    }
-
-    pub fn is_lazy(self) -> bool {
-        matches!(self, BinOpKind::And | BinOpKind::Or)
-    }
-
-    pub fn is_shift(self) -> bool {
-        matches!(self, BinOpKind::Shl | BinOpKind::Shr)
-    }
-
-    pub fn is_comparison(self) -> bool {
-        match self {
-            BinOpKind::Eq
-            | BinOpKind::Lt
-            | BinOpKind::Le
-            | BinOpKind::Ne
-            | BinOpKind::Gt
-            | BinOpKind::Ge => true,
-            BinOpKind::And
-            | BinOpKind::Or
-            | BinOpKind::Add
-            | BinOpKind::Sub
-            | BinOpKind::Mul
-            | BinOpKind::Div
-            | BinOpKind::Rem
-            | BinOpKind::BitXor
-            | BinOpKind::BitAnd
-            | BinOpKind::BitOr
-            | BinOpKind::Shl
-            | BinOpKind::Shr => false,
-        }
-    }
-
-    /// Returns `true` if the binary operator takes its arguments by value.
-    pub fn is_by_value(self) -> bool {
-        !self.is_comparison()
-    }
-}
-
-impl Into<ast::BinOpKind> for BinOpKind {
-    fn into(self) -> ast::BinOpKind {
-        match self {
-            BinOpKind::Add => ast::BinOpKind::Add,
-            BinOpKind::Sub => ast::BinOpKind::Sub,
-            BinOpKind::Mul => ast::BinOpKind::Mul,
-            BinOpKind::Div => ast::BinOpKind::Div,
-            BinOpKind::Rem => ast::BinOpKind::Rem,
-            BinOpKind::And => ast::BinOpKind::And,
-            BinOpKind::Or => ast::BinOpKind::Or,
-            BinOpKind::BitXor => ast::BinOpKind::BitXor,
-            BinOpKind::BitAnd => ast::BinOpKind::BitAnd,
-            BinOpKind::BitOr => ast::BinOpKind::BitOr,
-            BinOpKind::Shl => ast::BinOpKind::Shl,
-            BinOpKind::Shr => ast::BinOpKind::Shr,
-            BinOpKind::Eq => ast::BinOpKind::Eq,
-            BinOpKind::Lt => ast::BinOpKind::Lt,
-            BinOpKind::Le => ast::BinOpKind::Le,
-            BinOpKind::Ne => ast::BinOpKind::Ne,
-            BinOpKind::Ge => ast::BinOpKind::Ge,
-            BinOpKind::Gt => ast::BinOpKind::Gt,
-        }
-    }
-}
-
-pub type BinOp = Spanned<BinOpKind>;
-
-#[derive(Copy, Clone, PartialEq, Debug, HashStable_Generic)]
-pub enum UnOp {
-    /// The `*` operator (dereferencing).
-    Deref,
-    /// The `!` operator (logical negation).
-    Not,
-    /// The `-` operator (negation).
-    Neg,
-}
-
-impl UnOp {
-    pub fn as_str(self) -> &'static str {
-        match self {
-            Self::Deref => "*",
-            Self::Not => "!",
-            Self::Neg => "-",
-        }
-    }
-
-    /// Returns `true` if the unary operator takes its argument by value.
-    pub fn is_by_value(self) -> bool {
-        matches!(self, Self::Neg | Self::Not)
-    }
-}
-
 /// A statement.
 #[derive(Debug, Clone, Copy, HashStable_Generic)]
 pub struct Stmt<'hir> {
@@ -1516,15 +1350,18 @@ impl<'hir> Body<'hir> {
 }
 
 /// The type of source expression that caused this coroutine to be created.
-#[derive(Clone, PartialEq, Eq, Debug, Copy, Hash)]
-#[derive(HashStable_Generic, Encodable, Decodable)]
+#[derive(Clone, PartialEq, Eq, Debug, Copy, Hash, HashStable_Generic, Encodable, Decodable)]
 pub enum CoroutineKind {
-    /// An explicit `async` block or the body of an async function.
+    /// An explicit `async` block or the body of an `async` function.
     Async(CoroutineSource),
 
     /// An explicit `gen` block or the body of a `gen` function.
     Gen(CoroutineSource),
 
+    /// An explicit `async gen` block or the body of an `async gen` function,
+    /// which is able to both `yield` and `.await`.
+    AsyncGen(CoroutineSource),
+
     /// A coroutine literal created via a `yield` inside a closure.
     Coroutine,
 }
@@ -1549,6 +1386,14 @@ impl fmt::Display for CoroutineKind {
                 }
                 k.fmt(f)
             }
+            CoroutineKind::AsyncGen(k) => {
+                if f.alternate() {
+                    f.write_str("`async gen` ")?;
+                } else {
+                    f.write_str("async gen ")?
+                }
+                k.fmt(f)
+            }
         }
     }
 }
@@ -1558,8 +1403,7 @@ impl fmt::Display for CoroutineKind {
 ///
 /// This helps error messages but is also used to drive coercions in
 /// type-checking (see #60424).
-#[derive(Clone, PartialEq, Eq, Hash, Debug, Copy)]
-#[derive(HashStable_Generic, Encodable, Decodable)]
+#[derive(Clone, PartialEq, Eq, Hash, Debug, Copy, HashStable_Generic, Encodable, Decodable)]
 pub enum CoroutineSource {
     /// An explicit `async`/`gen` block written by the user.
     Block,
@@ -1712,7 +1556,7 @@ impl Expr<'_> {
             ExprKind::Call(..) => ExprPrecedence::Call,
             ExprKind::MethodCall(..) => ExprPrecedence::MethodCall,
             ExprKind::Tup(_) => ExprPrecedence::Tup,
-            ExprKind::Binary(op, ..) => ExprPrecedence::Binary(op.node.into()),
+            ExprKind::Binary(op, ..) => ExprPrecedence::Binary(op.node),
             ExprKind::Unary(..) => ExprPrecedence::Unary,
             ExprKind::Lit(_) => ExprPrecedence::Lit,
             ExprKind::Type(..) | ExprKind::Cast(..) => ExprPrecedence::Cast,
@@ -1852,11 +1696,9 @@ impl Expr<'_> {
                 // them being used only for its side-effects.
                 base.can_have_side_effects()
             }
-            ExprKind::Struct(_, fields, init) => fields
-                .iter()
-                .map(|field| field.expr)
-                .chain(init.into_iter())
-                .any(|e| e.can_have_side_effects()),
+            ExprKind::Struct(_, fields, init) => {
+                fields.iter().map(|field| field.expr).chain(init).any(|e| e.can_have_side_effects())
+            }
 
             ExprKind::Array(args)
             | ExprKind::Tup(args)
@@ -2094,8 +1936,8 @@ pub enum QPath<'hir> {
     /// the `X` and `Y` nodes each being a `TyKind::Path(QPath::TypeRelative(..))`.
     TypeRelative(&'hir Ty<'hir>, &'hir PathSegment<'hir>),
 
-    /// Reference to a `#[lang = "foo"]` item. `HirId` of the inner expr.
-    LangItem(LangItem, Span, Option<HirId>),
+    /// Reference to a `#[lang = "foo"]` item.
+    LangItem(LangItem, Span),
 }
 
 impl<'hir> QPath<'hir> {
@@ -2104,7 +1946,7 @@ impl<'hir> QPath<'hir> {
         match *self {
             QPath::Resolved(_, path) => path.span,
             QPath::TypeRelative(qself, ps) => qself.span.to(ps.ident.span),
-            QPath::LangItem(_, span, _) => span,
+            QPath::LangItem(_, span) => span,
         }
     }
 
@@ -2114,17 +1956,7 @@ impl<'hir> QPath<'hir> {
         match *self {
             QPath::Resolved(_, path) => path.span,
             QPath::TypeRelative(qself, _) => qself.span,
-            QPath::LangItem(_, span, _) => span,
-        }
-    }
-
-    /// Returns the span of the last segment of this `QPath`. For example, `method` in
-    /// `<() as Trait>::method`.
-    pub fn last_segment_span(&self) -> Span {
-        match *self {
-            QPath::Resolved(_, path) => path.segments.last().unwrap().ident.span,
-            QPath::TypeRelative(_, segment) => segment.ident.span,
-            QPath::LangItem(_, span, _) => span,
+            QPath::LangItem(_, span) => span,
         }
     }
 }
@@ -2153,8 +1985,7 @@ pub enum LocalSource {
 }
 
 /// Hints at the original code for a `match _ { .. }`.
-#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)]
-#[derive(HashStable_Generic, Encodable, Decodable)]
+#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic, Encodable, Decodable)]
 pub enum MatchSource {
     /// A `match _ { .. }`.
     Normal,
@@ -2256,17 +2087,6 @@ impl fmt::Display for YieldSource {
     }
 }
 
-impl From<CoroutineKind> for YieldSource {
-    fn from(kind: CoroutineKind) -> Self {
-        match kind {
-            // Guess based on the kind of the current coroutine.
-            CoroutineKind::Coroutine => Self::Yield,
-            CoroutineKind::Async(_) => Self::Await { expr: None },
-            CoroutineKind::Gen(_) => Self::Yield,
-        }
-    }
-}
-
 // N.B., if you change this, you'll probably want to change the corresponding
 // type structure in middle/ty.rs as well.
 #[derive(Debug, Clone, Copy, HashStable_Generic)]
@@ -2314,6 +2134,35 @@ pub struct TraitItem<'hir> {
     pub defaultness: Defaultness,
 }
 
+macro_rules! expect_methods_self_kind {
+    ( $( $name:ident, $ret_ty:ty, $pat:pat, $ret_val:expr; )* ) => {
+        $(
+            #[track_caller]
+            pub fn $name(&self) -> $ret_ty {
+                let $pat = &self.kind else { expect_failed(stringify!($ident), self) };
+                $ret_val
+            }
+        )*
+    }
+}
+
+macro_rules! expect_methods_self {
+    ( $( $name:ident, $ret_ty:ty, $pat:pat, $ret_val:expr; )* ) => {
+        $(
+            #[track_caller]
+            pub fn $name(&self) -> $ret_ty {
+                let $pat = self else { expect_failed(stringify!($ident), self) };
+                $ret_val
+            }
+        )*
+    }
+}
+
+#[track_caller]
+fn expect_failed<T: fmt::Debug>(ident: &'static str, found: T) -> ! {
+    panic!("{ident}: found {found:?}")
+}
+
 impl<'hir> TraitItem<'hir> {
     #[inline]
     pub fn hir_id(&self) -> HirId {
@@ -2325,30 +2174,15 @@ impl<'hir> TraitItem<'hir> {
         TraitItemId { owner_id: self.owner_id }
     }
 
-    /// Expect an [`TraitItemKind::Const`] or panic.
-    #[track_caller]
-    pub fn expect_const(&self) -> (&'hir Ty<'hir>, Option<BodyId>) {
-        let TraitItemKind::Const(ty, body) = self.kind else { self.expect_failed("a constant") };
-        (ty, body)
-    }
-
-    /// Expect an [`TraitItemKind::Fn`] or panic.
-    #[track_caller]
-    pub fn expect_fn(&self) -> (&FnSig<'hir>, &TraitFn<'hir>) {
-        let TraitItemKind::Fn(ty, trfn) = &self.kind else { self.expect_failed("a function") };
-        (ty, trfn)
-    }
+    expect_methods_self_kind! {
+        expect_const, (&'hir Ty<'hir>, Option<BodyId>),
+            TraitItemKind::Const(ty, body), (ty, *body);
 
-    /// Expect an [`TraitItemKind::Type`] or panic.
-    #[track_caller]
-    pub fn expect_type(&self) -> (GenericBounds<'hir>, Option<&'hir Ty<'hir>>) {
-        let TraitItemKind::Type(bounds, ty) = self.kind else { self.expect_failed("a type") };
-        (bounds, ty)
-    }
+        expect_fn, (&FnSig<'hir>, &TraitFn<'hir>),
+            TraitItemKind::Fn(ty, trfn), (ty, trfn);
 
-    #[track_caller]
-    fn expect_failed(&self, expected: &'static str) -> ! {
-        panic!("expected {expected} item, found {self:?}")
+        expect_type, (GenericBounds<'hir>, Option<&'hir Ty<'hir>>),
+            TraitItemKind::Type(bounds, ty), (bounds, *ty);
     }
 }
 
@@ -2413,30 +2247,10 @@ impl<'hir> ImplItem<'hir> {
         ImplItemId { owner_id: self.owner_id }
     }
 
-    /// Expect an [`ImplItemKind::Const`] or panic.
-    #[track_caller]
-    pub fn expect_const(&self) -> (&'hir Ty<'hir>, BodyId) {
-        let ImplItemKind::Const(ty, body) = self.kind else { self.expect_failed("a constant") };
-        (ty, body)
-    }
-
-    /// Expect an [`ImplItemKind::Fn`] or panic.
-    #[track_caller]
-    pub fn expect_fn(&self) -> (&FnSig<'hir>, BodyId) {
-        let ImplItemKind::Fn(ty, body) = &self.kind else { self.expect_failed("a function") };
-        (ty, *body)
-    }
-
-    /// Expect an [`ImplItemKind::Type`] or panic.
-    #[track_caller]
-    pub fn expect_type(&self) -> &'hir Ty<'hir> {
-        let ImplItemKind::Type(ty) = self.kind else { self.expect_failed("a type") };
-        ty
-    }
-
-    #[track_caller]
-    fn expect_failed(&self, expected: &'static str) -> ! {
-        panic!("expected {expected} item, found {self:?}")
+    expect_methods_self_kind! {
+        expect_const, (&'hir Ty<'hir>, BodyId), ImplItemKind::Const(ty, body), (ty, *body);
+        expect_fn,    (&FnSig<'hir>, BodyId),   ImplItemKind::Fn(ty, body),    (ty, *body);
+        expect_type,  &'hir Ty<'hir>,           ImplItemKind::Type(ty),        ty;
     }
 }
 
@@ -2452,9 +2266,6 @@ pub enum ImplItemKind<'hir> {
     Type(&'hir Ty<'hir>),
 }
 
-/// The name of the associated type for `Fn` return types.
-pub const FN_OUTPUT_NAME: Symbol = sym::Output;
-
 /// Bind a type to an associated type (i.e., `A = Foo`).
 ///
 /// Bindings like `A: Debug` are represented as a special type `A =
@@ -2579,8 +2390,7 @@ impl<'hir> Ty<'hir> {
 }
 
 /// Not represented directly in the AST; referred to by name through a `ty_path`.
-#[derive(Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, Debug)]
-#[derive(HashStable_Generic)]
+#[derive(Copy, Clone, PartialEq, Eq, Encodable, Decodable, Hash, Debug, HashStable_Generic)]
 pub enum PrimTy {
     Int(IntTy),
     Uint(UintTy),
@@ -2860,8 +2670,7 @@ impl ImplicitSelfKind {
     }
 }
 
-#[derive(Copy, Clone, PartialEq, Eq, Encodable, Decodable, Debug)]
-#[derive(HashStable_Generic)]
+#[derive(Copy, Clone, PartialEq, Eq, Encodable, Decodable, Debug, HashStable_Generic)]
 pub enum IsAsync {
     Async(Span),
     NotAsync,
@@ -3000,7 +2809,7 @@ impl TraitRef<'_> {
         match self.path.res {
             Res::Def(DefKind::Trait | DefKind::TraitAlias, did) => Some(did),
             Res::Err => None,
-            _ => unreachable!(),
+            res => panic!("{res:?} did not resolve to a trait or trait alias"),
         }
     }
 }
@@ -3039,7 +2848,11 @@ pub enum VariantData<'hir> {
     /// A struct variant.
     ///
     /// E.g., `Bar { .. }` as in `enum Foo { Bar { .. } }`.
-    Struct(&'hir [FieldDef<'hir>], /* recovered */ bool),
+    Struct {
+        fields: &'hir [FieldDef<'hir>],
+        // FIXME: investigate making this a `Option<ErrorGuaranteed>`
+        recovered: bool,
+    },
     /// A tuple variant.
     ///
     /// E.g., `Bar(..)` as in `enum Foo { Bar(..) }`.
@@ -3054,7 +2867,7 @@ impl<'hir> VariantData<'hir> {
     /// Return the fields of this variant.
     pub fn fields(&self) -> &'hir [FieldDef<'hir>] {
         match *self {
-            VariantData::Struct(ref fields, ..) | VariantData::Tuple(ref fields, ..) => fields,
+            VariantData::Struct { fields, .. } | VariantData::Tuple(fields, ..) => fields,
             _ => &[],
         }
     }
@@ -3063,7 +2876,7 @@ impl<'hir> VariantData<'hir> {
         match *self {
             VariantData::Tuple(_, hir_id, def_id) => Some((CtorKind::Fn, hir_id, def_id)),
             VariantData::Unit(hir_id, def_id) => Some((CtorKind::Const, hir_id, def_id)),
-            VariantData::Struct(..) => None,
+            VariantData::Struct { .. } => None,
         }
     }
 
@@ -3124,134 +2937,51 @@ impl<'hir> Item<'hir> {
         ItemId { owner_id: self.owner_id }
     }
 
-    /// Expect an [`ItemKind::ExternCrate`] or panic.
-    #[track_caller]
-    pub fn expect_extern_crate(&self) -> Option<Symbol> {
-        let ItemKind::ExternCrate(s) = self.kind else { self.expect_failed("an extern crate") };
-        s
-    }
+    expect_methods_self_kind! {
+        expect_extern_crate, Option<Symbol>, ItemKind::ExternCrate(s), *s;
 
-    /// Expect an [`ItemKind::Use`] or panic.
-    #[track_caller]
-    pub fn expect_use(&self) -> (&'hir UsePath<'hir>, UseKind) {
-        let ItemKind::Use(p, uk) = self.kind else { self.expect_failed("a use") };
-        (p, uk)
-    }
+        expect_use, (&'hir UsePath<'hir>, UseKind), ItemKind::Use(p, uk), (p, *uk);
 
-    /// Expect an [`ItemKind::Static`] or panic.
-    #[track_caller]
-    pub fn expect_static(&self) -> (&'hir Ty<'hir>, Mutability, BodyId) {
-        let ItemKind::Static(ty, mutbl, body) = self.kind else { self.expect_failed("a static") };
-        (ty, mutbl, body)
-    }
-    /// Expect an [`ItemKind::Const`] or panic.
-    #[track_caller]
-    pub fn expect_const(&self) -> (&'hir Ty<'hir>, &'hir Generics<'hir>, BodyId) {
-        let ItemKind::Const(ty, gen, body) = self.kind else { self.expect_failed("a constant") };
-        (ty, gen, body)
-    }
-    /// Expect an [`ItemKind::Fn`] or panic.
-    #[track_caller]
-    pub fn expect_fn(&self) -> (&FnSig<'hir>, &'hir Generics<'hir>, BodyId) {
-        let ItemKind::Fn(sig, gen, body) = &self.kind else { self.expect_failed("a function") };
-        (sig, gen, *body)
-    }
+        expect_static, (&'hir Ty<'hir>, Mutability, BodyId),
+            ItemKind::Static(ty, mutbl, body), (ty, *mutbl, *body);
 
-    /// Expect an [`ItemKind::Macro`] or panic.
-    #[track_caller]
-    pub fn expect_macro(&self) -> (&ast::MacroDef, MacroKind) {
-        let ItemKind::Macro(def, mk) = &self.kind else { self.expect_failed("a macro") };
-        (def, *mk)
-    }
+        expect_const, (&'hir Ty<'hir>, &'hir Generics<'hir>, BodyId),
+            ItemKind::Const(ty, gen, body), (ty, gen, *body);
 
-    /// Expect an [`ItemKind::Mod`] or panic.
-    #[track_caller]
-    pub fn expect_mod(&self) -> &'hir Mod<'hir> {
-        let ItemKind::Mod(m) = self.kind else { self.expect_failed("a module") };
-        m
-    }
+        expect_fn, (&FnSig<'hir>, &'hir Generics<'hir>, BodyId),
+            ItemKind::Fn(sig, gen, body), (sig, gen, *body);
 
-    /// Expect an [`ItemKind::ForeignMod`] or panic.
-    #[track_caller]
-    pub fn expect_foreign_mod(&self) -> (Abi, &'hir [ForeignItemRef]) {
-        let ItemKind::ForeignMod { abi, items } = self.kind else {
-            self.expect_failed("a foreign module")
-        };
-        (abi, items)
-    }
+        expect_macro, (&ast::MacroDef, MacroKind), ItemKind::Macro(def, mk), (def, *mk);
 
-    /// Expect an [`ItemKind::GlobalAsm`] or panic.
-    #[track_caller]
-    pub fn expect_global_asm(&self) -> &'hir InlineAsm<'hir> {
-        let ItemKind::GlobalAsm(asm) = self.kind else { self.expect_failed("a global asm") };
-        asm
-    }
+        expect_mod, &'hir Mod<'hir>, ItemKind::Mod(m), m;
 
-    /// Expect an [`ItemKind::TyAlias`] or panic.
-    #[track_caller]
-    pub fn expect_ty_alias(&self) -> (&'hir Ty<'hir>, &'hir Generics<'hir>) {
-        let ItemKind::TyAlias(ty, gen) = self.kind else { self.expect_failed("a type alias") };
-        (ty, gen)
-    }
+        expect_foreign_mod, (Abi, &'hir [ForeignItemRef]),
+            ItemKind::ForeignMod { abi, items }, (*abi, items);
 
-    /// Expect an [`ItemKind::OpaqueTy`] or panic.
-    #[track_caller]
-    pub fn expect_opaque_ty(&self) -> &OpaqueTy<'hir> {
-        let ItemKind::OpaqueTy(ty) = &self.kind else { self.expect_failed("an opaque type") };
-        ty
-    }
+        expect_global_asm, &'hir InlineAsm<'hir>, ItemKind::GlobalAsm(asm), asm;
 
-    /// Expect an [`ItemKind::Enum`] or panic.
-    #[track_caller]
-    pub fn expect_enum(&self) -> (&EnumDef<'hir>, &'hir Generics<'hir>) {
-        let ItemKind::Enum(def, gen) = &self.kind else { self.expect_failed("an enum") };
-        (def, gen)
-    }
+        expect_ty_alias, (&'hir Ty<'hir>, &'hir Generics<'hir>),
+            ItemKind::TyAlias(ty, gen), (ty, gen);
 
-    /// Expect an [`ItemKind::Struct`] or panic.
-    #[track_caller]
-    pub fn expect_struct(&self) -> (&VariantData<'hir>, &'hir Generics<'hir>) {
-        let ItemKind::Struct(data, gen) = &self.kind else { self.expect_failed("a struct") };
-        (data, gen)
-    }
+        expect_opaque_ty, &OpaqueTy<'hir>, ItemKind::OpaqueTy(ty), ty;
 
-    /// Expect an [`ItemKind::Union`] or panic.
-    #[track_caller]
-    pub fn expect_union(&self) -> (&VariantData<'hir>, &'hir Generics<'hir>) {
-        let ItemKind::Union(data, gen) = &self.kind else { self.expect_failed("a union") };
-        (data, gen)
-    }
+        expect_enum, (&EnumDef<'hir>, &'hir Generics<'hir>), ItemKind::Enum(def, gen), (def, gen);
 
-    /// Expect an [`ItemKind::Trait`] or panic.
-    #[track_caller]
-    pub fn expect_trait(
-        self,
-    ) -> (IsAuto, Unsafety, &'hir Generics<'hir>, GenericBounds<'hir>, &'hir [TraitItemRef]) {
-        let ItemKind::Trait(is_auto, unsafety, gen, bounds, items) = self.kind else {
-            self.expect_failed("a trait")
-        };
-        (is_auto, unsafety, gen, bounds, items)
-    }
+        expect_struct, (&VariantData<'hir>, &'hir Generics<'hir>),
+            ItemKind::Struct(data, gen), (data, gen);
 
-    /// Expect an [`ItemKind::TraitAlias`] or panic.
-    #[track_caller]
-    pub fn expect_trait_alias(&self) -> (&'hir Generics<'hir>, GenericBounds<'hir>) {
-        let ItemKind::TraitAlias(gen, bounds) = self.kind else {
-            self.expect_failed("a trait alias")
-        };
-        (gen, bounds)
-    }
+        expect_union, (&VariantData<'hir>, &'hir Generics<'hir>),
+            ItemKind::Union(data, gen), (data, gen);
 
-    /// Expect an [`ItemKind::Impl`] or panic.
-    #[track_caller]
-    pub fn expect_impl(&self) -> &'hir Impl<'hir> {
-        let ItemKind::Impl(imp) = self.kind else { self.expect_failed("an impl") };
-        imp
-    }
+        expect_trait,
+            (IsAuto, Unsafety, &'hir Generics<'hir>, GenericBounds<'hir>, &'hir [TraitItemRef]),
+            ItemKind::Trait(is_auto, unsafety, gen, bounds, items),
+            (*is_auto, *unsafety, gen, bounds, items);
+
+        expect_trait_alias, (&'hir Generics<'hir>, GenericBounds<'hir>),
+            ItemKind::TraitAlias(gen, bounds), (gen, bounds);
 
-    #[track_caller]
-    fn expect_failed(&self, expected: &'static str) -> ! {
-        panic!("expected {expected} item, found {self:?}")
+        expect_impl, &'hir Impl<'hir>, ItemKind::Impl(imp), imp;
     }
 }
 
@@ -3280,8 +3010,7 @@ impl fmt::Display for Unsafety {
     }
 }
 
-#[derive(Copy, Clone, PartialEq, Eq, Debug)]
-#[derive(Encodable, Decodable, HashStable_Generic)]
+#[derive(Copy, Clone, PartialEq, Eq, Debug, Encodable, Decodable, HashStable_Generic)]
 pub enum Constness {
     Const,
     NotConst,
@@ -3624,32 +3353,11 @@ impl<'hir> OwnerNode<'hir> {
         }
     }
 
-    pub fn expect_item(self) -> &'hir Item<'hir> {
-        match self {
-            OwnerNode::Item(n) => n,
-            _ => panic!(),
-        }
-    }
-
-    pub fn expect_foreign_item(self) -> &'hir ForeignItem<'hir> {
-        match self {
-            OwnerNode::ForeignItem(n) => n,
-            _ => panic!(),
-        }
-    }
-
-    pub fn expect_impl_item(self) -> &'hir ImplItem<'hir> {
-        match self {
-            OwnerNode::ImplItem(n) => n,
-            _ => panic!(),
-        }
-    }
-
-    pub fn expect_trait_item(self) -> &'hir TraitItem<'hir> {
-        match self {
-            OwnerNode::TraitItem(n) => n,
-            _ => panic!(),
-        }
+    expect_methods_self! {
+        expect_item,         &'hir Item<'hir>,        OwnerNode::Item(n),        n;
+        expect_foreign_item, &'hir ForeignItem<'hir>, OwnerNode::ForeignItem(n), n;
+        expect_impl_item,    &'hir ImplItem<'hir>,    OwnerNode::ImplItem(n),    n;
+        expect_trait_item,   &'hir TraitItem<'hir>,   OwnerNode::TraitItem(n),   n;
     }
 }
 
@@ -3902,196 +3610,33 @@ impl<'hir> Node<'hir> {
         }
     }
 
-    /// Get the fields for the tuple-constructor,
-    /// if this node is a tuple constructor, otherwise None
-    pub fn tuple_fields(&self) -> Option<&'hir [FieldDef<'hir>]> {
-        if let Node::Ctor(&VariantData::Tuple(fields, _, _)) = self { Some(fields) } else { None }
-    }
-
-    /// Expect a [`Node::Param`] or panic.
-    #[track_caller]
-    pub fn expect_param(self) -> &'hir Param<'hir> {
-        let Node::Param(this) = self else { self.expect_failed("a parameter") };
-        this
-    }
-
-    /// Expect a [`Node::Item`] or panic.
-    #[track_caller]
-    pub fn expect_item(self) -> &'hir Item<'hir> {
-        let Node::Item(this) = self else { self.expect_failed("a item") };
-        this
-    }
-
-    /// Expect a [`Node::ForeignItem`] or panic.
-    #[track_caller]
-    pub fn expect_foreign_item(self) -> &'hir ForeignItem<'hir> {
-        let Node::ForeignItem(this) = self else { self.expect_failed("a foreign item") };
-        this
-    }
-
-    /// Expect a [`Node::TraitItem`] or panic.
-    #[track_caller]
-    pub fn expect_trait_item(self) -> &'hir TraitItem<'hir> {
-        let Node::TraitItem(this) = self else { self.expect_failed("a trait item") };
-        this
-    }
-
-    /// Expect a [`Node::ImplItem`] or panic.
-    #[track_caller]
-    pub fn expect_impl_item(self) -> &'hir ImplItem<'hir> {
-        let Node::ImplItem(this) = self else { self.expect_failed("an implementation item") };
-        this
-    }
-
-    /// Expect a [`Node::Variant`] or panic.
-    #[track_caller]
-    pub fn expect_variant(self) -> &'hir Variant<'hir> {
-        let Node::Variant(this) = self else { self.expect_failed("a variant") };
-        this
-    }
-
-    /// Expect a [`Node::Field`] or panic.
-    #[track_caller]
-    pub fn expect_field(self) -> &'hir FieldDef<'hir> {
-        let Node::Field(this) = self else { self.expect_failed("a field definition") };
-        this
-    }
-
-    /// Expect a [`Node::AnonConst`] or panic.
-    #[track_caller]
-    pub fn expect_anon_const(self) -> &'hir AnonConst {
-        let Node::AnonConst(this) = self else { self.expect_failed("an anonymous constant") };
-        this
-    }
-
-    /// Expect a [`Node::ConstBlock`] or panic.
-    #[track_caller]
-    pub fn expect_inline_const(self) -> &'hir ConstBlock {
-        let Node::ConstBlock(this) = self else { self.expect_failed("an inline constant") };
-        this
-    }
-
-    /// Expect a [`Node::Expr`] or panic.
-    #[track_caller]
-    pub fn expect_expr(self) -> &'hir Expr<'hir> {
-        let Node::Expr(this) = self else { self.expect_failed("an expression") };
-        this
-    }
-    /// Expect a [`Node::ExprField`] or panic.
-    #[track_caller]
-    pub fn expect_expr_field(self) -> &'hir ExprField<'hir> {
-        let Node::ExprField(this) = self else { self.expect_failed("an expression field") };
-        this
-    }
-
-    /// Expect a [`Node::Stmt`] or panic.
-    #[track_caller]
-    pub fn expect_stmt(self) -> &'hir Stmt<'hir> {
-        let Node::Stmt(this) = self else { self.expect_failed("a statement") };
-        this
-    }
-
-    /// Expect a [`Node::PathSegment`] or panic.
-    #[track_caller]
-    pub fn expect_path_segment(self) -> &'hir PathSegment<'hir> {
-        let Node::PathSegment(this) = self else { self.expect_failed("a path segment") };
-        this
-    }
-
-    /// Expect a [`Node::Ty`] or panic.
-    #[track_caller]
-    pub fn expect_ty(self) -> &'hir Ty<'hir> {
-        let Node::Ty(this) = self else { self.expect_failed("a type") };
-        this
-    }
-
-    /// Expect a [`Node::TypeBinding`] or panic.
-    #[track_caller]
-    pub fn expect_type_binding(self) -> &'hir TypeBinding<'hir> {
-        let Node::TypeBinding(this) = self else { self.expect_failed("a type binding") };
-        this
-    }
-
-    /// Expect a [`Node::TraitRef`] or panic.
-    #[track_caller]
-    pub fn expect_trait_ref(self) -> &'hir TraitRef<'hir> {
-        let Node::TraitRef(this) = self else { self.expect_failed("a trait reference") };
-        this
-    }
-
-    /// Expect a [`Node::Pat`] or panic.
-    #[track_caller]
-    pub fn expect_pat(self) -> &'hir Pat<'hir> {
-        let Node::Pat(this) = self else { self.expect_failed("a pattern") };
-        this
-    }
-
-    /// Expect a [`Node::PatField`] or panic.
-    #[track_caller]
-    pub fn expect_pat_field(self) -> &'hir PatField<'hir> {
-        let Node::PatField(this) = self else { self.expect_failed("a pattern field") };
-        this
-    }
-
-    /// Expect a [`Node::Arm`] or panic.
-    #[track_caller]
-    pub fn expect_arm(self) -> &'hir Arm<'hir> {
-        let Node::Arm(this) = self else { self.expect_failed("an arm") };
-        this
-    }
-
-    /// Expect a [`Node::Block`] or panic.
-    #[track_caller]
-    pub fn expect_block(self) -> &'hir Block<'hir> {
-        let Node::Block(this) = self else { self.expect_failed("a block") };
-        this
-    }
-
-    /// Expect a [`Node::Local`] or panic.
-    #[track_caller]
-    pub fn expect_local(self) -> &'hir Local<'hir> {
-        let Node::Local(this) = self else { self.expect_failed("a local") };
-        this
-    }
-
-    /// Expect a [`Node::Ctor`] or panic.
-    #[track_caller]
-    pub fn expect_ctor(self) -> &'hir VariantData<'hir> {
-        let Node::Ctor(this) = self else { self.expect_failed("a constructor") };
-        this
-    }
-
-    /// Expect a [`Node::Lifetime`] or panic.
-    #[track_caller]
-    pub fn expect_lifetime(self) -> &'hir Lifetime {
-        let Node::Lifetime(this) = self else { self.expect_failed("a lifetime") };
-        this
-    }
-
-    /// Expect a [`Node::GenericParam`] or panic.
-    #[track_caller]
-    pub fn expect_generic_param(self) -> &'hir GenericParam<'hir> {
-        let Node::GenericParam(this) = self else { self.expect_failed("a generic parameter") };
-        this
-    }
-
-    /// Expect a [`Node::Crate`] or panic.
-    #[track_caller]
-    pub fn expect_crate(self) -> &'hir Mod<'hir> {
-        let Node::Crate(this) = self else { self.expect_failed("a crate") };
-        this
-    }
-
-    /// Expect a [`Node::Infer`] or panic.
-    #[track_caller]
-    pub fn expect_infer(self) -> &'hir InferArg {
-        let Node::Infer(this) = self else { self.expect_failed("an infer") };
-        this
-    }
-
-    #[track_caller]
-    fn expect_failed(&self, expected: &'static str) -> ! {
-        panic!("expected {expected} node, found {self:?}")
+    expect_methods_self! {
+        expect_param,         &'hir Param<'hir>,        Node::Param(n),        n;
+        expect_item,          &'hir Item<'hir>,         Node::Item(n),         n;
+        expect_foreign_item,  &'hir ForeignItem<'hir>,  Node::ForeignItem(n),  n;
+        expect_trait_item,    &'hir TraitItem<'hir>,    Node::TraitItem(n),    n;
+        expect_impl_item,     &'hir ImplItem<'hir>,     Node::ImplItem(n),     n;
+        expect_variant,       &'hir Variant<'hir>,      Node::Variant(n),      n;
+        expect_field,         &'hir FieldDef<'hir>,     Node::Field(n),        n;
+        expect_anon_const,    &'hir AnonConst,          Node::AnonConst(n),    n;
+        expect_inline_const,  &'hir ConstBlock,         Node::ConstBlock(n),   n;
+        expect_expr,          &'hir Expr<'hir>,         Node::Expr(n),         n;
+        expect_expr_field,    &'hir ExprField<'hir>,    Node::ExprField(n),    n;
+        expect_stmt,          &'hir Stmt<'hir>,         Node::Stmt(n),         n;
+        expect_path_segment,  &'hir PathSegment<'hir>,  Node::PathSegment(n),  n;
+        expect_ty,            &'hir Ty<'hir>,           Node::Ty(n),           n;
+        expect_type_binding,  &'hir TypeBinding<'hir>,  Node::TypeBinding(n),  n;
+        expect_trait_ref,     &'hir TraitRef<'hir>,     Node::TraitRef(n),     n;
+        expect_pat,           &'hir Pat<'hir>,          Node::Pat(n),          n;
+        expect_pat_field,     &'hir PatField<'hir>,     Node::PatField(n),     n;
+        expect_arm,           &'hir Arm<'hir>,          Node::Arm(n),          n;
+        expect_block,         &'hir Block<'hir>,        Node::Block(n),        n;
+        expect_local,         &'hir Local<'hir>,        Node::Local(n),        n;
+        expect_ctor,          &'hir VariantData<'hir>,  Node::Ctor(n),         n;
+        expect_lifetime,      &'hir Lifetime,           Node::Lifetime(n),     n;
+        expect_generic_param, &'hir GenericParam<'hir>, Node::GenericParam(n), n;
+        expect_crate,         &'hir Mod<'hir>,          Node::Crate(n),        n;
+        expect_infer,         &'hir InferArg,           Node::Infer(n),        n;
     }
 }
 
diff --git a/compiler/rustc_hir/src/hir_id.rs b/compiler/rustc_hir/src/hir_id.rs
index 34c615779..d339075c1 100644
--- a/compiler/rustc_hir/src/hir_id.rs
+++ b/compiler/rustc_hir/src/hir_id.rs
@@ -3,8 +3,7 @@ use rustc_data_structures::stable_hasher::{HashStable, StableHasher, StableOrd,
 use rustc_span::{def_id::DefPathHash, HashStableContext};
 use std::fmt::{self, Debug};
 
-#[derive(Copy, Clone, PartialEq, Eq, Hash)]
-#[derive(Encodable, Decodable)]
+#[derive(Copy, Clone, PartialEq, Eq, Hash, Encodable, Decodable)]
 pub struct OwnerId {
     pub def_id: LocalDefId,
 }
@@ -73,8 +72,7 @@ impl<CTX: HashStableContext> ToStableHashKey<CTX> for OwnerId {
 /// the `local_id` part of the `HirId` changing, which is a very useful property in
 /// incremental compilation where we have to persist things through changes to
 /// the code base.
-#[derive(Copy, Clone, PartialEq, Eq, Hash)]
-#[derive(Encodable, Decodable, HashStable_Generic)]
+#[derive(Copy, Clone, PartialEq, Eq, Hash, Encodable, Decodable, HashStable_Generic)]
 #[rustc_pass_by_value]
 pub struct HirId {
     pub owner: OwnerId,
@@ -156,6 +154,8 @@ rustc_index::newtype_index! {
     /// an "item-like" to something else can be implemented by a `Vec` instead of a
     /// tree or hash map.
     #[derive(HashStable_Generic)]
+    #[encodable]
+    #[orderable]
     pub struct ItemLocalId {}
 }
 
diff --git a/compiler/rustc_hir/src/intravisit.rs b/compiler/rustc_hir/src/intravisit.rs
index 8a6728559..67e058a32 100644
--- a/compiler/rustc_hir/src/intravisit.rs
+++ b/compiler/rustc_hir/src/intravisit.rs
@@ -660,7 +660,7 @@ pub fn walk_pat<'v, V: Visitor<'v>>(visitor: &mut V, pattern: &'v Pat<'v>) {
             walk_list!(visitor, visit_expr, lower_bound);
             walk_list!(visitor, visit_expr, upper_bound);
         }
-        PatKind::Wild => (),
+        PatKind::Never | PatKind::Wild => (),
         PatKind::Slice(prepatterns, ref slice_pattern, postpatterns) => {
             walk_list!(visitor, visit_pat, prepatterns);
             walk_list!(visitor, visit_pat, slice_pattern);
@@ -874,7 +874,7 @@ pub fn walk_generic_param<'v, V: Visitor<'v>>(visitor: &mut V, param: &'v Generi
     match param.kind {
         GenericParamKind::Lifetime { .. } => {}
         GenericParamKind::Type { ref default, .. } => walk_list!(visitor, visit_ty, default),
-        GenericParamKind::Const { ref ty, ref default } => {
+        GenericParamKind::Const { ref ty, ref default, is_host_effect: _ } => {
             visitor.visit_ty(ty);
             if let Some(ref default) = default {
                 visitor.visit_const_param_default(param.hir_id, default);
@@ -1075,10 +1075,6 @@ pub fn walk_param_bound<'v, V: Visitor<'v>>(visitor: &mut V, bound: &'v GenericB
         GenericBound::Trait(ref typ, _modifier) => {
             visitor.visit_poly_trait_ref(typ);
         }
-        GenericBound::LangItemTrait(_, _span, hir_id, args) => {
-            visitor.visit_id(hir_id);
-            visitor.visit_generic_args(args);
-        }
         GenericBound::Outlives(ref lifetime) => visitor.visit_lifetime(lifetime),
     }
 }
diff --git a/compiler/rustc_hir/src/lang_items.rs b/compiler/rustc_hir/src/lang_items.rs
index 1d1a1ee88..b0b53bb74 100644
--- a/compiler/rustc_hir/src/lang_items.rs
+++ b/compiler/rustc_hir/src/lang_items.rs
@@ -212,6 +212,7 @@ language_item_table! {
 
     Iterator,                sym::iterator,            iterator_trait,             Target::Trait,          GenericRequirement::Exact(0);
     Future,                  sym::future_trait,        future_trait,               Target::Trait,          GenericRequirement::Exact(0);
+    AsyncIterator,           sym::async_iterator,      async_iterator_trait,       Target::Trait,          GenericRequirement::Exact(0);
     CoroutineState,          sym::coroutine_state,     coroutine_state,            Target::Enum,           GenericRequirement::None;
     Coroutine,               sym::coroutine,           coroutine_trait,            Target::Trait,          GenericRequirement::Minimum(1);
     Unpin,                   sym::unpin,               unpin_trait,                Target::Trait,          GenericRequirement::None;
@@ -233,7 +234,7 @@ language_item_table! {
     PanicFmt,                sym::panic_fmt,           panic_fmt,                  Target::Fn,             GenericRequirement::None;
     ConstPanicFmt,           sym::const_panic_fmt,     const_panic_fmt,            Target::Fn,             GenericRequirement::None;
     PanicBoundsCheck,        sym::panic_bounds_check,  panic_bounds_check_fn,      Target::Fn,             GenericRequirement::Exact(0);
-    PanicMisalignedPointerDereference,        sym::panic_misaligned_pointer_dereference,  panic_misaligned_pointer_dereference_fn,      Target::Fn,             GenericRequirement::Exact(0);
+    PanicMisalignedPointerDereference, sym::panic_misaligned_pointer_dereference, panic_misaligned_pointer_dereference_fn, Target::Fn, GenericRequirement::Exact(0);
     PanicInfo,               sym::panic_info,          panic_info,                 Target::Struct,         GenericRequirement::None;
     PanicLocation,           sym::panic_location,      panic_location,             Target::Struct,         GenericRequirement::None;
     PanicImpl,               sym::panic_impl,          panic_impl,                 Target::Fn,             GenericRequirement::None;
@@ -294,6 +295,10 @@ language_item_table! {
     PollReady,               sym::Ready,               poll_ready_variant,         Target::Variant,        GenericRequirement::None;
     PollPending,             sym::Pending,             poll_pending_variant,       Target::Variant,        GenericRequirement::None;
 
+    AsyncGenReady,           sym::AsyncGenReady,       async_gen_ready,            Target::Method(MethodKind::Inherent), GenericRequirement::Exact(1);
+    AsyncGenPending,         sym::AsyncGenPending,     async_gen_pending,          Target::AssocConst,     GenericRequirement::Exact(1);
+    AsyncGenFinished,        sym::AsyncGenFinished,    async_gen_finished,         Target::AssocConst,     GenericRequirement::Exact(1);
+
     // FIXME(swatinem): the following lang items are used for async lowering and
     // should become obsolete eventually.
     ResumeTy,                sym::ResumeTy,            resume_ty,                  Target::Struct,         GenericRequirement::None;
diff --git a/compiler/rustc_hir/src/lib.rs b/compiler/rustc_hir/src/lib.rs
index 094d5b1e7..87de3c087 100644
--- a/compiler/rustc_hir/src/lib.rs
+++ b/compiler/rustc_hir/src/lib.rs
@@ -4,7 +4,6 @@
 
 #![feature(associated_type_defaults)]
 #![feature(closure_track_caller)]
-#![feature(const_btree_len)]
 #![feature(let_chains)]
 #![feature(min_specialization)]
 #![feature(never_type)]
diff --git a/compiler/rustc_hir/src/pat_util.rs b/compiler/rustc_hir/src/pat_util.rs
index 838c123f8..e60503271 100644
--- a/compiler/rustc_hir/src/pat_util.rs
+++ b/compiler/rustc_hir/src/pat_util.rs
@@ -1,7 +1,6 @@
 use crate::def::{CtorOf, DefKind, Res};
-use crate::def_id::DefId;
+use crate::def_id::{DefId, DefIdSet};
 use crate::hir::{self, BindingAnnotation, ByRef, HirId, PatKind};
-use rustc_data_structures::fx::FxHashSet;
 use rustc_span::symbol::Ident;
 use rustc_span::Span;
 
@@ -114,9 +113,9 @@ impl hir::Pat<'_> {
             }
             _ => true,
         });
-        // We remove duplicates by inserting into a `FxHashSet` to avoid re-ordering
+        // We remove duplicates by inserting into a hash set to avoid re-ordering
         // the bounds
-        let mut duplicates = FxHashSet::default();
+        let mut duplicates = DefIdSet::default();
         variants.retain(|def_id| duplicates.insert(*def_id));
         variants
     }
diff --git a/compiler/rustc_hir/src/target.rs b/compiler/rustc_hir/src/target.rs
index 0d65ddb56..8948a03e4 100644
--- a/compiler/rustc_hir/src/target.rs
+++ b/compiler/rustc_hir/src/target.rs
@@ -67,6 +67,42 @@ impl Display for Target {
 }
 
 impl Target {
+    pub fn is_associated_item(self) -> bool {
+        match self {
+            Target::AssocConst | Target::AssocTy | Target::Method(_) => true,
+            Target::ExternCrate
+            | Target::Use
+            | Target::Static
+            | Target::Const
+            | Target::Fn
+            | Target::Closure
+            | Target::Mod
+            | Target::ForeignMod
+            | Target::GlobalAsm
+            | Target::TyAlias
+            | Target::OpaqueTy
+            | Target::Enum
+            | Target::Variant
+            | Target::Struct
+            | Target::Field
+            | Target::Union
+            | Target::Trait
+            | Target::TraitAlias
+            | Target::Impl
+            | Target::Expression
+            | Target::Statement
+            | Target::Arm
+            | Target::ForeignFn
+            | Target::ForeignStatic
+            | Target::ForeignTy
+            | Target::GenericParam(_)
+            | Target::MacroDef
+            | Target::Param
+            | Target::PatField
+            | Target::ExprField => false,
+        }
+    }
+
     pub fn from_item(item: &Item<'_>) -> Target {
         match item.kind {
             ItemKind::ExternCrate(..) => Target::ExternCrate,
diff --git a/compiler/rustc_hir_analysis/messages.ftl b/compiler/rustc_hir_analysis/messages.ftl
index 8ab91bebc..139e1c0ac 100644
--- a/compiler/rustc_hir_analysis/messages.ftl
+++ b/compiler/rustc_hir_analysis/messages.ftl
@@ -1,8 +1,28 @@
+hir_analysis_ambiguous_assoc_item = ambiguous associated {$assoc_kind} `{$assoc_name}` in bounds of `{$ty_param_name}`
+    .label = ambiguous associated {$assoc_kind} `{$assoc_name}`
+
 hir_analysis_ambiguous_lifetime_bound =
     ambiguous lifetime bound, explicit lifetime bound required
 
-hir_analysis_assoc_bound_on_const = expected associated type, found {$descr}
-    .note = trait bounds not allowed on {$descr}
+hir_analysis_assoc_item_not_found = associated {$assoc_kind} `{$assoc_name}` not found for `{$ty_param_name}`
+
+hir_analysis_assoc_item_not_found_found_in_other_trait_label = there is {$identically_named ->
+        [true] an
+        *[false] a similarly named
+    } associated {$assoc_kind} `{$suggested_name}` in the trait `{$trait_name}`
+hir_analysis_assoc_item_not_found_label = associated {$assoc_kind} `{$assoc_name}` not found
+hir_analysis_assoc_item_not_found_other_sugg = `{$ty_param_name}` has the following associated {$assoc_kind}
+hir_analysis_assoc_item_not_found_similar_in_other_trait_sugg = change the associated {$assoc_kind} name to use `{$suggested_name}` from `{$trait_name}`
+hir_analysis_assoc_item_not_found_similar_in_other_trait_with_bound_sugg = and also change the associated {$assoc_kind} name
+hir_analysis_assoc_item_not_found_similar_sugg = there is an associated {$assoc_kind} with a similar name
+
+hir_analysis_assoc_kind_mismatch = expected {$expected}, found {$got}
+    .label = unexpected {$got}
+    .expected_because_label = expected a {$expected} because of this associated {$expected}
+    .note = the associated {$assoc_kind} is defined here
+    .bound_on_assoc_const_label = bounds are not allowed on associated constants
+
+hir_analysis_assoc_kind_mismatch_wrap_in_braces_sugg = consider adding braces here
 
 hir_analysis_assoc_type_binding_not_allowed =
     associated type bindings are not allowed here
@@ -280,10 +300,6 @@ hir_analysis_placeholder_not_allowed_item_signatures = the placeholder `_` is no
 
 hir_analysis_requires_note = the `{$trait_name}` impl for `{$ty}` requires that `{$error_predicate}`
 
-hir_analysis_return_type_notation_conflicting_bound =
-    ambiguous associated function `{$assoc_name}` for `{$ty_name}`
-    .note = `{$assoc_name}` is declared in two supertraits: `{$first_bound}` and `{$second_bound}`
-
 hir_analysis_return_type_notation_equality_bound =
     return type notation is not allowed to use type equality
 
@@ -294,9 +310,6 @@ hir_analysis_return_type_notation_illegal_param_type =
     return type notation is not allowed for functions that have type parameters
     .label = type parameter declared here
 
-hir_analysis_return_type_notation_missing_method =
-    cannot find associated function `{$assoc_name}` for `{$ty_name}`
-
 hir_analysis_return_type_notation_on_non_rpitit =
     return type notation used on function that is not `async` and does not return `impl Trait`
     .note = function returns `{$ty}`, which is not compatible with associated type return bounds
diff --git a/compiler/rustc_hir_analysis/src/astconv/bounds.rs b/compiler/rustc_hir_analysis/src/astconv/bounds.rs
index 3e700f2da..0748644cc 100644
--- a/compiler/rustc_hir_analysis/src/astconv/bounds.rs
+++ b/compiler/rustc_hir_analysis/src/astconv/bounds.rs
@@ -3,8 +3,7 @@ use rustc_errors::struct_span_err;
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
 use rustc_hir::def_id::{DefId, LocalDefId};
-use rustc_lint_defs::Applicability;
-use rustc_middle::ty::{self as ty, Ty, TypeVisitableExt};
+use rustc_middle::ty::{self as ty, Ty};
 use rustc_span::symbol::Ident;
 use rustc_span::{ErrorGuaranteed, Span};
 use rustc_trait_selection::traits;
@@ -135,17 +134,6 @@ impl<'tcx> dyn AstConv<'tcx> + '_ {
                         only_self_bounds,
                     );
                 }
-                &hir::GenericBound::LangItemTrait(lang_item, span, hir_id, args) => {
-                    self.instantiate_lang_item_trait_ref(
-                        lang_item,
-                        span,
-                        hir_id,
-                        args,
-                        param_ty,
-                        bounds,
-                        only_self_bounds,
-                    );
-                }
                 hir::GenericBound::Outlives(lifetime) => {
                     let region = self.ast_region_to_region(lifetime, None);
                     bounds.push_region_bound(
@@ -194,8 +182,7 @@ impl<'tcx> dyn AstConv<'tcx> + '_ {
 
         self.add_bounds(
             param_ty,
-            ast_bounds.iter().filter(|bound| {
-                match filter {
+            ast_bounds.iter().filter(|bound| match filter {
                 PredicateFilter::All
                 | PredicateFilter::SelfOnly
                 | PredicateFilter::SelfAndAssociatedTypeBounds => true,
@@ -209,7 +196,6 @@ impl<'tcx> dyn AstConv<'tcx> + '_ {
                         false
                     }
                 }
-            }
             }),
             &mut bounds,
             ty::List::empty(),
@@ -258,64 +244,49 @@ impl<'tcx> dyn AstConv<'tcx> + '_ {
 
         let tcx = self.tcx();
 
-        let return_type_notation =
-            binding.gen_args.parenthesized == hir::GenericArgsParentheses::ReturnTypeNotation;
-
-        let candidate = if return_type_notation {
-            if self.trait_defines_associated_item_named(
-                trait_ref.def_id(),
-                ty::AssocKind::Fn,
-                binding.item_name,
-            ) {
-                trait_ref
+        let assoc_kind =
+            if binding.gen_args.parenthesized == hir::GenericArgsParentheses::ReturnTypeNotation {
+                ty::AssocKind::Fn
+            } else if let ConvertedBindingKind::Equality(term) = binding.kind
+                && let ty::TermKind::Const(_) = term.node.unpack()
+            {
+                ty::AssocKind::Const
             } else {
-                self.one_bound_for_assoc_method(
-                    traits::supertraits(tcx, trait_ref),
-                    trait_ref.print_only_trait_path(),
-                    binding.item_name,
-                    path_span,
-                )?
-            }
-        } else if self.trait_defines_associated_item_named(
+                ty::AssocKind::Type
+            };
+
+        let candidate = if self.trait_defines_associated_item_named(
             trait_ref.def_id(),
-            ty::AssocKind::Type,
+            assoc_kind,
             binding.item_name,
         ) {
-            // Simple case: X is defined in the current trait.
+            // Simple case: The assoc item is defined in the current trait.
             trait_ref
         } else {
             // Otherwise, we have to walk through the supertraits to find
-            // those that do.
-            self.one_bound_for_assoc_type(
+            // one that does define it.
+            self.one_bound_for_assoc_item(
                 || traits::supertraits(tcx, trait_ref),
                 trait_ref.skip_binder().print_only_trait_name(),
                 None,
+                assoc_kind,
                 binding.item_name,
                 path_span,
-                match binding.kind {
-                    ConvertedBindingKind::Equality(term) => Some(term),
-                    _ => None,
-                },
+                Some(&binding),
             )?
         };
 
         let (assoc_ident, def_scope) =
             tcx.adjust_ident_and_get_scope(binding.item_name, candidate.def_id(), hir_ref_id);
 
-        // We have already adjusted the item name above, so compare with `ident.normalize_to_macros_2_0()` instead
-        // of calling `filter_by_name_and_kind`.
-        let find_item_of_kind = |kind| {
-            tcx.associated_items(candidate.def_id())
-                .filter_by_name_unhygienic(assoc_ident.name)
-                .find(|i| i.kind == kind && i.ident(tcx).normalize_to_macros_2_0() == assoc_ident)
-        };
-        let assoc_item = if return_type_notation {
-            find_item_of_kind(ty::AssocKind::Fn)
-        } else {
-            find_item_of_kind(ty::AssocKind::Type)
-                .or_else(|| find_item_of_kind(ty::AssocKind::Const))
-        }
-        .expect("missing associated type");
+        // We have already adjusted the item name above, so compare with `.normalize_to_macros_2_0()`
+        // instead of calling `filter_by_name_and_kind` which would needlessly normalize the
+        // `assoc_ident` again and again.
+        let assoc_item = tcx
+            .associated_items(candidate.def_id())
+            .filter_by_name_unhygienic(assoc_ident.name)
+            .find(|i| i.kind == assoc_kind && i.ident(tcx).normalize_to_macros_2_0() == assoc_ident)
+            .expect("missing associated item");
 
         if !assoc_item.visibility(tcx).is_accessible_from(def_scope, tcx) {
             tcx.sess
@@ -342,7 +313,7 @@ impl<'tcx> dyn AstConv<'tcx> + '_ {
                 .or_insert(binding.span);
         }
 
-        let projection_ty = if return_type_notation {
+        let projection_ty = if let ty::AssocKind::Fn = assoc_kind {
             let mut emitted_bad_param_err = false;
             // If we have an method return type bound, then we need to substitute
             // the method's early bound params with suitable late-bound params.
@@ -350,7 +321,7 @@ impl<'tcx> dyn AstConv<'tcx> + '_ {
             let args =
                 candidate.skip_binder().args.extend_to(tcx, assoc_item.def_id, |param, _| {
                     let subst = match param.kind {
-                        ty::GenericParamDefKind::Lifetime => ty::Region::new_late_bound(
+                        ty::GenericParamDefKind::Lifetime => ty::Region::new_bound(
                             tcx,
                             ty::INNERMOST,
                             ty::BoundRegion {
@@ -469,7 +440,7 @@ impl<'tcx> dyn AstConv<'tcx> + '_ {
                 let late_bound_in_trait_ref =
                     tcx.collect_constrained_late_bound_regions(&projection_ty);
                 let late_bound_in_ty =
-                    tcx.collect_referenced_late_bound_regions(&trait_ref.rebind(ty));
+                    tcx.collect_referenced_late_bound_regions(&trait_ref.rebind(ty.node));
                 debug!(?late_bound_in_trait_ref);
                 debug!(?late_bound_in_ty);
 
@@ -494,77 +465,27 @@ impl<'tcx> dyn AstConv<'tcx> + '_ {
             }
         }
 
-        let assoc_item_def_id = projection_ty.skip_binder().def_id;
-        let def_kind = tcx.def_kind(assoc_item_def_id);
         match binding.kind {
-            ConvertedBindingKind::Equality(..) if return_type_notation => {
+            ConvertedBindingKind::Equality(..) if let ty::AssocKind::Fn = assoc_kind => {
                 return Err(self.tcx().sess.emit_err(
                     crate::errors::ReturnTypeNotationEqualityBound { span: binding.span },
                 ));
             }
-            ConvertedBindingKind::Equality(mut term) => {
+            ConvertedBindingKind::Equality(term) => {
                 // "Desugar" a constraint like `T: Iterator<Item = u32>` this to
                 // the "projection predicate" for:
                 //
                 // `<T as Iterator>::Item = u32`
-                match (def_kind, term.unpack()) {
-                    (DefKind::AssocTy, ty::TermKind::Ty(_))
-                    | (DefKind::AssocConst, ty::TermKind::Const(_)) => (),
-                    (_, _) => {
-                        let got = if let Some(_) = term.ty() { "type" } else { "constant" };
-                        let expected = tcx.def_descr(assoc_item_def_id);
-                        let mut err = tcx.sess.struct_span_err(
-                            binding.span,
-                            format!("expected {expected} bound, found {got}"),
-                        );
-                        err.span_note(
-                            tcx.def_span(assoc_item_def_id),
-                            format!("{expected} defined here"),
-                        );
-
-                        if let DefKind::AssocConst = def_kind
-                            && let Some(t) = term.ty()
-                            && (t.is_enum() || t.references_error())
-                            && tcx.features().associated_const_equality
-                        {
-                            err.span_suggestion(
-                                binding.span,
-                                "if equating a const, try wrapping with braces",
-                                format!("{} = {{ const }}", binding.item_name),
-                                Applicability::HasPlaceholders,
-                            );
-                        }
-                        let reported = err.emit();
-                        term = match def_kind {
-                            DefKind::AssocTy => Ty::new_error(tcx, reported).into(),
-                            DefKind::AssocConst => ty::Const::new_error(
-                                tcx,
-                                reported,
-                                tcx.type_of(assoc_item_def_id)
-                                    .instantiate(tcx, projection_ty.skip_binder().args),
-                            )
-                            .into(),
-                            _ => unreachable!(),
-                        };
-                    }
-                }
                 bounds.push_projection_bound(
                     tcx,
-                    projection_ty
-                        .map_bound(|projection_ty| ty::ProjectionPredicate { projection_ty, term }),
+                    projection_ty.map_bound(|projection_ty| ty::ProjectionPredicate {
+                        projection_ty,
+                        term: term.node,
+                    }),
                     binding.span,
                 );
             }
             ConvertedBindingKind::Constraint(ast_bounds) => {
-                match def_kind {
-                    DefKind::AssocTy => {}
-                    _ => {
-                        return Err(tcx.sess.emit_err(errors::AssocBoundOnConst {
-                            span: assoc_ident.span,
-                            descr: tcx.def_descr(assoc_item_def_id),
-                        }));
-                    }
-                }
                 // "Desugar" a constraint like `T: Iterator<Item: Debug>` to
                 //
                 // `<T as Iterator>::Item: Debug`
diff --git a/compiler/rustc_hir_analysis/src/astconv/errors.rs b/compiler/rustc_hir_analysis/src/astconv/errors.rs
index 32be7e083..13ad9a453 100644
--- a/compiler/rustc_hir_analysis/src/astconv/errors.rs
+++ b/compiler/rustc_hir_analysis/src/astconv/errors.rs
@@ -1,23 +1,22 @@
-use crate::astconv::AstConv;
+use crate::astconv::{AstConv, ConvertedBindingKind};
 use crate::errors::{
-    AssocTypeBindingNotAllowed, ManualImplementation, MissingTypeParams,
+    self, AssocTypeBindingNotAllowed, ManualImplementation, MissingTypeParams,
     ParenthesizedFnTraitExpansion,
 };
+use crate::fluent_generated as fluent;
 use crate::traits::error_reporting::report_object_safety_error;
-use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::fx::{FxHashMap, FxIndexMap, FxIndexSet};
 use rustc_errors::{pluralize, struct_span_err, Applicability, Diagnostic, ErrorGuaranteed};
 use rustc_hir as hir;
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_infer::traits::FulfillmentError;
-use rustc_middle::ty::{self, suggest_constraining_type_param, Ty, TyCtxt};
+use rustc_middle::ty::{self, suggest_constraining_type_param, Ty, TyCtxt, TypeVisitableExt};
 use rustc_session::parse::feature_err;
 use rustc_span::edit_distance::find_best_match_for_name;
 use rustc_span::symbol::{sym, Ident};
 use rustc_span::{Span, Symbol, DUMMY_SP};
 use rustc_trait_selection::traits::object_safety_violations_for_assoc_item;
 
-use std::collections::BTreeSet;
-
 impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
     /// On missing type parameters, emit an E0393 error and provide a structured suggestion using
     /// the type parameter's name as a placeholder.
@@ -99,83 +98,88 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         }
     }
 
-    pub(crate) fn complain_about_assoc_type_not_found<I>(
+    pub(super) fn complain_about_assoc_item_not_found<I>(
         &self,
         all_candidates: impl Fn() -> I,
         ty_param_name: &str,
         ty_param_def_id: Option<LocalDefId>,
+        assoc_kind: ty::AssocKind,
         assoc_name: Ident,
         span: Span,
+        binding: Option<&super::ConvertedBinding<'_, 'tcx>>,
     ) -> ErrorGuaranteed
     where
         I: Iterator<Item = ty::PolyTraitRef<'tcx>>,
     {
+        let tcx = self.tcx();
+
+        // First and foremost, provide a more user-friendly & “intuitive” error on kind mismatches.
+        if let Some(assoc_item) = all_candidates().find_map(|r| {
+            tcx.associated_items(r.def_id())
+                .filter_by_name_unhygienic(assoc_name.name)
+                .find(|item| tcx.hygienic_eq(assoc_name, item.ident(tcx), r.def_id()))
+        }) {
+            return self.complain_about_assoc_kind_mismatch(
+                assoc_item, assoc_kind, assoc_name, span, binding,
+            );
+        }
+
+        let assoc_kind_str = super::assoc_kind_str(assoc_kind);
+
         // The fallback span is needed because `assoc_name` might be an `Fn()`'s `Output` without a
         // valid span, so we point at the whole path segment instead.
         let is_dummy = assoc_name.span == DUMMY_SP;
 
-        let mut err = struct_span_err!(
-            self.tcx().sess,
-            if is_dummy { span } else { assoc_name.span },
-            E0220,
-            "associated type `{}` not found for `{}`",
+        let mut err = errors::AssocItemNotFound {
+            span: if is_dummy { span } else { assoc_name.span },
             assoc_name,
-            ty_param_name
-        );
+            assoc_kind: assoc_kind_str,
+            ty_param_name,
+            label: None,
+            sugg: None,
+        };
 
         if is_dummy {
-            err.span_label(span, format!("associated type `{assoc_name}` not found"));
-            return err.emit();
+            err.label = Some(errors::AssocItemNotFoundLabel::NotFound { span });
+            return tcx.sess.emit_err(err);
         }
 
         let all_candidate_names: Vec<_> = all_candidates()
-            .flat_map(|r| self.tcx().associated_items(r.def_id()).in_definition_order())
+            .flat_map(|r| tcx.associated_items(r.def_id()).in_definition_order())
             .filter_map(|item| {
-                if !item.is_impl_trait_in_trait() && item.kind == ty::AssocKind::Type {
-                    Some(item.name)
-                } else {
-                    None
-                }
+                (!item.is_impl_trait_in_trait() && item.kind == assoc_kind).then_some(item.name)
             })
             .collect();
 
         if let Some(suggested_name) =
             find_best_match_for_name(&all_candidate_names, assoc_name.name, None)
         {
-            err.span_suggestion(
-                assoc_name.span,
-                "there is an associated type with a similar name",
+            err.sugg = Some(errors::AssocItemNotFoundSugg::Similar {
+                span: assoc_name.span,
+                assoc_kind: assoc_kind_str,
                 suggested_name,
-                Applicability::MaybeIncorrect,
-            );
-            return err.emit();
+            });
+            return tcx.sess.emit_err(err);
         }
 
         // If we didn't find a good item in the supertraits (or couldn't get
         // the supertraits), like in ItemCtxt, then look more generally from
         // all visible traits. If there's one clear winner, just suggest that.
 
-        let visible_traits: Vec<_> = self
-            .tcx()
+        let visible_traits: Vec<_> = tcx
             .all_traits()
             .filter(|trait_def_id| {
-                let viz = self.tcx().visibility(*trait_def_id);
+                let viz = tcx.visibility(*trait_def_id);
                 let def_id = self.item_def_id();
-                viz.is_accessible_from(def_id, self.tcx())
+                viz.is_accessible_from(def_id, tcx)
             })
             .collect();
 
         let wider_candidate_names: Vec<_> = visible_traits
             .iter()
-            .flat_map(|trait_def_id| {
-                self.tcx().associated_items(*trait_def_id).in_definition_order()
-            })
+            .flat_map(|trait_def_id| tcx.associated_items(*trait_def_id).in_definition_order())
             .filter_map(|item| {
-                if !item.is_impl_trait_in_trait() && item.kind == ty::AssocKind::Type {
-                    Some(item.name)
-                } else {
-                    None
-                }
+                (!item.is_impl_trait_in_trait() && item.kind == assoc_kind).then_some(item.name)
             })
             .collect();
 
@@ -184,96 +188,155 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         {
             if let [best_trait] = visible_traits
                 .iter()
+                .copied()
                 .filter(|trait_def_id| {
-                    self.tcx()
-                        .associated_items(*trait_def_id)
+                    tcx.associated_items(trait_def_id)
                         .filter_by_name_unhygienic(suggested_name)
-                        .any(|item| item.kind == ty::AssocKind::Type)
+                        .any(|item| item.kind == assoc_kind)
                 })
                 .collect::<Vec<_>>()[..]
             {
-                let trait_name = self.tcx().def_path_str(*best_trait);
-                let an = if suggested_name != assoc_name.name { "a similarly named" } else { "an" };
-                err.span_label(
-                    assoc_name.span,
-                    format!(
-                        "there is {an} associated type `{suggested_name}` in the \
-                         trait `{trait_name}`",
-                    ),
-                );
-                let hir = self.tcx().hir();
+                let trait_name = tcx.def_path_str(best_trait);
+                err.label = Some(errors::AssocItemNotFoundLabel::FoundInOtherTrait {
+                    span: assoc_name.span,
+                    assoc_kind: assoc_kind_str,
+                    trait_name: &trait_name,
+                    suggested_name,
+                    identically_named: suggested_name == assoc_name.name,
+                });
+                let hir = tcx.hir();
                 if let Some(def_id) = ty_param_def_id
-                    && let parent = hir.get_parent_item(hir.local_def_id_to_hir_id(def_id))
+                    && let parent = hir.get_parent_item(tcx.local_def_id_to_hir_id(def_id))
                     && let Some(generics) = hir.get_generics(parent.def_id)
                 {
-                    if generics.bounds_for_param(def_id)
-                        .flat_map(|pred| pred.bounds.iter())
-                        .any(|b| match b {
+                    if generics.bounds_for_param(def_id).flat_map(|pred| pred.bounds.iter()).any(
+                        |b| match b {
                             hir::GenericBound::Trait(t, ..) => {
-                                t.trait_ref.trait_def_id().as_ref() == Some(best_trait)
+                                t.trait_ref.trait_def_id() == Some(best_trait)
                             }
                             _ => false,
-                        })
-                    {
+                        },
+                    ) {
                         // The type param already has a bound for `trait_name`, we just need to
-                        // change the associated type.
-                        err.span_suggestion_verbose(
-                            assoc_name.span,
-                            format!(
-                                "change the associated type name to use `{suggested_name}` from \
-                                 `{trait_name}`",
-                            ),
-                            suggested_name.to_string(),
-                            Applicability::MaybeIncorrect,
-                        );
-                    } else if suggest_constraining_type_param(
-                            self.tcx(),
-                            generics,
-                            &mut err,
-                            &ty_param_name,
-                            &trait_name,
-                            None,
-                            None,
-                        )
-                        && suggested_name != assoc_name.name
+                        // change the associated item.
+                        err.sugg = Some(errors::AssocItemNotFoundSugg::SimilarInOtherTrait {
+                            span: assoc_name.span,
+                            assoc_kind: assoc_kind_str,
+                            suggested_name,
+                        });
+                        return tcx.sess.emit_err(err);
+                    }
+
+                    let mut err = tcx.sess.create_err(err);
+                    if suggest_constraining_type_param(
+                        tcx,
+                        generics,
+                        &mut err,
+                        &ty_param_name,
+                        &trait_name,
+                        None,
+                        None,
+                    ) && suggested_name != assoc_name.name
                     {
                         // We suggested constraining a type parameter, but the associated type on it
                         // was also not an exact match, so we also suggest changing it.
                         err.span_suggestion_verbose(
                             assoc_name.span,
-                            "and also change the associated type name",
+                            fluent::hir_analysis_assoc_item_not_found_similar_in_other_trait_with_bound_sugg,
                             suggested_name.to_string(),
                             Applicability::MaybeIncorrect,
                         );
                     }
+                    return err.emit();
                 }
-                return err.emit();
+                return tcx.sess.emit_err(err);
             }
         }
 
         // If we still couldn't find any associated type, and only one associated type exists,
         // suggests using it.
-
-        if all_candidate_names.len() == 1 {
+        if let [candidate_name] = all_candidate_names.as_slice() {
             // this should still compile, except on `#![feature(associated_type_defaults)]`
             // where it could suggests `type A = Self::A`, thus recursing infinitely
-            let applicability = if self.tcx().features().associated_type_defaults {
+            let applicability = if tcx.features().associated_type_defaults {
                 Applicability::Unspecified
             } else {
                 Applicability::MaybeIncorrect
             };
 
-            err.span_suggestion(
-                assoc_name.span,
-                format!("`{ty_param_name}` has the following associated type"),
-                all_candidate_names.first().unwrap().to_string(),
+            err.sugg = Some(errors::AssocItemNotFoundSugg::Other {
+                span: assoc_name.span,
                 applicability,
-            );
+                ty_param_name,
+                assoc_kind: assoc_kind_str,
+                suggested_name: *candidate_name,
+            });
         } else {
-            err.span_label(assoc_name.span, format!("associated type `{assoc_name}` not found"));
+            err.label = Some(errors::AssocItemNotFoundLabel::NotFound { span: assoc_name.span });
         }
 
-        err.emit()
+        tcx.sess.emit_err(err)
+    }
+
+    fn complain_about_assoc_kind_mismatch(
+        &self,
+        assoc_item: &ty::AssocItem,
+        assoc_kind: ty::AssocKind,
+        ident: Ident,
+        span: Span,
+        binding: Option<&super::ConvertedBinding<'_, 'tcx>>,
+    ) -> ErrorGuaranteed {
+        let tcx = self.tcx();
+
+        let bound_on_assoc_const_label = if let ty::AssocKind::Const = assoc_item.kind
+            && let Some(binding) = binding
+            && let ConvertedBindingKind::Constraint(_) = binding.kind
+        {
+            let lo = if binding.gen_args.span_ext.is_dummy() {
+                ident.span
+            } else {
+                binding.gen_args.span_ext
+            };
+            Some(lo.between(span.shrink_to_hi()))
+        } else {
+            None
+        };
+
+        // FIXME(associated_const_equality): This has quite a few false positives and negatives.
+        let wrap_in_braces_sugg = if let Some(binding) = binding
+            && let ConvertedBindingKind::Equality(term) = binding.kind
+            && let ty::TermKind::Ty(ty) = term.node.unpack()
+            && (ty.is_enum() || ty.references_error())
+            && tcx.features().associated_const_equality
+        {
+            Some(errors::AssocKindMismatchWrapInBracesSugg {
+                lo: term.span.shrink_to_lo(),
+                hi: term.span.shrink_to_hi(),
+            })
+        } else {
+            None
+        };
+
+        // For equality bounds, we want to blame the term (RHS) instead of the item (LHS) since
+        // one can argue that that's more “untuitive” to the user.
+        let (span, expected_because_label, expected, got) = if let Some(binding) = binding
+            && let ConvertedBindingKind::Equality(term) = binding.kind
+        {
+            (term.span, Some(ident.span), assoc_item.kind, assoc_kind)
+        } else {
+            (ident.span, None, assoc_kind, assoc_item.kind)
+        };
+
+        tcx.sess.emit_err(errors::AssocKindMismatch {
+            span,
+            expected: super::assoc_kind_str(expected),
+            got: super::assoc_kind_str(got),
+            expected_because_label,
+            assoc_kind: super::assoc_kind_str(assoc_item.kind),
+            def_span: tcx.def_span(assoc_item.def_id),
+            bound_on_assoc_const_label,
+            wrap_in_braces_sugg,
+        })
     }
 
     pub(crate) fn complain_about_ambiguous_inherent_assoc_type(
@@ -506,23 +569,24 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
     /// emit a generic note suggesting using a `where` clause to constraint instead.
     pub(crate) fn complain_about_missing_associated_types(
         &self,
-        associated_types: FxHashMap<Span, BTreeSet<DefId>>,
+        associated_types: FxIndexMap<Span, FxIndexSet<DefId>>,
         potential_assoc_types: Vec<Span>,
         trait_bounds: &[hir::PolyTraitRef<'_>],
     ) {
         if associated_types.values().all(|v| v.is_empty()) {
             return;
         }
+
         let tcx = self.tcx();
         // FIXME: Marked `mut` so that we can replace the spans further below with a more
         // appropriate one, but this should be handled earlier in the span assignment.
-        let mut associated_types: FxHashMap<Span, Vec<_>> = associated_types
+        let mut associated_types: FxIndexMap<Span, Vec<_>> = associated_types
             .into_iter()
             .map(|(span, def_ids)| {
                 (span, def_ids.into_iter().map(|did| tcx.associated_item(did)).collect())
             })
             .collect();
-        let mut names: FxHashMap<String, Vec<Symbol>> = Default::default();
+        let mut names: FxIndexMap<String, Vec<Symbol>> = Default::default();
         let mut names_len = 0;
 
         // Account for things like `dyn Foo + 'a`, like in tests `issue-22434.rs` and
@@ -585,6 +649,32 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
             }
         }
 
+        // We get all the associated items that _are_ set,
+        // so that we can check if any of their names match one of the ones we are missing.
+        // This would mean that they are shadowing the associated type we are missing,
+        // and we can then use their span to indicate this to the user.
+        let bound_names = trait_bounds
+            .iter()
+            .filter_map(|poly_trait_ref| {
+                let path = poly_trait_ref.trait_ref.path.segments.last()?;
+                let args = path.args?;
+
+                Some(args.bindings.iter().filter_map(|binding| {
+                    let ident = binding.ident;
+                    let trait_def = path.res.def_id();
+                    let assoc_item = tcx.associated_items(trait_def).find_by_name_and_kind(
+                        tcx,
+                        ident,
+                        ty::AssocKind::Type,
+                        trait_def,
+                    );
+
+                    Some((ident.name, assoc_item?))
+                }))
+            })
+            .flatten()
+            .collect::<FxHashMap<Symbol, &ty::AssocItem>>();
+
         let mut names = names
             .into_iter()
             .map(|(trait_, mut assocs)| {
@@ -625,16 +715,42 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                 *names.entry(item.name).or_insert(0) += 1;
             }
             let mut dupes = false;
+            let mut shadows = false;
             for item in assoc_items {
                 let prefix = if names[&item.name] > 1 {
                     let trait_def_id = item.container_id(tcx);
                     dupes = true;
                     format!("{}::", tcx.def_path_str(trait_def_id))
+                } else if bound_names.get(&item.name).is_some_and(|x| x != &item) {
+                    let trait_def_id = item.container_id(tcx);
+                    shadows = true;
+                    format!("{}::", tcx.def_path_str(trait_def_id))
                 } else {
                     String::new()
                 };
+
+                let mut is_shadowed = false;
+
+                if let Some(assoc_item) = bound_names.get(&item.name)
+                    && assoc_item != &item
+                {
+                    is_shadowed = true;
+
+                    let rename_message =
+                        if assoc_item.def_id.is_local() { ", consider renaming it" } else { "" };
+                    err.span_label(
+                        tcx.def_span(assoc_item.def_id),
+                        format!("`{}{}` shadowed here{}", prefix, item.name, rename_message),
+                    );
+                }
+
+                let rename_message = if is_shadowed { ", consider renaming it" } else { "" };
+
                 if let Some(sp) = tcx.hir().span_if_local(item.def_id) {
-                    err.span_label(sp, format!("`{}{}` defined here", prefix, item.name));
+                    err.span_label(
+                        sp,
+                        format!("`{}{}` defined here{}", prefix, item.name, rename_message),
+                    );
                 }
             }
             if potential_assoc_types.len() == assoc_items.len() {
@@ -642,8 +758,8 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                 // extra type arguments present. A suggesting to replace the generic args with
                 // associated types is already emitted.
                 already_has_generics_args_suggestion = true;
-            } else if let (Ok(snippet), false) =
-                (tcx.sess.source_map().span_to_snippet(*span), dupes)
+            } else if let (Ok(snippet), false, false) =
+                (tcx.sess.source_map().span_to_snippet(*span), dupes, shadows)
             {
                 let types: Vec<_> =
                     assoc_items.iter().map(|item| format!("{} = Type", item.name)).collect();
@@ -725,6 +841,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                 err.span_help(where_constraints, where_msg);
             }
         }
+
         err.emit();
     }
 }
diff --git a/compiler/rustc_hir_analysis/src/astconv/generics.rs b/compiler/rustc_hir_analysis/src/astconv/generics.rs
index d29a27ece..b495b00ec 100644
--- a/compiler/rustc_hir_analysis/src/astconv/generics.rs
+++ b/compiler/rustc_hir_analysis/src/astconv/generics.rs
@@ -243,6 +243,31 @@ pub fn create_args_for_parent_generic_args<'tcx, 'a>(
             match (args_iter.peek(), params.peek()) {
                 (Some(&arg), Some(&param)) => {
                     match (arg, &param.kind, arg_count.explicit_late_bound) {
+                        (
+                            GenericArg::Const(hir::ConstArg {
+                                is_desugared_from_effects: true,
+                                ..
+                            }),
+                            GenericParamDefKind::Const { is_host_effect: false, .. }
+                            | GenericParamDefKind::Type { .. }
+                            | GenericParamDefKind::Lifetime,
+                            _,
+                        ) => {
+                            // SPECIAL CASE FOR DESUGARED EFFECT PARAMS
+                            // This comes from the following example:
+                            //
+                            // ```
+                            // #[const_trait]
+                            // pub trait PartialEq<Rhs: ?Sized = Self> {}
+                            // impl const PartialEq for () {}
+                            // ```
+                            //
+                            // Since this is a const impl, we need to insert a host arg at the end of
+                            // `PartialEq`'s generics, but this errors since `Rhs` isn't specified.
+                            // To work around this, we infer all arguments until we reach the host param.
+                            args.push(ctx.inferred_kind(Some(&args), param, infer_args));
+                            params.next();
+                        }
                         (GenericArg::Lifetime(_), GenericParamDefKind::Lifetime, _)
                         | (
                             GenericArg::Type(_) | GenericArg::Infer(_),
@@ -636,7 +661,7 @@ pub(crate) fn prohibit_explicit_late_bound_lifetimes(
                 args.args[0].hir_id(),
                 multispan,
                 msg,
-                |lint| lint,
+                |_| {},
             );
         }
 
diff --git a/compiler/rustc_hir_analysis/src/astconv/lint.rs b/compiler/rustc_hir_analysis/src/astconv/lint.rs
index bc57bbcca..9afb04b74 100644
--- a/compiler/rustc_hir_analysis/src/astconv/lint.rs
+++ b/compiler/rustc_hir_analysis/src/astconv/lint.rs
@@ -24,7 +24,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                     ..
                 }),
             ..
-        }) = tcx.hir().get_by_def_id(parent_id)
+        }) = tcx.hir_node_by_def_id(parent_id)
             && self_ty.hir_id == impl_self_ty.hir_id
         {
             if !of_trait_ref.trait_def_id().is_some_and(|def_id| def_id.is_local()) {
@@ -106,7 +106,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                     );
                 }
                 // check if the impl trait that we are considering is a impl of a local trait
-                self.maybe_lint_blanket_trait_impl(&self_ty, &mut diag);
+                self.maybe_lint_blanket_trait_impl(self_ty, &mut diag);
                 diag.stash(self_ty.span, StashKey::TraitMissingMethod);
             } else {
                 let msg = "trait objects without an explicit `dyn` are deprecated";
@@ -121,8 +121,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                             sugg,
                             Applicability::MachineApplicable,
                         );
-                        self.maybe_lint_blanket_trait_impl(&self_ty, lint);
-                        lint
+                        self.maybe_lint_blanket_trait_impl(self_ty, lint);
                     },
                 );
             }
diff --git a/compiler/rustc_hir_analysis/src/astconv/mod.rs b/compiler/rustc_hir_analysis/src/astconv/mod.rs
index 2fcb45ef8..6f8e80172 100644
--- a/compiler/rustc_hir_analysis/src/astconv/mod.rs
+++ b/compiler/rustc_hir_analysis/src/astconv/mod.rs
@@ -18,29 +18,30 @@ use crate::require_c_abi_if_c_variadic;
 use rustc_ast::TraitObjectSyntax;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_errors::{
-    struct_span_err, Applicability, Diagnostic, DiagnosticBuilder, ErrorGuaranteed, FatalError,
-    MultiSpan,
+    error_code, struct_span_err, Applicability, Diagnostic, DiagnosticBuilder, ErrorGuaranteed,
+    FatalError, MultiSpan,
 };
 use rustc_hir as hir;
 use rustc_hir::def::{CtorOf, DefKind, Namespace, Res};
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::intravisit::{walk_generics, Visitor as _};
 use rustc_hir::{GenericArg, GenericArgs, OpaqueTyOrigin};
-use rustc_infer::infer::{InferCtxt, InferOk, TyCtxtInferExt};
+use rustc_infer::infer::{InferCtxt, TyCtxtInferExt};
 use rustc_infer::traits::ObligationCause;
 use rustc_middle::middle::stability::AllowUnstable;
 use rustc_middle::ty::GenericParamDefKind;
 use rustc_middle::ty::{
-    self, Const, GenericArgKind, GenericArgsRef, IsSuggestable, Ty, TyCtxt, TypeVisitableExt,
+    self, Const, GenericArgKind, GenericArgsRef, IsSuggestable, ParamEnv, Ty, TyCtxt,
+    TypeVisitableExt,
 };
 use rustc_session::lint::builtin::AMBIGUOUS_ASSOCIATED_ITEMS;
 use rustc_span::edit_distance::find_best_match_for_name;
+use rustc_span::source_map::{respan, Spanned};
 use rustc_span::symbol::{kw, Ident, Symbol};
 use rustc_span::{sym, BytePos, Span, DUMMY_SP};
 use rustc_target::spec::abi;
 use rustc_trait_selection::traits::wf::object_region_bounds;
-use rustc_trait_selection::traits::{self, NormalizeExt, ObligationCtxt};
-use rustc_type_ir::fold::{TypeFoldable, TypeFolder, TypeSuperFoldable};
+use rustc_trait_selection::traits::{self, ObligationCtxt};
 
 use std::fmt::Display;
 use std::slice;
@@ -162,7 +163,7 @@ struct ConvertedBinding<'a, 'tcx> {
 
 #[derive(Debug)]
 enum ConvertedBindingKind<'a, 'tcx> {
-    Equality(ty::Term<'tcx>),
+    Equality(Spanned<ty::Term<'tcx>>),
     Constraint(&'a [hir::GenericBound<'a>]),
 }
 
@@ -239,7 +240,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         def: Option<&ty::GenericParamDef>,
     ) -> ty::Region<'tcx> {
         let tcx = self.tcx();
-        let lifetime_name = |def_id| tcx.hir().name(tcx.hir().local_def_id_to_hir_id(def_id));
+        let lifetime_name = |def_id| tcx.hir().name(tcx.local_def_id_to_hir_id(def_id));
 
         match tcx.named_bound_var(lifetime.hir_id) {
             Some(rbv::ResolvedArg::StaticLifetime) => tcx.lifetimes.re_static,
@@ -250,7 +251,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                     var: ty::BoundVar::from_u32(index),
                     kind: ty::BrNamed(def_id, name),
                 };
-                ty::Region::new_late_bound(tcx, debruijn, br)
+                ty::Region::new_bound(tcx, debruijn, br)
             }
 
             Some(rbv::ResolvedArg::EarlyBound(def_id)) => {
@@ -258,12 +259,12 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                 let item_def_id = tcx.hir().ty_param_owner(def_id.expect_local());
                 let generics = tcx.generics_of(item_def_id);
                 let index = generics.param_def_id_to_index[&def_id];
-                ty::Region::new_early_bound(tcx, ty::EarlyBoundRegion { def_id, index, name })
+                ty::Region::new_early_param(tcx, ty::EarlyParamRegion { def_id, index, name })
             }
 
             Some(rbv::ResolvedArg::Free(scope, id)) => {
                 let name = lifetime_name(id.expect_local());
-                ty::Region::new_free(tcx, scope, ty::BrNamed(id, name))
+                ty::Region::new_late_param(tcx, scope, ty::BrNamed(id, name))
 
                 // (*) -- not late-bound, won't change
             }
@@ -476,7 +477,10 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                             ty::Const::new_misc_error(tcx, ty).into()
                         }
                     }
-                    _ => unreachable!(),
+                    (kind, arg) => span_bug!(
+                        self.span,
+                        "mismatched path argument for kind {kind:?}: found arg {arg:?}"
+                    ),
                 }
             }
 
@@ -595,12 +599,14 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
             .map(|binding| {
                 let kind = match &binding.kind {
                     hir::TypeBindingKind::Equality { term } => match term {
-                        hir::Term::Ty(ty) => {
-                            ConvertedBindingKind::Equality(self.ast_ty_to_ty(ty).into())
-                        }
+                        hir::Term::Ty(ty) => ConvertedBindingKind::Equality(respan(
+                            ty.span,
+                            self.ast_ty_to_ty(ty).into(),
+                        )),
                         hir::Term::Const(c) => {
+                            let span = self.tcx().def_span(c.def_id);
                             let c = Const::from_anon_const(self.tcx(), c.def_id);
-                            ConvertedBindingKind::Equality(c.into())
+                            ConvertedBindingKind::Equality(respan(span, c.into()))
                         }
                     },
                     hir::TypeBindingKind::Constraint { bounds } => {
@@ -672,36 +678,57 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         )
     }
 
-    fn instantiate_poly_trait_ref_inner(
+    /// Given a trait bound like `Debug`, applies that trait bound the given self-type to construct
+    /// a full trait reference. The resulting trait reference is returned. This may also generate
+    /// auxiliary bounds, which are added to `bounds`.
+    ///
+    /// Example:
+    ///
+    /// ```ignore (illustrative)
+    /// poly_trait_ref = Iterator<Item = u32>
+    /// self_ty = Foo
+    /// ```
+    ///
+    /// this would return `Foo: Iterator` and add `<Foo as Iterator>::Item = u32` into `bounds`.
+    ///
+    /// **A note on binders:** against our usual convention, there is an implied binder around
+    /// the `self_ty` and `poly_trait_ref` parameters here. So they may reference bound regions.
+    /// If for example you had `for<'a> Foo<'a>: Bar<'a>`, then the `self_ty` would be `Foo<'a>`
+    /// where `'a` is a bound region at depth 0. Similarly, the `poly_trait_ref` would be
+    /// `Bar<'a>`. The returned poly-trait-ref will have this binder instantiated explicitly,
+    /// however.
+    #[instrument(level = "debug", skip(self, span, constness, bounds, speculative))]
+    pub(crate) fn instantiate_poly_trait_ref(
         &self,
-        hir_id: hir::HirId,
+        trait_ref: &hir::TraitRef<'_>,
         span: Span,
-        binding_span: Option<Span>,
         constness: ty::BoundConstness,
         polarity: ty::ImplPolarity,
+        self_ty: Ty<'tcx>,
         bounds: &mut Bounds<'tcx>,
         speculative: bool,
-        trait_ref_span: Span,
-        trait_def_id: DefId,
-        trait_segment: &hir::PathSegment<'_>,
-        args: &GenericArgs<'_>,
-        infer_args: bool,
-        self_ty: Ty<'tcx>,
         only_self_bounds: OnlySelfBounds,
     ) -> GenericArgCountResult {
+        let trait_def_id = trait_ref.trait_def_id().unwrap_or_else(|| FatalError.raise());
+        let trait_segment = trait_ref.path.segments.last().unwrap();
+        let args = trait_segment.args();
+
+        self.prohibit_generics(trait_ref.path.segments.split_last().unwrap().1.iter(), |_| {});
+        self.complain_about_internal_fn_trait(span, trait_def_id, trait_segment, false);
+
         let (generic_args, arg_count) = self.create_args_for_ast_path(
-            trait_ref_span,
+            trait_ref.path.span,
             trait_def_id,
             &[],
             trait_segment,
             args,
-            infer_args,
+            trait_segment.infer_args,
             Some(self_ty),
             constness,
         );
 
         let tcx = self.tcx();
-        let bound_vars = tcx.late_bound_vars(hir_id);
+        let bound_vars = tcx.late_bound_vars(trait_ref.hir_ref_id);
         debug!(?bound_vars);
 
         let assoc_bindings = self.create_assoc_bindings_for_generic_args(args);
@@ -720,21 +747,22 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
             // since we should have emitten an error for them earlier, and they will
             // not be well-formed!
             if polarity == ty::ImplPolarity::Negative {
-                self.tcx()
-                    .sess
-                    .delay_span_bug(binding.span, "negative trait bounds should not have bindings");
+                self.tcx().sess.span_delayed_bug(
+                    binding.span,
+                    "negative trait bounds should not have bindings",
+                );
                 continue;
             }
 
             // Specify type to assert that error was already reported in `Err` case.
             let _: Result<_, ErrorGuaranteed> = self.add_predicates_for_ast_type_binding(
-                hir_id,
+                trait_ref.hir_ref_id,
                 poly_trait_ref,
                 binding,
                 bounds,
                 speculative,
                 &mut dup_bindings,
-                binding_span.unwrap_or(binding.span),
+                binding.span,
                 constness,
                 only_self_bounds,
                 polarity,
@@ -745,102 +773,6 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         arg_count
     }
 
-    /// Given a trait bound like `Debug`, applies that trait bound the given self-type to construct
-    /// a full trait reference. The resulting trait reference is returned. This may also generate
-    /// auxiliary bounds, which are added to `bounds`.
-    ///
-    /// Example:
-    ///
-    /// ```ignore (illustrative)
-    /// poly_trait_ref = Iterator<Item = u32>
-    /// self_ty = Foo
-    /// ```
-    ///
-    /// this would return `Foo: Iterator` and add `<Foo as Iterator>::Item = u32` into `bounds`.
-    ///
-    /// **A note on binders:** against our usual convention, there is an implied bounder around
-    /// the `self_ty` and `poly_trait_ref` parameters here. So they may reference bound regions.
-    /// If for example you had `for<'a> Foo<'a>: Bar<'a>`, then the `self_ty` would be `Foo<'a>`
-    /// where `'a` is a bound region at depth 0. Similarly, the `poly_trait_ref` would be
-    /// `Bar<'a>`. The returned poly-trait-ref will have this binder instantiated explicitly,
-    /// however.
-    #[instrument(level = "debug", skip(self, span, constness, bounds, speculative))]
-    pub(crate) fn instantiate_poly_trait_ref(
-        &self,
-        trait_ref: &hir::TraitRef<'_>,
-        span: Span,
-        constness: ty::BoundConstness,
-        polarity: ty::ImplPolarity,
-        self_ty: Ty<'tcx>,
-        bounds: &mut Bounds<'tcx>,
-        speculative: bool,
-        only_self_bounds: OnlySelfBounds,
-    ) -> GenericArgCountResult {
-        let hir_id = trait_ref.hir_ref_id;
-        let binding_span = None;
-        let trait_ref_span = trait_ref.path.span;
-        let trait_def_id = trait_ref.trait_def_id().unwrap_or_else(|| FatalError.raise());
-        let trait_segment = trait_ref.path.segments.last().unwrap();
-        let args = trait_segment.args();
-        let infer_args = trait_segment.infer_args;
-
-        self.prohibit_generics(trait_ref.path.segments.split_last().unwrap().1.iter(), |_| {});
-        self.complain_about_internal_fn_trait(span, trait_def_id, trait_segment, false);
-
-        self.instantiate_poly_trait_ref_inner(
-            hir_id,
-            span,
-            binding_span,
-            constness,
-            polarity,
-            bounds,
-            speculative,
-            trait_ref_span,
-            trait_def_id,
-            trait_segment,
-            args,
-            infer_args,
-            self_ty,
-            only_self_bounds,
-        )
-    }
-
-    pub(crate) fn instantiate_lang_item_trait_ref(
-        &self,
-        lang_item: hir::LangItem,
-        span: Span,
-        hir_id: hir::HirId,
-        args: &GenericArgs<'_>,
-        self_ty: Ty<'tcx>,
-        bounds: &mut Bounds<'tcx>,
-        only_self_bounds: OnlySelfBounds,
-    ) {
-        let binding_span = Some(span);
-        let constness = ty::BoundConstness::NotConst;
-        let speculative = false;
-        let trait_ref_span = span;
-        let trait_def_id = self.tcx().require_lang_item(lang_item, Some(span));
-        let trait_segment = &hir::PathSegment::invalid();
-        let infer_args = false;
-
-        self.instantiate_poly_trait_ref_inner(
-            hir_id,
-            span,
-            binding_span,
-            constness,
-            ty::ImplPolarity::Positive,
-            bounds,
-            speculative,
-            trait_ref_span,
-            trait_def_id,
-            trait_segment,
-            args,
-            infer_args,
-            self_ty,
-            only_self_bounds,
-        );
-    }
-
     fn ast_path_to_mono_trait_ref(
         &self,
         span: Span,
@@ -945,7 +877,11 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                 Applicability::MachineApplicable,
             );
         } else {
-            match (types, traits) {
+            let mut types = types.to_vec();
+            types.sort();
+            let mut traits = traits.to_vec();
+            traits.sort();
+            match (&types[..], &traits[..]) {
                 ([], []) => {
                     err.span_suggestion_verbose(
                         span,
@@ -1051,7 +987,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         debug!("find_bound_for_assoc_item: predicates={:#?}", predicates);
 
         let param_name = tcx.hir().ty_param_name(ty_param_def_id);
-        self.one_bound_for_assoc_type(
+        self.one_bound_for_assoc_item(
             || {
                 traits::transitive_bounds_that_define_assoc_item(
                     tcx,
@@ -1063,6 +999,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
             },
             param_name,
             Some(ty_param_def_id),
+            ty::AssocKind::Type,
             assoc_name,
             span,
             None,
@@ -1071,48 +1008,45 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
 
     // Checks that `bounds` contains exactly one element and reports appropriate
     // errors otherwise.
-    #[instrument(level = "debug", skip(self, all_candidates, ty_param_name, is_equality), ret)]
-    fn one_bound_for_assoc_type<I>(
+    #[instrument(level = "debug", skip(self, all_candidates, ty_param_name, binding), ret)]
+    fn one_bound_for_assoc_item<I>(
         &self,
         all_candidates: impl Fn() -> I,
         ty_param_name: impl Display,
         ty_param_def_id: Option<LocalDefId>,
+        assoc_kind: ty::AssocKind,
         assoc_name: Ident,
         span: Span,
-        is_equality: Option<ty::Term<'tcx>>,
+        binding: Option<&ConvertedBinding<'_, 'tcx>>,
     ) -> Result<ty::PolyTraitRef<'tcx>, ErrorGuaranteed>
     where
         I: Iterator<Item = ty::PolyTraitRef<'tcx>>,
     {
+        let tcx = self.tcx();
+
         let mut matching_candidates = all_candidates().filter(|r| {
-            self.trait_defines_associated_item_named(r.def_id(), ty::AssocKind::Type, assoc_name)
-        });
-        let mut const_candidates = all_candidates().filter(|r| {
-            self.trait_defines_associated_item_named(r.def_id(), ty::AssocKind::Const, assoc_name)
+            self.trait_defines_associated_item_named(r.def_id(), assoc_kind, assoc_name)
         });
 
-        let (mut bound, mut next_cand) = match (matching_candidates.next(), const_candidates.next())
-        {
-            (Some(bound), _) => (bound, matching_candidates.next()),
-            (None, Some(bound)) => (bound, const_candidates.next()),
-            (None, None) => {
-                let reported = self.complain_about_assoc_type_not_found(
-                    all_candidates,
-                    &ty_param_name.to_string(),
-                    ty_param_def_id,
-                    assoc_name,
-                    span,
-                );
-                return Err(reported);
-            }
+        let Some(mut bound) = matching_candidates.next() else {
+            let reported = self.complain_about_assoc_item_not_found(
+                all_candidates,
+                &ty_param_name.to_string(),
+                ty_param_def_id,
+                assoc_kind,
+                assoc_name,
+                span,
+                binding,
+            );
+            return Err(reported);
         };
         debug!(?bound);
 
         // look for a candidate that is not the same as our first bound, disregarding
         // whether the bound is const.
+        let mut next_cand = matching_candidates.next();
         while let Some(mut bound2) = next_cand {
             debug!(?bound2);
-            let tcx = self.tcx();
             if bound2.bound_vars() != bound.bound_vars() {
                 break;
             }
@@ -1133,7 +1067,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                 .map(|(n, arg)| if host_index == n { tcx.consts.true_.into() } else { arg });
 
             if unconsted_args.eq(bound2.skip_binder().args.iter()) {
-                next_cand = matching_candidates.next().or_else(|| const_candidates.next());
+                next_cand = matching_candidates.next();
             } else {
                 break;
             }
@@ -1142,51 +1076,53 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         if let Some(bound2) = next_cand {
             debug!(?bound2);
 
-            let bounds = IntoIterator::into_iter([bound, bound2]).chain(matching_candidates);
-            let mut err = if is_equality.is_some() {
-                // More specific Error Index entry.
-                struct_span_err!(
-                    self.tcx().sess,
-                    span,
-                    E0222,
-                    "ambiguous associated type `{}` in bounds of `{}`",
-                    assoc_name,
-                    ty_param_name
-                )
-            } else {
-                struct_span_err!(
-                    self.tcx().sess,
-                    span,
-                    E0221,
-                    "ambiguous associated type `{}` in bounds of `{}`",
-                    assoc_name,
-                    ty_param_name
-                )
-            };
-            err.span_label(span, format!("ambiguous associated type `{assoc_name}`"));
+            let assoc_kind_str = assoc_kind_str(assoc_kind);
+            let ty_param_name = &ty_param_name.to_string();
+            let mut err = tcx.sess.create_err(crate::errors::AmbiguousAssocItem {
+                span,
+                assoc_kind: assoc_kind_str,
+                assoc_name,
+                ty_param_name,
+            });
+            // Provide a more specific error code index entry for equality bindings.
+            err.code(
+                if let Some(binding) = binding
+                    && let ConvertedBindingKind::Equality(_) = binding.kind
+                {
+                    error_code!(E0222)
+                } else {
+                    error_code!(E0221)
+                },
+            );
 
+            // FIXME(#97583): Resugar equality bounds to type/const bindings.
+            // FIXME: Turn this into a structured, translateable & more actionable suggestion.
             let mut where_bounds = vec![];
-            for bound in bounds {
+            for bound in [bound, bound2].into_iter().chain(matching_candidates) {
                 let bound_id = bound.def_id();
-                let bound_span = self
-                    .tcx()
+                let bound_span = tcx
                     .associated_items(bound_id)
-                    .find_by_name_and_kind(self.tcx(), assoc_name, ty::AssocKind::Type, bound_id)
-                    .and_then(|item| self.tcx().hir().span_if_local(item.def_id));
+                    .find_by_name_and_kind(tcx, assoc_name, assoc_kind, bound_id)
+                    .and_then(|item| tcx.hir().span_if_local(item.def_id));
 
                 if let Some(bound_span) = bound_span {
                     err.span_label(
                         bound_span,
-                        format!(
-                            "ambiguous `{assoc_name}` from `{}`",
-                            bound.print_only_trait_path(),
-                        ),
+                        format!("ambiguous `{assoc_name}` from `{}`", bound.print_trait_sugared(),),
                     );
-                    if let Some(constraint) = &is_equality {
-                        where_bounds.push(format!(
-                            "        T: {trait}::{assoc_name} = {constraint}",
-                            trait=bound.print_only_trait_path(),
-                        ));
+                    if let Some(binding) = binding {
+                        match binding.kind {
+                            ConvertedBindingKind::Equality(term) => {
+                                // FIXME(#97583): This isn't syntactically well-formed!
+                                where_bounds.push(format!(
+                                    "        T: {trait}::{assoc_name} = {term}",
+                                    trait = bound.print_only_trait_path(),
+                                    term = term.node,
+                                ));
+                            }
+                            // FIXME: Provide a suggestion.
+                            ConvertedBindingKind::Constraint(_bounds) => {}
+                        }
                     } else {
                         err.span_suggestion_verbose(
                             span.with_hi(assoc_name.span.lo()),
@@ -1197,7 +1133,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                     }
                 } else {
                     err.note(format!(
-                        "associated type `{ty_param_name}` could derive from `{}`",
+                        "associated {assoc_kind_str} `{assoc_name}` could derive from `{}`",
                         bound.print_only_trait_path(),
                     ));
                 }
@@ -1218,46 +1154,6 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         Ok(bound)
     }
 
-    #[instrument(level = "debug", skip(self, all_candidates, ty_name), ret)]
-    fn one_bound_for_assoc_method(
-        &self,
-        all_candidates: impl Iterator<Item = ty::PolyTraitRef<'tcx>>,
-        ty_name: impl Display,
-        assoc_name: Ident,
-        span: Span,
-    ) -> Result<ty::PolyTraitRef<'tcx>, ErrorGuaranteed> {
-        let mut matching_candidates = all_candidates.filter(|r| {
-            self.trait_defines_associated_item_named(r.def_id(), ty::AssocKind::Fn, assoc_name)
-        });
-
-        let candidate = match matching_candidates.next() {
-            Some(candidate) => candidate,
-            None => {
-                return Err(self.tcx().sess.emit_err(
-                    crate::errors::ReturnTypeNotationMissingMethod {
-                        span,
-                        ty_name: ty_name.to_string(),
-                        assoc_name: assoc_name.name,
-                    },
-                ));
-            }
-        };
-
-        if let Some(conflicting_candidate) = matching_candidates.next() {
-            return Err(self.tcx().sess.emit_err(
-                crate::errors::ReturnTypeNotationConflictingBound {
-                    span,
-                    ty_name: ty_name.to_string(),
-                    assoc_name: assoc_name.name,
-                    first_bound: candidate.print_only_trait_path(),
-                    second_bound: conflicting_candidate.print_only_trait_path(),
-                },
-            ));
-        }
-
-        Ok(candidate)
-    }
-
     // Create a type from a path to an associated type or to an enum variant.
     // For a path `A::B::C::D`, `qself_ty` and `qself_def` are the type and def for `A::B::C`
     // and item_segment is the path segment for `D`. We return a type and a def for
@@ -1415,11 +1311,11 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                 // trait reference.
                 let Some(trait_ref) = tcx.impl_trait_ref(impl_def_id) else {
                     // A cycle error occurred, most likely.
-                    let guar = tcx.sess.delay_span_bug(span, "expected cycle error");
+                    let guar = tcx.sess.span_delayed_bug(span, "expected cycle error");
                     return Err(guar);
                 };
 
-                self.one_bound_for_assoc_type(
+                self.one_bound_for_assoc_item(
                     || {
                         traits::supertraits(
                             tcx,
@@ -1428,6 +1324,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                     },
                     kw::SelfUpper,
                     None,
+                    ty::AssocKind::Type,
                     assoc_ident,
                     span,
                     None,
@@ -1508,15 +1405,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         };
 
         let trait_did = bound.def_id();
-        let Some(assoc_ty_did) = self.lookup_assoc_ty(assoc_ident, hir_ref_id, span, trait_did)
-        else {
-            // Assume that if it's not matched, there must be a const defined with the same name
-            // but it was used in a type position.
-            let msg = format!("found associated const `{assoc_ident}` when type was expected");
-            let guar = tcx.sess.struct_span_err(span, msg).emit();
-            return Err(guar);
-        };
-
+        let assoc_ty_did = self.lookup_assoc_ty(assoc_ident, hir_ref_id, span, trait_did).unwrap();
         let ty = self.projected_ty_from_poly_trait_ref(span, assoc_ty_did, assoc_segment, bound);
 
         if let Some(variant_def_id) = variant_resolution {
@@ -1545,8 +1434,6 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                         format!("<{} as {}>::{}", qself_ty, tcx.item_name(trait_did), assoc_ident),
                         Applicability::MachineApplicable,
                     );
-
-                    lint
                 },
             );
         }
@@ -1602,134 +1489,110 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         // FIXME(inherent_associated_types): Acquiring the ParamEnv this early leads to cycle errors
         // when inside of an ADT (#108491) or where clause.
         let param_env = tcx.param_env(block.owner);
-        let cause = ObligationCause::misc(span, block.owner.def_id);
-
-        let mut fulfillment_errors = Vec::new();
-        let mut applicable_candidates: Vec<_> = infcx.probe(|_| {
-            // Regions are not considered during selection.
-            let self_ty = self_ty
-                .fold_with(&mut BoundVarEraser { tcx, universe: infcx.create_next_universe() });
-
-            struct BoundVarEraser<'tcx> {
-                tcx: TyCtxt<'tcx>,
-                universe: ty::UniverseIndex,
-            }
 
-            // FIXME(non_lifetime_binders): Don't assign the same universe to each placeholder.
-            impl<'tcx> TypeFolder<TyCtxt<'tcx>> for BoundVarEraser<'tcx> {
-                fn interner(&self) -> TyCtxt<'tcx> {
-                    self.tcx
-                }
+        let mut universes = if self_ty.has_escaping_bound_vars() {
+            vec![None; self_ty.outer_exclusive_binder().as_usize()]
+        } else {
+            vec![]
+        };
 
-                fn fold_region(&mut self, r: ty::Region<'tcx>) -> ty::Region<'tcx> {
-                    if r.is_late_bound() { self.tcx.lifetimes.re_erased } else { r }
-                }
+        let (impl_, (assoc_item, def_scope)) =
+            crate::traits::project::with_replaced_escaping_bound_vars(
+                infcx,
+                &mut universes,
+                self_ty,
+                |self_ty| {
+                    self.select_inherent_assoc_type_candidates(
+                        infcx, name, span, self_ty, param_env, candidates,
+                    )
+                },
+            )?;
+
+        self.check_assoc_ty(assoc_item, name, def_scope, block, span);
+
+        // FIXME(fmease): Currently creating throwaway `parent_args` to please
+        // `create_args_for_associated_item`. Modify the latter instead (or sth. similar) to
+        // not require the parent args logic.
+        let parent_args = ty::GenericArgs::identity_for_item(tcx, impl_);
+        let args = self.create_args_for_associated_item(span, assoc_item, segment, parent_args);
+        let args = tcx.mk_args_from_iter(
+            std::iter::once(ty::GenericArg::from(self_ty))
+                .chain(args.into_iter().skip(parent_args.len())),
+        );
 
-                fn fold_ty(&mut self, ty: Ty<'tcx>) -> Ty<'tcx> {
-                    match *ty.kind() {
-                        ty::Bound(_, bv) => Ty::new_placeholder(
-                            self.tcx,
-                            ty::PlaceholderType { universe: self.universe, bound: bv },
-                        ),
-                        _ => ty.super_fold_with(self),
-                    }
-                }
+        let ty = Ty::new_alias(tcx, ty::Inherent, ty::AliasTy::new(tcx, assoc_item, args));
 
-                fn fold_const(
-                    &mut self,
-                    ct: ty::Const<'tcx>,
-                ) -> <TyCtxt<'tcx> as rustc_type_ir::Interner>::Const {
-                    assert!(!ct.ty().has_escaping_bound_vars());
-
-                    match ct.kind() {
-                        ty::ConstKind::Bound(_, bv) => ty::Const::new_placeholder(
-                            self.tcx,
-                            ty::PlaceholderConst { universe: self.universe, bound: bv },
-                            ct.ty(),
-                        ),
-                        _ => ct.super_fold_with(self),
-                    }
-                }
-            }
+        Ok(Some((ty, assoc_item)))
+    }
 
-            let InferOk { value: self_ty, obligations } =
-                infcx.at(&cause, param_env).normalize(self_ty);
+    fn select_inherent_assoc_type_candidates(
+        &self,
+        infcx: &InferCtxt<'tcx>,
+        name: Ident,
+        span: Span,
+        self_ty: Ty<'tcx>,
+        param_env: ParamEnv<'tcx>,
+        candidates: Vec<(DefId, (DefId, DefId))>,
+    ) -> Result<(DefId, (DefId, DefId)), ErrorGuaranteed> {
+        let tcx = self.tcx();
+        let mut fulfillment_errors = Vec::new();
 
-            candidates
-                .iter()
-                .copied()
-                .filter(|&(impl_, _)| {
-                    infcx.probe(|_| {
-                        let ocx = ObligationCtxt::new(&infcx);
-                        ocx.register_obligations(obligations.clone());
-
-                        let impl_args = infcx.fresh_args_for_item(span, impl_);
-                        let impl_ty = tcx.type_of(impl_).instantiate(tcx, impl_args);
-                        let impl_ty = ocx.normalize(&cause, param_env, impl_ty);
-
-                        // Check that the self types can be related.
-                        // FIXME(inherent_associated_types): Should we use `eq` here? Method probing uses
-                        // `sup` for this situtation, too. What for? To constrain inference variables?
-                        if ocx.sup(&ObligationCause::dummy(), param_env, impl_ty, self_ty).is_err()
-                        {
-                            return false;
-                        }
+        let applicable_candidates: Vec<_> = candidates
+            .iter()
+            .copied()
+            .filter(|&(impl_, _)| {
+                infcx.probe(|_| {
+                    let ocx = ObligationCtxt::new(infcx);
+                    let self_ty = ocx.normalize(&ObligationCause::dummy(), param_env, self_ty);
+
+                    let impl_args = infcx.fresh_args_for_item(span, impl_);
+                    let impl_ty = tcx.type_of(impl_).instantiate(tcx, impl_args);
+                    let impl_ty = ocx.normalize(&ObligationCause::dummy(), param_env, impl_ty);
+
+                    // Check that the self types can be related.
+                    if ocx.eq(&ObligationCause::dummy(), param_env, impl_ty, self_ty).is_err() {
+                        return false;
+                    }
 
-                        // Check whether the impl imposes obligations we have to worry about.
-                        let impl_bounds = tcx.predicates_of(impl_).instantiate(tcx, impl_args);
-                        let impl_bounds = ocx.normalize(&cause, param_env, impl_bounds);
-                        let impl_obligations = traits::predicates_for_generics(
-                            |_, _| cause.clone(),
-                            param_env,
-                            impl_bounds,
-                        );
-                        ocx.register_obligations(impl_obligations);
+                    // Check whether the impl imposes obligations we have to worry about.
+                    let impl_bounds = tcx.predicates_of(impl_).instantiate(tcx, impl_args);
+                    let impl_bounds =
+                        ocx.normalize(&ObligationCause::dummy(), param_env, impl_bounds);
+                    let impl_obligations = traits::predicates_for_generics(
+                        |_, _| ObligationCause::dummy(),
+                        param_env,
+                        impl_bounds,
+                    );
+                    ocx.register_obligations(impl_obligations);
 
-                        let mut errors = ocx.select_where_possible();
-                        if !errors.is_empty() {
-                            fulfillment_errors.append(&mut errors);
-                            return false;
-                        }
+                    let mut errors = ocx.select_where_possible();
+                    if !errors.is_empty() {
+                        fulfillment_errors.append(&mut errors);
+                        return false;
+                    }
 
-                        true
-                    })
+                    true
                 })
-                .collect()
-        });
+            })
+            .collect();
 
-        if applicable_candidates.len() > 1 {
-            return Err(self.complain_about_ambiguous_inherent_assoc_type(
+        match &applicable_candidates[..] {
+            &[] => Err(self.complain_about_inherent_assoc_type_not_found(
                 name,
-                applicable_candidates.into_iter().map(|(_, (candidate, _))| candidate).collect(),
+                self_ty,
+                candidates,
+                fulfillment_errors,
                 span,
-            ));
-        }
+            )),
 
-        if let Some((impl_, (assoc_item, def_scope))) = applicable_candidates.pop() {
-            self.check_assoc_ty(assoc_item, name, def_scope, block, span);
-
-            // FIXME(fmease): Currently creating throwaway `parent_args` to please
-            // `create_args_for_associated_item`. Modify the latter instead (or sth. similar) to
-            // not require the parent args logic.
-            let parent_args = ty::GenericArgs::identity_for_item(tcx, impl_);
-            let args = self.create_args_for_associated_item(span, assoc_item, segment, parent_args);
-            let args = tcx.mk_args_from_iter(
-                std::iter::once(ty::GenericArg::from(self_ty))
-                    .chain(args.into_iter().skip(parent_args.len())),
-            );
+            &[applicable_candidate] => Ok(applicable_candidate),
 
-            let ty = Ty::new_alias(tcx, ty::Inherent, ty::AliasTy::new(tcx, assoc_item, args));
-
-            return Ok(Some((ty, assoc_item)));
+            &[_, ..] => Err(self.complain_about_ambiguous_inherent_assoc_type(
+                name,
+                applicable_candidates.into_iter().map(|(_, (candidate, _))| candidate).collect(),
+                span,
+            )),
         }
-
-        Err(self.complain_about_inherent_assoc_type_not_found(
-            name,
-            self_ty,
-            candidates,
-            fulfillment_errors,
-            span,
-        ))
     }
 
     fn lookup_assoc_ty(
@@ -1753,8 +1616,9 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         let tcx = self.tcx();
         let (ident, def_scope) = tcx.adjust_ident_and_get_scope(name, scope, block);
 
-        // We have already adjusted the item name above, so compare with `ident.normalize_to_macros_2_0()` instead
-        // of calling `find_by_name_and_kind`.
+        // We have already adjusted the item name above, so compare with `.normalize_to_macros_2_0()`
+        // instead of calling `filter_by_name_and_kind` which would needlessly normalize the
+        // `ident` again and again.
         let item = tcx.associated_items(scope).in_definition_order().find(|i| {
             i.kind.namespace() == Namespace::TypeNS
                 && i.ident(tcx).normalize_to_macros_2_0() == ident
@@ -1869,7 +1733,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
 
             let parent_def_id = def_id
                 .as_local()
-                .map(|def_id| tcx.hir().local_def_id_to_hir_id(def_id))
+                .map(|def_id| tcx.local_def_id_to_hir_id(def_id))
                 .map(|hir_id| tcx.hir().get_parent_item(hir_id).to_def_id());
 
             debug!("qpath_to_ty: parent_def_id={:?}", parent_def_id);
@@ -1973,7 +1837,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                     types_and_spans[..types_and_spans.len() - 1]
                         .iter()
                         .map(|(x, _)| x.as_str())
-                        .intersperse(&", ")
+                        .intersperse(", ")
                         .collect::<String>()
                 ),
                 [(only, _)] => only.to_string(),
@@ -2008,7 +1872,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                     "s",
                 ),
                 [only] => (only.to_string(), ""),
-                [] => unreachable!(),
+                [] => unreachable!("expected at least one generic to prohibit"),
             };
             let last_span = *arg_spans.last().unwrap();
             let span: MultiSpan = arg_spans.into();
@@ -2396,7 +2260,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                 let e = self
                     .tcx()
                     .sess
-                    .delay_span_bug(path.span, "path with `Res::Err` but no error emitted");
+                    .span_delayed_bug(path.span, "path with `Res::Err` but no error emitted");
                 self.set_tainted_by_errors(e);
                 Ty::new_error(self.tcx(), e)
             }
@@ -2543,7 +2407,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                     .map(|(ty, _, _)| ty)
                     .unwrap_or_else(|guar| Ty::new_error(tcx, guar))
             }
-            &hir::TyKind::Path(hir::QPath::LangItem(lang_item, span, _)) => {
+            &hir::TyKind::Path(hir::QPath::LangItem(lang_item, span)) => {
                 let def_id = tcx.require_lang_item(lang_item, Some(span));
                 let (args, _) = self.create_args_for_ast_path(
                     span,
@@ -2617,8 +2481,19 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
             if let Some(i) = (param.index as usize).checked_sub(generics.count() - lifetimes.len())
             {
                 // Resolve our own lifetime parameters.
-                let GenericParamDefKind::Lifetime { .. } = param.kind else { bug!() };
-                let hir::GenericArg::Lifetime(lifetime) = &lifetimes[i] else { bug!() };
+                let GenericParamDefKind::Lifetime { .. } = param.kind else {
+                    span_bug!(
+                        tcx.def_span(param.def_id),
+                        "only expected lifetime for opaque's own generics, got {:?}",
+                        param.kind
+                    );
+                };
+                let hir::GenericArg::Lifetime(lifetime) = &lifetimes[i] else {
+                    bug!(
+                        "expected lifetime argument for param {param:?}, found {:?}",
+                        &lifetimes[i]
+                    )
+                };
                 self.ast_region_to_region(lifetime, None).into()
             } else {
                 tcx.mk_param_from_def(param)
@@ -2777,7 +2652,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         let hir = tcx.hir();
 
         let hir::Node::ImplItem(hir::ImplItem { kind: hir::ImplItemKind::Fn(..), ident, .. }) =
-            hir.get(fn_hir_id)
+            tcx.hir_node(fn_hir_id)
         else {
             return None;
         };
@@ -2845,6 +2720,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
     /// provided, if they provided one, and otherwise search the supertypes of trait bounds
     /// for region bounds. It may be that we can derive no bound at all, in which case
     /// we return `None`.
+    #[instrument(level = "debug", skip(self, span), ret)]
     fn compute_object_lifetime_bound(
         &self,
         span: Span,
@@ -2853,8 +2729,6 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
     {
         let tcx = self.tcx();
 
-        debug!("compute_opt_region_bound(existential_predicates={:?})", existential_predicates);
-
         // No explicit region bound specified. Therefore, examine trait
         // bounds and see if we can derive region bounds from those.
         let derived_region_bounds = object_region_bounds(tcx, existential_predicates);
@@ -2881,3 +2755,11 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
         Some(r)
     }
 }
+
+fn assoc_kind_str(kind: ty::AssocKind) -> &'static str {
+    match kind {
+        ty::AssocKind::Fn => "function",
+        ty::AssocKind::Const => "constant",
+        ty::AssocKind::Type => "type",
+    }
+}
diff --git a/compiler/rustc_hir_analysis/src/astconv/object_safety.rs b/compiler/rustc_hir_analysis/src/astconv/object_safety.rs
index 00ff3f836..dd5deb6f2 100644
--- a/compiler/rustc_hir_analysis/src/astconv/object_safety.rs
+++ b/compiler/rustc_hir_analysis/src/astconv/object_safety.rs
@@ -1,7 +1,7 @@
 use crate::astconv::{GenericArgCountMismatch, GenericArgCountResult, OnlySelfBounds};
 use crate::bounds::Bounds;
 use crate::errors::TraitObjectDeclaredWithNoTraits;
-use rustc_data_structures::fx::{FxHashMap, FxHashSet};
+use rustc_data_structures::fx::{FxHashSet, FxIndexMap, FxIndexSet};
 use rustc_errors::struct_span_err;
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
@@ -14,7 +14,6 @@ use rustc_trait_selection::traits::error_reporting::report_object_safety_error;
 use rustc_trait_selection::traits::{self, astconv_object_safety_violations};
 
 use smallvec::{smallvec, SmallVec};
-use std::collections::BTreeSet;
 
 use super::AstConv;
 
@@ -74,7 +73,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                 | ty::ClauseKind::ConstArgHasType(..)
                 | ty::ClauseKind::WellFormed(_)
                 | ty::ClauseKind::ConstEvaluatable(_) => {
-                    bug!()
+                    span_bug!(span, "did not expect {pred} clause in object bounds");
                 }
             }
         }
@@ -107,6 +106,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
              trait here instead: `trait NewTrait: {} {{}}`",
                 regular_traits
                     .iter()
+                    // FIXME: This should `print_sugared`, but also needs to integrate projection bounds...
                     .map(|t| t.trait_ref().print_only_trait_path().to_string())
                     .collect::<Vec<_>>()
                     .join(" + "),
@@ -148,8 +148,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
             }
         }
 
-        // Use a `BTreeSet` to keep output in a more consistent order.
-        let mut associated_types: FxHashMap<Span, BTreeSet<DefId>> = FxHashMap::default();
+        let mut associated_types: FxIndexMap<Span, FxIndexSet<DefId>> = FxIndexMap::default();
 
         let regular_traits_refs_spans = trait_bounds
             .into_iter()
@@ -327,7 +326,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                         false
                     });
                     if references_self {
-                        let guar = tcx.sess.delay_span_bug(
+                        let guar = tcx.sess.span_delayed_bug(
                             span,
                             "trait object projection bounds reference `Self`",
                         );
diff --git a/compiler/rustc_hir_analysis/src/autoderef.rs b/compiler/rustc_hir_analysis/src/autoderef.rs
index 39db29504..5fc500f48 100644
--- a/compiler/rustc_hir_analysis/src/autoderef.rs
+++ b/compiler/rustc_hir_analysis/src/autoderef.rs
@@ -182,7 +182,7 @@ impl<'a, 'tcx> Autoderef<'a, 'tcx> {
             }
         };
 
-        let errors = fulfill_cx.select_where_possible(&self.infcx);
+        let errors = fulfill_cx.select_where_possible(self.infcx);
         if !errors.is_empty() {
             // This shouldn't happen, except for evaluate/fulfill mismatches,
             // but that's not a reason for an ICE (`predicate_may_hold` is conservative
diff --git a/compiler/rustc_hir_analysis/src/check/check.rs b/compiler/rustc_hir_analysis/src/check/check.rs
index e61ca232d..8413a1cc0 100644
--- a/compiler/rustc_hir_analysis/src/check/check.rs
+++ b/compiler/rustc_hir_analysis/src/check/check.rs
@@ -1,5 +1,5 @@
 use crate::check::intrinsicck::InlineAsmCtxt;
-use crate::errors::{self, LinkageType};
+use crate::errors::LinkageType;
 
 use super::compare_impl_item::check_type_bounds;
 use super::compare_impl_item::{compare_impl_method, compare_impl_ty};
@@ -8,9 +8,8 @@ use rustc_attr as attr;
 use rustc_errors::{ErrorGuaranteed, MultiSpan};
 use rustc_hir as hir;
 use rustc_hir::def::{CtorKind, DefKind};
-use rustc_hir::def_id::{DefId, LocalDefId, LocalModDefId};
+use rustc_hir::def_id::LocalModDefId;
 use rustc_hir::Node;
-use rustc_infer::infer::outlives::env::OutlivesEnvironment;
 use rustc_infer::infer::{RegionVariableOrigin, TyCtxtInferExt};
 use rustc_infer::traits::{Obligation, TraitEngineExt as _};
 use rustc_lint_defs::builtin::REPR_TRANSPARENT_EXTERNAL_PRIVATE_FIELDS;
@@ -21,18 +20,15 @@ use rustc_middle::ty::layout::{LayoutError, MAX_SIMD_LANES};
 use rustc_middle::ty::util::{Discr, IntTypeExt};
 use rustc_middle::ty::GenericArgKind;
 use rustc_middle::ty::{
-    self, AdtDef, ParamEnv, RegionKind, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable,
-    TypeVisitableExt,
+    AdtDef, ParamEnv, RegionKind, TypeSuperVisitable, TypeVisitable, TypeVisitableExt,
 };
 use rustc_session::lint::builtin::{UNINHABITED_STATIC, UNSUPPORTED_CALLING_CONVENTIONS};
 use rustc_span::symbol::sym;
-use rustc_span::{self, Span};
 use rustc_target::abi::FieldIdx;
-use rustc_target::spec::abi::Abi;
 use rustc_trait_selection::traits::error_reporting::on_unimplemented::OnUnimplementedDirective;
 use rustc_trait_selection::traits::error_reporting::TypeErrCtxtExt as _;
 use rustc_trait_selection::traits::outlives_bounds::InferCtxtExt as _;
-use rustc_trait_selection::traits::{self, ObligationCtxt, TraitEngine, TraitEngineExt as _};
+use rustc_trait_selection::traits::{self, TraitEngine, TraitEngineExt as _};
 use rustc_type_ir::fold::TypeFoldable;
 
 use std::ops::ControlFlow;
@@ -55,7 +51,7 @@ pub fn check_abi(tcx: TyCtxt<'_>, hir_id: hir::HirId, span: Span, abi: Abi) {
                 hir_id,
                 span,
                 "use of calling convention not supported on this target",
-                |lint| lint,
+                |_| {},
             );
         }
     }
@@ -130,7 +126,7 @@ fn check_union_fields(tcx: TyCtxt<'_>, span: Span, item_def_id: LocalDefId) -> b
         for field in &def.non_enum_variant().fields {
             let Ok(field_ty) = tcx.try_normalize_erasing_regions(param_env, field.ty(tcx, args))
             else {
-                tcx.sess.delay_span_bug(span, "could not normalize field type");
+                tcx.sess.span_delayed_bug(span, "could not normalize field type");
                 continue;
             };
 
@@ -151,7 +147,8 @@ fn check_union_fields(tcx: TyCtxt<'_>, span: Span, item_def_id: LocalDefId) -> b
                 return false;
             } else if field_ty.needs_drop(tcx, param_env) {
                 // This should never happen. But we can get here e.g. in case of name resolution errors.
-                tcx.sess.delay_span_bug(span, "we should never accept maybe-dropping union fields");
+                tcx.sess
+                    .span_delayed_bug(span, "we should never accept maybe-dropping union fields");
             }
         }
     } else {
@@ -181,19 +178,19 @@ fn check_static_inhabited(tcx: TyCtxt<'_>, def_id: LocalDefId) {
         }
         // Generic statics are rejected, but we still reach this case.
         Err(e) => {
-            tcx.sess.delay_span_bug(span, format!("{e:?}"));
+            tcx.sess.span_delayed_bug(span, format!("{e:?}"));
             return;
         }
     };
     if layout.abi.is_uninhabited() {
         tcx.struct_span_lint_hir(
             UNINHABITED_STATIC,
-            tcx.hir().local_def_id_to_hir_id(def_id),
+            tcx.local_def_id_to_hir_id(def_id),
             span,
             "static of uninhabited type",
             |lint| {
                 lint
-                .note("uninhabited statics cannot be initialized, and any access would be an immediate error")
+                .note("uninhabited statics cannot be initialized, and any access would be an immediate error");
             },
         );
     }
@@ -204,7 +201,7 @@ fn check_static_inhabited(tcx: TyCtxt<'_>, def_id: LocalDefId) {
 fn check_opaque(tcx: TyCtxt<'_>, id: hir::ItemId) {
     let item = tcx.hir().item(id);
     let hir::ItemKind::OpaqueTy(hir::OpaqueTy { origin, .. }) = item.kind else {
-        tcx.sess.delay_span_bug(item.span, "expected opaque item");
+        tcx.sess.span_delayed_bug(item.span, "expected opaque item");
         return;
     };
 
@@ -222,11 +219,11 @@ fn check_opaque(tcx: TyCtxt<'_>, id: hir::ItemId) {
     if tcx.type_of(item.owner_id.def_id).instantiate_identity().references_error() {
         return;
     }
-    if check_opaque_for_cycles(tcx, item.owner_id.def_id, args, span, &origin).is_err() {
+    if check_opaque_for_cycles(tcx, item.owner_id.def_id, args, span, origin).is_err() {
         return;
     }
 
-    let _ = check_opaque_meets_bounds(tcx, item.owner_id.def_id, span, &origin);
+    let _ = check_opaque_meets_bounds(tcx, item.owner_id.def_id, span, origin);
 }
 
 /// Checks that an opaque type does not contain cycles.
@@ -313,7 +310,7 @@ fn check_opaque_meets_bounds<'tcx>(
         Ok(()) => {}
         Err(ty_err) => {
             let ty_err = ty_err.to_string(tcx);
-            return Err(tcx.sess.delay_span_bug(
+            return Err(tcx.sess.span_delayed_bug(
                 span,
                 format!("could not unify `{hidden_ty}` with revealed type:\n{ty_err}"),
             ));
@@ -482,7 +479,7 @@ fn check_item_type(tcx: TyCtxt<'_>, id: hir::ItemId) {
                 match assoc_item.kind {
                     ty::AssocKind::Fn => {
                         let abi = tcx.fn_sig(assoc_item.def_id).skip_binder().abi();
-                        fn_maybe_err(tcx, assoc_item.ident(tcx).span, abi);
+                        forbid_intrinsic_abi(tcx, assoc_item.ident(tcx).span, abi);
                     }
                     ty::AssocKind::Type if assoc_item.defaultness(tcx).has_value() => {
                         let trait_args = GenericArgs::identity_for_item(tcx, id.owner_id);
@@ -507,7 +504,7 @@ fn check_item_type(tcx: TyCtxt<'_>, id: hir::ItemId) {
             let origin = tcx.opaque_type_origin(id.owner_id.def_id);
             if let hir::OpaqueTyOrigin::FnReturn(fn_def_id)
             | hir::OpaqueTyOrigin::AsyncFn(fn_def_id) = origin
-                && let hir::Node::TraitItem(trait_item) = tcx.hir().get_by_def_id(fn_def_id)
+                && let hir::Node::TraitItem(trait_item) = tcx.hir_node_by_def_id(fn_def_id)
                 && let (_, hir::TraitFn::Required(..)) = trait_item.expect_fn()
             {
                 // Skip opaques from RPIT in traits with no default body.
@@ -518,7 +515,7 @@ fn check_item_type(tcx: TyCtxt<'_>, id: hir::ItemId) {
         DefKind::TyAlias => {
             let pty_ty = tcx.type_of(id.owner_id).instantiate_identity();
             let generics = tcx.generics_of(id.owner_id);
-            check_type_params_are_used(tcx, &generics, pty_ty);
+            check_type_params_are_used(tcx, generics, pty_ty);
         }
         DefKind::ForeignMod => {
             let it = tcx.hir().item(id);
@@ -655,7 +652,7 @@ pub(super) fn check_specialization_validity<'tcx>(
         if !tcx.is_impl_trait_in_trait(impl_item) {
             report_forbidden_specialization(tcx, impl_item, parent_impl);
         } else {
-            tcx.sess.delay_span_bug(
+            tcx.sess.span_delayed_bug(
                 DUMMY_SP,
                 format!("parent item: {parent_impl:?} not marked as default"),
             );
@@ -703,7 +700,7 @@ fn check_impl_items_against_trait<'tcx>(
             tcx.associated_item(trait_item_id)
         } else {
             // Checked in `associated_item`.
-            tcx.sess.delay_span_bug(tcx.def_span(impl_item), "missing associated item in trait");
+            tcx.sess.span_delayed_bug(tcx.def_span(impl_item), "missing associated item in trait");
             continue;
         };
         match ty_impl_item.kind {
@@ -753,8 +750,7 @@ fn check_impl_items_against_trait<'tcx>(
                 leaf_def.as_ref().is_some_and(|node_item| !node_item.defining_node.is_from_trait());
 
             if !is_implemented_here {
-                let full_impl_span =
-                    tcx.hir().span_with_body(tcx.hir().local_def_id_to_hir_id(impl_id));
+                let full_impl_span = tcx.hir().span_with_body(tcx.local_def_id_to_hir_id(impl_id));
                 match tcx.eval_default_body_stability(trait_item_id, full_impl_span) {
                     EvalResult::Deny { feature, reason, issue, .. } => default_body_is_unstable(
                         tcx,
@@ -811,8 +807,7 @@ fn check_impl_items_against_trait<'tcx>(
         }
 
         if !missing_items.is_empty() {
-            let full_impl_span =
-                tcx.hir().span_with_body(tcx.hir().local_def_id_to_hir_id(impl_id));
+            let full_impl_span = tcx.hir().span_with_body(tcx.local_def_id_to_hir_id(impl_id));
             missing_items_err(tcx, impl_id, &missing_items, full_impl_span);
         }
 
@@ -900,7 +895,7 @@ pub(super) fn check_packed(tcx: TyCtxt<'_>, sp: Span, def: ty::AdtDef<'_>) {
     let repr = def.repr();
     if repr.packed() {
         for attr in tcx.get_attrs(def.did(), sym::repr) {
-            for r in attr::parse_repr_attr(&tcx.sess, attr) {
+            for r in attr::parse_repr_attr(tcx.sess, attr) {
                 if let attr::ReprPacked(pack) = r
                     && let Some(repr_pack) = repr.pack
                     && pack as u64 != repr_pack.bytes()
@@ -1083,7 +1078,7 @@ pub(super) fn check_transparent<'tcx>(tcx: TyCtxt<'tcx>, adt: ty::AdtDef<'tcx>)
             if non_trivial_count > 0 || prev_non_exhaustive_1zst {
                 tcx.struct_span_lint_hir(
                     REPR_TRANSPARENT_EXTERNAL_PRIVATE_FIELDS,
-                    tcx.hir().local_def_id_to_hir_id(adt.did().expect_local()),
+                    tcx.local_def_id_to_hir_id(adt.did().expect_local()),
                     span,
                     "zero-sized fields in `repr(transparent)` cannot \
                     contain external non-exhaustive types",
@@ -1098,7 +1093,7 @@ pub(super) fn check_transparent<'tcx>(tcx: TyCtxt<'tcx>, adt: ty::AdtDef<'tcx>)
                             "this {descr} contains `{field_ty}`, which {note}, \
                                 and makes it not a breaking change to become \
                                 non-zero-sized in the future."
-                        ))
+                        ));
                     },
                 )
             } else {
@@ -1150,8 +1145,8 @@ fn check_enum(tcx: TyCtxt<'_>, def_id: LocalDefId) {
         let has_disr = |var: &ty::VariantDef| matches!(var.discr, ty::VariantDiscr::Explicit(_));
 
         let has_non_units = def.variants().iter().any(|var| !is_unit(var));
-        let disr_units = def.variants().iter().any(|var| is_unit(&var) && has_disr(&var));
-        let disr_non_unit = def.variants().iter().any(|var| !is_unit(&var) && has_disr(&var));
+        let disr_units = def.variants().iter().any(|var| is_unit(var) && has_disr(var));
+        let disr_non_unit = def.variants().iter().any(|var| !is_unit(var) && has_disr(var));
 
         if disr_non_unit || (disr_units && has_non_units) {
             let mut err = struct_span_err!(
@@ -1178,7 +1173,7 @@ fn detect_discriminant_duplicate<'tcx>(tcx: TyCtxt<'tcx>, adt: ty::AdtDef<'tcx>)
             ty::VariantDiscr::Explicit(discr_def_id) => {
                 // In the case the discriminant is both a duplicate and overflowed, let the user know
                 if let hir::Node::AnonConst(expr) =
-                    tcx.hir().get_by_def_id(discr_def_id.expect_local())
+                    tcx.hir_node_by_def_id(discr_def_id.expect_local())
                     && let hir::ExprKind::Lit(lit) = &tcx.hir().body(expr.body).value.kind
                     && let rustc_ast::LitKind::Int(lit_value, _int_kind) = &lit.node
                     && *lit_value != dis.val
@@ -1451,7 +1446,7 @@ fn opaque_type_cycle_error(
                         label_match(capture.place.ty(), capture.get_path_span(tcx));
                     }
                     // Label any coroutine locals that capture the opaque
-                    if let DefKind::Coroutine = tcx.def_kind(closure_def_id)
+                    if tcx.is_coroutine(closure_def_id)
                         && let Some(coroutine_layout) = tcx.mir_coroutine_witnesses(closure_def_id)
                     {
                         for interior_ty in &coroutine_layout.field_tys {
@@ -1468,8 +1463,11 @@ fn opaque_type_cycle_error(
     err.emit()
 }
 
-pub(super) fn check_coroutine_obligations(tcx: TyCtxt<'_>, def_id: LocalDefId) {
-    debug_assert!(matches!(tcx.def_kind(def_id), DefKind::Coroutine));
+pub(super) fn check_coroutine_obligations(
+    tcx: TyCtxt<'_>,
+    def_id: LocalDefId,
+) -> Result<(), ErrorGuaranteed> {
+    debug_assert!(tcx.is_coroutine(def_id.to_def_id()));
 
     let typeck = tcx.typeck(def_id);
     let param_env = tcx.param_env(def_id);
@@ -1482,8 +1480,9 @@ pub(super) fn check_coroutine_obligations(tcx: TyCtxt<'_>, def_id: LocalDefId) {
         // typeck writeback gives us predicates with their regions erased.
         // As borrowck already has checked lifetimes, we do not need to do it again.
         .ignoring_regions()
-        // Bind opaque types to `def_id` as they should have been checked by borrowck.
-        .with_opaque_type_inference(DefiningAnchor::Bind(def_id))
+        // Bind opaque types to type checking root, as they should have been checked by borrowck,
+        // but may show up in some cases, like when (root) obligations are stalled in the new solver.
+        .with_opaque_type_inference(DefiningAnchor::Bind(typeck.hir_owner.def_id))
         .build();
 
     let mut fulfillment_cx = <dyn TraitEngine<'_>>::new(&infcx);
@@ -1513,6 +1512,16 @@ pub(super) fn check_coroutine_obligations(tcx: TyCtxt<'_>, def_id: LocalDefId) {
     let errors = fulfillment_cx.select_all_or_error(&infcx);
     debug!(?errors);
     if !errors.is_empty() {
-        infcx.err_ctxt().report_fulfillment_errors(errors);
+        return Err(infcx.err_ctxt().report_fulfillment_errors(errors));
     }
+
+    // Check that any hidden types found when checking these stalled coroutine obligations
+    // are valid.
+    for (key, ty) in infcx.take_opaque_types() {
+        let hidden_type = infcx.resolve_vars_if_possible(ty.hidden_type);
+        let key = infcx.resolve_vars_if_possible(key);
+        sanity_check_found_hidden_type(tcx, key, hidden_type)?;
+    }
+
+    Ok(())
 }
diff --git a/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs b/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs
index 857515f97..264868fdf 100644
--- a/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs
+++ b/compiler/rustc_hir_analysis/src/check/compare_impl_item.rs
@@ -2,9 +2,7 @@ use super::potentially_plural_count;
 use crate::errors::LifetimesOrBoundsMismatchOnTrait;
 use hir::def_id::{DefId, LocalDefId};
 use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet};
-use rustc_errors::{
-    pluralize, struct_span_err, Applicability, DiagnosticId, ErrorGuaranteed, MultiSpan,
-};
+use rustc_errors::{pluralize, struct_span_err, Applicability, DiagnosticId, ErrorGuaranteed};
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
 use rustc_hir::intravisit;
@@ -50,13 +48,7 @@ pub(super) fn compare_impl_method<'tcx>(
 
     let _: Result<_, ErrorGuaranteed> = try {
         check_method_is_structurally_compatible(tcx, impl_m, trait_m, impl_trait_ref, false)?;
-        compare_method_predicate_entailment(
-            tcx,
-            impl_m,
-            trait_m,
-            impl_trait_ref,
-            CheckImpliedWfMode::Check,
-        )?;
+        compare_method_predicate_entailment(tcx, impl_m, trait_m, impl_trait_ref)?;
         refine::check_refining_return_position_impl_trait_in_trait(
             tcx,
             impl_m,
@@ -170,7 +162,6 @@ fn compare_method_predicate_entailment<'tcx>(
     impl_m: ty::AssocItem,
     trait_m: ty::AssocItem,
     impl_trait_ref: ty::TraitRef<'tcx>,
-    check_implied_wf: CheckImpliedWfMode,
 ) -> Result<(), ErrorGuaranteed> {
     let trait_to_impl_args = impl_trait_ref.args;
 
@@ -307,7 +298,7 @@ fn compare_method_predicate_entailment<'tcx>(
         debug!(?impl_sig, ?trait_sig, ?terr, "sub_types failed");
 
         let emitted = report_trait_method_mismatch(
-            &infcx,
+            infcx,
             cause,
             terr,
             (trait_m, trait_sig),
@@ -317,10 +308,10 @@ fn compare_method_predicate_entailment<'tcx>(
         return Err(emitted);
     }
 
-    if check_implied_wf == CheckImpliedWfMode::Check && !(impl_sig, trait_sig).references_error() {
+    if !(impl_sig, trait_sig).references_error() {
         // Select obligations to make progress on inference before processing
         // the wf obligation below.
-        // FIXME(-Ztrait-solver=next): Not needed when the hack below is removed.
+        // FIXME(-Znext-solver): Not needed when the hack below is removed.
         let errors = ocx.select_where_possible();
         if !errors.is_empty() {
             let reported = infcx.err_ctxt().report_fulfillment_errors(errors);
@@ -333,8 +324,9 @@ fn compare_method_predicate_entailment<'tcx>(
         // trigger the lint. Instead, let's only consider type outlives and
         // region outlives obligations.
         //
-        // FIXME(-Ztrait-solver=next): Try removing this hack again once
-        // the new solver is stable.
+        // FIXME(-Znext-solver): Try removing this hack again once the new
+        // solver is stable. We should just be able to register a WF pred for
+        // the fn sig.
         let mut wf_args: smallvec::SmallVec<[_; 4]> =
             unnormalized_impl_sig.inputs_and_output.iter().map(|ty| ty.into()).collect();
         // Annoyingly, asking for the WF predicates of an array (with an unevaluated const (only?))
@@ -357,7 +349,7 @@ fn compare_method_predicate_entailment<'tcx>(
                     // We need to register Projection oblgiations too, because we may end up with
                     // an implied `X::Item: 'a`, which gets desugared into `X::Item = ?0`, `?0: 'a`.
                     // If we only register the region outlives obligation, this leads to an unconstrained var.
-                    // See `implied_bounds_entailment_alias_var` test.
+                    // See `implied_bounds_entailment_alias_var.rs` test.
                     ty::PredicateKind::Clause(
                         ty::ClauseKind::RegionOutlives(..)
                         | ty::ClauseKind::TypeOutlives(..)
@@ -378,26 +370,8 @@ fn compare_method_predicate_entailment<'tcx>(
     // version.
     let errors = ocx.select_all_or_error();
     if !errors.is_empty() {
-        match check_implied_wf {
-            CheckImpliedWfMode::Check => {
-                let impl_m_hir_id = tcx.hir().local_def_id_to_hir_id(impl_m_def_id);
-                return compare_method_predicate_entailment(
-                    tcx,
-                    impl_m,
-                    trait_m,
-                    impl_trait_ref,
-                    CheckImpliedWfMode::Skip,
-                )
-                .map(|()| {
-                    // If the skip-mode was successful, emit a lint.
-                    emit_implied_wf_lint(infcx.tcx, impl_m, impl_m_hir_id, vec![]);
-                });
-            }
-            CheckImpliedWfMode::Skip => {
-                let reported = infcx.err_ctxt().report_fulfillment_errors(errors);
-                return Err(reported);
-            }
-        }
+        let reported = infcx.err_ctxt().report_fulfillment_errors(errors);
+        return Err(reported);
     }
 
     // Finally, resolve all regions. This catches wily misuses of
@@ -408,119 +382,14 @@ fn compare_method_predicate_entailment<'tcx>(
     );
     let errors = infcx.resolve_regions(&outlives_env);
     if !errors.is_empty() {
-        // FIXME(compiler-errors): This can be simplified when IMPLIED_BOUNDS_ENTAILMENT
-        // becomes a hard error (i.e. ideally we'd just call `resolve_regions_and_report_errors`
-        let impl_m_hir_id = tcx.hir().local_def_id_to_hir_id(impl_m_def_id);
-        match check_implied_wf {
-            CheckImpliedWfMode::Check => {
-                return compare_method_predicate_entailment(
-                    tcx,
-                    impl_m,
-                    trait_m,
-                    impl_trait_ref,
-                    CheckImpliedWfMode::Skip,
-                )
-                .map(|()| {
-                    let bad_args = extract_bad_args_for_implies_lint(
-                        tcx,
-                        &errors,
-                        (trait_m, trait_sig),
-                        // Unnormalized impl sig corresponds to the HIR types written
-                        (impl_m, unnormalized_impl_sig),
-                        impl_m_hir_id,
-                    );
-                    // If the skip-mode was successful, emit a lint.
-                    emit_implied_wf_lint(tcx, impl_m, impl_m_hir_id, bad_args);
-                });
-            }
-            CheckImpliedWfMode::Skip => {
-                if infcx.tainted_by_errors().is_none() {
-                    infcx.err_ctxt().report_region_errors(impl_m_def_id, &errors);
-                }
-                return Err(tcx
-                    .sess
-                    .delay_span_bug(rustc_span::DUMMY_SP, "error should have been emitted"));
-            }
-        }
+        return Err(infcx
+            .tainted_by_errors()
+            .unwrap_or_else(|| infcx.err_ctxt().report_region_errors(impl_m_def_id, &errors)));
     }
 
     Ok(())
 }
 
-fn extract_bad_args_for_implies_lint<'tcx>(
-    tcx: TyCtxt<'tcx>,
-    errors: &[infer::RegionResolutionError<'tcx>],
-    (trait_m, trait_sig): (ty::AssocItem, ty::FnSig<'tcx>),
-    (impl_m, impl_sig): (ty::AssocItem, ty::FnSig<'tcx>),
-    hir_id: hir::HirId,
-) -> Vec<(Span, Option<String>)> {
-    let mut blame_generics = vec![];
-    for error in errors {
-        // Look for the subregion origin that contains an input/output type
-        let origin = match error {
-            infer::RegionResolutionError::ConcreteFailure(o, ..) => o,
-            infer::RegionResolutionError::GenericBoundFailure(o, ..) => o,
-            infer::RegionResolutionError::SubSupConflict(_, _, o, ..) => o,
-            infer::RegionResolutionError::UpperBoundUniverseConflict(.., o, _) => o,
-        };
-        // Extract (possible) input/output types from origin
-        match origin {
-            infer::SubregionOrigin::Subtype(trace) => {
-                if let Some((a, b)) = trace.values.ty() {
-                    blame_generics.extend([a, b]);
-                }
-            }
-            infer::SubregionOrigin::RelateParamBound(_, ty, _) => blame_generics.push(*ty),
-            infer::SubregionOrigin::ReferenceOutlivesReferent(ty, _) => blame_generics.push(*ty),
-            _ => {}
-        }
-    }
-
-    let fn_decl = tcx.hir().fn_decl_by_hir_id(hir_id).unwrap();
-    let opt_ret_ty = match fn_decl.output {
-        hir::FnRetTy::DefaultReturn(_) => None,
-        hir::FnRetTy::Return(ty) => Some(ty),
-    };
-
-    // Map late-bound regions from trait to impl, so the names are right.
-    let mapping = std::iter::zip(
-        tcx.fn_sig(trait_m.def_id).skip_binder().bound_vars(),
-        tcx.fn_sig(impl_m.def_id).skip_binder().bound_vars(),
-    )
-    .filter_map(|(impl_bv, trait_bv)| {
-        if let ty::BoundVariableKind::Region(impl_bv) = impl_bv
-            && let ty::BoundVariableKind::Region(trait_bv) = trait_bv
-        {
-            Some((impl_bv, trait_bv))
-        } else {
-            None
-        }
-    })
-    .collect();
-
-    // For each arg, see if it was in the "blame" of any of the region errors.
-    // If so, then try to produce a suggestion to replace the argument type with
-    // one from the trait.
-    let mut bad_args = vec![];
-    for (idx, (ty, hir_ty)) in
-        std::iter::zip(impl_sig.inputs_and_output, fn_decl.inputs.iter().chain(opt_ret_ty))
-            .enumerate()
-    {
-        let expected_ty = trait_sig.inputs_and_output[idx]
-            .fold_with(&mut RemapLateBound { tcx, mapping: &mapping });
-        if blame_generics.iter().any(|blame| ty.contains(*blame)) {
-            let expected_ty_sugg = expected_ty.to_string();
-            bad_args.push((
-                hir_ty.span,
-                // Only suggest something if it actually changed.
-                (expected_ty_sugg != ty.to_string()).then_some(expected_ty_sugg),
-            ));
-        }
-    }
-
-    bad_args
-}
-
 struct RemapLateBound<'a, 'tcx> {
     tcx: TyCtxt<'tcx>,
     mapping: &'a FxHashMap<ty::BoundRegionKind, ty::BoundRegionKind>,
@@ -532,8 +401,8 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for RemapLateBound<'_, 'tcx> {
     }
 
     fn fold_region(&mut self, r: ty::Region<'tcx>) -> ty::Region<'tcx> {
-        if let ty::ReFree(fr) = *r {
-            ty::Region::new_free(
+        if let ty::ReLateParam(fr) = *r {
+            ty::Region::new_late_param(
                 self.tcx,
                 fr.scope,
                 self.mapping.get(&fr.bound_region).copied().unwrap_or(fr.bound_region),
@@ -544,53 +413,6 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for RemapLateBound<'_, 'tcx> {
     }
 }
 
-fn emit_implied_wf_lint<'tcx>(
-    tcx: TyCtxt<'tcx>,
-    impl_m: ty::AssocItem,
-    hir_id: hir::HirId,
-    bad_args: Vec<(Span, Option<String>)>,
-) {
-    let span: MultiSpan = if bad_args.is_empty() {
-        tcx.def_span(impl_m.def_id).into()
-    } else {
-        bad_args.iter().map(|(span, _)| *span).collect::<Vec<_>>().into()
-    };
-    tcx.struct_span_lint_hir(
-        rustc_session::lint::builtin::IMPLIED_BOUNDS_ENTAILMENT,
-        hir_id,
-        span,
-        "impl method assumes more implied bounds than the corresponding trait method",
-        |lint| {
-            let bad_args: Vec<_> =
-                bad_args.into_iter().filter_map(|(span, sugg)| Some((span, sugg?))).collect();
-            if !bad_args.is_empty() {
-                lint.multipart_suggestion(
-                    format!(
-                        "replace {} type{} to make the impl signature compatible",
-                        pluralize!("this", bad_args.len()),
-                        pluralize!(bad_args.len())
-                    ),
-                    bad_args,
-                    Applicability::MaybeIncorrect,
-                );
-            }
-            lint
-        },
-    );
-}
-
-#[derive(Debug, PartialEq, Eq)]
-enum CheckImpliedWfMode {
-    /// Checks implied well-formedness of the impl method. If it fails, we will
-    /// re-check with `Skip`, and emit a lint if it succeeds.
-    Check,
-    /// Skips checking implied well-formedness of the impl method, but will emit
-    /// a lint if the `compare_method_predicate_entailment` succeeded. This means that
-    /// the reason that we had failed earlier during `Check` was due to the impl
-    /// having stronger requirements than the trait.
-    Skip,
-}
-
 fn compare_asyncness<'tcx>(
     tcx: TyCtxt<'tcx>,
     impl_m: ty::AssocItem,
@@ -667,7 +489,7 @@ pub(super) fn collect_return_position_impl_trait_in_trait_tys<'tcx>(
 
     let trait_to_impl_args = impl_trait_ref.args;
 
-    let impl_m_hir_id = tcx.hir().local_def_id_to_hir_id(impl_m_def_id);
+    let impl_m_hir_id = tcx.local_def_id_to_hir_id(impl_m_def_id);
     let return_span = tcx.hir().fn_decl_by_hir_id(impl_m_hir_id).unwrap().output.span();
     let cause = ObligationCause::new(
         return_span,
@@ -937,7 +759,7 @@ pub(super) fn collect_return_position_impl_trait_in_trait_tys<'tcx>(
                 remapped_types.insert(def_id, ty::EarlyBinder::bind(ty));
             }
             Err(err) => {
-                let reported = tcx.sess.delay_span_bug(
+                let reported = tcx.sess.span_delayed_bug(
                     return_span,
                     format!("could not fully resolve: {ty} => {err:?}"),
                 );
@@ -1078,20 +900,26 @@ impl<'tcx> ty::FallibleTypeFolder<TyCtxt<'tcx>> for RemapHiddenTyRegions<'tcx> {
         region: ty::Region<'tcx>,
     ) -> Result<ty::Region<'tcx>, Self::Error> {
         match region.kind() {
-            // Remap all free regions, which correspond to late-bound regions in the function.
-            ty::ReFree(_) => {}
+            // Remap late-bound regions from the function.
+            ty::ReLateParam(_) => {}
             // Remap early-bound regions as long as they don't come from the `impl` itself,
             // in which case we don't really need to renumber them.
-            ty::ReEarlyBound(ebr) if self.tcx.parent(ebr.def_id) != self.impl_def_id => {}
+            ty::ReEarlyParam(ebr) if self.tcx.parent(ebr.def_id) != self.impl_def_id => {}
             _ => return Ok(region),
         }
 
-        let e = if let Some(region) = self.map.get(&region) {
-            if let ty::ReEarlyBound(e) = region.kind() { e } else { bug!() }
+        let e = if let Some(id_region) = self.map.get(&region) {
+            if let ty::ReEarlyParam(e) = id_region.kind() {
+                e
+            } else {
+                bug!(
+                    "expected to map region {region} to early-bound identity region, but got {id_region}"
+                );
+            }
         } else {
             let guar = match region.kind() {
-                ty::ReEarlyBound(ty::EarlyBoundRegion { def_id, .. })
-                | ty::ReFree(ty::FreeRegion {
+                ty::ReEarlyParam(ty::EarlyParamRegion { def_id, .. })
+                | ty::ReLateParam(ty::LateParamRegion {
                     bound_region: ty::BoundRegionKind::BrNamed(def_id, _),
                     ..
                 }) => {
@@ -1114,14 +942,16 @@ impl<'tcx> ty::FallibleTypeFolder<TyCtxt<'tcx>> for RemapHiddenTyRegions<'tcx> {
                         .note(format!("hidden type inferred to be `{}`", self.ty))
                         .emit()
                 }
-                _ => self.tcx.sess.delay_span_bug(DUMMY_SP, "should've been able to remap region"),
+                _ => {
+                    self.tcx.sess.span_delayed_bug(DUMMY_SP, "should've been able to remap region")
+                }
             };
             return Err(guar);
         };
 
-        Ok(ty::Region::new_early_bound(
+        Ok(ty::Region::new_early_param(
             self.tcx,
-            ty::EarlyBoundRegion {
+            ty::EarlyParamRegion {
                 def_id: e.def_id,
                 name: e.name,
                 index: (e.index as usize - self.num_trait_args + self.num_impl_args) as u32,
@@ -1140,7 +970,7 @@ fn report_trait_method_mismatch<'tcx>(
 ) -> ErrorGuaranteed {
     let tcx = infcx.tcx;
     let (impl_err_span, trait_err_span) =
-        extract_spans_for_error_reporting(&infcx, terr, &cause, impl_m, trait_m);
+        extract_spans_for_error_reporting(infcx, terr, &cause, impl_m, trait_m);
 
     let mut diag = struct_span_err!(
         tcx.sess,
@@ -1473,7 +1303,7 @@ fn compare_number_of_generics<'tcx>(
     // inheriting the generics from will also have mismatched arguments, and
     // we'll report an error for that instead. Delay a bug for safety, though.
     if trait_.is_impl_trait_in_trait() {
-        return Err(tcx.sess.delay_span_bug(
+        return Err(tcx.sess.span_delayed_bug(
             rustc_span::DUMMY_SP,
             "errors comparing numbers of generics of trait/impl functions were not emitted",
         ));
@@ -1708,92 +1538,87 @@ fn compare_synthetic_generics<'tcx>(
                 trait_m.name
             );
             err.span_label(trait_span, "declaration in trait here");
-            match (impl_synthetic, trait_synthetic) {
+            if impl_synthetic {
                 // The case where the impl method uses `impl Trait` but the trait method uses
                 // explicit generics
-                (true, false) => {
-                    err.span_label(impl_span, "expected generic parameter, found `impl Trait`");
-                    let _: Option<_> = try {
-                        // try taking the name from the trait impl
-                        // FIXME: this is obviously suboptimal since the name can already be used
-                        // as another generic argument
-                        let new_name = tcx.opt_item_name(trait_def_id)?;
-                        let trait_m = trait_m.def_id.as_local()?;
-                        let trait_m = tcx.hir().expect_trait_item(trait_m);
-
-                        let impl_m = impl_m.def_id.as_local()?;
-                        let impl_m = tcx.hir().expect_impl_item(impl_m);
-
-                        // in case there are no generics, take the spot between the function name
-                        // and the opening paren of the argument list
-                        let new_generics_span = tcx.def_ident_span(impl_def_id)?.shrink_to_hi();
-                        // in case there are generics, just replace them
-                        let generics_span =
-                            impl_m.generics.span.substitute_dummy(new_generics_span);
-                        // replace with the generics from the trait
-                        let new_generics =
-                            tcx.sess.source_map().span_to_snippet(trait_m.generics.span).ok()?;
-
-                        err.multipart_suggestion(
-                            "try changing the `impl Trait` argument to a generic parameter",
-                            vec![
-                                // replace `impl Trait` with `T`
-                                (impl_span, new_name.to_string()),
-                                // replace impl method generics with trait method generics
-                                // This isn't quite right, as users might have changed the names
-                                // of the generics, but it works for the common case
-                                (generics_span, new_generics),
-                            ],
-                            Applicability::MaybeIncorrect,
-                        );
-                    };
-                }
+                err.span_label(impl_span, "expected generic parameter, found `impl Trait`");
+                let _: Option<_> = try {
+                    // try taking the name from the trait impl
+                    // FIXME: this is obviously suboptimal since the name can already be used
+                    // as another generic argument
+                    let new_name = tcx.opt_item_name(trait_def_id)?;
+                    let trait_m = trait_m.def_id.as_local()?;
+                    let trait_m = tcx.hir().expect_trait_item(trait_m);
+
+                    let impl_m = impl_m.def_id.as_local()?;
+                    let impl_m = tcx.hir().expect_impl_item(impl_m);
+
+                    // in case there are no generics, take the spot between the function name
+                    // and the opening paren of the argument list
+                    let new_generics_span = tcx.def_ident_span(impl_def_id)?.shrink_to_hi();
+                    // in case there are generics, just replace them
+                    let generics_span = impl_m.generics.span.substitute_dummy(new_generics_span);
+                    // replace with the generics from the trait
+                    let new_generics =
+                        tcx.sess.source_map().span_to_snippet(trait_m.generics.span).ok()?;
+
+                    err.multipart_suggestion(
+                        "try changing the `impl Trait` argument to a generic parameter",
+                        vec![
+                            // replace `impl Trait` with `T`
+                            (impl_span, new_name.to_string()),
+                            // replace impl method generics with trait method generics
+                            // This isn't quite right, as users might have changed the names
+                            // of the generics, but it works for the common case
+                            (generics_span, new_generics),
+                        ],
+                        Applicability::MaybeIncorrect,
+                    );
+                };
+            } else {
                 // The case where the trait method uses `impl Trait`, but the impl method uses
                 // explicit generics.
-                (false, true) => {
-                    err.span_label(impl_span, "expected `impl Trait`, found generic parameter");
-                    let _: Option<_> = try {
-                        let impl_m = impl_m.def_id.as_local()?;
-                        let impl_m = tcx.hir().expect_impl_item(impl_m);
-                        let (sig, _) = impl_m.expect_fn();
-                        let input_tys = sig.decl.inputs;
-
-                        struct Visitor(Option<Span>, hir::def_id::LocalDefId);
-                        impl<'v> intravisit::Visitor<'v> for Visitor {
-                            fn visit_ty(&mut self, ty: &'v hir::Ty<'v>) {
-                                intravisit::walk_ty(self, ty);
-                                if let hir::TyKind::Path(hir::QPath::Resolved(None, path)) = ty.kind
-                                    && let Res::Def(DefKind::TyParam, def_id) = path.res
-                                    && def_id == self.1.to_def_id()
-                                {
-                                    self.0 = Some(ty.span);
-                                }
+                err.span_label(impl_span, "expected `impl Trait`, found generic parameter");
+                let _: Option<_> = try {
+                    let impl_m = impl_m.def_id.as_local()?;
+                    let impl_m = tcx.hir().expect_impl_item(impl_m);
+                    let (sig, _) = impl_m.expect_fn();
+                    let input_tys = sig.decl.inputs;
+
+                    struct Visitor(Option<Span>, hir::def_id::LocalDefId);
+                    impl<'v> intravisit::Visitor<'v> for Visitor {
+                        fn visit_ty(&mut self, ty: &'v hir::Ty<'v>) {
+                            intravisit::walk_ty(self, ty);
+                            if let hir::TyKind::Path(hir::QPath::Resolved(None, path)) = ty.kind
+                                && let Res::Def(DefKind::TyParam, def_id) = path.res
+                                && def_id == self.1.to_def_id()
+                            {
+                                self.0 = Some(ty.span);
                             }
                         }
+                    }
 
-                        let mut visitor = Visitor(None, impl_def_id);
-                        for ty in input_tys {
-                            intravisit::Visitor::visit_ty(&mut visitor, ty);
-                        }
-                        let span = visitor.0?;
-
-                        let bounds = impl_m.generics.bounds_for_param(impl_def_id).next()?.bounds;
-                        let bounds = bounds.first()?.span().to(bounds.last()?.span());
-                        let bounds = tcx.sess.source_map().span_to_snippet(bounds).ok()?;
-
-                        err.multipart_suggestion(
-                            "try removing the generic parameter and using `impl Trait` instead",
-                            vec![
-                                // delete generic parameters
-                                (impl_m.generics.span, String::new()),
-                                // replace param usage with `impl Trait`
-                                (span, format!("impl {bounds}")),
-                            ],
-                            Applicability::MaybeIncorrect,
-                        );
-                    };
-                }
-                _ => unreachable!(),
+                    let mut visitor = Visitor(None, impl_def_id);
+                    for ty in input_tys {
+                        intravisit::Visitor::visit_ty(&mut visitor, ty);
+                    }
+                    let span = visitor.0?;
+
+                    let bounds = impl_m.generics.bounds_for_param(impl_def_id).next()?.bounds;
+                    let bounds = bounds.first()?.span().to(bounds.last()?.span());
+                    let bounds = tcx.sess.source_map().span_to_snippet(bounds).ok()?;
+
+                    err.multipart_suggestion(
+                        "try removing the generic parameter and using `impl Trait` instead",
+                        vec![
+                            // delete generic parameters
+                            (impl_m.generics.span, String::new()),
+                            // replace param usage with `impl Trait`
+                            (span, format!("impl {bounds}")),
+                        ],
+                        Applicability::MaybeIncorrect,
+                    );
+                };
             }
             error_found = Some(err.emit_unless(delay));
         }
@@ -1857,7 +1682,9 @@ fn compare_generic_param_kinds<'tcx>(
             // this is exhaustive so that anyone adding new generic param kinds knows
             // to make sure this error is reported for them.
             (Const { .. }, Const { .. }) | (Type { .. }, Type { .. }) => false,
-            (Lifetime { .. }, _) | (_, Lifetime { .. }) => unreachable!(),
+            (Lifetime { .. }, _) | (_, Lifetime { .. }) => {
+                bug!("lifetime params are expected to be filtered by `ty_const_params_of`")
+            }
         } {
             let param_impl_span = tcx.def_span(param_impl.def_id);
             let param_trait_span = tcx.def_span(param_trait.def_id);
@@ -1881,7 +1708,10 @@ fn compare_generic_param_kinds<'tcx>(
                     )
                 }
                 Type { .. } => format!("{prefix} type parameter"),
-                Lifetime { .. } => unreachable!(),
+                Lifetime { .. } => span_bug!(
+                    tcx.def_span(param.def_id),
+                    "lifetime params are expected to be filtered by `ty_const_params_of`"
+                ),
             };
 
             let trait_header_span = tcx.def_ident_span(tcx.parent(trait_item.def_id)).unwrap();
@@ -2179,13 +2009,16 @@ pub(super) fn check_type_bounds<'tcx>(
     let impl_ty_span = if impl_ty.is_impl_trait_in_trait() {
         tcx.def_span(impl_ty_def_id)
     } else {
-        match tcx.hir().get_by_def_id(impl_ty_def_id) {
+        match tcx.hir_node_by_def_id(impl_ty_def_id) {
             hir::Node::TraitItem(hir::TraitItem {
                 kind: hir::TraitItemKind::Type(_, Some(ty)),
                 ..
             }) => ty.span,
             hir::Node::ImplItem(hir::ImplItem { kind: hir::ImplItemKind::Type(ty), .. }) => ty.span,
-            _ => bug!(),
+            item => span_bug!(
+                tcx.def_span(impl_ty_def_id),
+                "cannot call `check_type_bounds` on item: {item:?}",
+            ),
         }
     };
     let assumed_wf_types = ocx.assumed_wf_types_and_report_errors(param_env, impl_ty_def_id)?;
@@ -2345,7 +2178,7 @@ fn param_env_with_gat_bounds<'tcx>(
                     let kind = ty::BoundRegionKind::BrNamed(param.def_id, param.name);
                     let bound_var = ty::BoundVariableKind::Region(kind);
                     bound_vars.push(bound_var);
-                    ty::Region::new_late_bound(
+                    ty::Region::new_bound(
                         tcx,
                         ty::INNERMOST,
                         ty::BoundRegion {
diff --git a/compiler/rustc_hir_analysis/src/check/compare_impl_item/refine.rs b/compiler/rustc_hir_analysis/src/check/compare_impl_item/refine.rs
index bc5029a1d..67796855e 100644
--- a/compiler/rustc_hir_analysis/src/check/compare_impl_item/refine.rs
+++ b/compiler/rustc_hir_analysis/src/check/compare_impl_item/refine.rs
@@ -91,7 +91,7 @@ pub(super) fn check_refining_return_position_impl_trait_in_trait<'tcx>(
 
         // This opaque also needs to be from the impl method -- otherwise,
         // it's a refinement to a TAIT.
-        if !tcx.hir().get_if_local(impl_opaque.def_id).map_or(false, |node| {
+        if !tcx.hir().get_if_local(impl_opaque.def_id).is_some_and(|node| {
             matches!(
                 node.expect_item().expect_opaque_ty().origin,
                 hir::OpaqueTyOrigin::AsyncFn(def_id)  | hir::OpaqueTyOrigin::FnReturn(def_id)
@@ -153,7 +153,7 @@ pub(super) fn check_refining_return_position_impl_trait_in_trait<'tcx>(
         trait_m_sig.inputs_and_output,
     ));
     if !ocx.select_all_or_error().is_empty() {
-        tcx.sess.delay_span_bug(
+        tcx.sess.span_delayed_bug(
             DUMMY_SP,
             "encountered errors when checking RPITIT refinement (selection)",
         );
@@ -165,7 +165,7 @@ pub(super) fn check_refining_return_position_impl_trait_in_trait<'tcx>(
     );
     let errors = infcx.resolve_regions(&outlives_env);
     if !errors.is_empty() {
-        tcx.sess.delay_span_bug(
+        tcx.sess.span_delayed_bug(
             DUMMY_SP,
             "encountered errors when checking RPITIT refinement (regions)",
         );
@@ -173,7 +173,7 @@ pub(super) fn check_refining_return_position_impl_trait_in_trait<'tcx>(
     }
     // Resolve any lifetime variables that may have been introduced during normalization.
     let Ok((trait_bounds, impl_bounds)) = infcx.fully_resolve((trait_bounds, impl_bounds)) else {
-        tcx.sess.delay_span_bug(
+        tcx.sess.span_delayed_bug(
             DUMMY_SP,
             "encountered errors when checking RPITIT refinement (resolution)",
         );
@@ -262,7 +262,10 @@ fn report_mismatched_rpitit_signature<'tcx>(
 
     if tcx.asyncness(impl_m_def_id).is_async() && tcx.asyncness(trait_m_def_id).is_async() {
         let ty::Alias(ty::Projection, future_ty) = return_ty.kind() else {
-            bug!();
+            span_bug!(
+                tcx.def_span(trait_m_def_id),
+                "expected return type of async fn in trait to be a AFIT projection"
+            );
         };
         let Some(future_output_ty) = tcx
             .explicit_item_bounds(future_ty.def_id)
@@ -272,13 +275,13 @@ fn report_mismatched_rpitit_signature<'tcx>(
                 _ => None,
             })
         else {
-            bug!()
+            span_bug!(tcx.def_span(trait_m_def_id), "expected `Future` projection bound in AFIT");
         };
         return_ty = future_output_ty;
     }
 
     let (span, impl_return_span, pre, post) =
-        match tcx.hir().get_by_def_id(impl_m_def_id.expect_local()).fn_decl().unwrap().output {
+        match tcx.hir_node_by_def_id(impl_m_def_id.expect_local()).fn_decl().unwrap().output {
             hir::FnRetTy::DefaultReturn(span) => (tcx.def_span(impl_m_def_id), span, "-> ", " "),
             hir::FnRetTy::Return(ty) => (ty.span, ty.span, "", ""),
         };
diff --git a/compiler/rustc_hir_analysis/src/check/dropck.rs b/compiler/rustc_hir_analysis/src/check/dropck.rs
index dda3f7425..58c77bb45 100644
--- a/compiler/rustc_hir_analysis/src/check/dropck.rs
+++ b/compiler/rustc_hir_analysis/src/check/dropck.rs
@@ -66,7 +66,7 @@ pub fn check_drop_impl(tcx: TyCtxt<'_>, drop_impl_did: DefId) -> Result<(), Erro
             // already checked by coherence, but compilation may
             // not have been terminated.
             let span = tcx.def_span(drop_impl_did);
-            let reported = tcx.sess.delay_span_bug(
+            let reported = tcx.sess.span_delayed_bug(
                 span,
                 format!("should have been rejected by coherence check: {dtor_self_type}"),
             );
@@ -81,8 +81,7 @@ fn ensure_drop_params_and_item_params_correspond<'tcx>(
     self_type_did: DefId,
     adt_to_impl_args: GenericArgsRef<'tcx>,
 ) -> Result<(), ErrorGuaranteed> {
-    let Err(arg) = tcx.uses_unique_generic_params(adt_to_impl_args, CheckRegions::OnlyEarlyBound)
-    else {
+    let Err(arg) = tcx.uses_unique_generic_params(adt_to_impl_args, CheckRegions::OnlyParam) else {
         return Ok(());
     };
 
diff --git a/compiler/rustc_hir_analysis/src/check/entry.rs b/compiler/rustc_hir_analysis/src/check/entry.rs
index 6681292c9..1d737e17e 100644
--- a/compiler/rustc_hir_analysis/src/check/entry.rs
+++ b/compiler/rustc_hir_analysis/src/check/entry.rs
@@ -42,8 +42,8 @@ fn check_main_fn_ty(tcx: TyCtxt<'_>, main_def_id: DefId) {
         if !def_id.is_local() {
             return None;
         }
-        let hir_id = tcx.hir().local_def_id_to_hir_id(def_id.expect_local());
-        match tcx.hir().find(hir_id) {
+        let hir_id = tcx.local_def_id_to_hir_id(def_id.expect_local());
+        match tcx.opt_hir_node(hir_id) {
             Some(Node::Item(hir::Item { kind: hir::ItemKind::Fn(_, generics, _), .. })) => {
                 generics.params.is_empty().not().then_some(generics.span)
             }
@@ -57,8 +57,8 @@ fn check_main_fn_ty(tcx: TyCtxt<'_>, main_def_id: DefId) {
         if !def_id.is_local() {
             return None;
         }
-        let hir_id = tcx.hir().local_def_id_to_hir_id(def_id.expect_local());
-        match tcx.hir().find(hir_id) {
+        let hir_id = tcx.local_def_id_to_hir_id(def_id.expect_local());
+        match tcx.opt_hir_node(hir_id) {
             Some(Node::Item(hir::Item { kind: hir::ItemKind::Fn(_, generics, _), .. })) => {
                 Some(generics.where_clause_span)
             }
@@ -79,8 +79,8 @@ fn check_main_fn_ty(tcx: TyCtxt<'_>, main_def_id: DefId) {
         if !def_id.is_local() {
             return None;
         }
-        let hir_id = tcx.hir().local_def_id_to_hir_id(def_id.expect_local());
-        match tcx.hir().find(hir_id) {
+        let hir_id = tcx.local_def_id_to_hir_id(def_id.expect_local());
+        match tcx.opt_hir_node(hir_id) {
             Some(Node::Item(hir::Item { kind: hir::ItemKind::Fn(fn_sig, _, _), .. })) => {
                 Some(fn_sig.decl.output.span())
             }
@@ -92,24 +92,6 @@ fn check_main_fn_ty(tcx: TyCtxt<'_>, main_def_id: DefId) {
 
     let mut error = false;
     let main_diagnostics_def_id = main_fn_diagnostics_def_id(tcx, main_def_id, main_span);
-    let main_fn_generics = tcx.generics_of(main_def_id);
-    let main_fn_predicates = tcx.predicates_of(main_def_id);
-    if main_fn_generics.count() != 0 || !main_fnsig.bound_vars().is_empty() {
-        let generics_param_span = main_fn_generics_params_span(tcx, main_def_id);
-        tcx.sess.emit_err(errors::MainFunctionGenericParameters {
-            span: generics_param_span.unwrap_or(main_span),
-            label_span: generics_param_span,
-        });
-        error = true;
-    } else if !main_fn_predicates.predicates.is_empty() {
-        // generics may bring in implicit predicates, so we skip this check if generics is present.
-        let generics_where_clauses_span = main_fn_where_clauses_span(tcx, main_def_id);
-        tcx.sess.emit_err(errors::WhereClauseOnMain {
-            span: generics_where_clauses_span.unwrap_or(main_span),
-            generics_span: generics_where_clauses_span,
-        });
-        error = true;
-    }
 
     let main_asyncness = tcx.asyncness(main_def_id);
     if main_asyncness.is_async() {
@@ -142,10 +124,6 @@ fn check_main_fn_ty(tcx: TyCtxt<'_>, main_def_id: DefId) {
     if let Some(term_did) = tcx.lang_items().termination() {
         let return_ty = main_fnsig.output();
         let return_ty_span = main_fn_return_type_span(tcx, main_def_id).unwrap_or(main_span);
-        if !return_ty.bound_vars().is_empty() {
-            tcx.sess.emit_err(errors::MainFunctionReturnTypeGeneric { span: return_ty_span });
-            error = true;
-        }
         let return_ty = return_ty.skip_binder();
         let infcx = tcx.infer_ctxt().build();
         let cause = traits::ObligationCause::new(
@@ -180,7 +158,7 @@ fn check_main_fn_ty(tcx: TyCtxt<'_>, main_def_id: DefId) {
         Abi::Rust,
     ));
 
-    check_function_signature(
+    if check_function_signature(
         tcx,
         ObligationCause::new(
             main_span,
@@ -189,17 +167,38 @@ fn check_main_fn_ty(tcx: TyCtxt<'_>, main_def_id: DefId) {
         ),
         main_def_id,
         expected_sig,
-    );
+    )
+    .is_err()
+    {
+        return;
+    }
+
+    let main_fn_generics = tcx.generics_of(main_def_id);
+    let main_fn_predicates = tcx.predicates_of(main_def_id);
+    if main_fn_generics.count() != 0 || !main_fnsig.bound_vars().is_empty() {
+        let generics_param_span = main_fn_generics_params_span(tcx, main_def_id);
+        tcx.sess.emit_err(errors::MainFunctionGenericParameters {
+            span: generics_param_span.unwrap_or(main_span),
+            label_span: generics_param_span,
+        });
+    } else if !main_fn_predicates.predicates.is_empty() {
+        // generics may bring in implicit predicates, so we skip this check if generics is present.
+        let generics_where_clauses_span = main_fn_where_clauses_span(tcx, main_def_id);
+        tcx.sess.emit_err(errors::WhereClauseOnMain {
+            span: generics_where_clauses_span.unwrap_or(main_span),
+            generics_span: generics_where_clauses_span,
+        });
+    }
 }
 
 fn check_start_fn_ty(tcx: TyCtxt<'_>, start_def_id: DefId) {
     let start_def_id = start_def_id.expect_local();
-    let start_id = tcx.hir().local_def_id_to_hir_id(start_def_id);
+    let start_id = tcx.local_def_id_to_hir_id(start_def_id);
     let start_span = tcx.def_span(start_def_id);
     let start_t = tcx.type_of(start_def_id).instantiate_identity();
     match start_t.kind() {
         ty::FnDef(..) => {
-            if let Some(Node::Item(it)) = tcx.hir().find(start_id) {
+            if let Some(Node::Item(it)) = tcx.opt_hir_node(start_id) {
                 if let hir::ItemKind::Fn(sig, generics, _) = &it.kind {
                     let mut error = false;
                     if !generics.params.is_empty() {
@@ -255,7 +254,7 @@ fn check_start_fn_ty(tcx: TyCtxt<'_>, start_def_id: DefId) {
                 Abi::Rust,
             ));
 
-            check_function_signature(
+            let _ = check_function_signature(
                 tcx,
                 ObligationCause::new(
                     start_span,
diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs
index c61719c1f..126bab68a 100644
--- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs
+++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs
@@ -55,7 +55,7 @@ fn equate_intrinsic_type<'tcx>(
         && gen_count_ok(own_counts.consts, n_cts, "const")
     {
         let it_def_id = it.owner_id.def_id;
-        check_function_signature(
+        let _ = check_function_signature(
             tcx,
             ObligationCause::new(it.span, it_def_id, ObligationCauseCode::IntrinsicType),
             it_def_id.into(),
@@ -143,12 +143,12 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
     ]);
     let mk_va_list_ty = |mutbl| {
         tcx.lang_items().va_list().map(|did| {
-            let region = ty::Region::new_late_bound(
+            let region = ty::Region::new_bound(
                 tcx,
                 ty::INNERMOST,
                 ty::BoundRegion { var: ty::BoundVar::from_u32(0), kind: ty::BrAnon },
             );
-            let env_region = ty::Region::new_late_bound(
+            let env_region = ty::Region::new_bound(
                 tcx,
                 ty::INNERMOST,
                 ty::BoundRegion { var: ty::BoundVar::from_u32(1), kind: ty::BrEnv },
@@ -225,25 +225,6 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
                 ],
                 Ty::new_ptr(tcx, ty::TypeAndMut { ty: param(0), mutbl: hir::Mutability::Not }),
             ),
-            sym::option_payload_ptr => {
-                let option_def_id = tcx.require_lang_item(hir::LangItem::Option, None);
-                let p0 = param(0);
-                (
-                    1,
-                    vec![Ty::new_ptr(
-                        tcx,
-                        ty::TypeAndMut {
-                            ty: Ty::new_adt(
-                                tcx,
-                                tcx.adt_def(option_def_id),
-                                tcx.mk_args_from_iter([ty::GenericArg::from(p0)].into_iter()),
-                            ),
-                            mutbl: hir::Mutability::Not,
-                        },
-                    )],
-                    Ty::new_ptr(tcx, ty::TypeAndMut { ty: p0, mutbl: hir::Mutability::Not }),
-                )
-            }
             sym::ptr_mask => (
                 1,
                 vec![
@@ -411,7 +392,7 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
                     1,
                     vec![Ty::new_imm_ref(
                         tcx,
-                        ty::Region::new_late_bound(tcx, ty::INNERMOST, br),
+                        ty::Region::new_bound(tcx, ty::INNERMOST, br),
                         param(0),
                     )],
                     Ty::new_projection(tcx, discriminant_def_id, tcx.mk_args(&[param(0).into()])),
@@ -465,11 +446,8 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
 
             sym::raw_eq => {
                 let br = ty::BoundRegion { var: ty::BoundVar::from_u32(0), kind: ty::BrAnon };
-                let param_ty = Ty::new_imm_ref(
-                    tcx,
-                    ty::Region::new_late_bound(tcx, ty::INNERMOST, br),
-                    param(0),
-                );
+                let param_ty =
+                    Ty::new_imm_ref(tcx, ty::Region::new_bound(tcx, ty::INNERMOST, br), param(0));
                 (1, vec![param_ty; 2], tcx.types.bool)
             }
 
@@ -543,6 +521,8 @@ pub fn check_platform_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>)
         sym::simd_fpowi => (1, 0, vec![param(0), tcx.types.i32], param(0)),
         sym::simd_fma => (1, 0, vec![param(0), param(0), param(0)], param(0)),
         sym::simd_gather => (3, 0, vec![param(0), param(1), param(2)], param(0)),
+        sym::simd_masked_load => (3, 0, vec![param(0), param(1), param(2)], param(2)),
+        sym::simd_masked_store => (3, 0, vec![param(0), param(1), param(2)], Ty::new_unit(tcx)),
         sym::simd_scatter => (3, 0, vec![param(0), param(1), param(2)], Ty::new_unit(tcx)),
         sym::simd_insert => (2, 0, vec![param(0), tcx.types.u32, param(1)], param(0)),
         sym::simd_extract => (2, 0, vec![param(0), tcx.types.u32], param(1)),
diff --git a/compiler/rustc_hir_analysis/src/check/intrinsicck.rs b/compiler/rustc_hir_analysis/src/check/intrinsicck.rs
index cd7e99172..d86ebc2c9 100644
--- a/compiler/rustc_hir_analysis/src/check/intrinsicck.rs
+++ b/compiler/rustc_hir_analysis/src/check/intrinsicck.rs
@@ -49,7 +49,7 @@ impl<'a, 'tcx> InlineAsmCtxt<'a, 'tcx> {
             16 => InlineAsmType::I16,
             32 => InlineAsmType::I32,
             64 => InlineAsmType::I64,
-            _ => unreachable!(),
+            width => bug!("unsupported pointer width: {width}"),
         };
 
         match *ty.kind() {
@@ -101,7 +101,7 @@ impl<'a, 'tcx> InlineAsmCtxt<'a, 'tcx> {
                             16 => InlineAsmType::VecI16(size),
                             32 => InlineAsmType::VecI32(size),
                             64 => InlineAsmType::VecI64(size),
-                            _ => unreachable!(),
+                            width => bug!("unsupported pointer width: {width}"),
                         })
                     }
                     ty::Float(FloatTy::F32) => Some(InlineAsmType::VecF32(size)),
@@ -109,7 +109,7 @@ impl<'a, 'tcx> InlineAsmCtxt<'a, 'tcx> {
                     _ => None,
                 }
             }
-            ty::Infer(_) => unreachable!(),
+            ty::Infer(_) => bug!("unexpected infer ty in asm operand"),
             _ => None,
         }
     }
@@ -136,8 +136,15 @@ impl<'a, 'tcx> InlineAsmCtxt<'a, 'tcx> {
             ty::Adt(adt, args) if Some(adt.did()) == self.tcx.lang_items().maybe_uninit() => {
                 let fields = &adt.non_enum_variant().fields;
                 let ty = fields[FieldIdx::from_u32(1)].ty(self.tcx, args);
-                let ty::Adt(ty, args) = ty.kind() else { unreachable!() };
-                assert!(ty.is_manually_drop());
+                // FIXME: Are we just trying to map to the `T` in `MaybeUninit<T>`?
+                // If so, just get it from the args.
+                let ty::Adt(ty, args) = ty.kind() else {
+                    unreachable!("expected first field of `MaybeUninit` to be an ADT")
+                };
+                assert!(
+                    ty.is_manually_drop(),
+                    "expected first field of `MaybeUnit` to be `ManuallyDrop`"
+                );
                 let fields = &ty.non_enum_variant().fields;
                 let ty = fields[FieldIdx::from_u32(0)].ty(self.tcx, args);
                 self.get_asm_ty(ty)
@@ -269,7 +276,6 @@ impl<'a, 'tcx> InlineAsmCtxt<'a, 'tcx> {
                         lint.help(format!(
                             "or use `{{{idx}:{default_modifier}}}` to keep the default formatting of `{default_result}`",
                         ));
-                        lint
                     },
                 );
             }
@@ -281,7 +287,7 @@ impl<'a, 'tcx> InlineAsmCtxt<'a, 'tcx> {
     pub fn check_asm(&self, asm: &hir::InlineAsm<'tcx>, enclosing_id: LocalDefId) {
         let target_features = self.tcx.asm_target_features(enclosing_id.to_def_id());
         let Some(asm_arch) = self.tcx.sess.asm_arch else {
-            self.tcx.sess.delay_span_bug(DUMMY_SP, "target architecture does not support asm");
+            self.tcx.sess.span_delayed_bug(DUMMY_SP, "target architecture does not support asm");
             return;
         };
         for (idx, (op, op_sp)) in asm.operands.iter().enumerate() {
@@ -307,7 +313,7 @@ impl<'a, 'tcx> InlineAsmCtxt<'a, 'tcx> {
                     if let Err(msg) = reg.validate(
                         asm_arch,
                         self.tcx.sess.relocation_model(),
-                        &target_features,
+                        target_features,
                         &self.tcx.sess.target,
                         op.is_clobber(),
                     ) {
@@ -382,7 +388,7 @@ impl<'a, 'tcx> InlineAsmCtxt<'a, 'tcx> {
                         asm.template,
                         true,
                         None,
-                        &target_features,
+                        target_features,
                     );
                 }
                 hir::InlineAsmOperand::Out { reg, late: _, expr } => {
@@ -394,7 +400,7 @@ impl<'a, 'tcx> InlineAsmCtxt<'a, 'tcx> {
                             asm.template,
                             false,
                             None,
-                            &target_features,
+                            target_features,
                         );
                     }
                 }
@@ -406,7 +412,7 @@ impl<'a, 'tcx> InlineAsmCtxt<'a, 'tcx> {
                         asm.template,
                         false,
                         None,
-                        &target_features,
+                        target_features,
                     );
                 }
                 hir::InlineAsmOperand::SplitInOut { reg, late: _, in_expr, out_expr } => {
@@ -417,7 +423,7 @@ impl<'a, 'tcx> InlineAsmCtxt<'a, 'tcx> {
                         asm.template,
                         true,
                         None,
-                        &target_features,
+                        target_features,
                     );
                     if let Some(out_expr) = out_expr {
                         self.check_asm_operand_type(
@@ -427,7 +433,7 @@ impl<'a, 'tcx> InlineAsmCtxt<'a, 'tcx> {
                             asm.template,
                             false,
                             Some((in_expr, in_ty)),
-                            &target_features,
+                            target_features,
                         );
                     }
                 }
diff --git a/compiler/rustc_hir_analysis/src/check/mod.rs b/compiler/rustc_hir_analysis/src/check/mod.rs
index 15c5558fc..e4904a043 100644
--- a/compiler/rustc_hir_analysis/src/check/mod.rs
+++ b/compiler/rustc_hir_analysis/src/check/mod.rs
@@ -77,6 +77,7 @@ use std::num::NonZeroU32;
 
 use check::check_mod_item_types;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
+use rustc_errors::ErrorGuaranteed;
 use rustc_errors::{pluralize, struct_span_err, Diagnostic, DiagnosticBuilder};
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::intravisit::Visitor;
@@ -128,9 +129,9 @@ fn get_owner_return_paths(
     tcx: TyCtxt<'_>,
     def_id: LocalDefId,
 ) -> Option<(LocalDefId, ReturnsVisitor<'_>)> {
-    let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
+    let hir_id = tcx.local_def_id_to_hir_id(def_id);
     let parent_id = tcx.hir().get_parent_item(hir_id).def_id;
-    tcx.hir().find_by_def_id(parent_id).and_then(|node| node.body_id()).map(|body_id| {
+    tcx.opt_hir_node_by_def_id(parent_id).and_then(|node| node.body_id()).map(|body_id| {
         let body = tcx.hir().body(body_id);
         let mut visitor = ReturnsVisitor::default();
         visitor.visit_body(body);
@@ -141,7 +142,7 @@ fn get_owner_return_paths(
 /// Forbid defining intrinsics in Rust code,
 /// as they must always be defined by the compiler.
 // FIXME: Move this to a more appropriate place.
-pub fn fn_maybe_err(tcx: TyCtxt<'_>, sp: Span, abi: Abi) {
+pub fn forbid_intrinsic_abi(tcx: TyCtxt<'_>, sp: Span, abi: Abi) {
     if let Abi::RustIntrinsic | Abi::PlatformIntrinsic = abi {
         tcx.sess.span_err(sp, "intrinsic must be in `extern \"rust-intrinsic\" { ... }` block");
     }
@@ -570,7 +571,26 @@ pub fn check_function_signature<'tcx>(
     mut cause: ObligationCause<'tcx>,
     fn_id: DefId,
     expected_sig: ty::PolyFnSig<'tcx>,
-) {
+) -> Result<(), ErrorGuaranteed> {
+    fn extract_span_for_error_reporting<'tcx>(
+        tcx: TyCtxt<'tcx>,
+        err: TypeError<'_>,
+        cause: &ObligationCause<'tcx>,
+        fn_id: LocalDefId,
+    ) -> rustc_span::Span {
+        let mut args = {
+            let node = tcx.hir().expect_owner(fn_id);
+            let decl = node.fn_decl().unwrap_or_else(|| bug!("expected fn decl, found {:?}", node));
+            decl.inputs.iter().map(|t| t.span).chain(std::iter::once(decl.output.span()))
+        };
+
+        match err {
+            TypeError::ArgumentMutability(i)
+            | TypeError::ArgumentSorts(ExpectedFound { .. }, i) => args.nth(i).unwrap(),
+            _ => cause.span(),
+        }
+    }
+
     let local_id = fn_id.as_local().unwrap_or(CRATE_DEF_ID);
 
     let param_env = ty::ParamEnv::empty();
@@ -587,8 +607,7 @@ pub fn check_function_signature<'tcx>(
         Ok(()) => {
             let errors = ocx.select_all_or_error();
             if !errors.is_empty() {
-                infcx.err_ctxt().report_fulfillment_errors(errors);
-                return;
+                return Err(infcx.err_ctxt().report_fulfillment_errors(errors));
             }
         }
         Err(err) => {
@@ -610,30 +629,14 @@ pub fn check_function_signature<'tcx>(
                 false,
                 false,
             );
-            diag.emit();
-            return;
+            return Err(diag.emit());
         }
     }
 
     let outlives_env = OutlivesEnvironment::new(param_env);
-    let _ = ocx.resolve_regions_and_report_errors(local_id, &outlives_env);
-
-    fn extract_span_for_error_reporting<'tcx>(
-        tcx: TyCtxt<'tcx>,
-        err: TypeError<'_>,
-        cause: &ObligationCause<'tcx>,
-        fn_id: LocalDefId,
-    ) -> rustc_span::Span {
-        let mut args = {
-            let node = tcx.hir().expect_owner(fn_id);
-            let decl = node.fn_decl().unwrap_or_else(|| bug!("expected fn decl, found {:?}", node));
-            decl.inputs.iter().map(|t| t.span).chain(std::iter::once(decl.output.span()))
-        };
-
-        match err {
-            TypeError::ArgumentMutability(i)
-            | TypeError::ArgumentSorts(ExpectedFound { .. }, i) => args.nth(i).unwrap(),
-            _ => cause.span(),
-        }
+    if let Err(e) = ocx.resolve_regions_and_report_errors(local_id, &outlives_env) {
+        return Err(e);
     }
+
+    Ok(())
 }
diff --git a/compiler/rustc_hir_analysis/src/check/region.rs b/compiler/rustc_hir_analysis/src/check/region.rs
index 40b33117f..37b308f9f 100644
--- a/compiler/rustc_hir_analysis/src/check/region.rs
+++ b/compiler/rustc_hir_analysis/src/check/region.rs
@@ -414,11 +414,11 @@ fn resolve_expr<'tcx>(visitor: &mut RegionResolutionVisitor<'tcx>, expr: &'tcx h
             // then we'll assign too low a count to any `yield` expressions
             // we encounter in 'right_expression' - they should really occur after all of the
             // expressions in 'left_expression'.
-            visitor.visit_expr(&right_expr);
+            visitor.visit_expr(right_expr);
             visitor.pessimistic_yield = prev_pessimistic;
 
             debug!("resolve_expr - restoring pessimistic_yield to {}", prev_pessimistic);
-            visitor.visit_expr(&left_expr);
+            visitor.visit_expr(left_expr);
             debug!("resolve_expr - fixing up counts to {}", visitor.expr_and_pat_count);
 
             // Remove and process any scopes pushed by the visitor
@@ -582,7 +582,7 @@ fn resolve_local<'tcx>(
     // due to rule C.
 
     if let Some(expr) = init {
-        record_rvalue_scope_if_borrow_expr(visitor, &expr, blk_scope);
+        record_rvalue_scope_if_borrow_expr(visitor, expr, blk_scope);
 
         if let Some(pat) = pat {
             if is_binding_pat(pat) {
@@ -645,25 +645,24 @@ fn resolve_local<'tcx>(
         match pat.kind {
             PatKind::Binding(hir::BindingAnnotation(hir::ByRef::Yes, _), ..) => true,
 
-            PatKind::Struct(_, field_pats, _) => {
-                field_pats.iter().any(|fp| is_binding_pat(&fp.pat))
-            }
+            PatKind::Struct(_, field_pats, _) => field_pats.iter().any(|fp| is_binding_pat(fp.pat)),
 
             PatKind::Slice(pats1, pats2, pats3) => {
-                pats1.iter().any(|p| is_binding_pat(&p))
-                    || pats2.iter().any(|p| is_binding_pat(&p))
-                    || pats3.iter().any(|p| is_binding_pat(&p))
+                pats1.iter().any(|p| is_binding_pat(p))
+                    || pats2.iter().any(|p| is_binding_pat(p))
+                    || pats3.iter().any(|p| is_binding_pat(p))
             }
 
             PatKind::Or(subpats)
             | PatKind::TupleStruct(_, subpats, _)
-            | PatKind::Tuple(subpats, _) => subpats.iter().any(|p| is_binding_pat(&p)),
+            | PatKind::Tuple(subpats, _) => subpats.iter().any(|p| is_binding_pat(p)),
 
-            PatKind::Box(subpat) => is_binding_pat(&subpat),
+            PatKind::Box(subpat) => is_binding_pat(subpat),
 
             PatKind::Ref(_, _)
             | PatKind::Binding(hir::BindingAnnotation(hir::ByRef::No, _), ..)
             | PatKind::Wild
+            | PatKind::Never
             | PatKind::Path(_)
             | PatKind::Lit(_)
             | PatKind::Range(_, _, _) => false,
@@ -700,20 +699,20 @@ fn resolve_local<'tcx>(
             }
             hir::ExprKind::Struct(_, fields, _) => {
                 for field in fields {
-                    record_rvalue_scope_if_borrow_expr(visitor, &field.expr, blk_id);
+                    record_rvalue_scope_if_borrow_expr(visitor, field.expr, blk_id);
                 }
             }
             hir::ExprKind::Array(subexprs) | hir::ExprKind::Tup(subexprs) => {
                 for subexpr in subexprs {
-                    record_rvalue_scope_if_borrow_expr(visitor, &subexpr, blk_id);
+                    record_rvalue_scope_if_borrow_expr(visitor, subexpr, blk_id);
                 }
             }
             hir::ExprKind::Cast(subexpr, _) => {
-                record_rvalue_scope_if_borrow_expr(visitor, &subexpr, blk_id)
+                record_rvalue_scope_if_borrow_expr(visitor, subexpr, blk_id)
             }
             hir::ExprKind::Block(block, _) => {
                 if let Some(subexpr) = block.expr {
-                    record_rvalue_scope_if_borrow_expr(visitor, &subexpr, blk_id);
+                    record_rvalue_scope_if_borrow_expr(visitor, subexpr, blk_id);
                 }
             }
             hir::ExprKind::Call(..) | hir::ExprKind::MethodCall(..) => {
@@ -795,13 +794,13 @@ impl<'tcx> Visitor<'tcx> for RegionResolutionVisitor<'tcx> {
         // The arguments and `self` are parented to the fn.
         self.cx.var_parent = self.cx.parent.take();
         for param in body.params {
-            self.visit_pat(&param.pat);
+            self.visit_pat(param.pat);
         }
 
         // The body of the every fn is a root scope.
         self.cx.parent = self.cx.var_parent;
         if self.tcx.hir().body_owner_kind(owner_id).is_fn_or_closure() {
-            self.visit_expr(&body.value)
+            self.visit_expr(body.value)
         } else {
             // Only functions have an outer terminating (drop) scope, while
             // temporaries in constant initializers may be 'static, but only
@@ -822,7 +821,7 @@ impl<'tcx> Visitor<'tcx> for RegionResolutionVisitor<'tcx> {
             // (i.e., `'static`), which means that after `g` returns, it drops,
             // and all the associated destruction scope rules apply.
             self.cx.var_parent = None;
-            resolve_local(self, None, Some(&body.value));
+            resolve_local(self, None, Some(body.value));
         }
 
         if body.coroutine_kind.is_some() {
@@ -849,7 +848,7 @@ impl<'tcx> Visitor<'tcx> for RegionResolutionVisitor<'tcx> {
         resolve_expr(self, ex);
     }
     fn visit_local(&mut self, l: &'tcx Local<'tcx>) {
-        resolve_local(self, Some(&l.pat), l.init)
+        resolve_local(self, Some(l.pat), l.init)
     }
 }
 
diff --git a/compiler/rustc_hir_analysis/src/check/wfcheck.rs b/compiler/rustc_hir_analysis/src/check/wfcheck.rs
index eb4491b89..b2ff79591 100644
--- a/compiler/rustc_hir_analysis/src/check/wfcheck.rs
+++ b/compiler/rustc_hir_analysis/src/check/wfcheck.rs
@@ -118,10 +118,10 @@ where
         if tcx.sess.err_count() > 0 {
             return Err(err);
         } else {
-            // HACK(oli-obk): tests/ui/specialization/min_specialization/specialize_on_type_error.rs causes an
-            // error (delay_span_bug) during normalization, without reporting an error, so we need to act as if
-            // no error happened, in order to let our callers continue and report an error later in
-            // check_impl_items_against_trait.
+            // HACK(oli-obk): tests/ui/specialization/min_specialization/specialize_on_type_error.rs
+            // causes an error (span_delayed_bug) during normalization, without reporting an error,
+            // so we need to act as if no error happened, in order to let our callers continue and
+            // report an error later in check_impl_items_against_trait.
             return Ok(());
         }
     }
@@ -204,11 +204,14 @@ fn check_item<'tcx>(tcx: TyCtxt<'tcx>, item: &'tcx hir::Item<'tcx>) -> Result<()
                 res = Err(err.emit());
             }
             // We match on both `ty::ImplPolarity` and `ast::ImplPolarity` just to get the `!` span.
-            match (tcx.impl_polarity(def_id), impl_.polarity) {
-                (ty::ImplPolarity::Positive, _) => {
+            match tcx.impl_polarity(def_id) {
+                ty::ImplPolarity::Positive => {
                     res = res.and(check_impl(tcx, item, impl_.self_ty, &impl_.of_trait));
                 }
-                (ty::ImplPolarity::Negative, ast::ImplPolarity::Negative(span)) => {
+                ty::ImplPolarity::Negative => {
+                    let ast::ImplPolarity::Negative(span) = impl_.polarity else {
+                        bug!("impl_polarity query disagrees with impl's polarity in AST");
+                    };
                     // FIXME(#27579): what amount of WF checking do we need for neg impls?
                     if let hir::Defaultness::Default { .. } = impl_.defaultness {
                         let mut spans = vec![span];
@@ -222,10 +225,9 @@ fn check_item<'tcx>(tcx: TyCtxt<'tcx>, item: &'tcx hir::Item<'tcx>) -> Result<()
                         .emit());
                     }
                 }
-                (ty::ImplPolarity::Reservation, _) => {
+                ty::ImplPolarity::Reservation => {
                     // FIXME: what amount of WF checking do we need for reservation impls?
                 }
-                _ => unreachable!(),
             }
             res
         }
@@ -584,7 +586,7 @@ fn gather_gat_bounds<'tcx, T: TypeFoldable<TyCtxt<'tcx>>>(
         // reflected in a where clause on the GAT itself.
         for (ty, ty_idx) in &types {
             // In our example, requires that `Self: 'a`
-            if ty_known_to_outlive(tcx, item_def_id, param_env, &wf_tys, *ty, *region_a) {
+            if ty_known_to_outlive(tcx, item_def_id, param_env, wf_tys, *ty, *region_a) {
                 debug!(?ty_idx, ?region_a_idx);
                 debug!("required clause: {ty} must outlive {region_a}");
                 // Translate into the generic parameters of the GAT. In
@@ -595,9 +597,9 @@ fn gather_gat_bounds<'tcx, T: TypeFoldable<TyCtxt<'tcx>>>(
                 // Same for the region. In our example, 'a corresponds
                 // to the 'me parameter.
                 let region_param = gat_generics.param_at(*region_a_idx, tcx);
-                let region_param = ty::Region::new_early_bound(
+                let region_param = ty::Region::new_early_param(
                     tcx,
-                    ty::EarlyBoundRegion {
+                    ty::EarlyParamRegion {
                         def_id: region_param.def_id,
                         index: region_param.index,
                         name: region_param.name,
@@ -623,14 +625,14 @@ fn gather_gat_bounds<'tcx, T: TypeFoldable<TyCtxt<'tcx>>>(
             if matches!(**region_b, ty::ReStatic | ty::ReError(_)) || region_a == region_b {
                 continue;
             }
-            if region_known_to_outlive(tcx, item_def_id, param_env, &wf_tys, *region_a, *region_b) {
+            if region_known_to_outlive(tcx, item_def_id, param_env, wf_tys, *region_a, *region_b) {
                 debug!(?region_a_idx, ?region_b_idx);
                 debug!("required clause: {region_a} must outlive {region_b}");
                 // Translate into the generic parameters of the GAT.
                 let region_a_param = gat_generics.param_at(*region_a_idx, tcx);
-                let region_a_param = ty::Region::new_early_bound(
+                let region_a_param = ty::Region::new_early_param(
                     tcx,
-                    ty::EarlyBoundRegion {
+                    ty::EarlyParamRegion {
                         def_id: region_a_param.def_id,
                         index: region_a_param.index,
                         name: region_a_param.name,
@@ -638,9 +640,9 @@ fn gather_gat_bounds<'tcx, T: TypeFoldable<TyCtxt<'tcx>>>(
                 );
                 // Same for the region.
                 let region_b_param = gat_generics.param_at(*region_b_idx, tcx);
-                let region_b_param = ty::Region::new_early_bound(
+                let region_b_param = ty::Region::new_early_param(
                     tcx,
-                    ty::EarlyBoundRegion {
+                    ty::EarlyParamRegion {
                         def_id: region_b_param.def_id,
                         index: region_b_param.index,
                         name: region_b_param.name,
@@ -671,7 +673,7 @@ fn ty_known_to_outlive<'tcx>(
     ty: Ty<'tcx>,
     region: ty::Region<'tcx>,
 ) -> bool {
-    resolve_regions_with_wf_tys(tcx, id, param_env, &wf_tys, |infcx, region_bound_pairs| {
+    resolve_regions_with_wf_tys(tcx, id, param_env, wf_tys, |infcx, region_bound_pairs| {
         let origin = infer::RelateParamBound(DUMMY_SP, ty, None);
         let outlives = &mut TypeOutlives::new(infcx, tcx, region_bound_pairs, None, param_env);
         outlives.type_must_outlive(origin, ty, region, ConstraintCategory::BoringNoLocation);
@@ -688,7 +690,7 @@ fn region_known_to_outlive<'tcx>(
     region_a: ty::Region<'tcx>,
     region_b: ty::Region<'tcx>,
 ) -> bool {
-    resolve_regions_with_wf_tys(tcx, id, param_env, &wf_tys, |mut infcx, _| {
+    resolve_regions_with_wf_tys(tcx, id, param_env, wf_tys, |mut infcx, _| {
         use rustc_infer::infer::outlives::obligations::TypeOutlivesDelegate;
         let origin = infer::RelateRegionParamBound(DUMMY_SP);
         // `region_a: region_b` -> `region_b <= region_a`
@@ -763,7 +765,7 @@ impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for GATSubstCollector<'tcx> {
             ty::Alias(ty::Projection, p) if p.def_id == self.gat => {
                 for (idx, subst) in p.args.iter().enumerate() {
                     match subst.unpack() {
-                        GenericArgKind::Lifetime(lt) if !lt.is_late_bound() => {
+                        GenericArgKind::Lifetime(lt) if !lt.is_bound() => {
                             self.regions.insert((lt, idx));
                         }
                         GenericArgKind::Type(t) => {
@@ -793,7 +795,7 @@ fn could_be_self(trait_def_id: LocalDefId, ty: &hir::Ty<'_>) -> bool {
 /// When this is done, suggest using `Self` instead.
 fn check_object_unsafe_self_trait_by_name(tcx: TyCtxt<'_>, item: &hir::TraitItem<'_>) {
     let (trait_name, trait_def_id) =
-        match tcx.hir().get_by_def_id(tcx.hir().get_parent_item(item.hir_id()).def_id) {
+        match tcx.hir_node_by_def_id(tcx.hir().get_parent_item(item.hir_id()).def_id) {
             hir::Node::Item(item) => match item.kind {
                 hir::ItemKind::Trait(..) => (item.ident, item.owner_id),
                 _ => return,
@@ -859,7 +861,7 @@ fn check_param_wf(tcx: TyCtxt<'_>, param: &hir::GenericParam<'_>) -> Result<(),
         hir::GenericParamKind::Lifetime { .. } | hir::GenericParamKind::Type { .. } => Ok(()),
 
         // Const parameters are well formed if their type is structural match.
-        hir::GenericParamKind::Const { ty: hir_ty, default: _ } => {
+        hir::GenericParamKind::Const { ty: hir_ty, default: _, is_host_effect: _ } => {
             let ty = tcx.type_of(param.def_id).instantiate_identity();
 
             if tcx.features().adt_const_params {
@@ -992,15 +994,6 @@ fn check_associated_item(
     })
 }
 
-fn item_adt_kind(kind: &ItemKind<'_>) -> Option<AdtKind> {
-    match kind {
-        ItemKind::Struct(..) => Some(AdtKind::Struct),
-        ItemKind::Union(..) => Some(AdtKind::Union),
-        ItemKind::Enum(..) => Some(AdtKind::Enum),
-        _ => None,
-    }
-}
-
 /// In a type definition, we check that to ensure that the types of the fields are well-formed.
 fn check_type_defn<'tcx>(
     tcx: TyCtxt<'tcx>,
@@ -1019,7 +1012,7 @@ fn check_type_defn<'tcx>(
             for field in &variant.fields {
                 let field_id = field.did.expect_local();
                 let hir::FieldDef { ty: hir_ty, .. } =
-                    tcx.hir().get_by_def_id(field_id).expect_field();
+                    tcx.hir_node_by_def_id(field_id).expect_field();
                 let ty = wfcx.normalize(
                     hir_ty.span,
                     None,
@@ -1040,7 +1033,7 @@ fn check_type_defn<'tcx>(
                     let ty = tcx.erase_regions(ty);
                     if ty.has_infer() {
                         tcx.sess
-                            .delay_span_bug(item.span, format!("inference variables in {ty:?}"));
+                            .span_delayed_bug(item.span, format!("inference variables in {ty:?}"));
                         // Just treat unresolved type expression as if it needs drop.
                         true
                     } else {
@@ -1057,7 +1050,7 @@ fn check_type_defn<'tcx>(
                 let last = idx == variant.fields.len() - 1;
                 let field_id = field.did.expect_local();
                 let hir::FieldDef { ty: hir_ty, .. } =
-                    tcx.hir().get_by_def_id(field_id).expect_field();
+                    tcx.hir_node_by_def_id(field_id).expect_field();
                 let ty = wfcx.normalize(
                     hir_ty.span,
                     None,
@@ -1068,9 +1061,14 @@ fn check_type_defn<'tcx>(
                         hir_ty.span,
                         wfcx.body_def_id,
                         traits::FieldSized {
-                            adt_kind: match item_adt_kind(&item.kind) {
-                                Some(i) => i,
-                                None => bug!(),
+                            adt_kind: match &item.kind {
+                                ItemKind::Struct(..) => AdtKind::Struct,
+                                ItemKind::Union(..) => AdtKind::Union,
+                                ItemKind::Enum(..) => AdtKind::Enum,
+                                kind => span_bug!(
+                                    item.span,
+                                    "should be wfchecking an ADT, got {kind:?}"
+                                ),
                             },
                             span: hir_ty.span,
                             last,
@@ -1302,7 +1300,9 @@ fn check_where_clauses<'tcx>(wfcx: &WfCheckingCtxt<'_, 'tcx>, span: Span, def_id
         | GenericParamDefKind::Const { has_default, .. } => {
             has_default && def.index >= generics.parent_count as u32
         }
-        GenericParamDefKind::Lifetime => unreachable!(),
+        GenericParamDefKind::Lifetime => {
+            span_bug!(tcx.def_span(def.def_id), "lifetime params can have no default")
+        }
     };
 
     // Check that concrete defaults are well-formed. See test `type-check-defaults.rs`.
@@ -1607,15 +1607,10 @@ fn check_method_receiver<'tcx>(
 }
 
 fn e0307(tcx: TyCtxt<'_>, span: Span, receiver_ty: Ty<'_>) -> ErrorGuaranteed {
-    struct_span_err!(
-        tcx.sess.diagnostic(),
-        span,
-        E0307,
-        "invalid `self` parameter type: {receiver_ty}"
-    )
-    .note("type of `self` must be `Self` or a type that dereferences to it")
-    .help(HELP_FOR_SELF_TYPE)
-    .emit()
+    struct_span_err!(tcx.sess.dcx(), span, E0307, "invalid `self` parameter type: {receiver_ty}")
+        .note("type of `self` must be `Self` or a type that dereferences to it")
+        .help(HELP_FOR_SELF_TYPE)
+        .emit()
 }
 
 /// Returns whether `receiver_ty` would be considered a valid receiver type for `self_ty`. If
@@ -1750,15 +1745,15 @@ fn check_variances_for_type_defn<'tcx>(
             }
         }
         ItemKind::TyAlias(..) => {
-            if tcx.type_alias_is_lazy(item.owner_id) {
-                if tcx.type_of(item.owner_id).skip_binder().references_error() {
-                    return;
-                }
-            } else {
-                bug!();
+            assert!(
+                tcx.type_alias_is_lazy(item.owner_id),
+                "should not be computing variance of non-weak type alias"
+            );
+            if tcx.type_of(item.owner_id).skip_binder().references_error() {
+                return;
             }
         }
-        _ => bug!(),
+        kind => span_bug!(item.span, "cannot compute the variances of {kind:?}"),
     }
 
     let ty_predicates = tcx.predicates_of(item.owner_id);
@@ -1812,8 +1807,10 @@ fn check_variances_for_type_defn<'tcx>(
             //
             // if they aren't in the same order, then the user has written invalid code, and already
             // got an error about it (or I'm wrong about this)
-            tcx.sess
-                .delay_span_bug(hir_param.span, "hir generics and ty generics in different order");
+            tcx.sess.span_delayed_bug(
+                hir_param.span,
+                "hir generics and ty generics in different order",
+            );
             continue;
         }
 
@@ -1880,7 +1877,7 @@ impl<'tcx> WfCheckingCtxt<'_, 'tcx> {
             // Match the existing behavior.
             if pred.is_global() && !pred.has_type_flags(TypeFlags::HAS_BINDER_VARS) {
                 let pred = self.normalize(span, None, pred);
-                let hir_node = tcx.hir().find_by_def_id(self.body_def_id);
+                let hir_node = tcx.opt_hir_node_by_def_id(self.body_def_id);
 
                 // only use the span of the predicate clause (#90869)
 
diff --git a/compiler/rustc_hir_analysis/src/check_unused.rs b/compiler/rustc_hir_analysis/src/check_unused.rs
index 9ad73eeff..36cb8f7a2 100644
--- a/compiler/rustc_hir_analysis/src/check_unused.rs
+++ b/compiler/rustc_hir_analysis/src/check_unused.rs
@@ -45,7 +45,7 @@ fn check_unused_traits(tcx: TyCtxt<'_>, (): ()) {
             item.hir_id(),
             path.span,
             msg,
-            |lint| lint,
+            |_| {},
         );
     }
 }
diff --git a/compiler/rustc_hir_analysis/src/coherence/builtin.rs b/compiler/rustc_hir_analysis/src/coherence/builtin.rs
index e5e192e00..f277badf2 100644
--- a/compiler/rustc_hir_analysis/src/coherence/builtin.rs
+++ b/compiler/rustc_hir_analysis/src/coherence/builtin.rs
@@ -550,7 +550,7 @@ fn infringing_fields_error(
                                 .entry((ty.clone(), predicate.clone()))
                                 .or_default()
                                 .push(origin.span());
-                            if let ty::RegionKind::ReEarlyBound(ebr) = *b
+                            if let ty::RegionKind::ReEarlyParam(ebr) = *b
                                 && ebr.has_name()
                             {
                                 bounds.push((b.to_string(), a.to_string(), None));
diff --git a/compiler/rustc_hir_analysis/src/coherence/inherent_impls_overlap.rs b/compiler/rustc_hir_analysis/src/coherence/inherent_impls_overlap.rs
index 7205b7a21..3f8c0db87 100644
--- a/compiler/rustc_hir_analysis/src/coherence/inherent_impls_overlap.rs
+++ b/compiler/rustc_hir_analysis/src/coherence/inherent_impls_overlap.rs
@@ -198,7 +198,7 @@ impl<'tcx> InherentOverlapChecker<'tcx> {
             // entire graph when there are many connected regions.
 
             rustc_index::newtype_index! {
-                #[custom_encodable]
+                #[orderable]
                 pub struct RegionId {}
             }
 
diff --git a/compiler/rustc_hir_analysis/src/coherence/orphan.rs b/compiler/rustc_hir_analysis/src/coherence/orphan.rs
index 7eeb78374..d33cfe4ad 100644
--- a/compiler/rustc_hir_analysis/src/coherence/orphan.rs
+++ b/compiler/rustc_hir_analysis/src/coherence/orphan.rs
@@ -58,7 +58,7 @@ fn do_orphan_check_impl<'tcx>(
                 tr.path.span,
                 trait_ref,
                 impl_.self_ty.span,
-                &impl_.generics,
+                impl_.generics,
                 err,
             )?
         }
@@ -452,7 +452,13 @@ fn lint_auto_trait_impl<'tcx>(
     trait_ref: ty::TraitRef<'tcx>,
     impl_def_id: LocalDefId,
 ) {
-    assert_eq!(trait_ref.args.len(), 1);
+    if trait_ref.args.len() != 1 {
+        tcx.sess.dcx().span_delayed_bug(
+            tcx.def_span(impl_def_id),
+            "auto traits cannot have generic parameters",
+        );
+        return;
+    }
     let self_ty = trait_ref.self_ty();
     let (self_type_did, args) = match self_ty.kind() {
         ty::Adt(def, args) => (def.did(), args),
@@ -491,7 +497,7 @@ fn lint_auto_trait_impl<'tcx>(
 
     tcx.struct_span_lint_hir(
         lint::builtin::SUSPICIOUS_AUTO_TRAIT_IMPLS,
-        tcx.hir().local_def_id_to_hir_id(impl_def_id),
+        tcx.local_def_id_to_hir_id(impl_def_id),
         tcx.def_span(impl_def_id),
         DelayDm(|| {
             format!(
@@ -516,7 +522,7 @@ fn lint_auto_trait_impl<'tcx>(
                 format!(
                     "try using the same sequence of generic parameters as the {self_descr} definition",
                 ),
-            )
+            );
         },
     );
 }
diff --git a/compiler/rustc_hir_analysis/src/coherence/unsafety.rs b/compiler/rustc_hir_analysis/src/coherence/unsafety.rs
index 6b18b0ebe..8a02bab92 100644
--- a/compiler/rustc_hir_analysis/src/coherence/unsafety.rs
+++ b/compiler/rustc_hir_analysis/src/coherence/unsafety.rs
@@ -23,7 +23,7 @@ pub(super) fn check_item(tcx: TyCtxt<'_>, def_id: LocalDefId) {
                     tcx.def_span(def_id),
                     E0199,
                     "implementing the trait `{}` is not unsafe",
-                    trait_ref.print_only_trait_path()
+                    trait_ref.print_trait_sugared()
                 )
                 .span_suggestion_verbose(
                     item.span.with_hi(item.span.lo() + rustc_span::BytePos(7)),
@@ -40,13 +40,13 @@ pub(super) fn check_item(tcx: TyCtxt<'_>, def_id: LocalDefId) {
                     tcx.def_span(def_id),
                     E0200,
                     "the trait `{}` requires an `unsafe impl` declaration",
-                    trait_ref.print_only_trait_path()
+                    trait_ref.print_trait_sugared()
                 )
                 .note(format!(
                     "the trait `{}` enforces invariants that the compiler can't check. \
                     Review the trait documentation and make sure this implementation \
                     upholds those invariants before adding the `unsafe` keyword",
-                    trait_ref.print_only_trait_path()
+                    trait_ref.print_trait_sugared()
                 ))
                 .span_suggestion_verbose(
                     item.span.shrink_to_lo(),
@@ -69,7 +69,7 @@ pub(super) fn check_item(tcx: TyCtxt<'_>, def_id: LocalDefId) {
                     "the trait `{}` enforces invariants that the compiler can't check. \
                     Review the trait documentation and make sure this implementation \
                     upholds those invariants before adding the `unsafe` keyword",
-                    trait_ref.print_only_trait_path()
+                    trait_ref.print_trait_sugared()
                 ))
                 .span_suggestion_verbose(
                     item.span.shrink_to_lo(),
@@ -82,7 +82,7 @@ pub(super) fn check_item(tcx: TyCtxt<'_>, def_id: LocalDefId) {
 
             (_, _, Unsafety::Unsafe, hir::ImplPolarity::Negative(_)) => {
                 // Reported in AST validation
-                tcx.sess.delay_span_bug(item.span, "unsafe negative impl");
+                tcx.sess.span_delayed_bug(item.span, "unsafe negative impl");
             }
             (_, _, Unsafety::Normal, hir::ImplPolarity::Negative(_))
             | (Unsafety::Unsafe, _, Unsafety::Unsafe, hir::ImplPolarity::Positive)
diff --git a/compiler/rustc_hir_analysis/src/collect.rs b/compiler/rustc_hir_analysis/src/collect.rs
index 9636c6144..d48535c82 100644
--- a/compiler/rustc_hir_analysis/src/collect.rs
+++ b/compiler/rustc_hir_analysis/src/collect.rs
@@ -221,7 +221,7 @@ pub(crate) fn placeholder_type_error_diag<'tcx>(
 
             // Check if parent is const or static
             let parent_id = tcx.hir().parent_id(hir_ty.hir_id);
-            let parent_node = tcx.hir().get(parent_id);
+            let parent_node = tcx.hir_node(parent_id);
 
             is_const_or_static = matches!(
                 parent_node,
@@ -350,11 +350,11 @@ impl<'tcx> ItemCtxt<'tcx> {
     }
 
     pub fn hir_id(&self) -> hir::HirId {
-        self.tcx.hir().local_def_id_to_hir_id(self.item_def_id)
+        self.tcx.local_def_id_to_hir_id(self.item_def_id)
     }
 
     pub fn node(&self) -> hir::Node<'tcx> {
-        self.tcx.hir().get(self.hir_id())
+        self.tcx.hir_node(self.hir_id())
     }
 }
 
@@ -440,10 +440,10 @@ impl<'tcx> AstConv<'tcx> for ItemCtxt<'tcx> {
                                 second: format!(
                                     "{}::",
                                     // Replace the existing lifetimes with a new named lifetime.
-                                    self.tcx.replace_late_bound_regions_uncached(
+                                    self.tcx.instantiate_bound_regions_uncached(
                                         poly_trait_ref,
                                         |_| {
-                                            ty::Region::new_early_bound(self.tcx, ty::EarlyBoundRegion {
+                                            ty::Region::new_early_param(self.tcx, ty::EarlyParamRegion {
                                                 def_id: item_def_id,
                                                 index: 0,
                                                 name: Symbol::intern(&lt_name),
@@ -672,7 +672,7 @@ fn convert_trait_item(tcx: TyCtxt<'_>, trait_item_id: hir::TraitItemId) {
 
         hir::TraitItemKind::Const(ty, body_id) => {
             tcx.ensure().type_of(def_id);
-            if !tcx.sess.diagnostic().has_stashed_diagnostic(ty.span, StashKey::ItemNoType)
+            if !tcx.sess.dcx().has_stashed_diagnostic(ty.span, StashKey::ItemNoType)
                 && !(is_suggestable_infer_ty(ty) && body_id.is_some())
             {
                 // Account for `const C: _;`.
@@ -814,7 +814,7 @@ fn convert_variant(
         })
         .collect();
     let recovered = match def {
-        hir::VariantData::Struct(_, r) => *r,
+        hir::VariantData::Struct { recovered, .. } => *recovered,
         _ => false,
     };
     ty::VariantDef::new(
@@ -835,9 +835,8 @@ fn convert_variant(
 fn adt_def(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::AdtDef<'_> {
     use rustc_hir::*;
 
-    let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
-    let Node::Item(item) = tcx.hir().get(hir_id) else {
-        bug!();
+    let Node::Item(item) = tcx.hir_node_by_def_id(def_id) else {
+        bug!("expected ADT to be an item");
     };
 
     let repr = tcx.repr_options_of_def(def_id.to_def_id());
@@ -888,7 +887,7 @@ fn adt_def(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::AdtDef<'_> {
 
             (adt_kind, variants)
         }
-        _ => bug!(),
+        _ => bug!("{:?} is not an ADT", item.owner_id.def_id),
     };
     tcx.mk_adt_def(def_id.to_def_id(), kind, variants, repr)
 }
@@ -1090,7 +1089,7 @@ fn is_suggestable_infer_ty(ty: &hir::Ty<'_>) -> bool {
 pub fn get_infer_ret_ty<'hir>(output: &'hir hir::FnRetTy<'hir>) -> Option<&'hir hir::Ty<'hir>> {
     if let hir::FnRetTy::Return(ty) = output {
         if is_suggestable_infer_ty(ty) {
-            return Some(&*ty);
+            return Some(*ty);
         }
     }
     None
@@ -1101,11 +1100,11 @@ fn fn_sig(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::EarlyBinder<ty::PolyFnSig<
     use rustc_hir::Node::*;
     use rustc_hir::*;
 
-    let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
+    let hir_id = tcx.local_def_id_to_hir_id(def_id);
 
     let icx = ItemCtxt::new(tcx, def_id);
 
-    let output = match tcx.hir().get(hir_id) {
+    let output = match tcx.hir_node(hir_id) {
         TraitItem(hir::TraitItem {
             kind: TraitItemKind::Fn(sig, TraitFn::Provided(_)),
             generics,
@@ -1186,7 +1185,7 @@ fn infer_return_ty_for_fn_sig<'tcx>(
     def_id: LocalDefId,
     icx: &ItemCtxt<'tcx>,
 ) -> ty::PolyFnSig<'tcx> {
-    let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
+    let hir_id = tcx.local_def_id_to_hir_id(def_id);
 
     match get_infer_ret_ty(&sig.decl.output) {
         Some(ty) => {
@@ -1373,7 +1372,7 @@ fn impl_trait_ref(
             if let Some(ErrorGuaranteed { .. }) = check_impl_constness(
                 tcx,
                 tcx.is_const_trait_impl_raw(def_id.to_def_id()),
-                &ast_trait_ref,
+                ast_trait_ref,
             ) {
                 // we have a const impl, but for a trait without `#[const_trait]`, so
                 // without the host param. If we continue with the HIR trait ref, we get
@@ -1383,7 +1382,7 @@ fn impl_trait_ref(
                 let last_segment = path_segments.len() - 1;
                 let mut args = *path_segments[last_segment].args();
                 let last_arg = args.args.len() - 1;
-                assert!(matches!(args.args[last_arg], hir::GenericArg::Const(anon_const) if tcx.has_attr(anon_const.value.def_id, sym::rustc_host)));
+                assert!(matches!(args.args[last_arg], hir::GenericArg::Const(anon_const) if anon_const.is_desugared_from_effects));
                 args.args = &args.args[..args.args.len() - 1];
                 path_segments[last_segment].args = Some(&args);
                 let path = hir::Path {
@@ -1394,7 +1393,7 @@ fn impl_trait_ref(
                 let trait_ref = hir::TraitRef { path: &path, hir_ref_id: ast_trait_ref.hir_ref_id };
                 icx.astconv().instantiate_mono_trait_ref(&trait_ref, selfty)
             } else {
-                icx.astconv().instantiate_mono_trait_ref(&ast_trait_ref, selfty)
+                icx.astconv().instantiate_mono_trait_ref(ast_trait_ref, selfty)
             }
         })
         .map(ty::EarlyBinder::bind)
@@ -1519,7 +1518,7 @@ fn compute_sig_of_foreign_fn_decl<'tcx>(
     } else {
         hir::Unsafety::Unsafe
     };
-    let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
+    let hir_id = tcx.local_def_id_to_hir_id(def_id);
     let fty =
         ItemCtxt::new(tcx, def_id).astconv().ty_of_fn(hir_id, unsafety, abi, decl, None, None);
 
@@ -1551,7 +1550,7 @@ fn compute_sig_of_foreign_fn_decl<'tcx>(
 }
 
 fn coroutine_kind(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Option<hir::CoroutineKind> {
-    match tcx.hir().get_by_def_id(def_id) {
+    match tcx.hir_node_by_def_id(def_id) {
         Node::Expr(&rustc_hir::Expr {
             kind: rustc_hir::ExprKind::Closure(&rustc_hir::Closure { body, .. }),
             ..
@@ -1561,7 +1560,7 @@ fn coroutine_kind(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Option<hir::CoroutineK
 }
 
 fn is_type_alias_impl_trait<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalDefId) -> bool {
-    match tcx.hir().get_by_def_id(def_id) {
+    match tcx.hir_node_by_def_id(def_id) {
         Node::Item(hir::Item { kind: hir::ItemKind::OpaqueTy(opaque), .. }) => {
             matches!(opaque.origin, hir::OpaqueTyOrigin::TyAlias { .. })
         }
diff --git a/compiler/rustc_hir_analysis/src/collect/generics_of.rs b/compiler/rustc_hir_analysis/src/collect/generics_of.rs
index 3d60c57b9..3ee2822ed 100644
--- a/compiler/rustc_hir_analysis/src/collect/generics_of.rs
+++ b/compiler/rustc_hir_analysis/src/collect/generics_of.rs
@@ -9,14 +9,14 @@ use rustc_hir::def_id::LocalDefId;
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_session::lint;
 use rustc_span::symbol::{kw, Symbol};
-use rustc_span::{sym, Span};
+use rustc_span::Span;
 
 pub(super) fn generics_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::Generics {
     use rustc_hir::*;
 
-    let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
+    let hir_id = tcx.local_def_id_to_hir_id(def_id);
 
-    let node = tcx.hir().get(hir_id);
+    let node = tcx.hir_node(hir_id);
     let parent_def_id = match node {
         Node::ImplItem(_)
         | Node::TraitItem(_)
@@ -182,7 +182,7 @@ pub(super) fn generics_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::Generics {
     }
 
     let no_generics = hir::Generics::empty();
-    let ast_generics = node.generics().unwrap_or(&no_generics);
+    let ast_generics = node.generics().unwrap_or(no_generics);
     let (opt_self, allow_defaults) = match node {
         Node::Item(item) => {
             match item.kind {
@@ -279,7 +279,7 @@ pub(super) fn generics_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::Generics {
                             param.hir_id,
                             param.span,
                             TYPE_DEFAULT_NOT_ALLOWED,
-                            |lint| lint,
+                            |_| {},
                         );
                     }
                     Defaults::Deny => {
@@ -298,13 +298,11 @@ pub(super) fn generics_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::Generics {
                 kind,
             })
         }
-        GenericParamKind::Const { default, .. } => {
-            let is_host_param = tcx.has_attr(param.def_id, sym::rustc_host);
-
+        GenericParamKind::Const { ty: _, default, is_host_effect } => {
             if !matches!(allow_defaults, Defaults::Allowed)
                 && default.is_some()
-                // `rustc_host` effect params are allowed to have defaults.
-                && !is_host_param
+                // `host` effect params are allowed to have defaults.
+                && !is_host_effect
             {
                 tcx.sess.span_err(
                     param.span,
@@ -315,7 +313,7 @@ pub(super) fn generics_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::Generics {
 
             let index = next_index();
 
-            if is_host_param {
+            if is_host_effect {
                 if let Some(idx) = host_effect_index {
                     bug!("parent also has host effect param? index: {idx}, def: {def_id:?}");
                 }
@@ -330,7 +328,7 @@ pub(super) fn generics_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::Generics {
                 pure_wrt_drop: param.pure_wrt_drop,
                 kind: ty::GenericParamDefKind::Const {
                     has_default: default.is_some(),
-                    is_host_effect: is_host_param,
+                    is_host_effect,
                 },
             })
         }
@@ -458,11 +456,11 @@ fn has_late_bound_regions<'tcx>(tcx: TyCtxt<'tcx>, node: Node<'tcx>) -> Option<S
 
     match node {
         Node::TraitItem(item) => match &item.kind {
-            hir::TraitItemKind::Fn(sig, _) => has_late_bound_regions(tcx, &item.generics, sig.decl),
+            hir::TraitItemKind::Fn(sig, _) => has_late_bound_regions(tcx, item.generics, sig.decl),
             _ => None,
         },
         Node::ImplItem(item) => match &item.kind {
-            hir::ImplItemKind::Fn(sig, _) => has_late_bound_regions(tcx, &item.generics, sig.decl),
+            hir::ImplItemKind::Fn(sig, _) => has_late_bound_regions(tcx, item.generics, sig.decl),
             _ => None,
         },
         Node::ForeignItem(item) => match item.kind {
@@ -489,7 +487,7 @@ struct AnonConstInParamTyDetector {
 
 impl<'v> Visitor<'v> for AnonConstInParamTyDetector {
     fn visit_generic_param(&mut self, p: &'v hir::GenericParam<'v>) {
-        if let GenericParamKind::Const { ty, default: _ } = p.kind {
+        if let GenericParamKind::Const { ty, default: _, is_host_effect: _ } = p.kind {
             let prev = self.in_param_ty;
             self.in_param_ty = true;
             self.visit_ty(ty);
diff --git a/compiler/rustc_hir_analysis/src/collect/item_bounds.rs b/compiler/rustc_hir_analysis/src/collect/item_bounds.rs
index d746e6dea..39ca1bba0 100644
--- a/compiler/rustc_hir_analysis/src/collect/item_bounds.rs
+++ b/compiler/rustc_hir_analysis/src/collect/item_bounds.rs
@@ -34,17 +34,14 @@ fn associated_type_bounds<'tcx>(
     let trait_def_id = tcx.local_parent(assoc_item_def_id);
     let trait_predicates = tcx.trait_explicit_predicates_and_bounds(trait_def_id);
 
-    let bounds_from_parent = trait_predicates
-        .predicates
-        .iter()
-        .copied()
-        .filter(|(pred, _)| match pred.kind().skip_binder() {
+    let bounds_from_parent = trait_predicates.predicates.iter().copied().filter(|(pred, _)| {
+        match pred.kind().skip_binder() {
             ty::ClauseKind::Trait(tr) => tr.self_ty() == item_ty,
             ty::ClauseKind::Projection(proj) => proj.projection_ty.self_ty() == item_ty,
             ty::ClauseKind::TypeOutlives(outlives) => outlives.0 == item_ty,
             _ => false,
-        })
-        .map(|(clause, span)| (clause, span));
+        }
+    });
 
     let all_bounds = tcx.arena.alloc_from_iter(bounds.clauses().chain(bounds_from_parent));
     debug!(
@@ -86,7 +83,7 @@ pub(super) fn explicit_item_bounds(
         // RPITIT's bounds are the same as opaque type bounds, but with
         // a projection self type.
         Some(ty::ImplTraitInTraitData::Trait { opaque_def_id, .. }) => {
-            let item = tcx.hir().get_by_def_id(opaque_def_id.expect_local()).expect_item();
+            let item = tcx.hir_node_by_def_id(opaque_def_id.expect_local()).expect_item();
             let opaque_ty = item.expect_opaque_ty();
             return ty::EarlyBinder::bind(opaque_type_bounds(
                 tcx,
@@ -100,13 +97,14 @@ pub(super) fn explicit_item_bounds(
                 item.span,
             ));
         }
-        // These should have been fed!
-        Some(ty::ImplTraitInTraitData::Impl { .. }) => unreachable!(),
+        Some(ty::ImplTraitInTraitData::Impl { .. }) => span_bug!(
+            tcx.def_span(def_id),
+            "item bounds for RPITIT in impl to be fed on def-id creation"
+        ),
         None => {}
     }
 
-    let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
-    let bounds = match tcx.hir().get(hir_id) {
+    let bounds = match tcx.hir_node_by_def_id(def_id) {
         hir::Node::TraitItem(hir::TraitItem {
             kind: hir::TraitItemKind::Type(bounds, _),
             span,
@@ -132,7 +130,7 @@ pub(super) fn explicit_item_bounds(
             let (hir::OpaqueTyOrigin::FnReturn(fn_def_id)
             | hir::OpaqueTyOrigin::AsyncFn(fn_def_id)) = *origin
             else {
-                bug!()
+                span_bug!(*span, "RPITIT cannot be a TAIT, but got origin {origin:?}");
             };
             let args = GenericArgs::identity_for_item(tcx, def_id);
             let item_ty = Ty::new_opaque(tcx, def_id.to_def_id(), args);
diff --git a/compiler/rustc_hir_analysis/src/collect/predicates_of.rs b/compiler/rustc_hir_analysis/src/collect/predicates_of.rs
index 104da581e..41520718a 100644
--- a/compiler/rustc_hir_analysis/src/collect/predicates_of.rs
+++ b/compiler/rustc_hir_analysis/src/collect/predicates_of.rs
@@ -134,8 +134,8 @@ fn gather_explicit_predicates_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::Gen
         None => {}
     }
 
-    let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
-    let node = tcx.hir().get(hir_id);
+    let hir_id = tcx.local_def_id_to_hir_id(def_id);
+    let node = tcx.hir_node(hir_id);
 
     let mut is_trait = None;
     let mut is_default_impl_trait = None;
@@ -290,13 +290,18 @@ fn gather_explicit_predicates_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::Gen
             }
 
             hir::WherePredicate::RegionPredicate(region_pred) => {
-                let r1 = icx.astconv().ast_region_to_region(&region_pred.lifetime, None);
+                let r1 = icx.astconv().ast_region_to_region(region_pred.lifetime, None);
                 predicates.extend(region_pred.bounds.iter().map(|bound| {
                     let (r2, span) = match bound {
                         hir::GenericBound::Outlives(lt) => {
                             (icx.astconv().ast_region_to_region(lt, None), lt.ident.span)
                         }
-                        _ => bug!(),
+                        bound => {
+                            span_bug!(
+                                bound.span(),
+                                "lifetime param bounds must be outlives, but found {bound:?}"
+                            )
+                        }
                     };
                     let pred = ty::ClauseKind::RegionOutlives(ty::OutlivesPredicate(r1, r2))
                         .to_predicate(tcx);
@@ -337,7 +342,7 @@ fn gather_explicit_predicates_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::Gen
     // and the duplicated parameter, to ensure that they do not get out of sync.
     if let Node::Item(&Item { kind: ItemKind::OpaqueTy(..), .. }) = node {
         let opaque_ty_id = tcx.hir().parent_id(hir_id);
-        let opaque_ty_node = tcx.hir().get(opaque_ty_id);
+        let opaque_ty_node = tcx.hir_node(opaque_ty_id);
         let Node::Ty(&Ty { kind: TyKind::OpaqueDef(_, lifetimes, _), .. }) = opaque_ty_node else {
             bug!("unexpected {opaque_ty_node:?}")
         };
@@ -362,10 +367,10 @@ fn compute_bidirectional_outlives_predicates<'tcx>(
 ) {
     for param in opaque_own_params {
         let orig_lifetime = tcx.map_rpit_lifetime_to_fn_lifetime(param.def_id.expect_local());
-        if let ty::ReEarlyBound(..) = *orig_lifetime {
-            let dup_lifetime = ty::Region::new_early_bound(
+        if let ty::ReEarlyParam(..) = *orig_lifetime {
+            let dup_lifetime = ty::Region::new_early_param(
                 tcx,
-                ty::EarlyBoundRegion { def_id: param.def_id, index: param.index, name: param.name },
+                ty::EarlyParamRegion { def_id: param.def_id, index: param.index, name: param.name },
             );
             let span = tcx.def_span(param.def_id);
             predicates.push((
@@ -412,8 +417,8 @@ fn const_evaluatable_predicates_of(
         }
     }
 
-    let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
-    let node = tcx.hir().get(hir_id);
+    let hir_id = tcx.local_def_id_to_hir_id(def_id);
+    let node = tcx.hir_node(hir_id);
 
     let mut collector = ConstCollector { tcx, preds: FxIndexSet::default() };
     if let hir::Node::Item(item) = node
@@ -503,7 +508,7 @@ pub(super) fn explicit_predicates_of<'tcx>(
         }
     } else {
         if matches!(def_kind, DefKind::AnonConst) && tcx.features().generic_const_exprs {
-            let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
+            let hir_id = tcx.local_def_id_to_hir_id(def_id);
             let parent_def_id = tcx.hir().get_parent_item(hir_id);
 
             if let Some(defaulted_param_def_id) =
@@ -571,7 +576,7 @@ pub(super) fn explicit_predicates_of<'tcx>(
                 // To fix this, we call `explicit_predicates_of` directly on `foo`, the parent's parent.
 
                 // In the above example this is `foo::{opaque#0}` or `impl Iterator`
-                let parent_hir_id = tcx.hir().local_def_id_to_hir_id(parent_def_id.def_id);
+                let parent_hir_id = tcx.local_def_id_to_hir_id(parent_def_id.def_id);
 
                 // In the above example this is the function `foo`
                 let item_def_id = tcx.hir().get_parent_item(parent_hir_id);
@@ -631,9 +636,9 @@ pub(super) fn implied_predicates_with_filter(
         return tcx.super_predicates_of(trait_def_id);
     };
 
-    let trait_hir_id = tcx.hir().local_def_id_to_hir_id(trait_def_id);
+    let trait_hir_id = tcx.local_def_id_to_hir_id(trait_def_id);
 
-    let Node::Item(item) = tcx.hir().get(trait_hir_id) else {
+    let Node::Item(item) = tcx.hir_node(trait_hir_id) else {
         bug!("trait_node_id {} is not an item", trait_hir_id);
     };
 
@@ -691,7 +696,7 @@ pub(super) fn type_param_predicates(
     // written inline like `<T: Foo>` or in a where-clause like
     // `where T: Foo`.
 
-    let param_id = tcx.hir().local_def_id_to_hir_id(def_id);
+    let param_id = tcx.local_def_id_to_hir_id(def_id);
     let param_owner = tcx.hir().ty_param_owner(def_id);
     let generics = tcx.generics_of(param_owner);
     let index = generics.param_def_id_to_index[&def_id.to_def_id()];
@@ -712,11 +717,11 @@ pub(super) fn type_param_predicates(
         .unwrap_or_default();
     let mut extend = None;
 
-    let item_hir_id = tcx.hir().local_def_id_to_hir_id(item_def_id);
-    let ast_generics = match tcx.hir().get(item_hir_id) {
-        Node::TraitItem(item) => &item.generics,
+    let item_hir_id = tcx.local_def_id_to_hir_id(item_def_id);
+    let ast_generics = match tcx.hir_node(item_hir_id) {
+        Node::TraitItem(item) => item.generics,
 
-        Node::ImplItem(item) => &item.generics,
+        Node::ImplItem(item) => item.generics,
 
         Node::Item(item) => {
             match item.kind {
diff --git a/compiler/rustc_hir_analysis/src/collect/resolve_bound_vars.rs b/compiler/rustc_hir_analysis/src/collect/resolve_bound_vars.rs
index 6424d1c79..9f0742dad 100644
--- a/compiler/rustc_hir_analysis/src/collect/resolve_bound_vars.rs
+++ b/compiler/rustc_hir_analysis/src/collect/resolve_bound_vars.rs
@@ -295,7 +295,7 @@ fn late_arg_as_bound_arg<'tcx>(
 ) -> ty::BoundVariableKind {
     match arg {
         ResolvedArg::LateBound(_, _, def_id) => {
-            let name = tcx.hir().name(tcx.hir().local_def_id_to_hir_id(def_id.expect_local()));
+            let name = tcx.hir().name(tcx.local_def_id_to_hir_id(def_id.expect_local()));
             match param.kind {
                 GenericParamKind::Lifetime { .. } => {
                     ty::BoundVariableKind::Region(ty::BrNamed(*def_id, name))
@@ -335,7 +335,7 @@ impl<'a, 'tcx> BoundVarContext<'a, 'tcx> {
                     // though this may happen when we call `poly_trait_ref_binder_info` with
                     // an (erroneous, #113423) associated return type bound in an impl header.
                     if !supertrait_bound_vars.is_empty() {
-                        self.tcx.sess.delay_span_bug(
+                        self.tcx.sess.span_delayed_bug(
                             DUMMY_SP,
                             format!(
                                 "found supertrait lifetimes without a binder to append \
@@ -350,7 +350,7 @@ impl<'a, 'tcx> BoundVarContext<'a, 'tcx> {
                     // Nested poly trait refs have the binders concatenated
                     let mut full_binders =
                         self.map.late_bound_vars.entry(*hir_id).or_default().clone();
-                    full_binders.extend(supertrait_bound_vars.into_iter());
+                    full_binders.extend(supertrait_bound_vars);
                     break (full_binders, BinderScopeType::Concatenating);
                 }
             }
@@ -565,7 +565,7 @@ impl<'a, 'tcx> Visitor<'tcx> for BoundVarContext<'a, 'tcx> {
                 let mut bound_vars = FxIndexMap::default();
                 debug!(?generics.params);
                 for param in generics.params {
-                    let (def_id, reg) = ResolvedArg::early(&param);
+                    let (def_id, reg) = ResolvedArg::early(param);
                     bound_vars.insert(def_id, reg);
                 }
 
@@ -684,7 +684,7 @@ impl<'a, 'tcx> Visitor<'tcx> for BoundVarContext<'a, 'tcx> {
                     lifetime: self.map.defs.get(&lifetime_ref.hir_id).cloned(),
                     s: self.scope,
                 };
-                self.with(scope, |this| this.visit_ty(&mt.ty));
+                self.with(scope, |this| this.visit_ty(mt.ty));
             }
             hir::TyKind::OpaqueDef(item_id, lifetimes, _in_trait) => {
                 // Resolve the lifetimes in the bounds to the lifetime defs in the generics.
@@ -733,7 +733,7 @@ impl<'a, 'tcx> Visitor<'tcx> for BoundVarContext<'a, 'tcx> {
                     let def = self.map.defs.get(&lifetime.hir_id).cloned();
                     let Some(ResolvedArg::LateBound(_, _, def_id)) = def else { continue };
                     let Some(def_id) = def_id.as_local() else { continue };
-                    let hir_id = self.tcx.hir().local_def_id_to_hir_id(def_id);
+                    let hir_id = self.tcx.local_def_id_to_hir_id(def_id);
                     // Ensure that the parent of the def is an item, not HRTB
                     let parent_id = self.tcx.hir().parent_id(hir_id);
                     if !parent_id.is_owner() {
@@ -748,7 +748,7 @@ impl<'a, 'tcx> Visitor<'tcx> for BoundVarContext<'a, 'tcx> {
                     }
                     if let hir::Node::Item(hir::Item {
                         kind: hir::ItemKind::OpaqueTy { .. }, ..
-                    }) = self.tcx.hir().get(parent_id)
+                    }) = self.tcx.hir_node(parent_id)
                     {
                         let mut err = self.tcx.sess.struct_span_err(
                             lifetime.ident.span,
@@ -775,7 +775,7 @@ impl<'a, 'tcx> Visitor<'tcx> for BoundVarContext<'a, 'tcx> {
             }
             Type(bounds, ty) => {
                 self.visit_early(trait_item.hir_id(), trait_item.generics, |this| {
-                    this.visit_generics(&trait_item.generics);
+                    this.visit_generics(trait_item.generics);
                     for bound in bounds {
                         this.visit_param_bound(bound);
                     }
@@ -847,7 +847,7 @@ impl<'a, 'tcx> Visitor<'tcx> for BoundVarContext<'a, 'tcx> {
             hir::FnRetTy::DefaultReturn(_) => None,
             hir::FnRetTy::Return(ty) => Some(ty),
         };
-        self.visit_fn_like_elision(&fd.inputs, output, matches!(fk, intravisit::FnKind::Closure));
+        self.visit_fn_like_elision(fd.inputs, output, matches!(fk, intravisit::FnKind::Closure));
         intravisit::walk_fn_kind(self, fk);
         self.visit_nested_body(body_id)
     }
@@ -894,7 +894,7 @@ impl<'a, 'tcx> Visitor<'tcx> for BoundVarContext<'a, 'tcx> {
                 };
                 self.with(scope, |this| {
                     walk_list!(this, visit_generic_param, bound_generic_params);
-                    this.visit_ty(&bounded_ty);
+                    this.visit_ty(bounded_ty);
                     walk_list!(this, visit_param_bound, bounds);
                 })
             }
@@ -925,7 +925,7 @@ impl<'a, 'tcx> Visitor<'tcx> for BoundVarContext<'a, 'tcx> {
                                     "you can use the `'static` lifetime directly, in place of `{}`",
                                     lifetime.ident,
                                 );
-                                lint.help(help)
+                                lint.help(help);
                             },
                         );
                     }
@@ -938,32 +938,6 @@ impl<'a, 'tcx> Visitor<'tcx> for BoundVarContext<'a, 'tcx> {
         }
     }
 
-    fn visit_param_bound(&mut self, bound: &'tcx hir::GenericBound<'tcx>) {
-        match bound {
-            hir::GenericBound::LangItemTrait(_, _, hir_id, _) => {
-                // FIXME(jackh726): This is pretty weird. `LangItemTrait` doesn't go
-                // through the regular poly trait ref code, so we don't get another
-                // chance to introduce a binder. For now, I'm keeping the existing logic
-                // of "if there isn't a Binder scope above us, add one", but I
-                // imagine there's a better way to go about this.
-                let (binders, scope_type) = self.poly_trait_ref_binder_info();
-
-                self.record_late_bound_vars(*hir_id, binders);
-                let scope = Scope::Binder {
-                    hir_id: *hir_id,
-                    bound_vars: FxIndexMap::default(),
-                    s: self.scope,
-                    scope_type,
-                    where_bound_origin: None,
-                };
-                self.with(scope, |this| {
-                    intravisit::walk_param_bound(this, bound);
-                });
-            }
-            _ => intravisit::walk_param_bound(self, bound),
-        }
-    }
-
     fn visit_poly_trait_ref(&mut self, trait_ref: &'tcx hir::PolyTraitRef<'tcx>) {
         self.visit_poly_trait_ref_inner(trait_ref, NonLifetimeBinderAllowed::Allow);
     }
@@ -992,7 +966,7 @@ impl<'a, 'tcx> Visitor<'tcx> for BoundVarContext<'a, 'tcx> {
                     self.visit_ty(ty);
                 }
             }
-            GenericParamKind::Const { ty, default } => {
+            GenericParamKind::Const { ty, default, is_host_effect: _ } => {
                 self.visit_ty(ty);
                 if let Some(default) = default {
                     self.visit_body(self.tcx.hir().body(default.body));
@@ -1004,7 +978,7 @@ impl<'a, 'tcx> Visitor<'tcx> for BoundVarContext<'a, 'tcx> {
 
 fn object_lifetime_default(tcx: TyCtxt<'_>, param_def_id: LocalDefId) -> ObjectLifetimeDefault {
     debug_assert_eq!(tcx.def_kind(param_def_id), DefKind::TyParam);
-    let hir::Node::GenericParam(param) = tcx.hir().get_by_def_id(param_def_id) else {
+    let hir::Node::GenericParam(param) = tcx.hir_node_by_def_id(param_def_id) else {
         bug!("expected GenericParam for object_lifetime_default");
     };
     match param.source {
@@ -1061,7 +1035,7 @@ impl<'a, 'tcx> BoundVarContext<'a, 'tcx> {
     {
         let BoundVarContext { tcx, map, .. } = self;
         let mut this = BoundVarContext { tcx: *tcx, map, scope: &wrap_scope };
-        let span = debug_span!("scope", scope = ?TruncatedScopeDebug(&this.scope));
+        let span = debug_span!("scope", scope = ?TruncatedScopeDebug(this.scope));
         {
             let _enter = span.enter();
             f(&mut this);
@@ -1300,12 +1274,16 @@ impl<'a, 'tcx> BoundVarContext<'a, 'tcx> {
                             what,
                         })
                     }
-                    _ => unreachable!(),
+                    kind => span_bug!(
+                        use_span,
+                        "did not expect to resolve lifetime to {}",
+                        kind.descr(param_def_id)
+                    ),
                 };
                 def = ResolvedArg::Error(guar);
             } else if let Some(body_id) = outermost_body {
                 let fn_id = self.tcx.hir().body_owner(body_id);
-                match self.tcx.hir().get(fn_id) {
+                match self.tcx.hir_node(fn_id) {
                     Node::Item(hir::Item { owner_id, kind: hir::ItemKind::Fn(..), .. })
                     | Node::TraitItem(hir::TraitItem {
                         owner_id,
@@ -1363,7 +1341,7 @@ impl<'a, 'tcx> BoundVarContext<'a, 'tcx> {
             }
         }
 
-        self.tcx.sess.delay_span_bug(
+        self.tcx.sess.span_delayed_bug(
             lifetime_ref.ident.span,
             format!("Could not resolve {:?} in scope {:#?}", lifetime_ref, self.scope,),
         );
@@ -1441,7 +1419,11 @@ impl<'a, 'tcx> BoundVarContext<'a, 'tcx> {
                             what,
                         })
                     }
-                    _ => unreachable!(),
+                    kind => span_bug!(
+                        use_span,
+                        "did not expect to resolve non-lifetime param to {}",
+                        kind.descr(param_def_id.to_def_id())
+                    ),
                 };
                 self.map.defs.insert(hir_id, ResolvedArg::Error(guar));
             } else {
@@ -1493,7 +1475,7 @@ impl<'a, 'tcx> BoundVarContext<'a, 'tcx> {
             }
         }
 
-        self.tcx.sess.delay_span_bug(
+        self.tcx.sess.span_delayed_bug(
             self.tcx.hir().span(hir_id),
             format!("could not resolve {param_def_id:?}"),
         );
@@ -1724,7 +1706,7 @@ impl<'a, 'tcx> BoundVarContext<'a, 'tcx> {
                 } else {
                     self.tcx
                         .sess
-                        .delay_span_bug(binding.ident.span, "bad return type notation here");
+                        .span_delayed_bug(binding.ident.span, "bad return type notation here");
                     vec![]
                 };
                 self.with(scope, |this| {
@@ -1848,8 +1830,8 @@ impl<'a, 'tcx> BoundVarContext<'a, 'tcx> {
         }
     }
 
+    #[instrument(level = "debug", skip(self))]
     fn resolve_object_lifetime_default(&mut self, lifetime_ref: &'tcx hir::Lifetime) {
-        debug!("resolve_object_lifetime_default(lifetime_ref={:?})", lifetime_ref);
         let mut late_depth = 0;
         let mut scope = self.scope;
         let lifetime = loop {
@@ -2012,7 +1994,7 @@ fn is_late_bound_map(
 
         fn visit_region(&mut self, r: ty::Region<'tcx>) -> ControlFlow<!> {
             debug!("r={:?}", r.kind());
-            if let ty::RegionKind::ReEarlyBound(region) = r.kind() {
+            if let ty::RegionKind::ReEarlyParam(region) = r.kind() {
                 self.arg_is_constrained[region.index as usize] = true;
             }
 
@@ -2057,7 +2039,7 @@ fn is_late_bound_map(
                                         Some(true) => Some(arg),
                                         Some(false) => None,
                                         None => {
-                                            tcx.sess.delay_span_bug(
+                                            tcx.sess.span_delayed_bug(
                                                 *span,
                                                 format!(
                                                     "Incorrect generic arg count for alias {alias_def:?}"
@@ -2122,8 +2104,8 @@ pub fn deny_non_region_late_bound(
     let mut first = true;
 
     for (var, arg) in bound_vars {
-        let Node::GenericParam(param) = tcx.hir().get_by_def_id(*var) else {
-            bug!();
+        let Node::GenericParam(param) = tcx.hir_node_by_def_id(*var) else {
+            span_bug!(tcx.def_span(*var), "expected bound-var def-id to resolve to param");
         };
 
         let what = match param.kind {
diff --git a/compiler/rustc_hir_analysis/src/collect/type_of.rs b/compiler/rustc_hir_analysis/src/collect/type_of.rs
index d7bd2a7b1..19e7fe388 100644
--- a/compiler/rustc_hir_analysis/src/collect/type_of.rs
+++ b/compiler/rustc_hir_analysis/src/collect/type_of.rs
@@ -18,12 +18,18 @@ mod opaque;
 fn anon_const_type_of<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalDefId) -> Ty<'tcx> {
     use hir::*;
     use rustc_middle::ty::Ty;
-    let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
+    let hir_id = tcx.local_def_id_to_hir_id(def_id);
 
-    let Node::AnonConst(_) = tcx.hir().get(hir_id) else { panic!() };
+    let node = tcx.hir_node(hir_id);
+    let Node::AnonConst(_) = node else {
+        span_bug!(
+            tcx.def_span(def_id),
+            "expected anon const in `anon_const_type_of`, got {node:?}"
+        );
+    };
 
     let parent_node_id = tcx.hir().parent_id(hir_id);
-    let parent_node = tcx.hir().get(parent_node_id);
+    let parent_node = tcx.hir_node(parent_node_id);
 
     let (generics, arg_idx) = match parent_node {
         // Easy case: arrays repeat expressions.
@@ -61,7 +67,7 @@ fn anon_const_type_of<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalDefId) -> Ty<'tcx> {
         }
 
         Node::TypeBinding(binding @ &TypeBinding { hir_id: binding_id, .. })
-            if let Node::TraitRef(trait_ref) = tcx.hir().get(tcx.hir().parent_id(binding_id)) =>
+            if let Node::TraitRef(trait_ref) = tcx.hir_node(tcx.hir().parent_id(binding_id)) =>
         {
             let Some(trait_def_id) = trait_ref.trait_def_id() else {
                 return Ty::new_error_with_message(
@@ -350,11 +356,11 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::EarlyBinder<Ty
         }
     }
 
-    let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
+    let hir_id = tcx.local_def_id_to_hir_id(def_id);
 
     let icx = ItemCtxt::new(tcx, def_id);
 
-    let output = match tcx.hir().get(hir_id) {
+    let output = match tcx.hir_node(hir_id) {
         Node::TraitItem(item) => match item.kind {
             TraitItemKind::Fn(..) => {
                 let args = ty::GenericArgs::identity_for_item(tcx, def_id);
@@ -475,7 +481,7 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::EarlyBinder<Ty
         },
 
         Node::Ctor(def) | Node::Variant(Variant { data: def, .. }) => match def {
-            VariantData::Unit(..) | VariantData::Struct(..) => {
+            VariantData::Unit(..) | VariantData::Struct { .. } => {
                 tcx.type_of(tcx.hir().get_parent_item(hir_id)).instantiate_identity()
             }
             VariantData::Tuple(..) => {
@@ -517,8 +523,7 @@ pub(super) fn type_of_opaque(
     if let Some(def_id) = def_id.as_local() {
         use rustc_hir::*;
 
-        let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
-        Ok(ty::EarlyBinder::bind(match tcx.hir().get(hir_id) {
+        Ok(ty::EarlyBinder::bind(match tcx.hir_node_by_def_id(def_id) {
             Node::Item(item) => match item.kind {
                 ItemKind::OpaqueTy(OpaqueTy {
                     origin: hir::OpaqueTyOrigin::TyAlias { .. },
@@ -569,7 +574,7 @@ fn infer_placeholder_type<'a>(
     // then the user may have written e.g. `const A = 42;`.
     // In this case, the parser has stashed a diagnostic for
     // us to improve in typeck so we do that now.
-    match tcx.sess.diagnostic().steal_diagnostic(span, StashKey::ItemNoType) {
+    match tcx.sess.dcx().steal_diagnostic(span, StashKey::ItemNoType) {
         Some(mut err) => {
             if !ty.references_error() {
                 // Only suggest adding `:` if it was missing (and suggested by parsing diagnostic)
diff --git a/compiler/rustc_hir_analysis/src/collect/type_of/opaque.rs b/compiler/rustc_hir_analysis/src/collect/type_of/opaque.rs
index e8ab2651d..3785b61f2 100644
--- a/compiler/rustc_hir_analysis/src/collect/type_of/opaque.rs
+++ b/compiler/rustc_hir_analysis/src/collect/type_of/opaque.rs
@@ -41,7 +41,7 @@ pub fn test_opaque_hidden_types(tcx: TyCtxt<'_>) {
 /// ```
 #[instrument(skip(tcx), level = "debug")]
 pub(super) fn find_opaque_ty_constraints_for_tait(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Ty<'_> {
-    let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
+    let hir_id = tcx.local_def_id_to_hir_id(def_id);
     let scope = tcx.hir().get_defining_scope(hir_id);
     let mut locator = TaitConstraintLocator { def_id, tcx, found: None, typeck_types: vec![] };
 
@@ -50,8 +50,8 @@ pub(super) fn find_opaque_ty_constraints_for_tait(tcx: TyCtxt<'_>, def_id: Local
     if scope == hir::CRATE_HIR_ID {
         tcx.hir().walk_toplevel_module(&mut locator);
     } else {
-        trace!("scope={:#?}", tcx.hir().get(scope));
-        match tcx.hir().get(scope) {
+        trace!("scope={:#?}", tcx.hir_node(scope));
+        match tcx.hir_node(scope) {
             // We explicitly call `visit_*` methods, instead of using `intravisit::walk_*` methods
             // This allows our visitor to process the defining item itself, causing
             // it to pick up any 'sibling' defining uses.
@@ -95,7 +95,7 @@ pub(super) fn find_opaque_ty_constraints_for_tait(tcx: TyCtxt<'_>, def_id: Local
         let reported = tcx.sess.emit_err(UnconstrainedOpaqueType {
             span: tcx.def_span(def_id),
             name: tcx.item_name(parent_def_id.to_def_id()),
-            what: match tcx.hir().get(scope) {
+            what: match tcx.hir_node(scope) {
                 _ if scope == hir::CRATE_HIR_ID => "module",
                 Node::Item(hir::Item { kind: hir::ItemKind::Mod(_), .. }) => "module",
                 Node::Item(hir::Item { kind: hir::ItemKind::Impl(_), .. }) => "impl",
@@ -278,11 +278,15 @@ pub(super) fn find_opaque_ty_constraints_for_rpit<'tcx>(
 
     let mir_opaque_ty = tcx.mir_borrowck(owner_def_id).concrete_opaque_types.get(&def_id).copied();
     if let Some(mir_opaque_ty) = mir_opaque_ty {
-        let scope = tcx.hir().local_def_id_to_hir_id(owner_def_id);
+        if mir_opaque_ty.references_error() {
+            return mir_opaque_ty.ty;
+        }
+
+        let scope = tcx.local_def_id_to_hir_id(owner_def_id);
         debug!(?scope);
         let mut locator = RpitConstraintChecker { def_id, tcx, found: mir_opaque_ty };
 
-        match tcx.hir().get(scope) {
+        match tcx.hir_node(scope) {
             Node::Item(it) => intravisit::walk_item(&mut locator, it),
             Node::ImplItem(it) => intravisit::walk_impl_item(&mut locator, it),
             Node::TraitItem(it) => intravisit::walk_trait_item(&mut locator, it),
diff --git a/compiler/rustc_hir_analysis/src/constrained_generic_params.rs b/compiler/rustc_hir_analysis/src/constrained_generic_params.rs
index ed5e9dd2b..65d1ffa40 100644
--- a/compiler/rustc_hir_analysis/src/constrained_generic_params.rs
+++ b/compiler/rustc_hir_analysis/src/constrained_generic_params.rs
@@ -13,8 +13,8 @@ impl From<ty::ParamTy> for Parameter {
     }
 }
 
-impl From<ty::EarlyBoundRegion> for Parameter {
-    fn from(param: ty::EarlyBoundRegion) -> Self {
+impl From<ty::EarlyParamRegion> for Parameter {
+    fn from(param: ty::EarlyParamRegion) -> Self {
         Parameter(param.index)
     }
 }
@@ -73,7 +73,7 @@ impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for ParameterCollector {
     }
 
     fn visit_region(&mut self, r: ty::Region<'tcx>) -> ControlFlow<Self::BreakTy> {
-        if let ty::ReEarlyBound(data) = *r {
+        if let ty::ReEarlyParam(data) = *r {
             self.parameters.push(Parameter::from(data));
         }
         ControlFlow::Continue(())
diff --git a/compiler/rustc_hir_analysis/src/errors.rs b/compiler/rustc_hir_analysis/src/errors.rs
index dd83b5b6f..f461b6a94 100644
--- a/compiler/rustc_hir_analysis/src/errors.rs
+++ b/compiler/rustc_hir_analysis/src/errors.rs
@@ -2,14 +2,126 @@
 
 use crate::fluent_generated as fluent;
 use rustc_errors::{
-    error_code, Applicability, DiagnosticBuilder, ErrorGuaranteed, Handler, IntoDiagnostic,
+    error_code, Applicability, DiagCtxt, DiagnosticBuilder, ErrorGuaranteed, IntoDiagnostic,
     MultiSpan,
 };
 use rustc_macros::{Diagnostic, LintDiagnostic, Subdiagnostic};
-use rustc_middle::ty::{self, print::TraitRefPrintOnlyTraitPath, Ty};
+use rustc_middle::ty::Ty;
 use rustc_span::{symbol::Ident, Span, Symbol};
 
 #[derive(Diagnostic)]
+#[diag(hir_analysis_ambiguous_assoc_item)]
+pub struct AmbiguousAssocItem<'a> {
+    #[primary_span]
+    #[label]
+    pub span: Span,
+    pub assoc_kind: &'static str,
+    pub assoc_name: Ident,
+    pub ty_param_name: &'a str,
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_assoc_kind_mismatch)]
+pub struct AssocKindMismatch {
+    #[primary_span]
+    #[label]
+    pub span: Span,
+    pub expected: &'static str,
+    pub got: &'static str,
+    #[label(hir_analysis_expected_because_label)]
+    pub expected_because_label: Option<Span>,
+    pub assoc_kind: &'static str,
+    #[note]
+    pub def_span: Span,
+    #[label(hir_analysis_bound_on_assoc_const_label)]
+    pub bound_on_assoc_const_label: Option<Span>,
+    #[subdiagnostic]
+    pub wrap_in_braces_sugg: Option<AssocKindMismatchWrapInBracesSugg>,
+}
+
+#[derive(Subdiagnostic)]
+#[multipart_suggestion(
+    hir_analysis_assoc_kind_mismatch_wrap_in_braces_sugg,
+    applicability = "maybe-incorrect"
+)]
+pub struct AssocKindMismatchWrapInBracesSugg {
+    #[suggestion_part(code = "{{ ")]
+    pub lo: Span,
+    #[suggestion_part(code = " }}")]
+    pub hi: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_assoc_item_not_found, code = "E0220")]
+pub struct AssocItemNotFound<'a> {
+    #[primary_span]
+    pub span: Span,
+    pub assoc_name: Ident,
+    pub assoc_kind: &'static str,
+    pub ty_param_name: &'a str,
+    #[subdiagnostic]
+    pub label: Option<AssocItemNotFoundLabel<'a>>,
+    #[subdiagnostic]
+    pub sugg: Option<AssocItemNotFoundSugg<'a>>,
+}
+
+#[derive(Subdiagnostic)]
+pub enum AssocItemNotFoundLabel<'a> {
+    #[label(hir_analysis_assoc_item_not_found_label)]
+    NotFound {
+        #[primary_span]
+        span: Span,
+    },
+    #[label(hir_analysis_assoc_item_not_found_found_in_other_trait_label)]
+    FoundInOtherTrait {
+        #[primary_span]
+        span: Span,
+        assoc_kind: &'static str,
+        trait_name: &'a str,
+        suggested_name: Symbol,
+        identically_named: bool,
+    },
+}
+
+#[derive(Subdiagnostic)]
+
+pub enum AssocItemNotFoundSugg<'a> {
+    #[suggestion(
+        hir_analysis_assoc_item_not_found_similar_sugg,
+        code = "{suggested_name}",
+        applicability = "maybe-incorrect"
+    )]
+    Similar {
+        #[primary_span]
+        span: Span,
+        assoc_kind: &'static str,
+        suggested_name: Symbol,
+    },
+    #[suggestion(
+        hir_analysis_assoc_item_not_found_similar_in_other_trait_sugg,
+        code = "{suggested_name}",
+        style = "verbose",
+        applicability = "maybe-incorrect"
+    )]
+    SimilarInOtherTrait {
+        #[primary_span]
+        span: Span,
+        assoc_kind: &'static str,
+        suggested_name: Symbol,
+    },
+    #[suggestion(hir_analysis_assoc_item_not_found_other_sugg, code = "{suggested_name}")]
+    Other {
+        #[primary_span]
+        span: Span,
+        #[applicability]
+        applicability: Applicability,
+        ty_param_name: &'a str,
+        assoc_kind: &'static str,
+        suggested_name: Symbol,
+    },
+}
+
+#[derive(Diagnostic)]
 #[diag(hir_analysis_unrecognized_atomic_operation, code = "E0092")]
 pub struct UnrecognizedAtomicOperation<'a> {
     #[primary_span]
@@ -205,8 +317,8 @@ pub struct MissingTypeParams {
 // Manual implementation of `IntoDiagnostic` to be able to call `span_to_snippet`.
 impl<'a> IntoDiagnostic<'a> for MissingTypeParams {
     #[track_caller]
-    fn into_diagnostic(self, handler: &'a Handler) -> DiagnosticBuilder<'a, ErrorGuaranteed> {
-        let mut err = handler.struct_span_err_with_code(
+    fn into_diagnostic(self, dcx: &'a DiagCtxt) -> DiagnosticBuilder<'a, ErrorGuaranteed> {
+        let mut err = dcx.struct_span_err_with_code(
             self.span,
             fluent::hir_analysis_missing_type_params,
             error_code!(E0393),
@@ -538,27 +650,6 @@ pub(crate) struct ReturnTypeNotationEqualityBound {
 }
 
 #[derive(Diagnostic)]
-#[diag(hir_analysis_return_type_notation_missing_method)]
-pub(crate) struct ReturnTypeNotationMissingMethod {
-    #[primary_span]
-    pub span: Span,
-    pub ty_name: String,
-    pub assoc_name: Symbol,
-}
-
-#[derive(Diagnostic)]
-#[diag(hir_analysis_return_type_notation_conflicting_bound)]
-#[note]
-pub(crate) struct ReturnTypeNotationConflictingBound<'tcx> {
-    #[primary_span]
-    pub span: Span,
-    pub ty_name: String,
-    pub assoc_name: Symbol,
-    pub first_bound: ty::Binder<'tcx, TraitRefPrintOnlyTraitPath<'tcx>>,
-    pub second_bound: ty::Binder<'tcx, TraitRefPrintOnlyTraitPath<'tcx>>,
-}
-
-#[derive(Diagnostic)]
 #[diag(hir_analysis_placeholder_not_allowed_item_signatures, code = "E0121")]
 pub(crate) struct PlaceholderNotAllowedItemSignatures {
     #[primary_span]
@@ -955,15 +1046,6 @@ pub(crate) struct ReturnPositionImplTraitInTraitRefined<'tcx> {
 }
 
 #[derive(Diagnostic)]
-#[diag(hir_analysis_assoc_bound_on_const)]
-#[note]
-pub struct AssocBoundOnConst {
-    #[primary_span]
-    pub span: Span,
-    pub descr: &'static str,
-}
-
-#[derive(Diagnostic)]
 #[diag(hir_analysis_inherent_ty_outside, code = "E0390")]
 #[help]
 pub struct InherentTyOutside {
diff --git a/compiler/rustc_hir_analysis/src/hir_wf_check.rs b/compiler/rustc_hir_analysis/src/hir_wf_check.rs
index ca7679cfb..78745fe47 100644
--- a/compiler/rustc_hir_analysis/src/hir_wf_check.rs
+++ b/compiler/rustc_hir_analysis/src/hir_wf_check.rs
@@ -5,7 +5,7 @@ use rustc_hir::{ForeignItem, ForeignItemKind};
 use rustc_infer::infer::TyCtxtInferExt;
 use rustc_infer::traits::{ObligationCause, WellFormedLoc};
 use rustc_middle::query::Providers;
-use rustc_middle::ty::{self, Region, TyCtxt, TypeFoldable, TypeFolder};
+use rustc_middle::ty::{self, TyCtxt};
 use rustc_span::def_id::LocalDefId;
 use rustc_trait_selection::traits::{self, ObligationCtxt};
 
@@ -25,11 +25,11 @@ fn diagnostic_hir_wf_check<'tcx>(
         WellFormedLoc::Ty(def_id) => def_id,
         WellFormedLoc::Param { function, param_idx: _ } => function,
     };
-    let hir_id = hir.local_def_id_to_hir_id(def_id);
+    let hir_id = tcx.local_def_id_to_hir_id(def_id);
 
     // HIR wfcheck should only ever happen as part of improving an existing error
     tcx.sess
-        .delay_span_bug(tcx.def_span(def_id), "Performed HIR wfcheck without an existing error!");
+        .span_delayed_bug(tcx.def_span(def_id), "Performed HIR wfcheck without an existing error!");
 
     let icx = ItemCtxt::new(tcx, def_id);
 
@@ -68,7 +68,13 @@ fn diagnostic_hir_wf_check<'tcx>(
             let infcx = self.tcx.infer_ctxt().build();
             let ocx = ObligationCtxt::new(&infcx);
 
-            let tcx_ty = self.icx.to_ty(ty).fold_with(&mut EraseAllBoundRegions { tcx: self.tcx });
+            let tcx_ty = self.icx.to_ty(ty);
+            // This visitor can walk into binders, resulting in the `tcx_ty` to
+            // potentially reference escaping bound variables. We simply erase
+            // those here.
+            let tcx_ty = self.tcx.fold_regions(tcx_ty, |r, _| {
+                if r.is_bound() { self.tcx.lifetimes.re_erased } else { r }
+            });
             let cause = traits::ObligationCause::new(
                 ty.span,
                 self.def_id,
@@ -116,7 +122,7 @@ fn diagnostic_hir_wf_check<'tcx>(
     // We will walk 'into' this type to try to find
     // a more precise span for our predicate.
     let tys = match loc {
-        WellFormedLoc::Ty(_) => match hir.get(hir_id) {
+        WellFormedLoc::Ty(_) => match tcx.hir_node(hir_id) {
             hir::Node::ImplItem(item) => match item.kind {
                 hir::ImplItemKind::Type(ty) => vec![ty],
                 hir::ImplItemKind::Const(ty, _) => vec![ty],
@@ -178,24 +184,3 @@ fn diagnostic_hir_wf_check<'tcx>(
     }
     visitor.cause
 }
-
-struct EraseAllBoundRegions<'tcx> {
-    tcx: TyCtxt<'tcx>,
-}
-
-// Higher ranked regions are complicated.
-// To make matters worse, the HIR WF check can instantiate them
-// outside of a `Binder`, due to the way we (ab)use
-// `ItemCtxt::to_ty`. To make things simpler, we just erase all
-// of them, regardless of depth. At worse, this will give
-// us an inaccurate span for an error message, but cannot
-// lead to unsoundness (we call `delay_span_bug` at the start
-// of `diagnostic_hir_wf_check`).
-impl<'tcx> TypeFolder<TyCtxt<'tcx>> for EraseAllBoundRegions<'tcx> {
-    fn interner(&self) -> TyCtxt<'tcx> {
-        self.tcx
-    }
-    fn fold_region(&mut self, r: Region<'tcx>) -> Region<'tcx> {
-        if r.is_late_bound() { self.tcx.lifetimes.re_erased } else { r }
-    }
-}
diff --git a/compiler/rustc_hir_analysis/src/impl_wf_check.rs b/compiler/rustc_hir_analysis/src/impl_wf_check.rs
index 788121f7a..fff4a919e 100644
--- a/compiler/rustc_hir_analysis/src/impl_wf_check.rs
+++ b/compiler/rustc_hir_analysis/src/impl_wf_check.rs
@@ -74,7 +74,7 @@ fn enforce_impl_params_are_constrained(tcx: TyCtxt<'_>, impl_def_id: LocalDefId)
     if impl_self_ty.references_error() {
         // Don't complain about unconstrained type params when self ty isn't known due to errors.
         // (#36836)
-        tcx.sess.delay_span_bug(
+        tcx.sess.span_delayed_bug(
             tcx.def_span(impl_def_id),
             format!(
                 "potentially unconstrained type parameters weren't evaluated: {impl_self_ty:?}",
diff --git a/compiler/rustc_hir_analysis/src/lib.rs b/compiler/rustc_hir_analysis/src/lib.rs
index 2b8219c01..91cdffbbe 100644
--- a/compiler/rustc_hir_analysis/src/lib.rs
+++ b/compiler/rustc_hir_analysis/src/lib.rs
@@ -57,9 +57,9 @@ This API is completely unstable and subject to change.
 
 #![allow(rustc::potential_query_instability)]
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
-#![cfg_attr(not(bootstrap), allow(internal_features))]
+#![doc(rust_logo)]
+#![feature(rustdoc_internals)]
+#![allow(internal_features)]
 #![feature(box_patterns)]
 #![feature(control_flow_enum)]
 #![feature(if_let_guard)]
@@ -99,8 +99,6 @@ pub mod structured_errors;
 mod variance;
 
 use rustc_errors::ErrorGuaranteed;
-use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
-use rustc_fluent_macro::fluent_messages;
 use rustc_hir as hir;
 use rustc_middle::middle;
 use rustc_middle::query::Providers;
@@ -115,7 +113,7 @@ use astconv::{AstConv, OnlySelfBounds};
 use bounds::Bounds;
 use rustc_hir::def::DefKind;
 
-fluent_messages! { "../messages.ftl" }
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
 
 fn require_c_abi_if_c_variadic(tcx: TyCtxt<'_>, decl: &hir::FnDecl<'_>, abi: Abi, span: Span) {
     const CONVENTIONS_UNSTABLE: &str = "`C`, `cdecl`, `aapcs`, `win64`, `sysv64` or `efiapi`";
@@ -211,8 +209,8 @@ pub fn check_crate(tcx: TyCtxt<'_>) -> Result<(), ErrorGuaranteed> {
         tcx.hir().for_each_module(|module| tcx.ensure().check_mod_item_types(module))
     });
 
-    // HACK: `check_mod_type_wf` may spuriously emit errors due to `delay_span_bug`, even if those errors
-    // only actually get emitted in `check_mod_item_types`.
+    // HACK: `check_mod_type_wf` may spuriously emit errors due to `span_delayed_bug`, even if
+    // those errors only actually get emitted in `check_mod_item_types`.
     errs?;
 
     if tcx.features().rustc_attrs {
diff --git a/compiler/rustc_hir_analysis/src/outlives/mod.rs b/compiler/rustc_hir_analysis/src/outlives/mod.rs
index be9d076bd..9541e5107 100644
--- a/compiler/rustc_hir_analysis/src/outlives/mod.rs
+++ b/compiler/rustc_hir_analysis/src/outlives/mod.rs
@@ -18,7 +18,7 @@ pub fn provide(providers: &mut Providers) {
 }
 
 fn inferred_outlives_of(tcx: TyCtxt<'_>, item_def_id: LocalDefId) -> &[(ty::Clause<'_>, Span)] {
-    let id = tcx.hir().local_def_id_to_hir_id(item_def_id);
+    let id = tcx.local_def_id_to_hir_id(item_def_id);
 
     if matches!(tcx.def_kind(item_def_id), hir::def::DefKind::AnonConst)
         && tcx.features().generic_const_exprs
@@ -41,7 +41,7 @@ fn inferred_outlives_of(tcx: TyCtxt<'_>, item_def_id: LocalDefId) -> &[(ty::Clau
         }
     }
 
-    match tcx.hir().get(id) {
+    match tcx.hir_node(id) {
         Node::Item(item) => match item.kind {
             hir::ItemKind::Struct(..) | hir::ItemKind::Enum(..) | hir::ItemKind::Union(..) => {
                 let crate_map = tcx.inferred_outlives_crate(());
diff --git a/compiler/rustc_hir_analysis/src/outlives/utils.rs b/compiler/rustc_hir_analysis/src/outlives/utils.rs
index a6410c944..8077acea5 100644
--- a/compiler/rustc_hir_analysis/src/outlives/utils.rs
+++ b/compiler/rustc_hir_analysis/src/outlives/utils.rs
@@ -80,6 +80,10 @@ pub(crate) fn insert_outlives_predicate<'tcx>(
                             .or_insert(span);
                     }
 
+                    Component::Placeholder(_) => {
+                        span_bug!(span, "Should not deduce placeholder outlives component");
+                    }
+
                     Component::Alias(alias_ty) => {
                         // This would either arise from something like:
                         //
@@ -146,11 +150,11 @@ fn is_free_region(region: Region<'_>) -> bool {
         // These correspond to `T: 'a` relationships:
         //
         //     struct Foo<'a, T> {
-        //         field: &'a T, // this would generate a ReEarlyBound referencing `'a`
+        //         field: &'a T, // this would generate a ReEarlyParam referencing `'a`
         //     }
         //
         // We care about these, so fall through.
-        ty::ReEarlyBound(_) => true,
+        ty::ReEarlyParam(_) => true,
 
         // These correspond to `T: 'static` relationships which can be
         // rather surprising.
@@ -167,13 +171,13 @@ fn is_free_region(region: Region<'_>) -> bool {
         //     }
         //
         // The type above might generate a `T: 'b` bound, but we can
-        // ignore it. We can't put it on the struct header anyway.
-        ty::ReLateBound(..) => false,
+        // ignore it. We can't name this lifetime pn the struct header anyway.
+        ty::ReBound(..) => false,
 
         ty::ReError(_) => false,
 
         // These regions don't appear in types from type declarations:
-        ty::ReErased | ty::ReVar(..) | ty::RePlaceholder(..) | ty::ReFree(..) => {
+        ty::ReErased | ty::ReVar(..) | ty::RePlaceholder(..) | ty::ReLateParam(..) => {
             bug!("unexpected region in outlives inference: {:?}", region);
         }
     }
diff --git a/compiler/rustc_hir_analysis/src/structured_errors/wrong_number_of_generic_args.rs b/compiler/rustc_hir_analysis/src/structured_errors/wrong_number_of_generic_args.rs
index 76bd370a6..fab841e36 100644
--- a/compiler/rustc_hir_analysis/src/structured_errors/wrong_number_of_generic_args.rs
+++ b/compiler/rustc_hir_analysis/src/structured_errors/wrong_number_of_generic_args.rs
@@ -137,10 +137,9 @@ impl<'a, 'tcx> WrongNumberOfGenericArgs<'a, 'tcx> {
             // is from the 'of_trait' field of the enclosing impl
 
             let parent = self.tcx.hir().get_parent(self.path_segment.hir_id);
-            let parent_item = self
-                .tcx
-                .hir()
-                .get_by_def_id(self.tcx.hir().get_parent_item(self.path_segment.hir_id).def_id);
+            let parent_item = self.tcx.hir_node_by_def_id(
+                self.tcx.hir().get_parent_item(self.path_segment.hir_id).def_id,
+            );
 
             // Get the HIR id of the trait ref
             let hir::Node::TraitRef(hir::TraitRef { hir_ref_id: trait_ref_id, .. }) = parent else {
@@ -774,7 +773,7 @@ impl<'a, 'tcx> WrongNumberOfGenericArgs<'a, 'tcx> {
         );
 
         if let Some(parent_node) = self.tcx.hir().opt_parent_id(self.path_segment.hir_id)
-            && let Some(parent_node) = self.tcx.hir().find(parent_node)
+            && let Some(parent_node) = self.tcx.opt_hir_node(parent_node)
             && let hir::Node::Expr(expr) = parent_node
         {
             match &expr.kind {
diff --git a/compiler/rustc_hir_analysis/src/variance/constraints.rs b/compiler/rustc_hir_analysis/src/variance/constraints.rs
index 5f8b1ace6..f09594cbb 100644
--- a/compiler/rustc_hir_analysis/src/variance/constraints.rs
+++ b/compiler/rustc_hir_analysis/src/variance/constraints.rs
@@ -413,20 +413,22 @@ impl<'a, 'tcx> ConstraintContext<'a, 'tcx> {
         variance: VarianceTermPtr<'a>,
     ) {
         match *region {
-            ty::ReEarlyBound(ref data) => {
+            ty::ReEarlyParam(ref data) => {
                 self.add_constraint(current, data.index, variance);
             }
 
             ty::ReStatic => {}
 
-            ty::ReLateBound(..) => {
-                // Late-bound regions do not get substituted the same
-                // way early-bound regions do, so we skip them here.
+            ty::ReBound(..) => {
+                // Either a higher-ranked region inside of a type or a
+                // late-bound function parameter.
+                //
+                // We do not compute constraints for either of these.
             }
 
             ty::ReError(_) => {}
 
-            ty::ReFree(..) | ty::ReVar(..) | ty::RePlaceholder(..) | ty::ReErased => {
+            ty::ReLateParam(..) | ty::ReVar(..) | ty::RePlaceholder(..) | ty::ReErased => {
                 // We don't expect to see anything but 'static or bound
                 // regions when visiting member types or method types.
                 bug!(
diff --git a/compiler/rustc_hir_analysis/src/variance/mod.rs b/compiler/rustc_hir_analysis/src/variance/mod.rs
index 9fb39a0e9..410706110 100644
--- a/compiler/rustc_hir_analysis/src/variance/mod.rs
+++ b/compiler/rustc_hir_analysis/src/variance/mod.rs
@@ -106,7 +106,7 @@ fn variance_of_opaque(tcx: TyCtxt<'_>, item_def_id: LocalDefId) -> &[ty::Varianc
     impl<'tcx> ty::TypeVisitor<TyCtxt<'tcx>> for OpaqueTypeLifetimeCollector<'tcx> {
         #[instrument(level = "trace", skip(self), ret)]
         fn visit_region(&mut self, r: ty::Region<'tcx>) -> ControlFlow<Self::BreakTy> {
-            if let ty::RegionKind::ReEarlyBound(ebr) = r.kind() {
+            if let ty::RegionKind::ReEarlyParam(ebr) = r.kind() {
                 self.variances[ebr.index as usize] = ty::Invariant;
             }
             ControlFlow::Continue(())
diff --git a/compiler/rustc_hir_pretty/src/lib.rs b/compiler/rustc_hir_pretty/src/lib.rs
index 5f82d9f06..6715d01c9 100644
--- a/compiler/rustc_hir_pretty/src/lib.rs
+++ b/compiler/rustc_hir_pretty/src/lib.rs
@@ -1,3 +1,6 @@
+//! HIR pretty-printing is layered on top of AST pretty-printing. A number of
+//! the definitions in this file have equivalents in `rustc_ast_pretty`.
+
 #![recursion_limit = "256"]
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
@@ -12,8 +15,8 @@ use rustc_hir::LifetimeParamKind;
 use rustc_hir::{BindingAnnotation, ByRef, GenericArg, GenericParam, GenericParamKind, Node, Term};
 use rustc_hir::{GenericBound, PatKind, RangeEnd, TraitBoundModifier};
 use rustc_span::source_map::SourceMap;
-use rustc_span::symbol::{kw, Ident, IdentPrinter, Symbol};
-use rustc_span::{self, FileName};
+use rustc_span::symbol::{kw, Ident, Symbol};
+use rustc_span::FileName;
 use rustc_target::spec::abi::Abi;
 
 use std::cell::Cell;
@@ -49,8 +52,8 @@ pub trait PpAnn {
 }
 
 pub struct NoAnn;
+
 impl PpAnn for NoAnn {}
-pub const NO_ANN: &dyn PpAnn = &NoAnn;
 
 impl PpAnn for &dyn rustc_hir::intravisit::Map<'_> {
     fn nested(&self, state: &mut State<'_>, nested: Nested) {
@@ -59,7 +62,7 @@ impl PpAnn for &dyn rustc_hir::intravisit::Map<'_> {
             Nested::TraitItem(id) => state.print_trait_item(self.trait_item(id)),
             Nested::ImplItem(id) => state.print_impl_item(self.impl_item(id)),
             Nested::ForeignItem(id) => state.print_foreign_item(self.foreign_item(id)),
-            Nested::Body(id) => state.print_expr(&self.body(id).value),
+            Nested::Body(id) => state.print_expr(self.body(id).value),
             Nested::BodyParamPat(id, i) => state.print_pat(self.body(id).params[i].pat),
         }
     }
@@ -88,14 +91,14 @@ impl<'a> State<'a> {
             Node::AnonConst(a) => self.print_anon_const(a),
             Node::ConstBlock(a) => self.print_inline_const(a),
             Node::Expr(a) => self.print_expr(a),
-            Node::ExprField(a) => self.print_expr_field(&a),
+            Node::ExprField(a) => self.print_expr_field(a),
             Node::Stmt(a) => self.print_stmt(a),
             Node::PathSegment(a) => self.print_path_segment(a),
             Node::Ty(a) => self.print_type(a),
             Node::TypeBinding(a) => self.print_type_binding(a),
             Node::TraitRef(a) => self.print_trait_ref(a),
             Node::Pat(a) => self.print_pat(a),
-            Node::PatField(a) => self.print_patfield(&a),
+            Node::PatField(a) => self.print_patfield(a),
             Node::Arm(a) => self.print_arm(a),
             Node::Infer(_) => self.word("_"),
             Node::Block(a) => {
@@ -136,9 +139,8 @@ impl<'a> PrintState<'a> for State<'a> {
         &mut self.comments
     }
 
-    fn print_ident(&mut self, ident: Ident) {
-        self.word(IdentPrinter::for_ast_ident(ident, ident.is_raw_guess()).to_string());
-        self.ann.post(self, AnnNode::Name(&ident.name))
+    fn ann_post(&mut self, ident: Ident) {
+        self.ann.post(self, AnnNode::Name(&ident.name));
     }
 
     fn print_generic_args(&mut self, _: &ast::GenericArgs, _colons_before_params: bool) {
@@ -183,15 +185,15 @@ where
 }
 
 pub fn ty_to_string(ty: &hir::Ty<'_>) -> String {
-    to_string(NO_ANN, |s| s.print_type(ty))
+    to_string(&NoAnn, |s| s.print_type(ty))
 }
 
 pub fn qpath_to_string(segment: &hir::QPath<'_>) -> String {
-    to_string(NO_ANN, |s| s.print_qpath(segment, false))
+    to_string(&NoAnn, |s| s.print_qpath(segment, false))
 }
 
 pub fn pat_to_string(pat: &hir::Pat<'_>) -> String {
-    to_string(NO_ANN, |s| s.print_pat(pat))
+    to_string(&NoAnn, |s| s.print_pat(pat))
 }
 
 impl<'a> State<'a> {
@@ -260,7 +262,7 @@ impl<'a> State<'a> {
                 self.word("*");
                 self.print_mt(mt, true);
             }
-            hir::TyKind::Ref(ref lifetime, ref mt) => {
+            hir::TyKind::Ref(lifetime, ref mt) => {
                 self.word("&");
                 self.print_opt_lifetime(lifetime);
                 self.print_mt(mt, false);
@@ -281,7 +283,7 @@ impl<'a> State<'a> {
             }
             hir::TyKind::OpaqueDef(..) => self.word("/*impl Trait*/"),
             hir::TyKind::Path(ref qpath) => self.print_qpath(qpath, false),
-            hir::TyKind::TraitObject(bounds, ref lifetime, syntax) => {
+            hir::TyKind::TraitObject(bounds, lifetime, syntax) => {
                 if syntax == ast::TraitObjectSyntax::Dyn {
                     self.word_space("dyn");
                 }
@@ -517,10 +519,10 @@ impl<'a> State<'a> {
                 self.end(); // need to close a box
                 self.ann.nested(self, Nested::Body(body));
             }
-            hir::ItemKind::Macro(ref macro_def, _) => {
+            hir::ItemKind::Macro(macro_def, _) => {
                 self.print_mac_def(macro_def, &item.ident, item.span, |_| {});
             }
-            hir::ItemKind::Mod(ref _mod) => {
+            hir::ItemKind::Mod(_mod) => {
                 self.head("mod");
                 self.print_ident(item.ident);
                 self.nbsp();
@@ -549,7 +551,7 @@ impl<'a> State<'a> {
                     state.print_type(ty);
                 });
             }
-            hir::ItemKind::OpaqueTy(ref opaque_ty) => {
+            hir::ItemKind::OpaqueTy(opaque_ty) => {
                 self.print_item_type(item, opaque_ty.generics, |state| {
                     let mut real_bounds = Vec::with_capacity(opaque_ty.bounds.len());
                     for b in opaque_ty.bounds {
@@ -739,7 +741,7 @@ impl<'a> State<'a> {
                 self.end();
                 self.end() // close the outer-box
             }
-            hir::VariantData::Struct(..) => {
+            hir::VariantData::Struct { .. } => {
                 self.print_where_clause(generics);
                 self.nbsp();
                 self.bopen();
@@ -1096,12 +1098,12 @@ impl<'a> State<'a> {
             self.space();
         }
         self.cbox(INDENT_UNIT);
-        self.print_outer_attributes(&self.attrs(field.hir_id));
+        self.print_outer_attributes(self.attrs(field.hir_id));
         if !field.is_shorthand {
             self.print_ident(field.ident);
             self.word_space(":");
         }
-        self.print_expr(&field.expr);
+        self.print_expr(field.expr);
         self.end()
     }
 
@@ -1131,7 +1133,7 @@ impl<'a> State<'a> {
         args: &[hir::Expr<'_>],
     ) {
         let base_args = args;
-        self.print_expr_maybe_paren(&receiver, parser::PREC_POSTFIX);
+        self.print_expr_maybe_paren(receiver, parser::PREC_POSTFIX);
         self.word(".");
         self.print_ident(segment.ident);
 
@@ -1217,7 +1219,7 @@ impl<'a> State<'a> {
         self.commasep(Consistent, &args, |s, arg| match *arg {
             AsmArg::Template(ref template) => s.print_string(template, ast::StrStyle::Cooked),
             AsmArg::Operand(op) => match *op {
-                hir::InlineAsmOperand::In { reg, ref expr } => {
+                hir::InlineAsmOperand::In { reg, expr } => {
                     s.word("in");
                     s.popen();
                     s.word(format!("{reg}"));
@@ -1236,7 +1238,7 @@ impl<'a> State<'a> {
                         None => s.word("_"),
                     }
                 }
-                hir::InlineAsmOperand::InOut { reg, late, ref expr } => {
+                hir::InlineAsmOperand::InOut { reg, late, expr } => {
                     s.word(if late { "inlateout" } else { "inout" });
                     s.popen();
                     s.word(format!("{reg}"));
@@ -1244,7 +1246,7 @@ impl<'a> State<'a> {
                     s.space();
                     s.print_expr(expr);
                 }
-                hir::InlineAsmOperand::SplitInOut { reg, late, ref in_expr, ref out_expr } => {
+                hir::InlineAsmOperand::SplitInOut { reg, late, in_expr, ref out_expr } => {
                     s.word(if late { "inlateout" } else { "inout" });
                     s.popen();
                     s.word(format!("{reg}"));
@@ -1350,7 +1352,7 @@ impl<'a> State<'a> {
             hir::ExprKind::AddrOf(k, m, expr) => {
                 self.print_expr_addr_of(k, m, expr);
             }
-            hir::ExprKind::Lit(ref lit) => {
+            hir::ExprKind::Lit(lit) => {
                 self.print_literal(lit);
             }
             hir::ExprKind::Cast(expr, ty) => {
@@ -1516,7 +1518,7 @@ impl<'a> State<'a> {
                 self.word("asm!");
                 self.print_inline_asm(asm);
             }
-            hir::ExprKind::OffsetOf(container, ref fields) => {
+            hir::ExprKind::OffsetOf(container, fields) => {
                 self.word("offset_of!(");
                 self.print_type(container);
                 self.word(",");
@@ -1621,7 +1623,7 @@ impl<'a> State<'a> {
                 self.print_ident(item_segment.ident);
                 self.print_generic_args(item_segment.args(), colons_before_params)
             }
-            hir::QPath::LangItem(lang_item, span, _) => {
+            hir::QPath::LangItem(lang_item, span) => {
                 self.word("#[lang = \"");
                 self.print_ident(Ident::new(lang_item.name(), span));
                 self.word("\"]");
@@ -1722,6 +1724,7 @@ impl<'a> State<'a> {
         // is that it doesn't matter
         match pat.kind {
             PatKind::Wild => self.word("_"),
+            PatKind::Never => self.word("!"),
             PatKind::Binding(BindingAnnotation(by_ref, mutbl), _, ident, sub) => {
                 if by_ref == ByRef::Yes {
                     self.word_nbsp("ref");
@@ -1764,7 +1767,7 @@ impl<'a> State<'a> {
                 if !empty {
                     self.space();
                 }
-                self.commasep_cmnt(Consistent, &fields, |s, f| s.print_patfield(f), |f| f.pat.span);
+                self.commasep_cmnt(Consistent, fields, |s, f| s.print_patfield(f), |f| f.pat.span);
                 if etc {
                     if !fields.is_empty() {
                         self.word_space(",");
@@ -1864,7 +1867,7 @@ impl<'a> State<'a> {
             self.space();
         }
         self.cbox(INDENT_UNIT);
-        self.print_outer_attributes(&self.attrs(field.hir_id));
+        self.print_outer_attributes(self.attrs(field.hir_id));
         if !field.is_shorthand {
             self.print_ident(field.ident);
             self.word_nbsp(":");
@@ -2000,18 +2003,14 @@ impl<'a> State<'a> {
         });
         self.word("|");
 
-        if let hir::FnRetTy::DefaultReturn(..) = decl.output {
-            return;
-        }
-
-        self.space_if_not_bol();
-        self.word_space("->");
         match decl.output {
             hir::FnRetTy::Return(ty) => {
+                self.space_if_not_bol();
+                self.word_space("->");
                 self.print_type(ty);
                 self.maybe_print_comment(ty.span.lo());
             }
-            hir::FnRetTy::DefaultReturn(..) => unreachable!(),
+            hir::FnRetTy::DefaultReturn(..) => {}
         }
     }
 
@@ -2085,11 +2084,6 @@ impl<'a> State<'a> {
                     }
                     self.print_poly_trait_ref(tref);
                 }
-                GenericBound::LangItemTrait(lang_item, span, ..) => {
-                    self.word("#[lang = \"");
-                    self.print_ident(Ident::new(lang_item.name(), *span));
-                    self.word("\"]");
-                }
                 GenericBound::Outlives(lt) => {
                     self.print_lifetime(lt);
                 }
@@ -2123,7 +2117,7 @@ impl<'a> State<'a> {
                     self.print_type(default);
                 }
             }
-            GenericParamKind::Const { ty, ref default } => {
+            GenericParamKind::Const { ty, ref default, is_host_effect: _ } => {
                 self.word_space(":");
                 self.print_type(ty);
                 if let Some(default) = default {
@@ -2164,7 +2158,7 @@ impl<'a> State<'a> {
                     self.print_bounds(":", bounds);
                 }
                 hir::WherePredicate::RegionPredicate(hir::WhereRegionPredicate {
-                    ref lifetime,
+                    lifetime,
                     bounds,
                     ..
                 }) => {
@@ -2176,7 +2170,7 @@ impl<'a> State<'a> {
                             GenericBound::Outlives(lt) => {
                                 self.print_lifetime(lt);
                             }
-                            _ => panic!(),
+                            _ => panic!("unexpected bound on lifetime param: {bound:?}"),
                         }
 
                         if i != 0 {
@@ -2213,16 +2207,14 @@ impl<'a> State<'a> {
     }
 
     fn print_fn_output(&mut self, decl: &hir::FnDecl<'_>) {
-        if let hir::FnRetTy::DefaultReturn(..) = decl.output {
-            return;
-        }
-
-        self.space_if_not_bol();
-        self.ibox(INDENT_UNIT);
-        self.word_space("->");
         match decl.output {
-            hir::FnRetTy::DefaultReturn(..) => unreachable!(),
-            hir::FnRetTy::Return(ty) => self.print_type(ty),
+            hir::FnRetTy::Return(ty) => {
+                self.space_if_not_bol();
+                self.ibox(INDENT_UNIT);
+                self.word_space("->");
+                self.print_type(ty);
+            }
+            hir::FnRetTy::DefaultReturn(..) => return,
         }
         self.end();
 
diff --git a/compiler/rustc_hir_typeck/Cargo.toml b/compiler/rustc_hir_typeck/Cargo.toml
index 0062889d2..b0c603044 100644
--- a/compiler/rustc_hir_typeck/Cargo.toml
+++ b/compiler/rustc_hir_typeck/Cargo.toml
@@ -19,7 +19,6 @@ rustc_infer = { path = "../rustc_infer" }
 rustc_lint = { path = "../rustc_lint" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_middle = { path = "../rustc_middle" }
-rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
diff --git a/compiler/rustc_hir_typeck/src/_match.rs b/compiler/rustc_hir_typeck/src/_match.rs
index 84e986488..181de3728 100644
--- a/compiler/rustc_hir_typeck/src/_match.rs
+++ b/compiler/rustc_hir_typeck/src/_match.rs
@@ -42,7 +42,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         // #55810: Type check patterns first so we get types for all bindings.
         let scrut_span = scrut.span.find_ancestor_inside(expr.span).unwrap_or(scrut.span);
         for arm in arms {
-            self.check_pat_top(&arm.pat, scrutinee_ty, Some(scrut_span), Some(scrut), None);
+            self.check_pat_top(arm.pat, scrutinee_ty, Some(scrut_span), Some(scrut), None);
         }
 
         // Now typecheck the blocks.
@@ -92,7 +92,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
             self.diverges.set(Diverges::Maybe);
 
-            let arm_ty = self.check_expr_with_expectation(&arm.body, expected);
+            let arm_ty = self.check_expr_with_expectation(arm.body, expected);
             all_arms_diverge &= self.diverges.get();
 
             let opt_suggest_box_span = prior_arm.and_then(|(_, prior_arm_ty, _)| {
@@ -137,9 +137,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             coercion.coerce_inner(
                 self,
                 &cause,
-                Some(&arm.body),
+                Some(arm.body),
                 arm_ty,
-                |err| self.suggest_removing_semicolon_for_coerce(err, expr, arm_ty, prior_arm),
+                |err| {
+                    self.explain_never_type_coerced_to_unit(err, arm, arm_ty, prior_arm, expr);
+                },
                 false,
             );
 
@@ -177,6 +179,38 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         coercion.complete(self)
     }
 
+    fn explain_never_type_coerced_to_unit(
+        &self,
+        err: &mut Diagnostic,
+        arm: &hir::Arm<'tcx>,
+        arm_ty: Ty<'tcx>,
+        prior_arm: Option<(Option<hir::HirId>, Ty<'tcx>, Span)>,
+        expr: &hir::Expr<'tcx>,
+    ) {
+        if let hir::ExprKind::Block(block, _) = arm.body.kind
+            && let Some(expr) = block.expr
+            && let arm_tail_ty = self.node_ty(expr.hir_id)
+            && arm_tail_ty.is_never()
+            && !arm_ty.is_never()
+        {
+            err.span_label(
+                expr.span,
+                format!(
+                    "this expression is of type `!`, but it is coerced to `{arm_ty}` due to its \
+                     surrounding expression",
+                ),
+            );
+            self.suggest_mismatched_types_on_tail(
+                err,
+                expr,
+                arm_ty,
+                prior_arm.map_or(arm_tail_ty, |(_, ty, _)| ty),
+                expr.hir_id,
+            );
+        }
+        self.suggest_removing_semicolon_for_coerce(err, expr, arm_ty, prior_arm)
+    }
+
     fn suggest_removing_semicolon_for_coerce(
         &self,
         diag: &mut Diagnostic,
@@ -204,8 +238,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         }
 
         // Next, make sure that we have no type expectation.
-        let Some(ret) = hir
-            .find_by_def_id(self.body_id)
+        let Some(ret) = self
+            .tcx
+            .opt_hir_node_by_def_id(self.body_id)
             .and_then(|owner| owner.fn_decl())
             .map(|decl| decl.output.span())
         else {
@@ -283,7 +318,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         hir_id: hir::HirId,
         sp: Span,
     ) -> Option<(Span, String)> {
-        let node = self.tcx.hir().get(hir_id);
+        let node = self.tcx.hir_node(hir_id);
         if let hir::Node::Block(block) = node {
             // check that the body's parent is an fn
             let parent = self.tcx.hir().get_parent(self.tcx.hir().parent_id(block.hir_id));
diff --git a/compiler/rustc_hir_typeck/src/callee.rs b/compiler/rustc_hir_typeck/src/callee.rs
index 6b6d1574b..5e6b54950 100644
--- a/compiler/rustc_hir_typeck/src/callee.rs
+++ b/compiler/rustc_hir_typeck/src/callee.rs
@@ -6,9 +6,8 @@ use crate::errors;
 use rustc_ast::util::parser::PREC_POSTFIX;
 use rustc_errors::{Applicability, Diagnostic, ErrorGuaranteed, StashKey};
 use rustc_hir as hir;
-use rustc_hir::def::{self, CtorKind, DefKind, Namespace, Res};
+use rustc_hir::def::{self, CtorKind, Namespace, Res};
 use rustc_hir::def_id::DefId;
-use rustc_hir::HirId;
 use rustc_hir_analysis::autoderef::Autoderef;
 use rustc_infer::{
     infer,
@@ -245,7 +244,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             ) {
                 // Check for `self` receiver on the method, otherwise we can't use this as a `Fn*` trait.
                 if !self.tcx.associated_item(ok.value.def_id).fn_has_self_parameter {
-                    self.tcx.sess.delay_span_bug(
+                    self.tcx.sess.span_delayed_bug(
                         call_expr.span,
                         "input to overloaded call fn is not a self receiver",
                     );
@@ -260,9 +259,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     let ty::Ref(region, _, mutbl) = method.sig.inputs()[0].kind() else {
                         // The `fn`/`fn_mut` lang item is ill-formed, which should have
                         // caused an error elsewhere.
-                        self.tcx
-                            .sess
-                            .delay_span_bug(call_expr.span, "input to call/call_mut is not a ref");
+                        self.tcx.sess.span_delayed_bug(
+                            call_expr.span,
+                            "input to call/call_mut is not a ref",
+                        );
                         return None;
                     };
 
@@ -295,7 +295,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     ) {
         let hir = self.tcx.hir();
         let parent_hir_id = hir.parent_id(hir_id);
-        let parent_node = hir.get(parent_hir_id);
+        let parent_node = self.tcx.hir_node(parent_hir_id);
         if let (
             hir::Node::Expr(hir::Expr {
                 kind: hir::ExprKind::Closure(&hir::Closure { fn_decl_span, body, .. }),
@@ -313,7 +313,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 if let hir::Node::Expr(hir::Expr {
                     kind: hir::ExprKind::Closure(&hir::Closure { fn_decl_span, .. }),
                     ..
-                }) = hir.get(async_closure)
+                }) = self.tcx.hir_node(async_closure)
                 {
                     fn_decl_span
                 } else {
@@ -343,7 +343,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         callee_expr: &'tcx hir::Expr<'tcx>,
     ) -> bool {
         let hir_id = self.tcx.hir().parent_id(call_expr.hir_id);
-        let parent_node = self.tcx.hir().get(hir_id);
+        let parent_node = self.tcx.hir_node(hir_id);
         if let (
             hir::Node::Expr(hir::Expr { kind: hir::ExprKind::Array(_), .. }),
             hir::ExprKind::Tup(exp),
@@ -373,7 +373,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     ) -> Ty<'tcx> {
         let (fn_sig, def_id) = match *callee_ty.kind() {
             ty::FnDef(def_id, args) => {
-                self.enforce_context_effects(call_expr.hir_id, call_expr.span, def_id, args);
+                self.enforce_context_effects(call_expr.span, def_id, args);
                 let fn_sig = self.tcx.fn_sig(def_id).instantiate(self.tcx, args);
 
                 // Unit testing: function items annotated with
@@ -417,7 +417,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     && let Some(mut diag) = self
                         .tcx
                         .sess
-                        .diagnostic()
+                        .dcx()
                         .steal_diagnostic(segment.ident.span, StashKey::CallIntoMethod)
                 {
                     // Try suggesting `foo(a)` -> `a.foo()` if possible.
@@ -471,6 +471,65 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             }
         }
 
+        if let Some(def_id) = def_id
+            && self.tcx.def_kind(def_id) == hir::def::DefKind::Fn
+            && self.tcx.is_intrinsic(def_id)
+            && self.tcx.item_name(def_id) == sym::const_eval_select
+        {
+            let fn_sig = self.resolve_vars_if_possible(fn_sig);
+            for idx in 0..=1 {
+                let arg_ty = fn_sig.inputs()[idx + 1];
+                let span = arg_exprs.get(idx + 1).map_or(call_expr.span, |arg| arg.span);
+                // Check that second and third argument of `const_eval_select` must be `FnDef`, and additionally that
+                // the second argument must be `const fn`. The first argument must be a tuple, but this is already expressed
+                // in the function signature (`F: FnOnce<ARG>`), so I did not bother to add another check here.
+                //
+                // This check is here because there is currently no way to express a trait bound for `FnDef` types only.
+                if let ty::FnDef(def_id, _args) = *arg_ty.kind() {
+                    let fn_once_def_id =
+                        self.tcx.require_lang_item(hir::LangItem::FnOnce, Some(span));
+                    let fn_once_output_def_id =
+                        self.tcx.require_lang_item(hir::LangItem::FnOnceOutput, Some(span));
+                    if self.tcx.generics_of(fn_once_def_id).host_effect_index.is_none() {
+                        if idx == 0 && !self.tcx.is_const_fn_raw(def_id) {
+                            self.tcx.sess.emit_err(errors::ConstSelectMustBeConst { span });
+                        }
+                    } else {
+                        let const_param: ty::GenericArg<'tcx> =
+                            ([self.tcx.consts.false_, self.tcx.consts.true_])[idx].into();
+                        self.register_predicate(traits::Obligation::new(
+                            self.tcx,
+                            self.misc(span),
+                            self.param_env,
+                            ty::TraitRef::new(
+                                self.tcx,
+                                fn_once_def_id,
+                                [arg_ty.into(), fn_sig.inputs()[0].into(), const_param],
+                            ),
+                        ));
+
+                        self.register_predicate(traits::Obligation::new(
+                            self.tcx,
+                            self.misc(span),
+                            self.param_env,
+                            ty::ProjectionPredicate {
+                                projection_ty: ty::AliasTy::new(
+                                    self.tcx,
+                                    fn_once_output_def_id,
+                                    [arg_ty.into(), fn_sig.inputs()[0].into(), const_param],
+                                ),
+                                term: fn_sig.output().into(),
+                            },
+                        ));
+
+                        self.select_obligations_where_possible(|_| {});
+                    }
+                } else {
+                    self.tcx.sess.emit_err(errors::ConstSelectMustBeFn { span, ty: arg_ty });
+                }
+            }
+        }
+
         fn_sig.output()
     }
 
@@ -625,12 +684,31 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             );
         }
 
+        if let hir::ExprKind::Path(hir::QPath::Resolved(None, path)) = callee_expr.kind
+            && let Res::Local(_) = path.res
+            && let [segment] = &path.segments[..]
+        {
+            for id in self.tcx.hir().items() {
+                if let Some(node) = self.tcx.hir().get_if_local(id.owner_id.into())
+                    && let hir::Node::Item(item) = node
+                    && let hir::ItemKind::Fn(..) = item.kind
+                    && item.ident.name == segment.ident.name
+                {
+                    err.span_label(
+                        self.tcx.def_span(id.owner_id),
+                        "this function of the same name is available here, but it's shadowed by \
+                         the local binding",
+                    );
+                }
+            }
+        }
+
         let mut inner_callee_path = None;
         let def = match callee_expr.kind {
             hir::ExprKind::Path(ref qpath) => {
                 self.typeck_results.borrow().qpath_res(qpath, callee_expr.hir_id)
             }
-            hir::ExprKind::Call(ref inner_callee, _) => {
+            hir::ExprKind::Call(inner_callee, _) => {
                 if let hir::ExprKind::Path(ref inner_qpath) = inner_callee.kind {
                     inner_callee_path = Some(inner_qpath);
                     self.typeck_results.borrow().qpath_res(inner_qpath, inner_callee.hir_id)
@@ -751,7 +829,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     #[tracing::instrument(level = "debug", skip(self, span))]
     pub(super) fn enforce_context_effects(
         &self,
-        call_expr_hir: HirId,
         span: Span,
         callee_did: DefId,
         callee_args: GenericArgsRef<'tcx>,
@@ -762,38 +839,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let generics = tcx.generics_of(callee_did);
         let Some(host_effect_index) = generics.host_effect_index else { return };
 
-        // if the callee does have the param, we need to equate the param to some const
-        // value no matter whether the effects feature is enabled in the local crate,
-        // because inference will fail if we don't.
-        let mut host_always_on =
-            !tcx.features().effects || tcx.sess.opts.unstable_opts.unleash_the_miri_inside_of_you;
-
-        // Compute the constness required by the context.
-        let context = tcx.hir().enclosing_body_owner(call_expr_hir);
-        let const_context = tcx.hir().body_const_context(context);
-
-        let kind = tcx.def_kind(context.to_def_id());
-        debug_assert_ne!(kind, DefKind::ConstParam);
-
-        if tcx.has_attr(context.to_def_id(), sym::rustc_do_not_const_check) {
-            trace!("do not const check this context");
-            host_always_on = true;
-        }
-
-        let effect = match const_context {
-            _ if host_always_on => tcx.consts.true_,
-            Some(hir::ConstContext::Static(_) | hir::ConstContext::Const { .. }) => {
-                tcx.consts.false_
-            }
-            Some(hir::ConstContext::ConstFn) => {
-                let host_idx = tcx
-                    .generics_of(context)
-                    .host_effect_index
-                    .expect("ConstContext::Maybe must have host effect param");
-                ty::GenericArgs::identity_for_item(tcx, context).const_at(host_idx)
-            }
-            None => tcx.consts.true_,
-        };
+        let effect = tcx.expected_host_effect_param_for_body(self.body_id);
 
         trace!(?effect, ?generics, ?callee_args);
 
@@ -826,7 +872,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             expected,
         );
 
-        self.write_method_call(call_expr.hir_id, method_callee);
+        self.write_method_call_and_enforce_effects(call_expr.hir_id, call_expr.span, method_callee);
         output_type
     }
 }
@@ -876,7 +922,11 @@ impl<'a, 'tcx> DeferredCallResolution<'tcx> {
                 adjustments.extend(autoref);
                 fcx.apply_adjustments(self.callee_expr, adjustments);
 
-                fcx.write_method_call(self.call_expr.hir_id, method_callee);
+                fcx.write_method_call_and_enforce_effects(
+                    self.call_expr.hir_id,
+                    self.call_expr.span,
+                    method_callee,
+                );
             }
             None => {
                 // This can happen if `#![no_core]` is used and the `fn/fn_mut/fn_once`
diff --git a/compiler/rustc_hir_typeck/src/cast.rs b/compiler/rustc_hir_typeck/src/cast.rs
index ee23f47c2..0de036536 100644
--- a/compiler/rustc_hir_typeck/src/cast.rs
+++ b/compiler/rustc_hir_typeck/src/cast.rs
@@ -101,7 +101,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         Ok(match *t.kind() {
             ty::Slice(_) | ty::Str => Some(PointerKind::Length),
-            ty::Dynamic(ref tty, _, ty::Dyn) => Some(PointerKind::VTable(tty.principal_def_id())),
+            ty::Dynamic(tty, _, ty::Dyn) => Some(PointerKind::VTable(tty.principal_def_id())),
             ty::Adt(def, args) if def.is_struct() => match def.non_enum_variant().tail_opt() {
                 None => Some(PointerKind::Thin),
                 Some(f) => {
@@ -142,14 +142,14 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 let reported = self
                     .tcx
                     .sess
-                    .delay_span_bug(span, format!("`{t:?}` should be sized but is not?"));
+                    .span_delayed_bug(span, format!("`{t:?}` should be sized but is not?"));
                 return Err(reported);
             }
         })
     }
 }
 
-#[derive(Copy, Clone)]
+#[derive(Copy, Clone, Debug)]
 pub enum CastError {
     ErrorGuaranteed(ErrorGuaranteed),
 
@@ -271,7 +271,7 @@ impl<'a, 'tcx> CastCheck<'tcx> {
                         match e {
                             CastError::NeedViaPtr => "a raw pointer",
                             CastError::NeedViaThinPtr => "a thin pointer",
-                            _ => bug!(),
+                            e => unreachable!("control flow means we should never encounter a {e:?}"),
                         }
                     ));
                 }
@@ -288,13 +288,7 @@ impl<'a, 'tcx> CastCheck<'tcx> {
                     self.cast_ty,
                     fcx,
                 )
-                .help(format!(
-                    "cast through {} first",
-                    match e {
-                        CastError::NeedViaInt => "an integer",
-                        _ => bug!(),
-                    }
-                ))
+                .help("cast through an integer first")
                 .emit();
             }
             CastError::IllegalCast => {
@@ -506,7 +500,7 @@ impl<'a, 'tcx> CastCheck<'tcx> {
                 };
 
                 SizedUnsizedCast {
-                    sess: &fcx.tcx.sess,
+                    sess: fcx.tcx.sess,
                     span: self.span,
                     expr_ty: self.expr_ty,
                     cast_ty: fcx.ty_to_string(self.cast_ty),
@@ -534,7 +528,7 @@ impl<'a, 'tcx> CastCheck<'tcx> {
                 let unknown_cast_to = match e {
                     CastError::UnknownCastPtrKind => true,
                     CastError::UnknownExprPtrKind => false,
-                    _ => bug!(),
+                    e => unreachable!("control flow means we should never encounter a {e:?}"),
                 };
                 let (span, sub) = if unknown_cast_to {
                     (self.cast_span, errors::CastUnknownPointerSub::To(self.cast_span))
diff --git a/compiler/rustc_hir_typeck/src/check.rs b/compiler/rustc_hir_typeck/src/check.rs
index b8a265d49..2855cea80 100644
--- a/compiler/rustc_hir_typeck/src/check.rs
+++ b/compiler/rustc_hir_typeck/src/check.rs
@@ -8,7 +8,7 @@ use rustc_hir as hir;
 use rustc_hir::def::DefKind;
 use rustc_hir::intravisit::Visitor;
 use rustc_hir::lang_items::LangItem;
-use rustc_hir_analysis::check::{check_function_signature, fn_maybe_err};
+use rustc_hir_analysis::check::{check_function_signature, forbid_intrinsic_abi};
 use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
 use rustc_infer::infer::RegionVariableOrigin;
 use rustc_middle::ty::{self, Binder, Ty, TyCtxt};
@@ -34,7 +34,7 @@ pub(super) fn check_fn<'a, 'tcx>(
     can_be_coroutine: Option<hir::Movability>,
     params_can_be_unsized: bool,
 ) -> Option<CoroutineTypes<'tcx>> {
-    let fn_id = fcx.tcx.hir().local_def_id_to_hir_id(fn_def_id);
+    let fn_id = fcx.tcx.local_def_id_to_hir_id(fn_def_id);
 
     let tcx = fcx.tcx;
     let hir = tcx.hir();
@@ -53,7 +53,7 @@ pub(super) fn check_fn<'a, 'tcx>(
 
     let span = body.value.span;
 
-    fn_maybe_err(tcx, span, fn_sig.abi);
+    forbid_intrinsic_abi(tcx, span, fn_sig.abi);
 
     if let Some(kind) = body.coroutine_kind
         && can_be_coroutine.is_some()
@@ -67,6 +67,28 @@ pub(super) fn check_fn<'a, 'tcx>(
                 fcx.require_type_is_sized(yield_ty, span, traits::SizedYieldType);
                 yield_ty
             }
+            // HACK(-Ztrait-solver=next): In the *old* trait solver, we must eagerly
+            // guide inference on the yield type so that we can handle `AsyncIterator`
+            // in this block in projection correctly. In the new trait solver, it is
+            // not a problem.
+            hir::CoroutineKind::AsyncGen(..) => {
+                let yield_ty = fcx.next_ty_var(TypeVariableOrigin {
+                    kind: TypeVariableOriginKind::TypeInference,
+                    span,
+                });
+                fcx.require_type_is_sized(yield_ty, span, traits::SizedYieldType);
+
+                Ty::new_adt(
+                    tcx,
+                    tcx.adt_def(tcx.require_lang_item(hir::LangItem::Poll, Some(span))),
+                    tcx.mk_args(&[Ty::new_adt(
+                        tcx,
+                        tcx.adt_def(tcx.require_lang_item(hir::LangItem::Option, Some(span))),
+                        tcx.mk_args(&[yield_ty.into()]),
+                    )
+                    .into()]),
+                )
+            }
             hir::CoroutineKind::Async(..) => Ty::new_unit(tcx),
         };
 
@@ -76,7 +98,7 @@ pub(super) fn check_fn<'a, 'tcx>(
         fcx.resume_yield_tys = Some((resume_ty, yield_ty));
     }
 
-    GatherLocalsVisitor::new(&fcx).visit_body(body);
+    GatherLocalsVisitor::new(fcx).visit_body(body);
 
     // C-variadic fns also have a `VaList` input that's not listed in `fn_sig`
     // (as it's created inside the body itself, not passed in from outside).
@@ -94,7 +116,7 @@ pub(super) fn check_fn<'a, 'tcx>(
     for (idx, (param_ty, param)) in inputs_fn.chain(maybe_va_list).zip(body.params).enumerate() {
         // Check the pattern.
         let ty_span = try { inputs_hir?.get(idx)?.span };
-        fcx.check_pat_top(&param.pat, param_ty, ty_span, None, None);
+        fcx.check_pat_top(param.pat, param_ty, ty_span, None, None);
 
         // Check that argument is Sized.
         if !params_can_be_unsized {
@@ -123,7 +145,7 @@ pub(super) fn check_fn<'a, 'tcx>(
         hir::FnRetTy::Return(ty) => ty.span,
     };
     fcx.require_type_is_sized(declared_ret_ty, return_or_body_span, traits::SizedReturnType);
-    fcx.check_return_expr(&body.value, false);
+    fcx.check_return_expr(body.value, false);
 
     // We insert the deferred_coroutine_interiors entry after visiting the body.
     // This ensures that all nested coroutines appear before the entry of this coroutine.
@@ -156,7 +178,7 @@ pub(super) fn check_fn<'a, 'tcx>(
     // really expected to fail, since the coercions would have failed
     // earlier when trying to find a LUB.
     let coercion = fcx.ret_coercion.take().unwrap().into_inner();
-    let mut actual_return_ty = coercion.complete(&fcx);
+    let mut actual_return_ty = coercion.complete(fcx);
     debug!("actual_return_ty = {:?}", actual_return_ty);
     if let ty::Dynamic(..) = declared_ret_ty.kind() {
         // We have special-cased the case where the function is declared
@@ -214,7 +236,7 @@ fn check_panic_info_fn(tcx: TyCtxt<'_>, fn_id: LocalDefId, fn_sig: ty::FnSig<'_>
     // build type `for<'a, 'b> fn(&'a PanicInfo<'b>) -> !`
     let panic_info_ty = tcx.type_of(panic_info_did).instantiate(
         tcx,
-        &[ty::GenericArg::from(ty::Region::new_late_bound(
+        &[ty::GenericArg::from(ty::Region::new_bound(
             tcx,
             ty::INNERMOST,
             ty::BoundRegion { var: ty::BoundVar::from_u32(1), kind: ty::BrAnon },
@@ -222,7 +244,7 @@ fn check_panic_info_fn(tcx: TyCtxt<'_>, fn_id: LocalDefId, fn_sig: ty::FnSig<'_>
     );
     let panic_info_ref_ty = Ty::new_imm_ref(
         tcx,
-        ty::Region::new_late_bound(
+        ty::Region::new_bound(
             tcx,
             ty::INNERMOST,
             ty::BoundRegion { var: ty::BoundVar::from_u32(0), kind: ty::BrAnon },
@@ -239,7 +261,7 @@ fn check_panic_info_fn(tcx: TyCtxt<'_>, fn_id: LocalDefId, fn_sig: ty::FnSig<'_>
         bounds,
     );
 
-    check_function_signature(
+    let _ = check_function_signature(
         tcx,
         ObligationCause::new(
             tcx.def_span(fn_id),
@@ -278,7 +300,7 @@ fn check_lang_start_fn<'tcx>(tcx: TyCtxt<'tcx>, fn_sig: ty::FnSig<'tcx>, def_id:
         Abi::Rust,
     ));
 
-    check_function_signature(
+    let _ = check_function_signature(
         tcx,
         ObligationCause::new(
             tcx.def_span(def_id),
diff --git a/compiler/rustc_hir_typeck/src/closure.rs b/compiler/rustc_hir_typeck/src/closure.rs
index a70ead8e5..d19d30412 100644
--- a/compiler/rustc_hir_typeck/src/closure.rs
+++ b/compiler/rustc_hir_typeck/src/closure.rs
@@ -7,7 +7,7 @@ use rustc_hir as hir;
 use rustc_hir::lang_items::LangItem;
 use rustc_hir_analysis::astconv::AstConv;
 use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
-use rustc_infer::infer::{DefineOpaqueTypes, LateBoundRegionConversionTime};
+use rustc_infer::infer::{BoundRegionConversionTime, DefineOpaqueTypes};
 use rustc_infer::infer::{InferOk, InferResult};
 use rustc_macros::{TypeFoldable, TypeVisitable};
 use rustc_middle::ty::visit::{TypeVisitable, TypeVisitableExt};
@@ -141,7 +141,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         debug!(?sig, ?opt_kind);
 
         let closure_kind_ty = match opt_kind {
-            Some(kind) => kind.to_ty(self.tcx),
+            Some(kind) => Ty::from_closure_kind(self.tcx, kind),
 
             // Create a type variable (for now) to represent the closure kind.
             // It will be unified during the upvar inference phase (`upvar.rs`)
@@ -181,7 +181,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         .iter_instantiated_copied(self.tcx, args)
                         .map(|(c, s)| (c.as_predicate(), s)),
                 ),
-            ty::Dynamic(ref object_type, ..) => {
+            ty::Dynamic(object_type, ..) => {
                 let sig = object_type.projection_bounds().find_map(|pb| {
                     let pb = pb.with_self_ty(self.tcx, self.tcx.types.trait_object_dummy_self);
                     self.deduce_sig_from_projection(None, pb)
@@ -483,8 +483,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         body: &hir::Body<'_>,
         expected_sig: ExpectedSig<'tcx>,
     ) -> ClosureSignatures<'tcx> {
-        let hir = self.tcx.hir();
-        let expr_map_node = hir.get_by_def_id(expr_def_id);
+        let expr_map_node = self.tcx.hir_node_by_def_id(expr_def_id);
         let expected_args: Vec<_> = expected_sig
             .sig
             .skip_binder()
@@ -558,7 +557,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 // Instantiate (this part of..) S to S', i.e., with fresh variables.
                 self.instantiate_binder_with_fresh_vars(
                     hir_ty.span,
-                    LateBoundRegionConversionTime::FnCall,
+                    BoundRegionConversionTime::FnCall,
                     // (*) binder moved to here
                     supplied_sig.inputs().rebind(supplied_ty),
                 )
@@ -583,7 +582,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
             let supplied_output_ty = self.instantiate_binder_with_fresh_vars(
                 decl.output.span(),
-                LateBoundRegionConversionTime::FnCall,
+                BoundRegionConversionTime::FnCall,
                 supplied_sig.output(),
             );
             let cause = &self.misc(decl.output.span());
@@ -624,13 +623,13 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         trace!("decl = {:#?}", decl);
         debug!(?body.coroutine_kind);
 
-        let hir_id = self.tcx.hir().local_def_id_to_hir_id(expr_def_id);
+        let hir_id = self.tcx.local_def_id_to_hir_id(expr_def_id);
         let bound_vars = self.tcx.late_bound_vars(hir_id);
 
         // First, convert the types that the user supplied (if any).
         let supplied_arguments = decl.inputs.iter().map(|a| astconv.ast_ty_to_ty(a));
         let supplied_return = match decl.output {
-            hir::FnRetTy::Return(ref output) => astconv.ast_ty_to_ty(&output),
+            hir::FnRetTy::Return(ref output) => astconv.ast_ty_to_ty(output),
             hir::FnRetTy::DefaultReturn(_) => match body.coroutine_kind {
                 // In the case of the async block that we create for a function body,
                 // we expect the return type of the block to match that of the enclosing
@@ -651,8 +650,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         },
                     )
                 }
-                Some(hir::CoroutineKind::Gen(hir::CoroutineSource::Fn)) => {
-                    todo!("gen closures do not exist yet")
+                // All `gen {}` and `async gen {}` must return unit.
+                Some(hir::CoroutineKind::Gen(_) | hir::CoroutineKind::AsyncGen(_)) => {
+                    self.tcx.types.unit
                 }
 
                 _ => astconv.ty_infer(None, decl.output.span()),
@@ -819,7 +819,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         });
 
         if let hir::FnRetTy::Return(ref output) = decl.output {
-            astconv.ast_ty_to_ty(&output);
+            astconv.ast_ty_to_ty(output);
         }
 
         let result = ty::Binder::dummy(self.tcx.mk_fn_sig(
diff --git a/compiler/rustc_hir_typeck/src/coercion.rs b/compiler/rustc_hir_typeck/src/coercion.rs
index 6c03bc3b5..f08af9edc 100644
--- a/compiler/rustc_hir_typeck/src/coercion.rs
+++ b/compiler/rustc_hir_typeck/src/coercion.rs
@@ -36,7 +36,9 @@
 //! ```
 
 use crate::FnCtxt;
-use rustc_errors::{struct_span_err, Diagnostic, DiagnosticBuilder, ErrorGuaranteed, MultiSpan};
+use rustc_errors::{
+    struct_span_err, Applicability, Diagnostic, DiagnosticBuilder, ErrorGuaranteed, MultiSpan,
+};
 use rustc_hir as hir;
 use rustc_hir::def_id::DefId;
 use rustc_hir::intravisit::{self, Visitor};
@@ -53,11 +55,10 @@ use rustc_middle::ty::adjustment::{
 use rustc_middle::ty::error::TypeError;
 use rustc_middle::ty::relate::RelateResult;
 use rustc_middle::ty::visit::TypeVisitableExt;
-use rustc_middle::ty::GenericArgsRef;
-use rustc_middle::ty::{self, Ty, TypeAndMut};
+use rustc_middle::ty::{self, GenericArgsRef, Ty, TyCtxt, TypeAndMut};
 use rustc_session::parse::feature_err;
 use rustc_span::symbol::sym;
-use rustc_span::{self, DesugaringKind};
+use rustc_span::DesugaringKind;
 use rustc_target::spec::abi::Abi;
 use rustc_trait_selection::infer::InferCtxtExt as _;
 use rustc_trait_selection::traits::error_reporting::TypeErrCtxtExt as _;
@@ -85,7 +86,7 @@ struct Coerce<'a, 'tcx> {
 impl<'a, 'tcx> Deref for Coerce<'a, 'tcx> {
     type Target = FnCtxt<'a, 'tcx>;
     fn deref(&self) -> &Self::Target {
-        &self.fcx
+        self.fcx
     }
 }
 
@@ -158,7 +159,7 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> {
             if self.next_trait_solver() {
                 if let Ok(res) = &res {
                     for obligation in &res.obligations {
-                        if !self.predicate_may_hold(&obligation) {
+                        if !self.predicate_may_hold(obligation) {
                             return Err(TypeError::Mismatch);
                         }
                     }
@@ -1041,7 +1042,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     /// Returns false if the coercion creates any obligations that result in
     /// errors.
     pub fn can_coerce(&self, expr_ty: Ty<'tcx>, target: Ty<'tcx>) -> bool {
-        // FIXME(-Ztrait-solver=next): We need to structurally resolve both types here.
+        // FIXME(-Znext-solver): We need to structurally resolve both types here.
         let source = self.resolve_vars_with_obligations(expr_ty);
         debug!("coercion::can_with_predicates({:?} -> {:?})", source, target);
 
@@ -1204,14 +1205,14 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     Adjust::Pointer(PointerCoercion::ClosureFnPointer(a_sig.unsafety()))
                 }
                 ty::FnDef(..) => Adjust::Pointer(PointerCoercion::ReifyFnPointer),
-                _ => unreachable!(),
+                _ => span_bug!(cause.span, "should not try to coerce a {prev_ty} to a fn pointer"),
             };
             let next_adjustment = match new_ty.kind() {
                 ty::Closure(..) => {
                     Adjust::Pointer(PointerCoercion::ClosureFnPointer(b_sig.unsafety()))
                 }
                 ty::FnDef(..) => Adjust::Pointer(PointerCoercion::ReifyFnPointer),
-                _ => unreachable!(),
+                _ => span_bug!(new.span, "should not try to coerce a {new_ty} to a fn pointer"),
             };
             for expr in exprs.iter().map(|e| e.as_coercion_site()) {
                 self.apply_adjustments(
@@ -1510,7 +1511,7 @@ impl<'tcx, 'exprs, E: AsCoercionSite> CoerceMany<'tcx, 'exprs, E> {
                         expression,
                         expression_ty,
                     ),
-                    Expressions::UpFront(ref coercion_sites) => fcx.try_find_coercion_lub(
+                    Expressions::UpFront(coercion_sites) => fcx.try_find_coercion_lub(
                         cause,
                         &coercion_sites[0..self.pushed],
                         self.merged_ty(),
@@ -1572,7 +1573,9 @@ impl<'tcx, 'exprs, E: AsCoercionSite> CoerceMany<'tcx, 'exprs, E> {
                 // any superfluous errors we might encounter while trying to
                 // emit or provide suggestions on how to fix the initial error.
                 fcx.set_tainted_by_errors(
-                    fcx.tcx.sess.delay_span_bug(cause.span, "coercion error but no error emitted"),
+                    fcx.tcx
+                        .sess
+                        .span_delayed_bug(cause.span, "coercion error but no error emitted"),
                 );
                 let (expected, found) = if label_expression_as_expected {
                     // In the case where this is a "forced unit", like
@@ -1660,12 +1663,15 @@ impl<'tcx, 'exprs, E: AsCoercionSite> CoerceMany<'tcx, 'exprs, E> {
                         None,
                         Some(coercion_error),
                     );
-                }
-
-                if visitor.ret_exprs.len() > 0
-                    && let Some(expr) = expression
-                {
-                    self.note_unreachable_loop_return(&mut err, &expr, &visitor.ret_exprs);
+                    if visitor.ret_exprs.len() > 0 {
+                        self.note_unreachable_loop_return(
+                            &mut err,
+                            fcx.tcx,
+                            &expr,
+                            &visitor.ret_exprs,
+                            expected,
+                        );
+                    }
                 }
 
                 let reported = err.emit_unless(unsized_return);
@@ -1678,8 +1684,10 @@ impl<'tcx, 'exprs, E: AsCoercionSite> CoerceMany<'tcx, 'exprs, E> {
     fn note_unreachable_loop_return(
         &self,
         err: &mut Diagnostic,
+        tcx: TyCtxt<'tcx>,
         expr: &hir::Expr<'tcx>,
         ret_exprs: &Vec<&'tcx hir::Expr<'tcx>>,
+        ty: Ty<'tcx>,
     ) {
         let hir::ExprKind::Loop(_, _, _, loop_span) = expr.kind else {
             return;
@@ -1704,10 +1712,77 @@ impl<'tcx, 'exprs, E: AsCoercionSite> CoerceMany<'tcx, 'exprs, E> {
                 ret_exprs.len() - MAXITER
             ));
         }
-        err.help(
-            "return a value for the case when the loop has zero elements to iterate on, or \
-           consider changing the return type to account for that possibility",
-        );
+        let hir = tcx.hir();
+        let item = hir.get_parent_item(expr.hir_id);
+        let ret_msg = "return a value for the case when the loop has zero elements to iterate on";
+        let ret_ty_msg =
+            "otherwise consider changing the return type to account for that possibility";
+        if let Some(node) = tcx.opt_hir_node(item.into())
+            && let Some(body_id) = node.body_id()
+            && let Some(sig) = node.fn_sig()
+            && let hir::ExprKind::Block(block, _) = hir.body(body_id).value.kind
+            && !ty.is_never()
+        {
+            let indentation = if let None = block.expr
+                && let [.., last] = &block.stmts
+            {
+                tcx.sess.source_map().indentation_before(last.span).unwrap_or_else(String::new)
+            } else if let Some(expr) = block.expr {
+                tcx.sess.source_map().indentation_before(expr.span).unwrap_or_else(String::new)
+            } else {
+                String::new()
+            };
+            if let None = block.expr
+                && let [.., last] = &block.stmts
+            {
+                err.span_suggestion_verbose(
+                    last.span.shrink_to_hi(),
+                    ret_msg,
+                    format!("\n{indentation}/* `{ty}` value */"),
+                    Applicability::MaybeIncorrect,
+                );
+            } else if let Some(expr) = block.expr {
+                err.span_suggestion_verbose(
+                    expr.span.shrink_to_hi(),
+                    ret_msg,
+                    format!("\n{indentation}/* `{ty}` value */"),
+                    Applicability::MaybeIncorrect,
+                );
+            }
+            let mut sugg = match sig.decl.output {
+                hir::FnRetTy::DefaultReturn(span) => {
+                    vec![(span, " -> Option<()>".to_string())]
+                }
+                hir::FnRetTy::Return(ty) => {
+                    vec![
+                        (ty.span.shrink_to_lo(), "Option<".to_string()),
+                        (ty.span.shrink_to_hi(), ">".to_string()),
+                    ]
+                }
+            };
+            for ret_expr in ret_exprs {
+                match ret_expr.kind {
+                    hir::ExprKind::Ret(Some(expr)) => {
+                        sugg.push((expr.span.shrink_to_lo(), "Some(".to_string()));
+                        sugg.push((expr.span.shrink_to_hi(), ")".to_string()));
+                    }
+                    hir::ExprKind::Ret(None) => {
+                        sugg.push((ret_expr.span.shrink_to_hi(), " Some(())".to_string()));
+                    }
+                    _ => {}
+                }
+            }
+            if let None = block.expr
+                && let [.., last] = &block.stmts
+            {
+                sugg.push((last.span.shrink_to_hi(), format!("\n{indentation}None")));
+            } else if let Some(expr) = block.expr {
+                sugg.push((expr.span.shrink_to_hi(), format!("\n{indentation}None")));
+            }
+            err.multipart_suggestion(ret_ty_msg, sugg, Applicability::MaybeIncorrect);
+        } else {
+            err.help(format!("{ret_msg}, {ret_ty_msg}"));
+        }
     }
 
     fn report_return_mismatched_types<'a>(
@@ -1724,7 +1799,7 @@ impl<'tcx, 'exprs, E: AsCoercionSite> CoerceMany<'tcx, 'exprs, E> {
         let mut err = fcx.err_ctxt().report_mismatched_types(cause, expected, found, ty_err);
 
         let parent_id = fcx.tcx.hir().parent_id(id);
-        let parent = fcx.tcx.hir().get(parent_id);
+        let parent = fcx.tcx.hir_node(parent_id);
         if let Some(expr) = expression
             && let hir::Node::Expr(hir::Expr {
                 kind: hir::ExprKind::Closure(&hir::Closure { body, .. }),
@@ -1738,6 +1813,7 @@ impl<'tcx, 'exprs, E: AsCoercionSite> CoerceMany<'tcx, 'exprs, E> {
         // label pointing out the cause for the type coercion will be wrong
         // as prior return coercions would not be relevant (#57664).
         let fn_decl = if let (Some(expr), Some(blk_id)) = (expression, blk_id) {
+            fcx.suggest_missing_semicolon(&mut err, expr, expected, false);
             let pointing_at_return_type =
                 fcx.suggest_mismatched_types_on_tail(&mut err, expr, expected, found, blk_id);
             if let (Some(cond_expr), true, false) = (
@@ -1772,7 +1848,7 @@ impl<'tcx, 'exprs, E: AsCoercionSite> CoerceMany<'tcx, 'exprs, E> {
             if blk_id.is_none() {
                 fcx.suggest_missing_return_type(
                     &mut err,
-                    &fn_decl,
+                    fn_decl,
                     expected,
                     found,
                     can_suggest,
@@ -1782,7 +1858,7 @@ impl<'tcx, 'exprs, E: AsCoercionSite> CoerceMany<'tcx, 'exprs, E> {
         }
 
         let parent_id = fcx.tcx.hir().get_parent_item(id);
-        let parent_item = fcx.tcx.hir().get_by_def_id(parent_id.def_id);
+        let parent_item = fcx.tcx.hir_node_by_def_id(parent_id.def_id);
 
         if let (Some(expr), Some(_), Some((fn_id, fn_decl, _, _))) =
             (expression, blk_id, fcx.get_node_fn_decl(parent_item))
@@ -1865,12 +1941,12 @@ where
 
 impl AsCoercionSite for ! {
     fn as_coercion_site(&self) -> &hir::Expr<'_> {
-        unreachable!()
+        *self
     }
 }
 
 impl AsCoercionSite for hir::Arm<'_> {
     fn as_coercion_site(&self) -> &hir::Expr<'_> {
-        &self.body
+        self.body
     }
 }
diff --git a/compiler/rustc_hir_typeck/src/demand.rs b/compiler/rustc_hir_typeck/src/demand.rs
index acaa3e02f..8b666c634 100644
--- a/compiler/rustc_hir_typeck/src/demand.rs
+++ b/compiler/rustc_hir_typeck/src/demand.rs
@@ -31,9 +31,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         }
 
         self.annotate_alternative_method_deref(err, expr, error);
+        self.explain_self_literal(err, expr, expected, expr_ty);
 
         // Use `||` to give these suggestions a precedence
         let suggested = self.suggest_missing_parentheses(err, expr)
+            || self.suggest_missing_unwrap_expect(err, expr, expected, expr_ty)
             || self.suggest_remove_last_method_call(err, expr, expected)
             || self.suggest_associated_const(err, expr, expected)
             || self.suggest_deref_ref_or_into(err, expr, expected, expr_ty, expected_ty_expr)
@@ -49,8 +51,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             || self.suggest_into(err, expr, expr_ty, expected)
             || self.suggest_floating_point_literal(err, expr, expected)
             || self.suggest_null_ptr_for_literal_zero_given_to_ptr_arg(err, expr, expected)
-            || self.suggest_coercing_result_via_try_operator(err, expr, expected, expr_ty)
-            || self.suggest_missing_unwrap_expect(err, expr, expected, expr_ty);
+            || self.suggest_coercing_result_via_try_operator(err, expr, expected, expr_ty);
 
         if !suggested {
             self.note_source_of_type_mismatch_constraint(
@@ -183,7 +184,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 self.register_predicates(obligations);
                 None
             }
-            Err(e) => Some(self.err_ctxt().report_mismatched_types(&cause, expected, actual, e)),
+            Err(e) => Some(self.err_ctxt().report_mismatched_types(cause, expected, actual, e)),
         }
     }
 
@@ -255,7 +256,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         self.adjust_expr_for_assert_eq_macro(&mut expr, &mut expected_ty_expr);
 
-        self.set_tainted_by_errors(self.tcx.sess.delay_span_bug(
+        self.set_tainted_by_errors(self.tcx.sess.span_delayed_bug(
             expr.span,
             "`TypeError` when attempting coercion but no error emitted",
         ));
@@ -288,7 +289,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let hir::def::Res::Local(local_hir_id) = p.res else {
             return false;
         };
-        let hir::Node::Pat(pat) = hir.get(local_hir_id) else {
+        let hir::Node::Pat(pat) = self.tcx.hir_node(local_hir_id) else {
             return false;
         };
         let (init_ty_hir_id, init) = match hir.get_parent(pat.hir_id) {
@@ -330,13 +331,15 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             ty_op: |ty| {
                 if let ty::Infer(infer) = ty.kind() {
                     match infer {
-                        ty::InferTy::TyVar(_) => self.next_ty_var(TypeVariableOrigin {
+                        ty::TyVar(_) => self.next_ty_var(TypeVariableOrigin {
                             kind: TypeVariableOriginKind::MiscVariable,
                             span: DUMMY_SP,
                         }),
-                        ty::InferTy::IntVar(_) => self.next_int_var(),
-                        ty::InferTy::FloatVar(_) => self.next_float_var(),
-                        _ => bug!(),
+                        ty::IntVar(_) => self.next_int_var(),
+                        ty::FloatVar(_) => self.next_float_var(),
+                        ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_) => {
+                            bug!("unexpected fresh ty outside of the trait solver")
+                        }
                     }
                 } else {
                     ty
@@ -556,7 +559,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 hir::Node::Stmt(hir::Stmt { kind: hir::StmtKind::Semi(&ref p), .. })
                 | hir::Node::Block(hir::Block { expr: Some(&ref p), .. })
                 | hir::Node::Expr(&ref p),
-            ) = self.tcx.hir().find(parent_id)
+            ) = self.tcx.opt_hir_node(parent_id)
             else {
                 break;
             };
@@ -569,7 +572,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             let mut direct = false;
             loop {
                 // Climb the HIR tree to find the (desugared) `loop` this `break` corresponds to.
-                let parent = match self.tcx.hir().find(parent_id) {
+                let parent = match self.tcx.opt_hir_node(parent_id) {
                     Some(hir::Node::Expr(&ref parent)) => {
                         parent_id = self.tcx.hir().parent_id(parent.hir_id);
                         parent
@@ -671,7 +674,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         error: Option<TypeError<'tcx>>,
     ) {
         let parent = self.tcx.hir().parent_id(expr.hir_id);
-        match (self.tcx.hir().find(parent), error) {
+        match (self.tcx.opt_hir_node(parent), error) {
             (Some(hir::Node::Local(hir::Local { ty: Some(ty), init: Some(init), .. })), _)
                 if init.hir_id == expr.hir_id =>
             {
@@ -716,7 +719,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         None,
                         hir::Path { res: hir::def::Res::Local(hir_id), .. },
                     )) => {
-                        if let Some(hir::Node::Pat(pat)) = self.tcx.hir().find(*hir_id) {
+                        if let Some(hir::Node::Pat(pat)) = self.tcx.opt_hir_node(*hir_id) {
                             primary_span = pat.span;
                             secondary_span = pat.span;
                             match self.tcx.hir().find_parent(pat.hir_id) {
@@ -789,7 +792,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             return;
         };
         let Some(hir::Node::Expr(hir::Expr { kind: hir::ExprKind::Assign(lhs, rhs, _), .. })) =
-            self.tcx.hir().find(parent)
+            self.tcx.opt_hir_node(parent)
         else {
             return;
         };
@@ -861,7 +864,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             mutability,
                         ),
                     ),
-                    match &args[..] {
+                    match &args {
                         [] => (base.span.shrink_to_hi().with_hi(deref.span.hi()), ")".to_string()),
                         [first, ..] => (base.span.between(first.span), ", ".to_string()),
                     },
@@ -882,7 +885,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let container_id = pick.item.container_id(self.tcx);
         let container = with_no_trimmed_paths!(self.tcx.def_path_str(container_id));
         for def_id in pick.import_ids {
-            let hir_id = self.tcx.hir().local_def_id_to_hir_id(def_id);
+            let hir_id = self.tcx.local_def_id_to_hir_id(def_id);
             path_span.push_span_label(
                 self.tcx.hir().span(hir_id),
                 format!("`{container}` imported here"),
@@ -1013,8 +1016,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             hir::Path { res: hir::def::Res::Local(bind_hir_id), .. },
         )) = expr.kind
         {
-            let bind = self.tcx.hir().find(*bind_hir_id);
-            let parent = self.tcx.hir().find(self.tcx.hir().parent_id(*bind_hir_id));
+            let bind = self.tcx.opt_hir_node(*bind_hir_id);
+            let parent = self.tcx.opt_hir_node(self.tcx.hir().parent_id(*bind_hir_id));
             if let Some(hir::Node::Pat(hir::Pat {
                 kind: hir::PatKind::Binding(_, _hir_id, _, _),
                 ..
@@ -1027,6 +1030,59 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         return false;
     }
 
+    fn explain_self_literal(
+        &self,
+        err: &mut Diagnostic,
+        expr: &hir::Expr<'tcx>,
+        expected: Ty<'tcx>,
+        found: Ty<'tcx>,
+    ) {
+        match expr.peel_drop_temps().kind {
+            hir::ExprKind::Struct(
+                hir::QPath::Resolved(
+                    None,
+                    hir::Path { res: hir::def::Res::SelfTyAlias { alias_to, .. }, span, .. },
+                ),
+                ..,
+            )
+            | hir::ExprKind::Call(
+                hir::Expr {
+                    kind:
+                        hir::ExprKind::Path(hir::QPath::Resolved(
+                            None,
+                            hir::Path {
+                                res: hir::def::Res::SelfTyAlias { alias_to, .. },
+                                span,
+                                ..
+                            },
+                        )),
+                    ..
+                },
+                ..,
+            ) => {
+                if let Some(hir::Node::Item(hir::Item {
+                    kind: hir::ItemKind::Impl(hir::Impl { self_ty, .. }),
+                    ..
+                })) = self.tcx.hir().get_if_local(*alias_to)
+                {
+                    err.span_label(self_ty.span, "this is the type of the `Self` literal");
+                }
+                if let ty::Adt(e_def, e_args) = expected.kind()
+                    && let ty::Adt(f_def, _f_args) = found.kind()
+                    && e_def == f_def
+                {
+                    err.span_suggestion_verbose(
+                        *span,
+                        "use the type name directly",
+                        self.tcx.value_path_str_with_args(*alias_to, e_args),
+                        Applicability::MaybeIncorrect,
+                    );
+                }
+            }
+            _ => {}
+        }
+    }
+
     fn note_wrong_return_ty_due_to_generic_arg(
         &self,
         err: &mut Diagnostic,
diff --git a/compiler/rustc_hir_typeck/src/diverges.rs b/compiler/rustc_hir_typeck/src/diverges.rs
index 29fcc61cb..0b559a085 100644
--- a/compiler/rustc_hir_typeck/src/diverges.rs
+++ b/compiler/rustc_hir_typeck/src/diverges.rs
@@ -1,4 +1,4 @@
-use rustc_span::{self, Span, DUMMY_SP};
+use rustc_span::{Span, DUMMY_SP};
 use std::{cmp, ops};
 
 /// Tracks whether executing a node may exit normally (versus
diff --git a/compiler/rustc_hir_typeck/src/errors.rs b/compiler/rustc_hir_typeck/src/errors.rs
index aff1baa19..ff03cf16a 100644
--- a/compiler/rustc_hir_typeck/src/errors.rs
+++ b/compiler/rustc_hir_typeck/src/errors.rs
@@ -573,13 +573,13 @@ impl rustc_errors::AddToDiagnostic for CastUnknownPointerSub {
     {
         match self {
             CastUnknownPointerSub::To(span) => {
-                let msg = f(diag, crate::fluent_generated::hir_typeck_label_to.into());
+                let msg = f(diag, crate::fluent_generated::hir_typeck_label_to);
                 diag.span_label(span, msg);
-                let msg = f(diag, crate::fluent_generated::hir_typeck_note.into());
+                let msg = f(diag, crate::fluent_generated::hir_typeck_note);
                 diag.note(msg);
             }
             CastUnknownPointerSub::From(span) => {
-                let msg = f(diag, crate::fluent_generated::hir_typeck_label_from.into());
+                let msg = f(diag, crate::fluent_generated::hir_typeck_label_from);
                 diag.span_label(span, msg);
             }
         }
@@ -626,7 +626,7 @@ pub struct SuggestConvertViaMethod<'tcx> {
     pub span: Span,
     #[suggestion_part(code = "")]
     pub borrow_removal_span: Option<Span>,
-    pub sugg: &'static str,
+    pub sugg: String,
     pub expected: Ty<'tcx>,
     pub found: Ty<'tcx>,
 }
diff --git a/compiler/rustc_hir_typeck/src/expectation.rs b/compiler/rustc_hir_typeck/src/expectation.rs
index 35e5fb769..ff84e753d 100644
--- a/compiler/rustc_hir_typeck/src/expectation.rs
+++ b/compiler/rustc_hir_typeck/src/expectation.rs
@@ -1,6 +1,6 @@
 use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
 use rustc_middle::ty::{self, Ty};
-use rustc_span::{self, Span};
+use rustc_span::Span;
 
 use super::Expectation::*;
 use super::FnCtxt;
diff --git a/compiler/rustc_hir_typeck/src/expr.rs b/compiler/rustc_hir_typeck/src/expr.rs
index 9f439a2b3..7bd2c3f8b 100644
--- a/compiler/rustc_hir_typeck/src/expr.rs
+++ b/compiler/rustc_hir_typeck/src/expr.rs
@@ -21,7 +21,7 @@ use crate::{
     TupleArgumentsFlag::DontTupleArguments,
 };
 use rustc_ast as ast;
-use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_errors::{
     pluralize, struct_span_err, AddToDiagnostic, Applicability, Diagnostic, DiagnosticBuilder,
@@ -75,7 +75,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         // coercions from ! to `expected`.
         if ty.is_never() {
             if let Some(adjustments) = self.typeck_results.borrow().adjustments().get(expr.hir_id) {
-                let reported = self.tcx().sess.delay_span_bug(
+                let reported = self.tcx().sess.span_delayed_bug(
                     expr.span,
                     "expression with never type wound up being adjusted",
                 );
@@ -277,7 +277,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         let tcx = self.tcx;
         match expr.kind {
-            ExprKind::Lit(ref lit) => self.check_lit(&lit, expected),
+            ExprKind::Lit(ref lit) => self.check_lit(lit, expected),
             ExprKind::Binary(op, lhs, rhs) => self.check_binop(expr, op, lhs, rhs, expected),
             ExprKind::Assign(lhs, rhs, span) => {
                 self.check_expr_assign(expr, expected, lhs, rhs, span)
@@ -289,8 +289,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             ExprKind::AddrOf(kind, mutbl, oprnd) => {
                 self.check_expr_addr_of(kind, mutbl, oprnd, expected, expr)
             }
-            ExprKind::Path(QPath::LangItem(lang_item, _, hir_id)) => {
-                self.check_lang_item_path(lang_item, expr, hir_id)
+            ExprKind::Path(QPath::LangItem(lang_item, _)) => {
+                self.check_lang_item_path(lang_item, expr)
             }
             ExprKind::Path(ref qpath) => self.check_expr_path(qpath, expr, &[]),
             ExprKind::InlineAsm(asm) => {
@@ -298,9 +298,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 self.deferred_asm_checks.borrow_mut().push((asm, expr.hir_id));
                 self.check_expr_asm(asm)
             }
-            ExprKind::OffsetOf(container, ref fields) => {
-                self.check_offset_of(container, fields, expr)
-            }
+            ExprKind::OffsetOf(container, fields) => self.check_offset_of(container, fields, expr),
             ExprKind::Break(destination, ref expr_opt) => {
                 self.check_expr_break(destination, expr_opt.as_deref(), expr)
             }
@@ -319,17 +317,17 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 self.check_expr_loop(body, source, expected, expr)
             }
             ExprKind::Match(discrim, arms, match_src) => {
-                self.check_match(expr, &discrim, arms, expected, match_src)
+                self.check_match(expr, discrim, arms, expected, match_src)
             }
             ExprKind::Closure(closure) => self.check_expr_closure(closure, expr.span, expected),
-            ExprKind::Block(body, _) => self.check_block_with_expected(&body, expected),
-            ExprKind::Call(callee, args) => self.check_call(expr, &callee, args, expected),
+            ExprKind::Block(body, _) => self.check_block_with_expected(body, expected),
+            ExprKind::Call(callee, args) => self.check_call(expr, callee, args, expected),
             ExprKind::MethodCall(segment, receiver, args, _) => {
                 self.check_method_call(expr, segment, receiver, args, expected)
             }
             ExprKind::Cast(e, t) => self.check_expr_cast(e, t, expr),
             ExprKind::Type(e, t) => {
-                let ascribed_ty = self.to_ty_saving_user_provided_ty(&t);
+                let ascribed_ty = self.to_ty_saving_user_provided_ty(t);
                 let ty = self.check_expr_with_hint(e, ascribed_ty);
                 self.demand_eqtype(e.span, ascribed_ty, ty);
                 ascribed_ty
@@ -347,7 +345,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             ExprKind::Struct(qpath, fields, ref base_expr) => {
                 self.check_expr_struct(expr, expected, qpath, fields, base_expr)
             }
-            ExprKind::Field(base, field) => self.check_field(expr, &base, field, expected),
+            ExprKind::Field(base, field) => self.check_field(expr, base, field, expected),
             ExprKind::Index(base, idx, brackets_span) => {
                 self.check_expr_index(base, idx, expr, brackets_span)
             }
@@ -368,7 +366,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             hir::UnOp::Not | hir::UnOp::Neg => expected,
             hir::UnOp::Deref => NoExpectation,
         };
-        let mut oprnd_t = self.check_expr_with_expectation(&oprnd, expected_inner);
+        let mut oprnd_t = self.check_expr_with_expectation(oprnd, expected_inner);
 
         if !oprnd_t.references_error() {
             oprnd_t = self.structurally_resolve_type(expr.span, oprnd_t);
@@ -436,7 +434,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             }
         });
         let ty =
-            self.check_expr_with_expectation_and_needs(&oprnd, hint, Needs::maybe_mut_place(mutbl));
+            self.check_expr_with_expectation_and_needs(oprnd, hint, Needs::maybe_mut_place(mutbl));
 
         let tm = ty::TypeAndMut { ty, mutbl };
         match kind {
@@ -499,9 +497,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         &self,
         lang_item: hir::LangItem,
         expr: &'tcx hir::Expr<'tcx>,
-        hir_id: Option<hir::HirId>,
     ) -> Ty<'tcx> {
-        self.resolve_lang_item_path(lang_item, expr.span, expr.hir_id, hir_id).1
+        self.resolve_lang_item_path(lang_item, expr.span, expr.hir_id).1
     }
 
     pub(crate) fn check_expr_path(
@@ -517,7 +514,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             Res::Err => {
                 self.suggest_assoc_method_call(segs);
                 let e =
-                    self.tcx.sess.delay_span_bug(qpath.span(), "`Res::Err` but no error emitted");
+                    self.tcx.sess.span_delayed_bug(qpath.span(), "`Res::Err` but no error emitted");
                 self.set_tainted_by_errors(e);
                 Ty::new_error(tcx, e)
             }
@@ -528,14 +525,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             _ => self.instantiate_value_path(segs, opt_ty, res, expr.span, expr.hir_id).0,
         };
 
-        if let ty::FnDef(did, callee_args) = *ty.kind() {
+        if let ty::FnDef(did, _) = *ty.kind() {
             let fn_sig = ty.fn_sig(tcx);
 
-            // HACK: whenever we get a FnDef in a non-const context, enforce effects to get the
-            // default `host = true` to avoid inference errors later.
-            if tcx.hir().body_const_context(self.body_id).is_none() {
-                self.enforce_context_effects(expr.hir_id, qpath.span(), did, callee_args);
-            }
             if tcx.fn_sig(did).skip_binder().abi() == RustIntrinsic
                 && tcx.item_name(did) == sym::transmute
             {
@@ -564,7 +556,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     let span = args.get(i).map(|a| a.span).unwrap_or(expr.span);
                     let input = self.instantiate_binder_with_fresh_vars(
                         span,
-                        infer::LateBoundRegionConversionTime::FnCall,
+                        infer::BoundRegionConversionTime::FnCall,
                         fn_sig.input(i),
                     );
                     self.require_type_is_sized_deferred(
@@ -582,7 +574,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             // with fresh vars.
             let output = self.instantiate_binder_with_fresh_vars(
                 expr.span,
-                infer::LateBoundRegionConversionTime::FnCall,
+                infer::BoundRegionConversionTime::FnCall,
                 fn_sig.output(),
             );
             self.require_type_is_sized_deferred(output, expr.span, traits::SizedReturnType);
@@ -626,15 +618,18 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     }
                 };
 
-                let coerce_to = match opt_coerce_to {
-                    Some(c) => c,
-                    None => {
-                        // If the loop context is not a `loop { }`, then break with
-                        // a value is illegal, and `opt_coerce_to` will be `None`.
-                        // Return error in that case (#114529).
-                        return Ty::new_misc_error(tcx);
-                    }
-                };
+                // If the loop context is not a `loop { }`, then break with
+                // a value is illegal, and `opt_coerce_to` will be `None`.
+                // Set expectation to error in that case and set tainted
+                // by error (#114529)
+                let coerce_to = opt_coerce_to.unwrap_or_else(|| {
+                    let guar = tcx.sess.span_delayed_bug(
+                        expr.span,
+                        "illegal break with value found but no error reported",
+                    );
+                    self.set_tainted_by_errors(guar);
+                    Ty::new_error(tcx, guar)
+                });
 
                 // Recurse without `enclosing_breakables` borrowed.
                 e_ty = self.check_expr_with_hint(e, coerce_to);
@@ -660,7 +655,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             };
 
             if let Some(ref mut coerce) = ctxt.coerce {
-                if let Some(ref e) = expr_opt {
+                if let Some(e) = expr_opt {
                     coerce.coerce(self, &cause, e, e_ty);
                 } else {
                     assert!(e_ty.is_unit());
@@ -669,11 +664,12 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         self,
                         &cause,
                         |mut err| {
+                            self.suggest_missing_semicolon(&mut err, expr, e_ty, false);
                             self.suggest_mismatched_types_on_tail(
                                 &mut err, expr, ty, e_ty, target_id,
                             );
                             let error = Some(Sorts(ExpectedFound { expected: ty, found: e_ty }));
-                            self.annotate_loop_expected_due_to_inference(&mut err, expr, error);
+                            self.annotate_loop_expected_due_to_inference(err, expr, error);
                             if let Some(val) = ty_kind_suggestion(ty) {
                                 err.span_suggestion_verbose(
                                     expr.span.shrink_to_hi(),
@@ -887,7 +883,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             kind: hir::ImplItemKind::Fn(..),
             span: encl_fn_span,
             ..
-        })) = self.tcx.hir().find_by_def_id(encl_item_id.def_id)
+        })) = self.tcx.opt_hir_node_by_def_id(encl_item_id.def_id)
         {
             // We are inside a function body, so reporting "return statement
             // outside of function body" needs an explanation.
@@ -1001,7 +997,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         then: impl FnOnce(&hir::Expr<'_>),
     ) {
         let mut parent = self.tcx.hir().parent_id(original_expr_id);
-        while let Some(node) = self.tcx.hir().find(parent) {
+        while let Some(node) = self.tcx.opt_hir_node(parent) {
             match node {
                 hir::Node::Expr(hir::Expr {
                     kind:
@@ -1133,8 +1129,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             // The likely cause of this is `if foo = bar { .. }`.
             let actual_ty = Ty::new_unit(self.tcx);
             let mut err = self.demand_suptype_diag(expr.span, expected_ty, actual_ty).unwrap();
-            let lhs_ty = self.check_expr(&lhs);
-            let rhs_ty = self.check_expr(&rhs);
+            let lhs_ty = self.check_expr(lhs);
+            let rhs_ty = self.check_expr(rhs);
             let (applicability, eq) = if self.can_coerce(rhs_ty, lhs_ty) {
                 (Applicability::MachineApplicable, true)
             } else if let ExprKind::Binary(
@@ -1145,7 +1141,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             {
                 // if x == 1 && y == 2 { .. }
                 //                 +
-                let actual_lhs_ty = self.check_expr(&rhs_expr);
+                let actual_lhs_ty = self.check_expr(rhs_expr);
                 (Applicability::MaybeIncorrect, self.can_coerce(rhs_ty, actual_lhs_ty))
             } else if let ExprKind::Binary(
                 Spanned { node: hir::BinOpKind::And | hir::BinOpKind::Or, .. },
@@ -1155,7 +1151,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             {
                 // if x == 1 && y == 2 { .. }
                 //       +
-                let actual_rhs_ty = self.check_expr(&lhs_expr);
+                let actual_rhs_ty = self.check_expr(lhs_expr);
                 (Applicability::MaybeIncorrect, self.can_coerce(actual_rhs_ty, lhs_ty))
             } else {
                 (Applicability::MaybeIncorrect, false)
@@ -1192,7 +1188,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             return Ty::new_error(self.tcx, reported);
         }
 
-        let lhs_ty = self.check_expr_with_needs(&lhs, Needs::MutPlace);
+        let lhs_ty = self.check_expr_with_needs(lhs, Needs::MutPlace);
 
         let suggest_deref_binop = |err: &mut Diagnostic, rhs_ty: Ty<'tcx>| {
             if let Some(lhs_deref_ty) = self.deref_once_mutably_for_diagnostic(lhs_ty) {
@@ -1219,7 +1215,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         // This is (basically) inlined `check_expr_coercible_to_type`, but we want
         // to suggest an additional fixup here in `suggest_deref_binop`.
-        let rhs_ty = self.check_expr_with_hint(&rhs, lhs_ty);
+        let rhs_ty = self.check_expr_with_hint(rhs, lhs_ty);
         if let (_, Some(mut diag)) =
             self.demand_coerce_diag(rhs, rhs_ty, lhs_ty, Some(lhs), AllowTwoPhase::No)
         {
@@ -1280,7 +1276,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         };
 
         let (ctxt, ()) = self.with_breakable_ctxt(expr.hir_id, ctxt, || {
-            self.check_block_no_value(&body);
+            self.check_block_no_value(body);
         });
 
         if ctxt.may_break {
@@ -1296,7 +1292,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         // permit break with a value [1].
         if ctxt.coerce.is_none() && !ctxt.may_break {
             // [1]
-            self.tcx.sess.delay_span_bug(body.span, "no coercion, but loop may not break");
+            self.tcx.sess.span_delayed_bug(body.span, "no coercion, but loop may not break");
         }
         ctxt.coerce.map(|c| c.complete(self)).unwrap_or_else(|| Ty::new_unit(self.tcx))
     }
@@ -1310,7 +1306,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         args: &'tcx [hir::Expr<'tcx>],
         expected: Expectation<'tcx>,
     ) -> Ty<'tcx> {
-        let rcvr_t = self.check_expr(&rcvr);
+        let rcvr_t = self.check_expr(rcvr);
         // no need to check for bot/err -- callee does that
         let rcvr_t = self.structurally_resolve_type(rcvr.span, rcvr_t);
         let span = segment.ident.span;
@@ -1319,9 +1315,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             Ok(method) => {
                 // We could add a "consider `foo::<params>`" suggestion here, but I wasn't able to
                 // trigger this codepath causing `structurally_resolve_type` to emit an error.
-
-                self.enforce_context_effects(expr.hir_id, expr.span, method.def_id, method.args);
-                self.write_method_call(expr.hir_id, method);
+                self.write_method_call_and_enforce_effects(expr.hir_id, expr.span, method);
                 Ok(method)
             }
             Err(error) => {
@@ -1344,7 +1338,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         };
 
         // Call the generic checker.
-        self.check_method_argument_types(span, expr, method, &args, DontTupleArguments, expected)
+        self.check_method_argument_types(span, expr, method, args, DontTupleArguments, expected)
     }
 
     fn check_expr_cast(
@@ -1442,12 +1436,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             && let hir::ArrayLen::Body(hir::AnonConst { hir_id, .. }) = length
             && let Some(span) = self.tcx.hir().opt_span(hir_id)
         {
-            match self
-                .tcx
-                .sess
-                .diagnostic()
-                .steal_diagnostic(span, StashKey::UnderscoreForArrayLengths)
-            {
+            match self.tcx.sess.dcx().steal_diagnostic(span, StashKey::UnderscoreForArrayLengths) {
                 Some(mut err) => {
                     err.span_suggestion(
                         span,
@@ -1475,7 +1464,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let fcx = FnCtxt::new(self, self.param_env, def_id);
         crate::GatherLocalsVisitor::new(&fcx).visit_body(body);
 
-        let ty = fcx.check_expr_with_expectation(&body.value, expected);
+        let ty = fcx.check_expr_with_expectation(body.value, expected);
         fcx.require_type_is_sized(ty, body.value.span, traits::ConstSized);
         fcx.write_ty(block.hir_id, ty);
         ty
@@ -1504,7 +1493,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         let (element_ty, t) = match uty {
             Some(uty) => {
-                self.check_expr_coercible_to_type(&element, uty, None);
+                self.check_expr_coercible_to_type(element, uty, None);
                 (uty, uty)
             }
             None => {
@@ -1512,7 +1501,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     kind: TypeVariableOriginKind::MiscVariable,
                     span: element.span,
                 });
-                let element_ty = self.check_expr_has_type_or_error(&element, ty, |_| {});
+                let element_ty = self.check_expr_has_type_or_error(element, ty, |_| {});
                 (element_ty, ty)
             }
         };
@@ -1608,10 +1597,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let elt_ts_iter = elts.iter().enumerate().map(|(i, e)| match flds {
             Some(fs) if i < fs.len() => {
                 let ety = fs[i];
-                self.check_expr_coercible_to_type(&e, ety, None);
+                self.check_expr_coercible_to_type(e, ety, None);
                 ety
             }
-            _ => self.check_expr_with_expectation(&e, NoExpectation),
+            _ => self.check_expr_with_expectation(e, NoExpectation),
         });
         let tuple = Ty::new_tup_from_iter(self.tcx, elt_ts_iter);
         if let Err(guar) = tuple.error_reported() {
@@ -1737,9 +1726,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
             // Make sure to give a type to the field even if there's
             // an error, so we can continue type-checking.
-            let ty = self.check_expr_with_hint(&field.expr, field_type);
+            let ty = self.check_expr_with_hint(field.expr, field_type);
             let (_, diag) =
-                self.demand_coerce_diag(&field.expr, ty, field_type, None, AllowTwoPhase::No);
+                self.demand_coerce_diag(field.expr, ty, field_type, None, AllowTwoPhase::No);
 
             if let Some(mut diag) = diag {
                 if idx == ast_fields.len() - 1 {
@@ -1897,7 +1886,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 .collect();
 
             if !private_fields.is_empty() {
-                self.report_private_fields(adt_ty, span, private_fields, ast_fields);
+                self.report_private_fields(adt_ty, span, expr.span, private_fields, ast_fields);
             } else {
                 self.report_missing_fields(
                     adt_ty,
@@ -1917,10 +1906,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         base_expr: &'tcx Option<&'tcx hir::Expr<'tcx>>,
     ) {
         for field in fields {
-            self.check_expr(&field.expr);
+            self.check_expr(field.expr);
         }
         if let Some(base) = *base_expr {
-            self.check_expr(&base);
+            self.check_expr(base);
         }
     }
 
@@ -1933,7 +1922,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     /// 8 |     foo::Foo {};
     ///   |     ^^^^^^^^ missing `you_can_use_this_field`
     ///
-    /// error: aborting due to previous error
+    /// error: aborting due to 1 previous error
     /// ```
     fn report_missing_fields(
         &self,
@@ -2008,11 +1997,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 != range_def_id
         {
             // Suppress any range expr type mismatches
-            if let Some(mut diag) = self
-                .tcx
-                .sess
-                .diagnostic()
-                .steal_diagnostic(last_expr_field.span, StashKey::MaybeFruTypo)
+            if let Some(mut diag) =
+                self.tcx.sess.dcx().steal_diagnostic(last_expr_field.span, StashKey::MaybeFruTypo)
             {
                 diag.delay_as_bug();
             }
@@ -2050,12 +2036,13 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     /// 8 |     foo::Foo {};
     ///   |     ^^^^^^^^
     ///
-    /// error: aborting due to previous error
+    /// error: aborting due to 1 previous error
     /// ```
     fn report_private_fields(
         &self,
         adt_ty: Ty<'tcx>,
         span: Span,
+        expr_span: Span,
         private_fields: Vec<&ty::FieldDef>,
         used_fields: &'tcx [hir::ExprField<'tcx>],
     ) {
@@ -2092,14 +2079,90 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     let names = names.iter().map(|name| format!("`{name}`")).collect::<Vec<_>>();
                     format!("{} and `{last}` ", names.join(", "))
                 }
-                [] => unreachable!(),
+                [] => bug!("expected at least one private field to report"),
             };
             err.note(format!(
-                "... and other private field{s} {names}that {were} not provided",
+                "{}private field{s} {names}that {were} not provided",
+                if used_fields.is_empty() { "" } else { "...and other " },
                 s = pluralize!(remaining_private_fields_len),
                 were = pluralize!("was", remaining_private_fields_len),
             ));
         }
+
+        if let ty::Adt(def, _) = adt_ty.kind() {
+            let def_id = def.did();
+            let mut items = self
+                .tcx
+                .inherent_impls(def_id)
+                .iter()
+                .flat_map(|i| self.tcx.associated_items(i).in_definition_order())
+                // Only assoc fn with no receivers.
+                .filter(|item| {
+                    matches!(item.kind, ty::AssocKind::Fn) && !item.fn_has_self_parameter
+                })
+                .filter_map(|item| {
+                    // Only assoc fns that return `Self`
+                    let fn_sig = self.tcx.fn_sig(item.def_id).skip_binder();
+                    let ret_ty = fn_sig.output();
+                    let ret_ty =
+                        self.tcx.normalize_erasing_late_bound_regions(self.param_env, ret_ty);
+                    if !self.can_eq(self.param_env, ret_ty, adt_ty) {
+                        return None;
+                    }
+                    let input_len = fn_sig.inputs().skip_binder().len();
+                    let order = !item.name.as_str().starts_with("new");
+                    Some((order, item.name, input_len))
+                })
+                .collect::<Vec<_>>();
+            items.sort_by_key(|(order, _, _)| *order);
+            let suggestion = |name, args| {
+                format!(
+                    "::{name}({})",
+                    std::iter::repeat("_").take(args).collect::<Vec<_>>().join(", ")
+                )
+            };
+            match &items[..] {
+                [] => {}
+                [(_, name, args)] => {
+                    err.span_suggestion_verbose(
+                        span.shrink_to_hi().with_hi(expr_span.hi()),
+                        format!("you might have meant to use the `{name}` associated function"),
+                        suggestion(name, *args),
+                        Applicability::MaybeIncorrect,
+                    );
+                }
+                _ => {
+                    err.span_suggestions(
+                        span.shrink_to_hi().with_hi(expr_span.hi()),
+                        "you might have meant to use an associated function to build this type",
+                        items
+                            .iter()
+                            .map(|(_, name, args)| suggestion(name, *args))
+                            .collect::<Vec<String>>(),
+                        Applicability::MaybeIncorrect,
+                    );
+                }
+            }
+            if let Some(default_trait) = self.tcx.get_diagnostic_item(sym::Default)
+                && self
+                    .infcx
+                    .type_implements_trait(default_trait, [adt_ty], self.param_env)
+                    .may_apply()
+            {
+                err.multipart_suggestion(
+                    "consider using the `Default` trait",
+                    vec![
+                        (span.shrink_to_lo(), "<".to_string()),
+                        (
+                            span.shrink_to_hi().with_hi(expr_span.hi()),
+                            " as std::default::Default>::default()".to_string(),
+                        ),
+                    ],
+                    Applicability::MaybeIncorrect,
+                );
+            }
+        }
+
         err.emit();
     }
 
@@ -2116,7 +2179,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             let guar = self
                 .tcx
                 .sess
-                .delay_span_bug(expr.span, "parser recovered but no error was emitted");
+                .span_delayed_bug(expr.span, "parser recovered but no error was emitted");
             self.set_tainted_by_errors(guar);
             return guar;
         }
@@ -2191,7 +2254,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 if let Some(field_name) =
                     find_best_match_for_name(&available_field_names, field.ident.name, None)
                 {
-                    err.span_suggestion(
+                    err.span_label(field.ident.span, "unknown field");
+                    err.span_suggestion_verbose(
                         field.ident.span,
                         "a field with a similar name exists",
                         field_name,
@@ -2274,7 +2338,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             match deref_base_ty.kind() {
                 ty::Adt(base_def, args) if !base_def.is_enum() => {
                     debug!("struct named {:?}", deref_base_ty);
-                    let body_hir_id = self.tcx.hir().local_def_id_to_hir_id(self.body_id);
+                    let body_hir_id = self.tcx.local_def_id_to_hir_id(self.body_id);
                     let (ident, def_scope) =
                         self.tcx.adjust_ident_and_get_scope(field, base_def.did(), body_hir_id);
                     let fields = &base_def.non_enum_variant().fields;
@@ -2331,7 +2395,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         }
 
         let guar = if field.name == kw::Empty {
-            self.tcx.sess.delay_span_bug(field.span, "field name with no name")
+            self.tcx.sess.span_delayed_bug(field.span, "field name with no name")
         } else if self.method_exists(field, base_ty, expr.hir_id, expected.only_has_type(self)) {
             self.ban_take_value_of_method(expr, base_ty, field)
         } else if !base_ty.is_primitive_ty() {
@@ -2420,35 +2484,32 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         ty: Ty<'tcx>,
     ) {
         let Some(output_ty) = self.get_impl_future_output_ty(ty) else {
+            err.span_label(field_ident.span, "unknown field");
             return;
         };
-        let mut add_label = true;
-        if let ty::Adt(def, _) = output_ty.kind() {
-            // no field access on enum type
-            if !def.is_enum() {
-                if def
-                    .non_enum_variant()
-                    .fields
-                    .iter()
-                    .any(|field| field.ident(self.tcx) == field_ident)
-                {
-                    add_label = false;
-                    err.span_label(
-                        field_ident.span,
-                        "field not available in `impl Future`, but it is available in its `Output`",
-                    );
-                    err.span_suggestion_verbose(
-                        base.span.shrink_to_hi(),
-                        "consider `await`ing on the `Future` and access the field of its `Output`",
-                        ".await",
-                        Applicability::MaybeIncorrect,
-                    );
-                }
-            }
+        let ty::Adt(def, _) = output_ty.kind() else {
+            err.span_label(field_ident.span, "unknown field");
+            return;
+        };
+        // no field access on enum type
+        if def.is_enum() {
+            err.span_label(field_ident.span, "unknown field");
+            return;
         }
-        if add_label {
-            err.span_label(field_ident.span, format!("field not found in `{ty}`"));
+        if !def.non_enum_variant().fields.iter().any(|field| field.ident(self.tcx) == field_ident) {
+            err.span_label(field_ident.span, "unknown field");
+            return;
         }
+        err.span_label(
+            field_ident.span,
+            "field not available in `impl Future`, but it is available in its `Output`",
+        );
+        err.span_suggestion_verbose(
+            base.span.shrink_to_hi(),
+            "consider `await`ing on the `Future` and access the field of its `Output`",
+            ".await",
+            Applicability::MaybeIncorrect,
+        );
     }
 
     fn ban_nonexisting_field(
@@ -2471,16 +2532,16 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             ty::RawPtr(..) => {
                 self.suggest_first_deref_field(&mut err, expr, base, ident);
             }
-            ty::Adt(def, _) if !def.is_enum() => {
-                self.suggest_fields_on_recordish(&mut err, expr, def, ident);
-            }
             ty::Param(param_ty) => {
+                err.span_label(ident.span, "unknown field");
                 self.point_at_param_definition(&mut err, param_ty);
             }
             ty::Alias(ty::Opaque, _) => {
                 self.suggest_await_on_field_access(&mut err, ident, base, base_ty.peel_refs());
             }
-            _ => {}
+            _ => {
+                err.span_label(ident.span, "unknown field");
+            }
         }
 
         self.suggest_fn_call(&mut err, base, base_ty, |output_ty| {
@@ -2624,7 +2685,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         }
         let param_def_id = generic_param.def_id;
         let param_hir_id = match param_def_id.as_local() {
-            Some(x) => self.tcx.hir().local_def_id_to_hir_id(x),
+            Some(x) => self.tcx.local_def_id_to_hir_id(x),
             None => return,
         };
         let param_span = self.tcx.hir().span(param_hir_id);
@@ -2633,34 +2694,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         err.span_label(param_span, format!("type parameter '{param_name}' declared here"));
     }
 
-    fn suggest_fields_on_recordish(
-        &self,
-        err: &mut Diagnostic,
-        expr: &hir::Expr<'_>,
-        def: ty::AdtDef<'tcx>,
-        field: Ident,
-    ) {
-        let available_field_names = self.available_field_names(def.non_enum_variant(), expr, &[]);
-        if let Some(suggested_field_name) =
-            find_best_match_for_name(&available_field_names, field.name, None)
-        {
-            err.span_suggestion(
-                field.span,
-                "a field with a similar name exists",
-                suggested_field_name,
-                Applicability::MaybeIncorrect,
-            );
-        } else {
-            err.span_label(field.span, "unknown field");
-            if !available_field_names.is_empty() {
-                err.note(format!(
-                    "available fields are: {}",
-                    self.name_series_display(available_field_names),
-                ));
-            }
-        }
-    }
-
     fn maybe_suggest_array_indexing(
         &self,
         err: &mut Diagnostic,
@@ -2669,6 +2702,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         field: Ident,
         len: ty::Const<'tcx>,
     ) {
+        err.span_label(field.span, "unknown field");
         if let (Some(len), Ok(user_index)) =
             (len.try_eval_target_usize(self.tcx, self.param_env), field.as_str().parse::<u64>())
             && let Ok(base) = self.tcx.sess.source_map().span_to_snippet(base.span)
@@ -2691,6 +2725,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         base: &hir::Expr<'_>,
         field: Ident,
     ) {
+        err.span_label(field.span, "unknown field");
         if let Ok(base) = self.tcx.sess.source_map().span_to_snippet(base.span) {
             let msg = format!("`{base}` is a raw pointer; try dereferencing it");
             let suggestion = format!("(*{base}).{field}");
@@ -2709,7 +2744,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         let mut err = type_error_struct!(
             self.tcx().sess,
-            field.span,
+            span,
             expr_t,
             E0609,
             "no field `{field}` on type `{expr_t}`",
@@ -2717,10 +2752,22 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         // try to add a suggestion in case the field is a nested field of a field of the Adt
         let mod_id = self.tcx.parent_module(id).to_def_id();
-        if let Some((fields, args)) =
-            self.get_field_candidates_considering_privacy(span, expr_t, mod_id)
+        let (ty, unwrap) = if let ty::Adt(def, args) = expr_t.kind()
+            && (self.tcx.is_diagnostic_item(sym::Result, def.did())
+                || self.tcx.is_diagnostic_item(sym::Option, def.did()))
+            && let Some(arg) = args.get(0)
+            && let Some(ty) = arg.as_type()
+        {
+            (ty, "unwrap().")
+        } else {
+            (expr_t, "")
+        };
+        for (found_fields, args) in
+            self.get_field_candidates_considering_privacy(span, ty, mod_id, id)
         {
-            let candidate_fields: Vec<_> = fields
+            let field_names = found_fields.iter().map(|field| field.name).collect::<Vec<_>>();
+            let mut candidate_fields: Vec<_> = found_fields
+                .into_iter()
                 .filter_map(|candidate_field| {
                     self.check_for_nested_field_satisfying(
                         span,
@@ -2729,17 +2776,18 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         args,
                         vec![],
                         mod_id,
+                        id,
                     )
                 })
                 .map(|mut field_path| {
                     field_path.pop();
                     field_path
                         .iter()
-                        .map(|id| id.name.to_ident_string())
-                        .collect::<Vec<String>>()
-                        .join(".")
+                        .map(|id| format!("{}.", id.name.to_ident_string()))
+                        .collect::<String>()
                 })
                 .collect::<Vec<_>>();
+            candidate_fields.sort();
 
             let len = candidate_fields.len();
             if len > 0 {
@@ -2750,9 +2798,24 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         if len > 1 { "some" } else { "one" },
                         if len > 1 { "have" } else { "has" },
                     ),
-                    candidate_fields.iter().map(|path| format!("{path}.")),
+                    candidate_fields.iter().map(|path| format!("{unwrap}{path}")),
                     Applicability::MaybeIncorrect,
                 );
+            } else {
+                if let Some(field_name) = find_best_match_for_name(&field_names, field.name, None) {
+                    err.span_suggestion_verbose(
+                        field.span,
+                        "a field with a similar name exists",
+                        format!("{unwrap}{}", field_name),
+                        Applicability::MaybeIncorrect,
+                    );
+                } else if !field_names.is_empty() {
+                    let is = if field_names.len() == 1 { " is" } else { "s are" };
+                    err.note(format!(
+                        "available field{is}: {}",
+                        self.name_series_display(field_names),
+                    ));
+                }
             }
         }
         err
@@ -2781,33 +2844,39 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         span: Span,
         base_ty: Ty<'tcx>,
         mod_id: DefId,
-    ) -> Option<(impl Iterator<Item = &'tcx ty::FieldDef> + 'tcx, GenericArgsRef<'tcx>)> {
+        hir_id: hir::HirId,
+    ) -> Vec<(Vec<&'tcx ty::FieldDef>, GenericArgsRef<'tcx>)> {
         debug!("get_field_candidates(span: {:?}, base_t: {:?}", span, base_ty);
 
-        for (base_t, _) in self.autoderef(span, base_ty) {
-            match base_t.kind() {
-                ty::Adt(base_def, args) if !base_def.is_enum() => {
-                    let tcx = self.tcx;
-                    let fields = &base_def.non_enum_variant().fields;
-                    // Some struct, e.g. some that impl `Deref`, have all private fields
-                    // because you're expected to deref them to access the _real_ fields.
-                    // This, for example, will help us suggest accessing a field through a `Box<T>`.
-                    if fields.iter().all(|field| !field.vis.is_accessible_from(mod_id, tcx)) {
-                        continue;
+        self.autoderef(span, base_ty)
+            .filter_map(move |(base_t, _)| {
+                match base_t.kind() {
+                    ty::Adt(base_def, args) if !base_def.is_enum() => {
+                        let tcx = self.tcx;
+                        let fields = &base_def.non_enum_variant().fields;
+                        // Some struct, e.g. some that impl `Deref`, have all private fields
+                        // because you're expected to deref them to access the _real_ fields.
+                        // This, for example, will help us suggest accessing a field through a `Box<T>`.
+                        if fields.iter().all(|field| !field.vis.is_accessible_from(mod_id, tcx)) {
+                            return None;
+                        }
+                        return Some((
+                            fields
+                                .iter()
+                                .filter(move |field| {
+                                    field.vis.is_accessible_from(mod_id, tcx)
+                                        && self.is_field_suggestable(field, hir_id, span)
+                                })
+                                // For compile-time reasons put a limit on number of fields we search
+                                .take(100)
+                                .collect::<Vec<_>>(),
+                            *args,
+                        ));
                     }
-                    return Some((
-                        fields
-                            .iter()
-                            .filter(move |field| field.vis.is_accessible_from(mod_id, tcx))
-                            // For compile-time reasons put a limit on number of fields we search
-                            .take(100),
-                        args,
-                    ));
+                    _ => None,
                 }
-                _ => {}
-            }
-        }
-        None
+            })
+            .collect()
     }
 
     /// This method is called after we have encountered a missing field error to recursively
@@ -2820,6 +2889,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         subst: GenericArgsRef<'tcx>,
         mut field_path: Vec<Ident>,
         mod_id: DefId,
+        hir_id: HirId,
     ) -> Option<Vec<Ident>> {
         debug!(
             "check_for_nested_field_satisfying(span: {:?}, candidate_field: {:?}, field_path: {:?}",
@@ -2835,20 +2905,23 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             let field_ty = candidate_field.ty(self.tcx, subst);
             if matches(candidate_field, field_ty) {
                 return Some(field_path);
-            } else if let Some((nested_fields, subst)) =
-                self.get_field_candidates_considering_privacy(span, field_ty, mod_id)
-            {
-                // recursively search fields of `candidate_field` if it's a ty::Adt
-                for field in nested_fields {
-                    if let Some(field_path) = self.check_for_nested_field_satisfying(
-                        span,
-                        matches,
-                        field,
-                        subst,
-                        field_path.clone(),
-                        mod_id,
-                    ) {
-                        return Some(field_path);
+            } else {
+                for (nested_fields, subst) in
+                    self.get_field_candidates_considering_privacy(span, field_ty, mod_id, hir_id)
+                {
+                    // recursively search fields of `candidate_field` if it's a ty::Adt
+                    for field in nested_fields {
+                        if let Some(field_path) = self.check_for_nested_field_satisfying(
+                            span,
+                            matches,
+                            field,
+                            subst,
+                            field_path.clone(),
+                            mod_id,
+                            hir_id,
+                        ) {
+                            return Some(field_path);
+                        }
                     }
                 }
             }
@@ -2863,8 +2936,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         expr: &'tcx hir::Expr<'tcx>,
         brackets_span: Span,
     ) -> Ty<'tcx> {
-        let base_t = self.check_expr(&base);
-        let idx_t = self.check_expr(&idx);
+        let base_t = self.check_expr(base);
+        let idx_t = self.check_expr(idx);
 
         if base_t.references_error() {
             base_t
@@ -2877,7 +2950,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     // two-phase not needed because index_ty is never mutable
                     self.demand_coerce(idx, idx_t, index_ty, None, AllowTwoPhase::No);
                     self.select_obligations_where_possible(|errors| {
-                        self.point_at_index_if_possible(errors, idx.span)
+                        self.point_at_index(errors, idx.span);
                     });
                     element_ty
                 }
@@ -2905,7 +2978,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         let mut needs_note = true;
                         // If the index is an integer, we can show the actual
                         // fixed expression:
-                        if let ExprKind::Lit(ref lit) = idx.kind
+                        if let ExprKind::Lit(lit) = idx.kind
                             && let ast::LitKind::Int(i, ast::LitIntType::Unsuffixed) = lit.node
                             && i < types
                                 .len()
@@ -2978,7 +3051,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             return None;
         };
 
-        self.commit_if_ok(|_| {
+        self.commit_if_ok(|snapshot| {
+            let outer_universe = self.universe();
+
             let ocx = ObligationCtxt::new(self);
             let impl_args = self.fresh_args_for_item(base_expr.span, impl_def_id);
             let impl_trait_ref =
@@ -2988,7 +3063,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             // Match the impl self type against the base ty. If this fails,
             // we just skip this impl, since it's not particularly useful.
             let impl_trait_ref = ocx.normalize(&cause, self.param_env, impl_trait_ref);
-            ocx.eq(&cause, self.param_env, impl_trait_ref.self_ty(), base_ty)?;
+            ocx.eq(&cause, self.param_env, base_ty, impl_trait_ref.self_ty())?;
 
             // Register the impl's predicates. One of these predicates
             // must be unsatisfied, or else we wouldn't have gotten here
@@ -3024,11 +3099,23 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 Ty::new_projection(self.tcx, index_trait_output_def_id, impl_trait_ref.args),
             );
 
-            let errors = ocx.select_where_possible();
+            let true_errors = ocx.select_where_possible();
+
+            // Do a leak check -- we can't really report report a useful error here,
+            // but it at least avoids an ICE when the error has to do with higher-ranked
+            // lifetimes.
+            self.leak_check(outer_universe, Some(snapshot))?;
+
+            // Bail if we have ambiguity errors, which we can't report in a useful way.
+            let ambiguity_errors = ocx.select_all_or_error();
+            if true_errors.is_empty() && !ambiguity_errors.is_empty() {
+                return Err(NoSolution);
+            }
+
             // There should be at least one error reported. If not, we
             // will still delay a span bug in `report_fulfillment_errors`.
             Ok::<_, NoSolution>((
-                self.err_ctxt().report_fulfillment_errors(errors),
+                self.err_ctxt().report_fulfillment_errors(true_errors),
                 impl_trait_ref.args.type_at(1),
                 element_ty,
             ))
@@ -3036,16 +3123,28 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         .ok()
     }
 
-    fn point_at_index_if_possible(
-        &self,
-        errors: &mut Vec<traits::FulfillmentError<'tcx>>,
-        span: Span,
-    ) {
+    fn point_at_index(&self, errors: &mut Vec<traits::FulfillmentError<'tcx>>, span: Span) {
+        let mut seen_preds = FxHashSet::default();
+        // We re-sort here so that the outer most root obligations comes first, as we have the
+        // subsequent weird logic to identify *every* relevant obligation for proper deduplication
+        // of diagnostics.
+        errors.sort_by_key(|error| error.root_obligation.recursion_depth);
         for error in errors {
-            match error.obligation.predicate.kind().skip_binder() {
-                ty::PredicateKind::Clause(ty::ClauseKind::Trait(predicate))
-                    if self.tcx.is_diagnostic_item(sym::SliceIndex, predicate.trait_ref.def_id) => {
+            match (
+                error.root_obligation.predicate.kind().skip_binder(),
+                error.obligation.predicate.kind().skip_binder(),
+            ) {
+                (ty::PredicateKind::Clause(ty::ClauseKind::Trait(predicate)), _)
+                    if self.tcx.lang_items().index_trait() == Some(predicate.trait_ref.def_id) =>
+                {
+                    seen_preds.insert(error.obligation.predicate.kind().skip_binder());
+                }
+                (_, ty::PredicateKind::Clause(ty::ClauseKind::Trait(predicate)))
+                    if self.tcx.is_diagnostic_item(sym::SliceIndex, predicate.trait_ref.def_id) =>
+                {
+                    seen_preds.insert(error.obligation.predicate.kind().skip_binder());
                 }
+                (root, pred) if seen_preds.contains(&pred) || seen_preds.contains(&root) => {}
                 _ => continue,
             }
             error.obligation.cause.span = span;
@@ -3060,7 +3159,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     ) -> Ty<'tcx> {
         match self.resume_yield_tys {
             Some((resume_ty, yield_ty)) => {
-                self.check_expr_coercible_to_type(&value, yield_ty, None);
+                self.check_expr_coercible_to_type(value, yield_ty, None);
 
                 resume_ty
             }
@@ -3069,7 +3168,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             // information. Hence, we check the source of the yield expression here and check its
             // value's type against `()` (this check should always hold).
             None if src.is_await() => {
-                self.check_expr_coercible_to_type(&value, Ty::new_unit(self.tcx), None);
+                self.check_expr_coercible_to_type(value, Ty::new_unit(self.tcx), None);
                 Ty::new_unit(self.tcx)
             }
             _ => {
@@ -3163,7 +3262,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
             match container.kind() {
                 ty::Adt(container_def, args) if container_def.is_enum() => {
-                    let block = self.tcx.hir().local_def_id_to_hir_id(self.body_id);
+                    let block = self.tcx.local_def_id_to_hir_id(self.body_id);
                     let (ident, _def_scope) =
                         self.tcx.adjust_ident_and_get_scope(field, container_def.did(), block);
 
@@ -3173,19 +3272,22 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             sym::offset_of_enum,
                             ident.span,
                             "using enums in offset_of is experimental",
-                        ).emit();
+                        )
+                        .emit();
                     }
 
-                    let Some((index, variant)) = container_def.variants()
+                    let Some((index, variant)) = container_def
+                        .variants()
                         .iter_enumerated()
-                        .find(|(_, v)| v.ident(self.tcx).normalize_to_macros_2_0() == ident) else {
+                        .find(|(_, v)| v.ident(self.tcx).normalize_to_macros_2_0() == ident)
+                    else {
                         let mut err = type_error_struct!(
                             self.tcx().sess,
                             ident.span,
                             container,
                             E0599,
                             "no variant named `{ident}` found for enum `{container}`",
-                            );
+                        );
                         err.span_label(field.span, "variant not found");
                         err.emit();
                         break;
@@ -3197,7 +3299,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             container,
                             E0795,
                             "`{ident}` is an enum variant; expected field at end of `offset_of`",
-                            );
+                        );
                         err.span_label(field.span, "enum variant");
                         err.emit();
                         break;
@@ -3205,16 +3307,18 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     let (subident, sub_def_scope) =
                         self.tcx.adjust_ident_and_get_scope(subfield, variant.def_id, block);
 
-                    let Some((subindex, field)) = variant.fields
+                    let Some((subindex, field)) = variant
+                        .fields
                         .iter_enumerated()
-                        .find(|(_, f)| f.ident(self.tcx).normalize_to_macros_2_0() == subident) else {
+                        .find(|(_, f)| f.ident(self.tcx).normalize_to_macros_2_0() == subident)
+                    else {
                         let mut err = type_error_struct!(
                             self.tcx().sess,
                             ident.span,
                             container,
                             E0609,
                             "no field named `{subfield}` on enum variant `{container}::{ident}`",
-                            );
+                        );
                         err.span_label(field.span, "this enum variant...");
                         err.span_label(subident.span, "...does not have this field");
                         err.emit();
@@ -3240,7 +3344,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     continue;
                 }
                 ty::Adt(container_def, args) => {
-                    let block = self.tcx.hir().local_def_id_to_hir_id(self.body_id);
+                    let block = self.tcx.local_def_id_to_hir_id(self.body_id);
                     let (ident, def_scope) =
                         self.tcx.adjust_ident_and_get_scope(field, container_def.did(), block);
 
diff --git a/compiler/rustc_hir_typeck/src/expr_use_visitor.rs b/compiler/rustc_hir_typeck/src/expr_use_visitor.rs
index 6be3ad657..c22b15231 100644
--- a/compiler/rustc_hir_typeck/src/expr_use_visitor.rs
+++ b/compiler/rustc_hir_typeck/src/expr_use_visitor.rs
@@ -147,7 +147,7 @@ impl<'a, 'tcx> ExprUseVisitor<'a, 'tcx> {
             self.walk_irrefutable_pat(&param_place, param.pat);
         }
 
-        self.consume_expr(&body.value);
+        self.consume_expr(body.value);
     }
 
     fn tcx(&self) -> TyCtxt<'tcx> {
@@ -237,10 +237,10 @@ impl<'a, 'tcx> ExprUseVisitor<'a, 'tcx> {
                 self.consume_exprs(exprs);
             }
 
-            hir::ExprKind::If(ref cond_expr, ref then_expr, ref opt_else_expr) => {
+            hir::ExprKind::If(cond_expr, then_expr, ref opt_else_expr) => {
                 self.consume_expr(cond_expr);
                 self.consume_expr(then_expr);
-                if let Some(ref else_expr) = *opt_else_expr {
+                if let Some(else_expr) = *opt_else_expr {
                     self.consume_expr(else_expr);
                 }
             }
@@ -249,7 +249,7 @@ impl<'a, 'tcx> ExprUseVisitor<'a, 'tcx> {
                 self.walk_local(init, pat, None, |t| t.borrow_expr(init, ty::ImmBorrow))
             }
 
-            hir::ExprKind::Match(ref discr, arms, _) => {
+            hir::ExprKind::Match(discr, arms, _) => {
                 let discr_place = return_if_err!(self.mc.cat_expr(discr));
                 return_if_err!(self.maybe_read_scrutinee(
                     discr,
@@ -267,7 +267,7 @@ impl<'a, 'tcx> ExprUseVisitor<'a, 'tcx> {
                 self.consume_exprs(exprs);
             }
 
-            hir::ExprKind::AddrOf(_, m, ref base) => {
+            hir::ExprKind::AddrOf(_, m, base) => {
                 // &base
                 // make sure that the thing we are pointing out stays valid
                 // for the lifetime `scope_r` of the resulting ptr:
@@ -379,7 +379,7 @@ impl<'a, 'tcx> ExprUseVisitor<'a, 'tcx> {
                 // only the fn body we were given.
             }
 
-            hir::StmtKind::Expr(ref expr) | hir::StmtKind::Semi(ref expr) => {
+            hir::StmtKind::Expr(expr) | hir::StmtKind::Semi(expr) => {
                 self.consume_expr(expr);
             }
         }
@@ -401,12 +401,17 @@ impl<'a, 'tcx> ExprUseVisitor<'a, 'tcx> {
             mc.cat_pattern(discr_place.clone(), pat, |place, pat| {
                 match &pat.kind {
                     PatKind::Binding(.., opt_sub_pat) => {
-                        // If the opt_sub_pat is None, than the binding does not count as
+                        // If the opt_sub_pat is None, then the binding does not count as
                         // a wildcard for the purpose of borrowing discr.
                         if opt_sub_pat.is_none() {
                             needs_to_be_read = true;
                         }
                     }
+                    PatKind::Never => {
+                        // A never pattern reads the value.
+                        // FIXME(never_patterns): does this do what I expect?
+                        needs_to_be_read = true;
+                    }
                     PatKind::Path(qpath) => {
                         // A `Path` pattern is just a name like `Foo`. This is either a
                         // named constant or else it refers to an ADT variant
@@ -505,7 +510,7 @@ impl<'a, 'tcx> ExprUseVisitor<'a, 'tcx> {
             ));
             self.walk_block(els)
         }
-        self.walk_irrefutable_pat(&expr_place, &pat);
+        self.walk_irrefutable_pat(&expr_place, pat);
     }
 
     /// Indicates that the value of `blk` will be consumed, meaning either copied or moved
@@ -517,7 +522,7 @@ impl<'a, 'tcx> ExprUseVisitor<'a, 'tcx> {
             self.walk_stmt(stmt);
         }
 
-        if let Some(ref tail_expr) = blk.expr {
+        if let Some(tail_expr) = blk.expr {
             self.consume_expr(tail_expr);
         }
     }
@@ -533,7 +538,7 @@ impl<'a, 'tcx> ExprUseVisitor<'a, 'tcx> {
 
             // The struct path probably didn't resolve
             if self.mc.typeck_results.opt_field_index(field.hir_id).is_none() {
-                self.tcx().sess.delay_span_bug(field.span, "couldn't resolve index for field");
+                self.tcx().sess.span_delayed_bug(field.span, "couldn't resolve index for field");
             }
         }
 
@@ -665,8 +670,8 @@ impl<'a, 'tcx> ExprUseVisitor<'a, 'tcx> {
         self.walk_pat(discr_place, arm.pat, arm.guard.is_some());
 
         match arm.guard {
-            Some(hir::Guard::If(ref e)) => self.consume_expr(e),
-            Some(hir::Guard::IfLet(ref l)) => {
+            Some(hir::Guard::If(e)) => self.consume_expr(e),
+            Some(hir::Guard::IfLet(l)) => {
                 self.walk_local(l.init, l.pat, None, |t| t.borrow_expr(l.init, ty::ImmBorrow))
             }
             None => {}
@@ -848,7 +853,7 @@ impl<'a, 'tcx> ExprUseVisitor<'a, 'tcx> {
                         // be a local variable
                         PlaceBase::Local(*var_hir_id)
                     };
-                    let closure_hir_id = tcx.hir().local_def_id_to_hir_id(closure_def_id);
+                    let closure_hir_id = tcx.local_def_id_to_hir_id(closure_def_id);
                     let place_with_id = PlaceWithHirId::new(
                         capture_info
                             .path_expr_id
diff --git a/compiler/rustc_hir_typeck/src/fallback.rs b/compiler/rustc_hir_typeck/src/fallback.rs
index 38b780367..023bd70be 100644
--- a/compiler/rustc_hir_typeck/src/fallback.rs
+++ b/compiler/rustc_hir_typeck/src/fallback.rs
@@ -24,9 +24,9 @@ impl<'tcx> FnCtxt<'_, 'tcx> {
             self.fulfillment_cx.borrow_mut().pending_obligations()
         );
 
-        let fallback_occured = self.fallback_types() | self.fallback_effects();
+        let fallback_occurred = self.fallback_types() | self.fallback_effects();
 
-        if !fallback_occured {
+        if !fallback_occurred {
             return;
         }
 
@@ -57,24 +57,25 @@ impl<'tcx> FnCtxt<'_, 'tcx> {
     }
 
     fn fallback_types(&self) -> bool {
-        // Check if we have any unsolved variables. If not, no need for fallback.
-        let unsolved_variables = self.unsolved_variables();
+        // Check if we have any unresolved variables. If not, no need for fallback.
+        let unresolved_variables = self.unresolved_variables();
 
-        if unsolved_variables.is_empty() {
+        if unresolved_variables.is_empty() {
             return false;
         }
 
-        let diverging_fallback = self.calculate_diverging_fallback(&unsolved_variables);
+        let diverging_fallback = self.calculate_diverging_fallback(&unresolved_variables);
 
         // We do fallback in two passes, to try to generate
         // better error messages.
         // The first time, we do *not* replace opaque types.
-        for ty in unsolved_variables {
+        let mut fallback_occurred = false;
+        for ty in unresolved_variables {
             debug!("unsolved_variable = {:?}", ty);
-            self.fallback_if_possible(ty, &diverging_fallback);
+            fallback_occurred |= self.fallback_if_possible(ty, &diverging_fallback);
         }
 
-        true
+        fallback_occurred
     }
 
     fn fallback_effects(&self) -> bool {
@@ -84,9 +85,8 @@ impl<'tcx> FnCtxt<'_, 'tcx> {
             return false;
         }
 
-        // not setting `fallback_has_occured` here because that field is only used for type fallback
-        // diagnostics.
-
+        // not setting the `fallback_has_occured` field here because
+        // that field is only used for type fallback diagnostics.
         for effect in unsolved_effects {
             let expected = self.tcx.consts.true_;
             let cause = self.misc(rustc_span::DUMMY_SP);
@@ -122,7 +122,7 @@ impl<'tcx> FnCtxt<'_, 'tcx> {
         &self,
         ty: Ty<'tcx>,
         diverging_fallback: &UnordMap<Ty<'tcx>, Ty<'tcx>>,
-    ) {
+    ) -> bool {
         // Careful: we do NOT shallow-resolve `ty`. We know that `ty`
         // is an unsolved variable, and we determine its fallback
         // based solely on how it was created, not what other type
@@ -147,7 +147,7 @@ impl<'tcx> FnCtxt<'_, 'tcx> {
             ty::Infer(ty::FloatVar(_)) => self.tcx.types.f64,
             _ => match diverging_fallback.get(&ty) {
                 Some(&fallback_ty) => fallback_ty,
-                None => return,
+                None => return false,
             },
         };
         debug!("fallback_if_possible(ty={:?}): defaulting to `{:?}`", ty, fallback);
@@ -159,6 +159,7 @@ impl<'tcx> FnCtxt<'_, 'tcx> {
             .unwrap_or(rustc_span::DUMMY_SP);
         self.demand_eqtype(span, ty, fallback);
         self.fallback_has_occurred.set(true);
+        true
     }
 
     /// The "diverging fallback" system is rather complicated. This is
@@ -230,9 +231,9 @@ impl<'tcx> FnCtxt<'_, 'tcx> {
     ///   any variable that has an edge into `D`.
     fn calculate_diverging_fallback(
         &self,
-        unsolved_variables: &[Ty<'tcx>],
+        unresolved_variables: &[Ty<'tcx>],
     ) -> UnordMap<Ty<'tcx>, Ty<'tcx>> {
-        debug!("calculate_diverging_fallback({:?})", unsolved_variables);
+        debug!("calculate_diverging_fallback({:?})", unresolved_variables);
 
         // Construct a coercion graph where an edge `A -> B` indicates
         // a type variable is that is coerced
@@ -240,7 +241,7 @@ impl<'tcx> FnCtxt<'_, 'tcx> {
 
         // Extract the unsolved type inference variable vids; note that some
         // unsolved variables are integer/float variables and are excluded.
-        let unsolved_vids = unsolved_variables.iter().filter_map(|ty| ty.ty_vid());
+        let unsolved_vids = unresolved_variables.iter().filter_map(|ty| ty.ty_vid());
 
         // Compute the diverging root vids D -- that is, the root vid of
         // those type variables that (a) are the target of a coercion from
diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs
index 750ed2c34..4bc237c23 100644
--- a/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs
+++ b/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs
@@ -72,7 +72,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             orig_span,
                             custom_note
                                 .unwrap_or("any code following this expression is unreachable"),
-                        )
+                        );
                     },
                 )
             }
@@ -83,7 +83,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     /// version (resolve_vars_if_possible), this version will
     /// also select obligations if it seems useful, in an effort
     /// to get more type information.
-    // FIXME(-Ztrait-solver=next): A lot of the calls to this method should
+    // FIXME(-Znext-solver): A lot of the calls to this method should
     // probably be `try_structurally_resolve_type` or `structurally_resolve_type` instead.
     #[instrument(skip(self), level = "debug", ret)]
     pub(in super::super) fn resolve_vars_with_obligations(&self, mut ty: Ty<'tcx>) -> Ty<'tcx> {
@@ -159,7 +159,13 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     }
 
     #[instrument(level = "debug", skip(self))]
-    pub fn write_method_call(&self, hir_id: hir::HirId, method: MethodCallee<'tcx>) {
+    pub fn write_method_call_and_enforce_effects(
+        &self,
+        hir_id: hir::HirId,
+        span: Span,
+        method: MethodCallee<'tcx>,
+    ) {
+        self.enforce_context_effects(span, method.def_id, method.args);
         self.write_resolution(hir_id, Ok((DefKind::AssocFn, method.def_id)));
         self.write_args(hir_id, method.args);
     }
@@ -271,7 +277,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     // FIXME: currently we never try to compose autoderefs
                     // and ReifyFnPointer/UnsafeFnPointer, but we could.
                     _ => {
-                        self.tcx.sess.delay_span_bug(
+                        self.tcx.sess.span_delayed_bug(
                             expr.span,
                             format!(
                                 "while adjusting {:?}, can't compose {:?} and {:?}",
@@ -505,7 +511,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
     pub(in super::super) fn resolve_rvalue_scopes(&self, def_id: DefId) {
         let scope_tree = self.tcx.region_scope_tree(def_id);
-        let rvalue_scopes = { rvalue_scopes::resolve_rvalue_scopes(self, &scope_tree, def_id) };
+        let rvalue_scopes = { rvalue_scopes::resolve_rvalue_scopes(self, scope_tree, def_id) };
         let mut typeck_results = self.inh.typeck_results.borrow_mut();
         typeck_results.rvalue_scopes = rvalue_scopes;
     }
@@ -651,20 +657,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 | ty::PredicateKind::Clause(ty::ClauseKind::TypeOutlives(..))
                 | ty::PredicateKind::Clause(ty::ClauseKind::WellFormed(..))
                 | ty::PredicateKind::ObjectSafe(..)
+                | ty::PredicateKind::NormalizesTo(..)
                 | ty::PredicateKind::AliasRelate(..)
                 | ty::PredicateKind::Clause(ty::ClauseKind::ConstEvaluatable(..))
                 | ty::PredicateKind::ConstEquate(..)
-                // N.B., this predicate is created by breaking down a
-                // `ClosureType: FnFoo()` predicate, where
-                // `ClosureType` represents some `Closure`. It can't
-                // possibly be referring to the current closure,
-                // because we haven't produced the `Closure` for
-                // this closure yet; this is exactly why the other
-                // code is looking for a self type of an unresolved
-                // inference variable.
-                | ty::PredicateKind::ClosureKind(..)
-                | ty::PredicateKind::Ambiguous
-                 => None,
+                | ty::PredicateKind::Ambiguous => None,
             },
         )
     }
@@ -754,7 +751,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         lang_item: hir::LangItem,
         span: Span,
         hir_id: hir::HirId,
-        expr_hir_id: Option<hir::HirId>,
     ) -> (Res, Ty<'tcx>) {
         let def_id = self.tcx.require_lang_item(lang_item, Some(span));
         let def_kind = self.tcx.def_kind(def_id);
@@ -767,11 +763,18 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let args = self.fresh_args_for_item(span, def_id);
         let ty = item_ty.instantiate(self.tcx, args);
 
+        self.write_args(hir_id, args);
         self.write_resolution(hir_id, Ok((def_kind, def_id)));
 
         let code = match lang_item {
             hir::LangItem::IntoFutureIntoFuture => {
-                Some(ObligationCauseCode::AwaitableExpr(expr_hir_id))
+                if let hir::Node::Expr(into_future_call) = self.tcx.hir().get_parent(hir_id)
+                    && let hir::ExprKind::Call(_, [arg0]) = &into_future_call.kind
+                {
+                    Some(ObligationCauseCode::AwaitableExpr(arg0.hir_id))
+                } else {
+                    None
+                }
             }
             hir::LangItem::IteratorNext | hir::LangItem::IntoIterIntoIter => {
                 Some(ObligationCauseCode::ForLoopIterator)
@@ -804,7 +807,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             qpath, hir_id, span
         );
         let (ty, qself, item_segment) = match *qpath {
-            QPath::Resolved(ref opt_qself, ref path) => {
+            QPath::Resolved(ref opt_qself, path) => {
                 return (
                     path.res,
                     opt_qself.as_ref().map(|qself| self.to_ty(qself)),
@@ -845,7 +848,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     && let Some(mut diag) = self
                         .tcx
                         .sess
-                        .diagnostic()
+                        .dcx()
                         .steal_diagnostic(qself.span, StashKey::TraitMissingMethod)
                 {
                     diag.emit();
@@ -856,7 +859,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 let guar = self
                     .tcx
                     .sess
-                    .delay_span_bug(span, "method resolution should've emitted an error");
+                    .span_delayed_bug(span, "method resolution should've emitted an error");
                 let result = match error {
                     method::MethodError::PrivateMatch(kind, def_id, _) => Ok((kind, def_id)),
                     _ => Err(guar),
@@ -881,7 +884,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     && let Some(mut diag) = self
                         .tcx
                         .sess
-                        .diagnostic()
+                        .dcx()
                         .steal_diagnostic(qself.span, StashKey::TraitMissingMethod)
                 {
                     if trait_missing_method {
@@ -991,7 +994,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         // Get enclosing Fn, if it is a function or a trait method, unless there's a `loop` or
         // `while` before reaching it, as block tail returns are not available in them.
         self.tcx.hir().get_return_block(blk_id).and_then(|blk_id| {
-            let parent = self.tcx.hir().get(blk_id);
+            let parent = self.tcx.hir_node(blk_id);
             self.get_node_fn_decl(parent)
                 .map(|(fn_id, fn_decl, _, is_main)| (fn_id, fn_decl, is_main))
         })
@@ -1186,7 +1189,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 tcx,
                 span,
                 def_id,
-                &generics,
+                generics,
                 seg,
                 IsMethodCall::No,
             );
@@ -1278,7 +1281,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     // going to infer the arguments for better error messages.
                     if !self.infer_args_for_err.contains(&index) {
                         // Check whether the user has provided generic arguments.
-                        if let Some(ref data) = self.segments[index].args {
+                        if let Some(data) = self.segments[index].args {
                             return (Some(data), self.segments[index].infer_args);
                         }
                     }
@@ -1406,7 +1409,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         // Normalize only after registering type annotations.
         let args = self.normalize(span, args_raw);
 
-        self.add_required_obligations_for_hir(span, def_id, &args, hir_id);
+        self.add_required_obligations_for_hir(span, def_id, args, hir_id);
 
         // Substitute the values for the type parameters into the type of
         // the referenced item.
@@ -1430,7 +1433,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             ) {
                 Ok(ok) => self.register_infer_ok_obligations(ok),
                 Err(_) => {
-                    self.tcx.sess.delay_span_bug(
+                    self.tcx.sess.span_delayed_bug(
                         span,
                         format!(
                         "instantiate_value_path: (UFCS) {self_ty:?} was a subtype of {impl_ty:?} but now is not?",
@@ -1473,7 +1476,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     ) {
         let param_env = self.param_env;
 
-        let bounds = self.instantiate_bounds(span, def_id, &args);
+        let bounds = self.instantiate_bounds(span, def_id, args);
 
         for obligation in traits::predicates_for_generics(
             |idx, predicate_span| {
@@ -1495,7 +1498,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let ty = self.resolve_vars_with_obligations(ty);
 
         if self.next_trait_solver()
-            && let ty::Alias(ty::Projection | ty::Inherent | ty::Weak, _) = ty.kind()
+            && let ty::Alias(..) = ty.kind()
         {
             match self
                 .at(&self.misc(sp), self.param_env)
diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/adjust_fulfillment_errors.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/adjust_fulfillment_errors.rs
index facbeb8ba..76360239c 100644
--- a/compiler/rustc_hir_typeck/src/fn_ctxt/adjust_fulfillment_errors.rs
+++ b/compiler/rustc_hir_typeck/src/fn_ctxt/adjust_fulfillment_errors.rs
@@ -4,7 +4,7 @@ use rustc_hir::def::{DefKind, Res};
 use rustc_hir::def_id::DefId;
 use rustc_infer::{infer::type_variable::TypeVariableOriginKind, traits::ObligationCauseCode};
 use rustc_middle::ty::{self, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable, TypeVisitor};
-use rustc_span::{self, symbol::kw, Span};
+use rustc_span::{symbol::kw, Span};
 use rustc_trait_selection::traits;
 
 use std::ops::ControlFlow;
@@ -94,7 +94,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         }
 
         let hir = self.tcx.hir();
-        let (expr, qpath) = match hir.get(hir_id) {
+        let (expr, qpath) = match self.tcx.hir_node(hir_id) {
             hir::Node::Expr(expr) => {
                 if self.closure_span_overlaps_error(error, expr.span) {
                     return false;
@@ -454,7 +454,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 .find_ancestor_in_same_ctxt(error.obligation.cause.span)
                 .unwrap_or(arg.span);
 
-            if let hir::Node::Expr(arg_expr) = self.tcx.hir().get(arg.hir_id) {
+            if let hir::Node::Expr(arg_expr) = self.tcx.hir_node(arg.hir_id) {
                 // This is more specific than pointing at the entire argument.
                 self.blame_specific_expr_if_possible(error, arg_expr)
             }
@@ -495,7 +495,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         // Whether it succeeded or failed, it likely made some amount of progress.
         // In the very worst case, it's just the same `expr` we originally passed in.
         let expr = match self.blame_specific_expr_if_possible_for_obligation_cause_code(
-            &error.obligation.cause.code(),
+            error.obligation.cause.code(),
             expr,
         ) {
             Ok(expr) => expr,
diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/arg_matrix.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/arg_matrix.rs
index 7aadb95d9..566d407d2 100644
--- a/compiler/rustc_hir_typeck/src/fn_ctxt/arg_matrix.rs
+++ b/compiler/rustc_hir_typeck/src/fn_ctxt/arg_matrix.rs
@@ -4,11 +4,13 @@ use rustc_middle::ty::error::TypeError;
 use std::cmp;
 
 rustc_index::newtype_index! {
+    #[orderable]
     #[debug_format = "ExpectedIdx({})"]
     pub(crate) struct ExpectedIdx {}
 }
 
 rustc_index::newtype_index! {
+    #[orderable]
     #[debug_format = "ProvidedIdx({})"]
     pub(crate) struct ProvidedIdx {}
 }
diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs
index 33dfa16a6..4caa0df58 100644
--- a/compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs
+++ b/compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs
@@ -33,7 +33,7 @@ use rustc_middle::ty::visit::TypeVisitableExt;
 use rustc_middle::ty::{self, IsSuggestable, Ty, TyCtxt};
 use rustc_session::Session;
 use rustc_span::symbol::{kw, Ident};
-use rustc_span::{self, sym, BytePos, Span};
+use rustc_span::{sym, BytePos, Span};
 use rustc_trait_selection::traits::{self, ObligationCauseCode, SelectionContext};
 
 use std::iter;
@@ -230,11 +230,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let minimum_input_count = expected_input_tys.len();
         let provided_arg_count = provided_args.len();
 
-        let is_const_eval_select = matches!(fn_def_id, Some(def_id) if
-            self.tcx.def_kind(def_id) == hir::def::DefKind::Fn
-            && self.tcx.is_intrinsic(def_id)
-            && self.tcx.item_name(def_id) == sym::const_eval_select);
-
         // We introduce a helper function to demand that a given argument satisfy a given input
         // This is more complicated than just checking type equality, as arguments could be coerced
         // This version writes those types back so further type checking uses the narrowed types
@@ -269,35 +264,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 return Compatibility::Incompatible(coerce_error);
             }
 
-            // Check that second and third argument of `const_eval_select` must be `FnDef`, and additionally that
-            // the second argument must be `const fn`. The first argument must be a tuple, but this is already expressed
-            // in the function signature (`F: FnOnce<ARG>`), so I did not bother to add another check here.
-            //
-            // This check is here because there is currently no way to express a trait bound for `FnDef` types only.
-            if is_const_eval_select && (1..=2).contains(&idx) {
-                if let ty::FnDef(def_id, args) = *checked_ty.kind() {
-                    if idx == 1 {
-                        if !self.tcx.is_const_fn_raw(def_id) {
-                            self.tcx.sess.emit_err(errors::ConstSelectMustBeConst {
-                                span: provided_arg.span,
-                            });
-                        } else {
-                            self.enforce_context_effects(
-                                provided_arg.hir_id,
-                                provided_arg.span,
-                                def_id,
-                                args,
-                            )
-                        }
-                    }
-                } else {
-                    self.tcx.sess.emit_err(errors::ConstSelectMustBeFn {
-                        span: provided_arg.span,
-                        ty: checked_ty,
-                    });
-                }
-            }
-
             // 3. Check if the formal type is a supertype of the checked one
             //    and register any such obligations for future type checks
             let supertype_error = self.at(&self.misc(provided_arg.span), self.param_env).sup(
@@ -397,7 +363,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         for arg in provided_args.iter().skip(minimum_input_count) {
             // Make sure we've checked this expr at least once.
-            let arg_ty = self.check_expr(&arg);
+            let arg_ty = self.check_expr(arg);
 
             // If the function is c-style variadic, we skipped a bunch of arguments
             // so we need to check those, and write out the types
@@ -522,7 +488,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let tcx = self.tcx;
         // FIXME: taint after emitting errors and pass through an `ErrorGuaranteed`
         self.set_tainted_by_errors(
-            tcx.sess.delay_span_bug(call_span, "no errors reported for args"),
+            tcx.sess.span_delayed_bug(call_span, "no errors reported for args"),
         );
 
         // Get the argument span in the context of the call span so that
@@ -796,7 +762,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             let mut err = self.err_ctxt().report_and_explain_type_error(trace, *err);
             self.emit_coerce_suggestions(
                 &mut err,
-                &provided_args[*provided_idx],
+                provided_args[*provided_idx],
                 provided_ty,
                 Expectation::rvalue_hint(self, expected_ty)
                     .only_has_type(self)
@@ -925,7 +891,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
                     self.emit_coerce_suggestions(
                         &mut err,
-                        &provided_args[provided_idx],
+                        provided_args[provided_idx],
                         provided_ty,
                         Expectation::rvalue_hint(self, expected_ty)
                             .only_has_type(self)
@@ -998,7 +964,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     }) {
                         match e {
                             Error::Missing(expected_idx) => missing_idxs.push(expected_idx),
-                            _ => unreachable!(),
+                            _ => unreachable!(
+                                "control flow ensures that we should always get an `Error::Missing`"
+                            ),
                         }
                     }
 
@@ -1366,7 +1334,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let variant = match def {
             Res::Err => {
                 let guar =
-                    self.tcx.sess.delay_span_bug(path_span, "`Res::Err` but no error emitted");
+                    self.tcx.sess.span_delayed_bug(path_span, "`Res::Err` but no error emitted");
                 self.set_tainted_by_errors(guar);
                 return Err(guar);
             }
@@ -1469,7 +1437,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         // Type check the initializer.
         if let Some(ref init) = decl.init {
-            let init_ty = self.check_decl_initializer(decl.hir_id, decl.pat, &init);
+            let init_ty = self.check_decl_initializer(decl.hir_id, decl.pat, init);
             self.overwrite_local_ty_if_err(decl.hir_id, decl.pat, init_ty);
         }
 
@@ -1483,7 +1451,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         };
 
         // Type check the pattern. Override if necessary to avoid knock-on errors.
-        self.check_pat_top(&decl.pat, decl_ty, ty_span, origin_expr, Some(decl.origin));
+        self.check_pat_top(decl.pat, decl_ty, ty_span, origin_expr, Some(decl.origin));
         let pat_ty = self.node_ty(decl.pat.hir_id);
         self.overwrite_local_ty_if_err(decl.hir_id, decl.pat, pat_ty);
 
@@ -1525,13 +1493,13 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             hir::StmtKind::Item(_) => {}
             hir::StmtKind::Expr(ref expr) => {
                 // Check with expected type of `()`.
-                self.check_expr_has_type_or_error(&expr, Ty::new_unit(self.tcx), |err| {
+                self.check_expr_has_type_or_error(expr, Ty::new_unit(self.tcx), |err| {
                     if expr.can_have_side_effects() {
                         self.suggest_semicolon_at_end(expr.span, err);
                     }
                 });
             }
-            hir::StmtKind::Semi(ref expr) => {
+            hir::StmtKind::Semi(expr) => {
                 self.check_expr(expr);
             }
         }
@@ -1739,7 +1707,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     }
 
     fn parent_item_span(&self, id: hir::HirId) -> Option<Span> {
-        let node = self.tcx.hir().get_by_def_id(self.tcx.hir().get_parent_item(id).def_id);
+        let node = self.tcx.hir_node_by_def_id(self.tcx.hir().get_parent_item(id).def_id);
         match node {
             Node::Item(&hir::Item { kind: hir::ItemKind::Fn(_, _, body_id), .. })
             | Node::ImplItem(&hir::ImplItem { kind: hir::ImplItemKind::Fn(_, body_id), .. }) => {
@@ -1755,7 +1723,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
     /// Given a function block's `HirId`, returns its `FnDecl` if it exists, or `None` otherwise.
     fn get_parent_fn_decl(&self, blk_id: hir::HirId) -> Option<(&'tcx hir::FnDecl<'tcx>, Ident)> {
-        let parent = self.tcx.hir().get_by_def_id(self.tcx.hir().get_parent_item(blk_id).def_id);
+        let parent = self.tcx.hir_node_by_def_id(self.tcx.hir().get_parent_item(blk_id).def_id);
         self.get_node_fn_decl(parent).map(|(_, fn_decl, ident, _)| (fn_decl, ident))
     }
 
@@ -1820,12 +1788,12 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         hir_id: hir::HirId,
     ) -> (Res, RawTy<'tcx>) {
         match *qpath {
-            QPath::Resolved(ref maybe_qself, ref path) => {
+            QPath::Resolved(ref maybe_qself, path) => {
                 let self_ty = maybe_qself.as_ref().map(|qself| self.to_ty(qself).raw);
                 let ty = self.astconv().res_to_ty(self_ty, path, hir_id, true);
                 (path.res, self.handle_raw_ty(path_span, ty))
             }
-            QPath::TypeRelative(ref qself, ref segment) => {
+            QPath::TypeRelative(qself, segment) => {
                 let ty = self.to_ty(qself);
 
                 let result = self
@@ -1842,8 +1810,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
                 (result.map_or(Res::Err, |(kind, def_id)| Res::Def(kind, def_id)), ty)
             }
-            QPath::LangItem(lang_item, span, id) => {
-                let (res, ty) = self.resolve_lang_item_path(lang_item, span, hir_id, id);
+            QPath::LangItem(lang_item, span) => {
+                let (res, ty) = self.resolve_lang_item_path(lang_item, span, hir_id);
                 (res, self.handle_raw_ty(path_span, ty))
             }
         }
@@ -1855,7 +1823,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     ) {
         for (span, code) in errors_causecode {
             let Some(mut diag) =
-                self.tcx.sess.diagnostic().steal_diagnostic(span, StashKey::MaybeForgetReturn)
+                self.tcx.sess.dcx().steal_diagnostic(span, StashKey::MaybeForgetReturn)
             else {
                 continue;
             };
@@ -1864,35 +1832,42 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 && let ExprBindingObligation(_, _, hir_id, ..) = code
                 && !fn_sig.output().is_unit()
             {
-                    let mut block_num = 0;
-                    let mut found_semi = false;
-                    for (_, node) in self.tcx.hir().parent_iter(hir_id) {
-                        match node {
-                            hir::Node::Stmt(stmt) => if let hir::StmtKind::Semi(ref expr) = stmt.kind {
+                let mut block_num = 0;
+                let mut found_semi = false;
+                for (_, node) in self.tcx.hir().parent_iter(hir_id) {
+                    match node {
+                        hir::Node::Stmt(stmt) => {
+                            if let hir::StmtKind::Semi(expr) = stmt.kind {
                                 let expr_ty = self.typeck_results.borrow().expr_ty(expr);
                                 let return_ty = fn_sig.output();
-                                if !matches!(expr.kind, hir::ExprKind::Ret(..)) &&
-                                    self.can_coerce(expr_ty, return_ty) {
+                                if !matches!(expr.kind, hir::ExprKind::Ret(..))
+                                    && self.can_coerce(expr_ty, return_ty)
+                                {
                                     found_semi = true;
                                 }
-                            },
-                            hir::Node::Block(_block) => if found_semi {
+                            }
+                        }
+                        hir::Node::Block(_block) => {
+                            if found_semi {
                                 block_num += 1;
                             }
-                            hir::Node::Item(item) => if let hir::ItemKind::Fn(..) = item.kind {
+                        }
+                        hir::Node::Item(item) => {
+                            if let hir::ItemKind::Fn(..) = item.kind {
                                 break;
                             }
-                            _ => {}
                         }
+                        _ => {}
                     }
-                    if block_num > 1 && found_semi {
-                        diag.span_suggestion_verbose(
-                            span.shrink_to_lo(),
-                            "you might have meant to return this to infer its type parameters",
-                            "return ",
-                            Applicability::MaybeIncorrect,
-                        );
-                    }
+                }
+                if block_num > 1 && found_semi {
+                    diag.span_suggestion_verbose(
+                        span.shrink_to_lo(),
+                        "you might have meant to return this to infer its type parameters",
+                        "return ",
+                        Applicability::MaybeIncorrect,
+                    );
+                }
             }
             diag.emit();
         }
@@ -2016,7 +1991,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     let new_def_id = self.probe(|_| {
                         let trait_ref = ty::TraitRef::new(
                             self.tcx,
-                            call_kind.to_def_id(self.tcx),
+                            self.tcx.fn_trait_kind_to_def_id(call_kind)?,
                             [
                                 callee_ty,
                                 self.next_ty_var(TypeVariableOrigin {
@@ -2031,7 +2006,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             self.param_env,
                             trait_ref,
                         );
-                        match SelectionContext::new(&self).select(&obligation) {
+                        match SelectionContext::new(self).select(&obligation) {
                             Ok(Some(traits::ImplSource::UserDefined(impl_source))) => {
                                 Some(impl_source.impl_def_id)
                             }
@@ -2082,7 +2057,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 let node = self
                     .tcx
                     .opt_local_def_id_to_hir_id(self.tcx.hir().get_parent_item(call_expr.hir_id))
-                    .and_then(|hir_id| self.tcx.hir().find(hir_id));
+                    .and_then(|hir_id| self.tcx.opt_hir_node(hir_id));
                 match node {
                     Some(hir::Node::Item(item)) => call_finder.visit_item(item),
                     Some(hir::Node::TraitItem(item)) => call_finder.visit_trait_item(item),
diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/mod.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/mod.rs
index e93d180fc..072df1343 100644
--- a/compiler/rustc_hir_typeck/src/fn_ctxt/mod.rs
+++ b/compiler/rustc_hir_typeck/src/fn_ctxt/mod.rs
@@ -143,7 +143,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     }
 
     pub fn sess(&self) -> &Session {
-        &self.tcx.sess
+        self.tcx.sess
     }
 
     /// Creates an `TypeErrCtxt` with a reference to the in-progress
@@ -196,7 +196,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 impl<'a, 'tcx> Deref for FnCtxt<'a, 'tcx> {
     type Target = Inherited<'tcx>;
     fn deref(&self) -> &Self::Target {
-        &self.inh
+        self.inh
     }
 }
 
@@ -238,7 +238,7 @@ impl<'a, 'tcx> AstConv<'tcx> for FnCtxt<'a, 'tcx> {
 
     fn re_infer(&self, def: Option<&ty::GenericParamDef>, span: Span) -> Option<ty::Region<'tcx>> {
         let v = match def {
-            Some(def) => infer::EarlyBoundRegion(span, def.name),
+            Some(def) => infer::RegionParameterDefinition(span, def.name),
             None => infer::MiscVariable(span),
         };
         Some(self.next_region_var(v))
@@ -289,7 +289,7 @@ impl<'a, 'tcx> AstConv<'tcx> for FnCtxt<'a, 'tcx> {
     ) -> Ty<'tcx> {
         let trait_ref = self.instantiate_binder_with_fresh_vars(
             span,
-            infer::LateBoundRegionConversionTime::AssocTypeProjection(item_def_id),
+            infer::BoundRegionConversionTime::AssocTypeProjection(item_def_id),
             poly_trait_ref,
         );
 
diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs
index c43d4932f..668e54757 100644
--- a/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs
+++ b/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs
@@ -43,7 +43,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         self.typeck_results
             .borrow()
             .liberated_fn_sigs()
-            .get(self.tcx.hir().local_def_id_to_hir_id(self.body_id))
+            .get(self.tcx.local_def_id_to_hir_id(self.body_id))
             .copied()
     }
 
@@ -72,23 +72,16 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         blk_id: hir::HirId,
     ) -> bool {
         let expr = expr.peel_drop_temps();
-        self.suggest_missing_semicolon(err, expr, expected, false);
         let mut pointing_at_return_type = false;
         if let hir::ExprKind::Break(..) = expr.kind {
             // `break` type mismatches provide better context for tail `loop` expressions.
             return false;
         }
         if let Some((fn_id, fn_decl, can_suggest)) = self.get_fn_decl(blk_id) {
-            pointing_at_return_type = self.suggest_missing_return_type(
-                err,
-                &fn_decl,
-                expected,
-                found,
-                can_suggest,
-                fn_id,
-            );
+            pointing_at_return_type =
+                self.suggest_missing_return_type(err, fn_decl, expected, found, can_suggest, fn_id);
             self.suggest_missing_break_or_return_expr(
-                err, expr, &fn_decl, expected, found, blk_id, fn_id,
+                err, expr, fn_decl, expected, found, blk_id, fn_id,
             );
         }
         pointing_at_return_type
@@ -449,12 +442,22 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     expected,
                 )
             });
+
+            let prefix_wrap = |sugg: &str| {
+                if let Some(name) = self.tcx.hir().maybe_get_struct_pattern_shorthand_field(expr) {
+                    format!(": {}{}", name, sugg)
+                } else {
+                    sugg.to_string()
+                }
+            };
+
             // FIXME: This could/should be extended to suggest `as_mut` and `as_deref_mut`,
             // but those checks need to be a bit more delicate and the benefit is diminishing.
             if self.can_eq(self.param_env, found_ty_inner, peeled) && error_tys_equate_as_ref {
+                let sugg = prefix_wrap(".as_ref()");
                 err.subdiagnostic(errors::SuggestConvertViaMethod {
                     span: expr.span.shrink_to_hi(),
-                    sugg: ".as_ref()",
+                    sugg,
                     expected,
                     found,
                     borrow_removal_span,
@@ -465,9 +468,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 && self.can_eq(self.param_env, deref_ty, peeled)
                 && error_tys_equate_as_ref
             {
+                let sugg = prefix_wrap(".as_deref()");
                 err.subdiagnostic(errors::SuggestConvertViaMethod {
                     span: expr.span.shrink_to_hi(),
-                    sugg: ".as_deref()",
+                    sugg,
                     expected,
                     found,
                     borrow_removal_span,
@@ -481,10 +485,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     self.can_eq(self.param_env, found, expected)
                 })
             {
+                let sugg = prefix_wrap(".map(|x| x.as_str())");
                 err.span_suggestion_verbose(
                     expr.span.shrink_to_hi(),
                     fluent::hir_typeck_convert_to_str,
-                    ".map(|x| x.as_str())",
+                    sugg,
                     Applicability::MachineApplicable,
                 );
                 return true;
@@ -615,7 +620,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             return false;
         }
         let pin_did = self.tcx.lang_items().pin_type();
-        // This guards the `unwrap` and `mk_box` below.
+        // This guards the `new_box` below.
         if pin_did.is_none() || self.tcx.lang_items().owned_box().is_none() {
             return false;
         }
@@ -635,12 +640,20 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             err.help("use `Box::pin`");
                         }
                         _ => {
+                            let prefix = if let Some(name) =
+                                self.tcx.hir().maybe_get_struct_pattern_shorthand_field(expr)
+                            {
+                                format!("{}: ", name)
+                            } else {
+                                String::new()
+                            };
+                            let suggestion = vec![
+                                (expr.span.shrink_to_lo(), format!("{prefix}Box::pin(")),
+                                (expr.span.shrink_to_hi(), ")".to_string()),
+                            ];
                             err.multipart_suggestion(
                                 "you need to pin and box this expression",
-                                vec![
-                                    (expr.span.shrink_to_lo(), "Box::pin(".to_string()),
-                                    (expr.span.shrink_to_hi(), ")".to_string()),
-                                ],
+                                suggestion,
                                 Applicability::MaybeIncorrect,
                             );
                         }
@@ -664,7 +677,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 // can suggest Box::pin.
                 let parent = self.tcx.hir().parent_id(expr.hir_id);
                 let Some(Node::Expr(Expr { kind: ExprKind::Call(fn_name, _), .. })) =
-                    self.tcx.hir().find(parent)
+                    self.tcx.opt_hir_node(parent)
                 else {
                     return false;
                 };
@@ -811,10 +824,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 if let hir::TyKind::OpaqueDef(item_id, ..) = hir_ty.kind
                     && let hir::Node::Item(hir::Item {
                         kind: hir::ItemKind::OpaqueTy(op_ty), ..
-                    }) = self.tcx.hir().get(item_id.hir_id())
-                    && let [
-                        hir::GenericBound::LangItemTrait(hir::LangItem::Future, _, _, generic_args),
-                    ] = op_ty.bounds
+                    }) = self.tcx.hir_node(item_id.hir_id())
+                    && let [hir::GenericBound::Trait(trait_ref, _)] = op_ty.bounds
+                    && let Some(hir::PathSegment { args: Some(generic_args), .. }) =
+                        trait_ref.trait_ref.path.segments.last()
                     && let hir::GenericArgs { bindings: [ty_binding], .. } = generic_args
                     && let hir::TypeBindingKind::Equality { term: hir::Term::Ty(term) } =
                         ty_binding.kind
@@ -846,7 +859,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     let bound_vars = self.tcx.late_bound_vars(hir_ty.hir_id.owner.into());
                     let ty = Binder::bind_with_vars(ty, bound_vars);
                     let ty = self.normalize(hir_ty.span, ty);
-                    let ty = self.tcx.erase_late_bound_regions(ty);
+                    let ty = self.tcx.instantiate_bound_regions_with_erased(ty);
                     if self.can_coerce(expected, ty) {
                         err.subdiagnostic(errors::ExpectedReturnTypeLabel::Other {
                             span: hir_ty.span,
@@ -889,7 +902,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         let ty::Param(expected_ty_as_param) = expected.kind() else { return };
 
-        let fn_node = self.tcx.hir().find(fn_id);
+        let fn_node = self.tcx.opt_hir_node(fn_id);
 
         let Some(hir::Node::Item(hir::Item {
             kind:
@@ -1023,7 +1036,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         if let hir::FnRetTy::Return(ty) = fn_decl.output {
             let ty = self.astconv().ast_ty_to_ty(ty);
             let bound_vars = self.tcx.late_bound_vars(fn_id);
-            let ty = self.tcx.erase_late_bound_regions(Binder::bind_with_vars(ty, bound_vars));
+            let ty = self
+                .tcx
+                .instantiate_bound_regions_with_erased(Binder::bind_with_vars(ty, bound_vars));
             let ty = match self.tcx.asyncness(fn_id.owner) {
                 ty::Asyncness::Yes => self.get_impl_future_output_ty(ty).unwrap_or_else(|| {
                     span_bug!(fn_decl.output.span(), "failed to get output type of async function")
@@ -1032,7 +1047,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             };
             let ty = self.normalize(expr.span, ty);
             if self.can_coerce(found, ty) {
-                if let Some(node) = self.tcx.hir().find(fn_id)
+                if let Some(node) = self.tcx.opt_hir_node(fn_id)
                     && let Some(owner_node) = node.as_owner()
                     && let Some(span) = expr.span.find_ancestor_inside(owner_node.span())
                 {
@@ -1219,7 +1234,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 span = parent_callsite;
             }
 
-            let sugg = if expr.precedence().order() >= PREC_POSTFIX {
+            let mut sugg = if expr.precedence().order() >= PREC_POSTFIX {
                 vec![(span.shrink_to_hi(), ".into()".to_owned())]
             } else {
                 vec![
@@ -1227,6 +1242,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     (span.shrink_to_hi(), ").into()".to_owned()),
                 ]
             };
+            if let Some(name) = self.tcx.hir().maybe_get_struct_pattern_shorthand_field(expr) {
+                sugg.insert(0, (expr.span.shrink_to_lo(), format!("{}: ", name)));
+            }
             diag.multipart_suggestion(
                 format!("call `Into::into` on this expression to convert `{expr_ty}` into `{expected_ty}`"),
                 sugg,
@@ -1527,12 +1545,12 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     }
 
     fn is_loop(&self, id: hir::HirId) -> bool {
-        let node = self.tcx.hir().get(id);
+        let node = self.tcx.hir_node(id);
         matches!(node, Node::Expr(Expr { kind: ExprKind::Loop(..), .. }))
     }
 
     fn is_local_statement(&self, id: hir::HirId) -> bool {
-        let node = self.tcx.hir().get(id);
+        let node = self.tcx.hir_node(id);
         matches!(node, Node::Stmt(Stmt { kind: StmtKind::Local(..), .. }))
     }
 
@@ -1601,6 +1619,46 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     None,
                 );
             } else {
+                if let Some(errors) =
+                    self.could_impl_trait(clone_trait_did, expected_ty, self.param_env)
+                {
+                    match &errors[..] {
+                        [] => {}
+                        [error] => {
+                            diag.help(format!(
+                                "`Clone` is not implemented because the trait bound `{}` is \
+                                 not satisfied",
+                                error.obligation.predicate,
+                            ));
+                        }
+                        [errors @ .., last] => {
+                            diag.help(format!(
+                                "`Clone` is not implemented because the following trait bounds \
+                                 could not be satisfied: {} and `{}`",
+                                errors
+                                    .iter()
+                                    .map(|e| format!("`{}`", e.obligation.predicate))
+                                    .collect::<Vec<_>>()
+                                    .join(", "),
+                                last.obligation.predicate,
+                            ));
+                        }
+                    }
+                    for error in errors {
+                        if let traits::FulfillmentErrorCode::CodeSelectionError(
+                            traits::SelectionError::Unimplemented,
+                        ) = error.code
+                            && let ty::PredicateKind::Clause(ty::ClauseKind::Trait(pred)) =
+                                error.obligation.predicate.kind().skip_binder()
+                        {
+                            self.infcx.err_ctxt().suggest_derive(
+                                &error.obligation,
+                                diag,
+                                error.obligation.predicate.kind().rebind(pred),
+                            );
+                        }
+                    }
+                }
                 self.suggest_derive(diag, &[(trait_ref.to_predicate(self.tcx), None, None)]);
             }
         }
@@ -1619,11 +1677,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 None,
                 hir::Path { segments: [_], res: crate::Res::Local(binding), .. },
             )) => {
-                let Some(hir::Node::Pat(hir::Pat { hir_id, .. })) = self.tcx.hir().find(*binding)
+                let Some(hir::Node::Pat(hir::Pat { hir_id, .. })) = self.tcx.opt_hir_node(*binding)
                 else {
                     return expr;
                 };
-                let Some(parent) = self.tcx.hir().find(self.tcx.hir().parent_id(*hir_id)) else {
+                let Some(parent) = self.tcx.opt_hir_node(self.tcx.hir().parent_id(*hir_id)) else {
                     return expr;
                 };
 
@@ -1639,7 +1697,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         ..
                     }) => {
                         let Some(hir::Node::Local(hir::Local { init: Some(init), .. })) =
-                            self.tcx.hir().find(self.tcx.hir().parent_id(*pat_hir_id))
+                            self.tcx.opt_hir_node(self.tcx.hir().parent_id(*pat_hir_id))
                         else {
                             return expr;
                         };
@@ -1672,8 +1730,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     && let hir::Path { segments: [_], res: crate::Res::Local(binding), .. } =
                         call_expr_path
                     && let Some(hir::Node::Pat(hir::Pat { hir_id, .. })) =
-                        self.tcx.hir().find(*binding)
-                    && let Some(closure) = self.tcx.hir().find(self.tcx.hir().parent_id(*hir_id))
+                        self.tcx.opt_hir_node(*binding)
+                    && let Some(closure) = self.tcx.opt_hir_node(self.tcx.hir().parent_id(*hir_id))
                     && let hir::Node::Local(hir::Local { init: Some(init), .. }) = closure
                     && let Expr {
                         kind: hir::ExprKind::Closure(hir::Closure { body: body_id, .. }),
@@ -1816,6 +1874,12 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 ".expect(\"REASON\")",
             )
         };
+
+        let sugg = match self.tcx.hir().maybe_get_struct_pattern_shorthand_field(expr) {
+            Some(ident) => format!(": {ident}{sugg}"),
+            None => sugg.to_string(),
+        };
+
         err.span_suggestion_verbose(
             expr.span.shrink_to_hi(),
             msg,
@@ -1921,7 +1985,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
                 if let Some(hir::Node::Block(&hir::Block {
                     span: block_span, expr: Some(e), ..
-                })) = self.tcx.hir().find(parent)
+                })) = self.tcx.opt_hir_node(parent)
                 {
                     if e.hir_id == id {
                         if let Some(span) = expr.span.find_ancestor_inside(block_span) {
@@ -2008,8 +2072,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     Some(CtorKind::Fn) => ("(".to_owned(), ")"),
                     None => (format!(" {{ {field_name}: "), " }"),
 
-                    // unit variants don't have fields
-                    Some(CtorKind::Const) => unreachable!(),
+                    Some(CtorKind::Const) => unreachable!("unit variants don't have fields"),
                 };
 
                 // Suggest constructor as deep into the block tree as possible.
@@ -2137,7 +2200,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     /// opt.map(|param| { takes_ref(param) });
     /// ```
     fn can_use_as_ref(&self, expr: &hir::Expr<'_>) -> Option<(Vec<(Span, String)>, &'static str)> {
-        let hir::ExprKind::Path(hir::QPath::Resolved(_, ref path)) = expr.kind else {
+        let hir::ExprKind::Path(hir::QPath::Resolved(_, path)) = expr.kind else {
             return None;
         };
 
@@ -2147,7 +2210,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         let local_parent = self.tcx.hir().parent_id(local_id);
         let Some(Node::Param(hir::Param { hir_id: param_hir_id, .. })) =
-            self.tcx.hir().find(local_parent)
+            self.tcx.opt_hir_node(local_parent)
         else {
             return None;
         };
@@ -2157,13 +2220,13 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             hir_id: expr_hir_id,
             kind: hir::ExprKind::Closure(hir::Closure { fn_decl: closure_fn_decl, .. }),
             ..
-        })) = self.tcx.hir().find(param_parent)
+        })) = self.tcx.opt_hir_node(param_parent)
         else {
             return None;
         };
 
         let expr_parent = self.tcx.hir().parent_id(*expr_hir_id);
-        let hir = self.tcx.hir().find(expr_parent);
+        let hir = self.tcx.opt_hir_node(expr_parent);
         let closure_params_len = closure_fn_decl.inputs.len();
         let (
             Some(Node::Expr(hir::Expr {
@@ -2293,16 +2356,18 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 };
                 if self.can_coerce(ref_ty, expected) {
                     let mut sugg_sp = sp;
-                    if let hir::ExprKind::MethodCall(ref segment, receiver, args, _) = expr.kind {
+                    if let hir::ExprKind::MethodCall(segment, receiver, args, _) = expr.kind {
                         let clone_trait =
                             self.tcx.require_lang_item(LangItem::Clone, Some(segment.ident.span));
                         if args.is_empty()
-                            && self.typeck_results.borrow().type_dependent_def_id(expr.hir_id).map(
-                                |did| {
+                            && self
+                                .typeck_results
+                                .borrow()
+                                .type_dependent_def_id(expr.hir_id)
+                                .is_some_and(|did| {
                                     let ai = self.tcx.associated_item(did);
                                     ai.trait_container(self.tcx) == Some(clone_trait)
-                                },
-                            ) == Some(true)
+                                })
                             && segment.ident.name == sym::clone
                         {
                             // If this expression had a clone call when suggesting borrowing
@@ -2311,7 +2376,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         }
                     }
 
-                    if let hir::ExprKind::Unary(hir::UnOp::Deref, ref inner) = expr.kind
+                    if let hir::ExprKind::Unary(hir::UnOp::Deref, inner) = expr.kind
                         && let Some(1) = self.deref_steps(expected, checked_ty)
                     {
                         // We have `*&T`, check if what was expected was `&T`.
@@ -2326,14 +2391,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         ));
                     }
 
-                    let needs_parens = match expr.kind {
-                        // parenthesize if needed (Issue #46756)
-                        hir::ExprKind::Cast(_, _) | hir::ExprKind::Binary(_, _, _) => true,
-                        // parenthesize borrows of range literals (Issue #54505)
-                        _ if is_range_literal(expr) => true,
-                        _ => false,
-                    };
-
                     if let Some((sugg, msg)) = self.can_use_as_ref(expr) {
                         return Some((
                             sugg,
@@ -2361,18 +2418,48 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         }
                     }
 
-                    let sugg = mutability.ref_prefix_str();
-                    let (sugg, verbose) = if needs_parens {
-                        (
-                            vec![
-                                (sp.shrink_to_lo(), format!("{prefix}{sugg}(")),
-                                (sp.shrink_to_hi(), ")".to_string()),
-                            ],
-                            false,
-                        )
-                    } else {
-                        (vec![(sp.shrink_to_lo(), format!("{prefix}{sugg}"))], true)
+                    let make_sugg = |expr: &Expr<'_>, span: Span, sugg: &str| {
+                        let needs_parens = match expr.kind {
+                            // parenthesize if needed (Issue #46756)
+                            hir::ExprKind::Cast(_, _) | hir::ExprKind::Binary(_, _, _) => true,
+                            // parenthesize borrows of range literals (Issue #54505)
+                            _ if is_range_literal(expr) => true,
+                            _ => false,
+                        };
+
+                        if needs_parens {
+                            (
+                                vec![
+                                    (span.shrink_to_lo(), format!("{prefix}{sugg}(")),
+                                    (span.shrink_to_hi(), ")".to_string()),
+                                ],
+                                false,
+                            )
+                        } else {
+                            (vec![(span.shrink_to_lo(), format!("{prefix}{sugg}"))], true)
+                        }
                     };
+
+                    // Suggest dereferencing the lhs for expressions such as `&T == T`
+                    if let Some(hir::Node::Expr(hir::Expr {
+                        kind: hir::ExprKind::Binary(_, lhs, ..),
+                        ..
+                    })) = self.tcx.hir().find_parent(expr.hir_id)
+                        && let &ty::Ref(..) = self.check_expr(lhs).kind()
+                    {
+                        let (sugg, verbose) = make_sugg(lhs, lhs.span, "*");
+
+                        return Some((
+                            sugg,
+                            "consider dereferencing the borrow".to_string(),
+                            Applicability::MachineApplicable,
+                            verbose,
+                            false,
+                        ));
+                    }
+
+                    let sugg = mutability.ref_prefix_str();
+                    let (sugg, verbose) = make_sugg(expr, sp, sugg);
                     return Some((
                         sugg,
                         format!("consider {}borrowing here", mutability.mutably_str()),
@@ -2382,11 +2469,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     ));
                 }
             }
-            (
-                hir::ExprKind::AddrOf(hir::BorrowKind::Ref, _, ref expr),
-                _,
-                &ty::Ref(_, checked, _),
-            ) if self.can_sub(self.param_env, checked, expected) => {
+            (hir::ExprKind::AddrOf(hir::BorrowKind::Ref, _, expr), _, &ty::Ref(_, checked, _))
+                if self.can_sub(self.param_env, checked, expected) =>
+            {
                 let make_sugg = |start: Span, end: BytePos| {
                     // skip `(` for tuples such as `(c) = (&123)`.
                     // make sure we won't suggest like `(c) = 123)` which is incorrect.
@@ -2581,7 +2666,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             if let Some(Node::Expr(hir::Expr {
                 kind: hir::ExprKind::If(_, _, Some(else_expr)),
                 ..
-            })) = self.tcx.hir().find(parent_id)
+            })) = self.tcx.opt_hir_node(parent_id)
             {
                 return else_expr.hir_id == expr.hir_id;
             }
@@ -2971,7 +3056,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             return;
         };
         let parent = self.tcx.hir().parent_id(expr.hir_id);
-        if let Some(hir::Node::ExprField(_)) = self.tcx.hir().find(parent) {
+        if let Some(hir::Node::ExprField(_)) = self.tcx.opt_hir_node(parent) {
             // Ignore `Foo { field: a..Default::default() }`
             return;
         }
@@ -3050,7 +3135,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let hir::def::Res::Local(hir_id) = path.res else {
             return;
         };
-        let Some(hir::Node::Pat(pat)) = self.tcx.hir().find(hir_id) else {
+        let Some(hir::Node::Pat(pat)) = self.tcx.opt_hir_node(hir_id) else {
             return;
         };
         let Some(hir::Node::Local(hir::Local { ty: None, init: Some(init), .. })) =
diff --git a/compiler/rustc_hir_typeck/src/gather_locals.rs b/compiler/rustc_hir_typeck/src/gather_locals.rs
index 0ad2c1d92..0cca779b1 100644
--- a/compiler/rustc_hir_typeck/src/gather_locals.rs
+++ b/compiler/rustc_hir_typeck/src/gather_locals.rs
@@ -93,7 +93,7 @@ impl<'a, 'tcx> GatherLocalsVisitor<'a, 'tcx> {
     fn declare(&mut self, decl: Declaration<'tcx>) {
         let local_ty = match decl.ty {
             Some(ref ty) => {
-                let o_ty = self.fcx.to_ty(&ty);
+                let o_ty = self.fcx.to_ty(ty);
 
                 let c_ty =
                     self.fcx.inh.infcx.canonicalize_user_type_annotation(UserType::Ty(o_ty.raw));
diff --git a/compiler/rustc_hir_typeck/src/inherited.rs b/compiler/rustc_hir_typeck/src/inherited.rs
index efd0b8577..7a6a2b2a0 100644
--- a/compiler/rustc_hir_typeck/src/inherited.rs
+++ b/compiler/rustc_hir_typeck/src/inherited.rs
@@ -9,7 +9,7 @@ use rustc_middle::traits::DefiningAnchor;
 use rustc_middle::ty::visit::TypeVisitableExt;
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_span::def_id::LocalDefIdMap;
-use rustc_span::{self, Span};
+use rustc_span::Span;
 use rustc_trait_selection::traits::query::evaluate_obligation::InferCtxtExt;
 use rustc_trait_selection::traits::{self, PredicateObligation, TraitEngine, TraitEngineExt as _};
 
@@ -75,7 +75,7 @@ impl<'tcx> Deref for Inherited<'tcx> {
 
 impl<'tcx> Inherited<'tcx> {
     pub fn new(tcx: TyCtxt<'tcx>, def_id: LocalDefId) -> Self {
-        let hir_owner = tcx.hir().local_def_id_to_hir_id(def_id).owner;
+        let hir_owner = tcx.local_def_id_to_hir_id(def_id).owner;
 
         let infcx = tcx
             .infer_ctxt()
diff --git a/compiler/rustc_hir_typeck/src/intrinsicck.rs b/compiler/rustc_hir_typeck/src/intrinsicck.rs
index 5d516eaf5..a3c77af62 100644
--- a/compiler/rustc_hir_typeck/src/intrinsicck.rs
+++ b/compiler/rustc_hir_typeck/src/intrinsicck.rs
@@ -48,7 +48,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let to = normalize(to);
         trace!(?from, ?to);
         if from.has_non_region_infer() || to.has_non_region_infer() {
-            tcx.sess.delay_span_bug(span, "argument to transmute has inference variables");
+            tcx.sess.span_delayed_bug(span, "argument to transmute has inference variables");
             return;
         }
         // Transmutes that are only changing lifetimes are always ok.
diff --git a/compiler/rustc_hir_typeck/src/lib.rs b/compiler/rustc_hir_typeck/src/lib.rs
index 46dcc4555..13a249486 100644
--- a/compiler/rustc_hir_typeck/src/lib.rs
+++ b/compiler/rustc_hir_typeck/src/lib.rs
@@ -52,11 +52,7 @@ use crate::expectation::Expectation;
 use crate::fn_ctxt::RawTy;
 use crate::gather_locals::GatherLocalsVisitor;
 use rustc_data_structures::unord::UnordSet;
-use rustc_errors::{
-    struct_span_err, DiagnosticId, DiagnosticMessage, ErrorGuaranteed, MultiSpan,
-    SubdiagnosticMessage,
-};
-use rustc_fluent_macro::fluent_messages;
+use rustc_errors::{struct_span_err, DiagnosticId, ErrorGuaranteed, MultiSpan};
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
 use rustc_hir::intravisit::Visitor;
@@ -71,7 +67,7 @@ use rustc_session::config;
 use rustc_span::def_id::{DefId, LocalDefId};
 use rustc_span::Span;
 
-fluent_messages! { "../messages.ftl" }
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
 
 #[macro_export]
 macro_rules! type_error_struct {
@@ -131,7 +127,7 @@ fn has_typeck_results(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
     }
 
     if let Some(def_id) = def_id.as_local() {
-        primary_body_of(tcx.hir().get_by_def_id(def_id)).is_some()
+        primary_body_of(tcx.hir_node_by_def_id(def_id)).is_some()
     } else {
         false
     }
@@ -150,7 +146,7 @@ fn typeck<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalDefId) -> &ty::TypeckResults<'tc
 /// Currently only used for type inference of `static`s and `const`s to avoid type cycle errors.
 fn diagnostic_only_typeck<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalDefId) -> &ty::TypeckResults<'tcx> {
     let fallback = move || {
-        let span = tcx.hir().span(tcx.hir().local_def_id_to_hir_id(def_id));
+        let span = tcx.hir().span(tcx.local_def_id_to_hir_id(def_id));
         Ty::new_error_with_message(tcx, span, "diagnostic only typeck table used")
     };
     typeck_with_fallback(tcx, def_id, fallback)
@@ -169,8 +165,8 @@ fn typeck_with_fallback<'tcx>(
         return tcx.typeck(typeck_root_def_id);
     }
 
-    let id = tcx.hir().local_def_id_to_hir_id(def_id);
-    let node = tcx.hir().get(id);
+    let id = tcx.local_def_id_to_hir_id(def_id);
+    let node = tcx.hir_node(id);
     let span = tcx.hir().span(id);
 
     // Figure out what primary body this item has.
@@ -205,7 +201,7 @@ fn typeck_with_fallback<'tcx>(
                 span,
             }))
         } else if let Node::AnonConst(_) = node {
-            match tcx.hir().get(tcx.hir().parent_id(id)) {
+            match tcx.hir_node(tcx.hir().parent_id(id)) {
                 Node::Ty(&hir::Ty { kind: hir::TyKind::Typeof(ref anon_const), .. })
                     if anon_const.hir_id == id =>
                 {
@@ -243,7 +239,7 @@ fn typeck_with_fallback<'tcx>(
         // Gather locals in statics (because of block expressions).
         GatherLocalsVisitor::new(&fcx).visit_body(body);
 
-        fcx.check_expr_coercible_to_type(&body.value, expected_type, None);
+        fcx.check_expr_coercible_to_type(body.value, expected_type, None);
 
         fcx.write_ty(id, expected_type);
     };
@@ -287,8 +283,6 @@ fn typeck_with_fallback<'tcx>(
 
     fcx.check_asms();
 
-    fcx.infcx.skip_region_resolution();
-
     let typeck_results = fcx.resolve_type_vars_in_body(body);
 
     // Consistency check our TypeckResults instance can hold all ItemLocalIds
@@ -419,32 +413,32 @@ enum TupleArgumentsFlag {
 }
 
 fn fatally_break_rust(tcx: TyCtxt<'_>) {
-    let handler = tcx.sess.diagnostic();
-    handler.span_bug_no_panic(
+    let dcx = tcx.sess.dcx();
+    dcx.span_bug_no_panic(
         MultiSpan::new(),
         "It looks like you're trying to break rust; would you like some ICE?",
     );
-    handler.note_without_error("the compiler expectedly panicked. this is a feature.");
-    handler.note_without_error(
+    dcx.note("the compiler expectedly panicked. this is a feature.");
+    dcx.note(
         "we would appreciate a joke overview: \
          https://github.com/rust-lang/rust/issues/43162#issuecomment-320764675",
     );
-    handler.note_without_error(format!(
-        "rustc {} running on {}",
-        tcx.sess.cfg_version,
-        config::host_triple(),
-    ));
+    dcx.note(format!("rustc {} running on {}", tcx.sess.cfg_version, config::host_triple(),));
     if let Some((flags, excluded_cargo_defaults)) = rustc_session::utils::extra_compiler_flags() {
-        handler.note_without_error(format!("compiler flags: {}", flags.join(" ")));
+        dcx.note(format!("compiler flags: {}", flags.join(" ")));
         if excluded_cargo_defaults {
-            handler.note_without_error("some of the compiler flags provided by cargo are hidden");
+            dcx.note("some of the compiler flags provided by cargo are hidden");
         }
     }
 }
 
+/// `expected` here is the expected number of explicit generic arguments on the trait.
 fn has_expected_num_generic_args(tcx: TyCtxt<'_>, trait_did: DefId, expected: usize) -> bool {
     let generics = tcx.generics_of(trait_did);
-    generics.count() == expected + if generics.has_self { 1 } else { 0 }
+    generics.count()
+        == expected
+            + if generics.has_self { 1 } else { 0 }
+            + if generics.host_effect_index.is_some() { 1 } else { 0 }
 }
 
 pub fn provide(providers: &mut Providers) {
diff --git a/compiler/rustc_hir_typeck/src/mem_categorization.rs b/compiler/rustc_hir_typeck/src/mem_categorization.rs
index 337d12b2d..ebb15e072 100644
--- a/compiler/rustc_hir_typeck/src/mem_categorization.rs
+++ b/compiler/rustc_hir_typeck/src/mem_categorization.rs
@@ -66,25 +66,18 @@ use rustc_trait_selection::infer::InferCtxtExt;
 
 pub(crate) trait HirNode {
     fn hir_id(&self) -> hir::HirId;
-    fn span(&self) -> Span;
 }
 
 impl HirNode for hir::Expr<'_> {
     fn hir_id(&self) -> hir::HirId {
         self.hir_id
     }
-    fn span(&self) -> Span {
-        self.span
-    }
 }
 
 impl HirNode for hir::Pat<'_> {
     fn hir_id(&self) -> hir::HirId {
         self.hir_id
     }
-    fn span(&self) -> Span {
-        self.span
-    }
 }
 
 #[derive(Clone)]
@@ -304,7 +297,7 @@ impl<'a, 'tcx> MemCategorizationContext<'a, 'tcx> {
     ) -> McResult<PlaceWithHirId<'tcx>> {
         let expr_ty = self.expr_ty(expr)?;
         match expr.kind {
-            hir::ExprKind::Unary(hir::UnOp::Deref, ref e_base) => {
+            hir::ExprKind::Unary(hir::UnOp::Deref, e_base) => {
                 if self.typeck_results.is_method_call(expr) {
                     self.cat_overloaded_place(expr, e_base)
                 } else {
@@ -313,7 +306,7 @@ impl<'a, 'tcx> MemCategorizationContext<'a, 'tcx> {
                 }
             }
 
-            hir::ExprKind::Field(ref base, _) => {
+            hir::ExprKind::Field(base, _) => {
                 let base = self.cat_expr(base)?;
                 debug!(?base);
 
@@ -332,7 +325,7 @@ impl<'a, 'tcx> MemCategorizationContext<'a, 'tcx> {
                 ))
             }
 
-            hir::ExprKind::Index(ref base, _, _) => {
+            hir::ExprKind::Index(base, _, _) => {
                 if self.typeck_results.is_method_call(expr) {
                     // If this is an index implemented by a method call, then it
                     // will include an implicit deref of the result.
@@ -351,7 +344,7 @@ impl<'a, 'tcx> MemCategorizationContext<'a, 'tcx> {
                 self.cat_res(expr.hir_id, expr.span, expr_ty, res)
             }
 
-            hir::ExprKind::Type(ref e, _) => self.cat_expr(e),
+            hir::ExprKind::Type(e, _) => self.cat_expr(e),
 
             hir::ExprKind::AddrOf(..)
             | hir::ExprKind::Call(..)
@@ -547,7 +540,7 @@ impl<'a, 'tcx> MemCategorizationContext<'a, 'tcx> {
         let ty::Adt(adt_def, _) = ty.kind() else {
             self.tcx()
                 .sess
-                .delay_span_bug(span, "struct or tuple struct pattern not applied to an ADT");
+                .span_delayed_bug(span, "struct or tuple struct pattern not applied to an ADT");
             return Err(());
         };
 
@@ -582,7 +575,7 @@ impl<'a, 'tcx> MemCategorizationContext<'a, 'tcx> {
             _ => {
                 self.tcx()
                     .sess
-                    .delay_span_bug(span, "struct or tuple struct pattern not applied to an ADT");
+                    .span_delayed_bug(span, "struct or tuple struct pattern not applied to an ADT");
                 Err(())
             }
         }
@@ -595,7 +588,7 @@ impl<'a, 'tcx> MemCategorizationContext<'a, 'tcx> {
         match ty.kind() {
             ty::Tuple(args) => Ok(args.len()),
             _ => {
-                self.tcx().sess.delay_span_bug(span, "tuple pattern not applied to a tuple");
+                self.tcx().sess.span_delayed_bug(span, "tuple pattern not applied to a tuple");
                 Err(())
             }
         }
@@ -728,11 +721,11 @@ impl<'a, 'tcx> MemCategorizationContext<'a, 'tcx> {
                 }
             }
 
-            PatKind::Binding(.., Some(ref subpat)) => {
+            PatKind::Binding(.., Some(subpat)) => {
                 self.cat_pattern_(place_with_id, subpat, op)?;
             }
 
-            PatKind::Box(ref subpat) | PatKind::Ref(ref subpat, _) => {
+            PatKind::Box(subpat) | PatKind::Ref(subpat, _) => {
                 // box p1, &p1, &mut p1. we can ignore the mutability of
                 // PatKind::Ref since that information is already contained
                 // in the type.
@@ -754,7 +747,7 @@ impl<'a, 'tcx> MemCategorizationContext<'a, 'tcx> {
                 for before_pat in before {
                     self.cat_pattern_(elt_place.clone(), before_pat, op)?;
                 }
-                if let Some(ref slice_pat) = *slice {
+                if let Some(slice_pat) = *slice {
                     let slice_pat_ty = self.pat_ty_adjusted(slice_pat)?;
                     let slice_place = self.cat_projection(
                         pat,
@@ -773,6 +766,7 @@ impl<'a, 'tcx> MemCategorizationContext<'a, 'tcx> {
             | PatKind::Binding(.., None)
             | PatKind::Lit(..)
             | PatKind::Range(..)
+            | PatKind::Never
             | PatKind::Wild => {
                 // always ok
             }
diff --git a/compiler/rustc_hir_typeck/src/method/confirm.rs b/compiler/rustc_hir_typeck/src/method/confirm.rs
index 7c73f6a89..b2ead3cd4 100644
--- a/compiler/rustc_hir_typeck/src/method/confirm.rs
+++ b/compiler/rustc_hir_typeck/src/method/confirm.rs
@@ -93,16 +93,16 @@ impl<'a, 'tcx> ConfirmContext<'a, 'tcx> {
         segment: &hir::PathSegment<'_>,
     ) -> ConfirmResult<'tcx> {
         // Adjust the self expression the user provided and obtain the adjusted type.
-        let self_ty = self.adjust_self_ty(unadjusted_self_ty, &pick);
+        let self_ty = self.adjust_self_ty(unadjusted_self_ty, pick);
 
         // Create substitutions for the method's type parameters.
-        let rcvr_args = self.fresh_receiver_args(self_ty, &pick);
-        let all_args = self.instantiate_method_args(&pick, segment, rcvr_args);
+        let rcvr_args = self.fresh_receiver_args(self_ty, pick);
+        let all_args = self.instantiate_method_args(pick, segment, rcvr_args);
 
         debug!("rcvr_args={rcvr_args:?}, all_args={all_args:?}");
 
         // Create the final signature for the method, replacing late-bound regions.
-        let (method_sig, method_predicates) = self.instantiate_method_sig(&pick, all_args);
+        let (method_sig, method_predicates) = self.instantiate_method_sig(pick, all_args);
 
         // If there is a `Self: Sized` bound and `Self` is a trait object, it is possible that
         // something which derefs to `Self` actually implements the trait and the caller
@@ -129,14 +129,14 @@ impl<'a, 'tcx> ConfirmContext<'a, 'tcx> {
             "confirm: self_ty={:?} method_sig_rcvr={:?} method_sig={:?} method_predicates={:?}",
             self_ty, method_sig_rcvr, method_sig, method_predicates
         );
-        self.unify_receivers(self_ty, method_sig_rcvr, &pick, all_args);
+        self.unify_receivers(self_ty, method_sig_rcvr, pick, all_args);
 
         let (method_sig, method_predicates) =
             self.normalize(self.span, (method_sig, method_predicates));
         let method_sig = ty::Binder::dummy(method_sig);
 
         // Make sure nobody calls `drop()` explicitly.
-        self.enforce_illegal_method_limitations(&pick);
+        self.enforce_illegal_method_limitations(pick);
 
         // Add any trait/regions obligations specified on the method's type parameters.
         // We won't add these if we encountered an illegal sized bound, so that we can use
@@ -415,7 +415,9 @@ impl<'a, 'tcx> ConfirmContext<'a, 'tcx> {
                             )
                             .into()
                     }
-                    _ => unreachable!(),
+                    (kind, arg) => {
+                        bug!("mismatched method arg kind {kind:?} in turbofish: {arg:?}")
+                    }
                 }
             }
 
diff --git a/compiler/rustc_hir_typeck/src/method/mod.rs b/compiler/rustc_hir_typeck/src/method/mod.rs
index d69d2529b..f820835ac 100644
--- a/compiler/rustc_hir_typeck/src/method/mod.rs
+++ b/compiler/rustc_hir_typeck/src/method/mod.rs
@@ -385,7 +385,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         // type parameters or early-bound regions.
         let tcx = self.tcx;
         let Some(method_item) = self.associated_value(trait_def_id, m_name) else {
-            tcx.sess.delay_span_bug(
+            tcx.sess.span_delayed_bug(
                 obligation.cause.span,
                 "operator trait does not have corresponding operator method",
             );
@@ -393,7 +393,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         };
 
         if method_item.kind != ty::AssocKind::Fn {
-            self.tcx.sess.delay_span_bug(tcx.def_span(method_item.def_id), "not a method");
+            self.tcx.sess.span_delayed_bug(tcx.def_span(method_item.def_id), "not a method");
             return None;
         }
 
diff --git a/compiler/rustc_hir_typeck/src/method/prelude2021.rs b/compiler/rustc_hir_typeck/src/method/prelude2021.rs
index 3f1dca5b1..43d258de6 100644
--- a/compiler/rustc_hir_typeck/src/method/prelude2021.rs
+++ b/compiler/rustc_hir_typeck/src/method/prelude2021.rs
@@ -122,8 +122,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             format!("disambiguate the method call with `({self_adjusted})`",),
                         );
                     }
-
-                    lint
                 },
             );
         } else {
@@ -187,8 +185,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             ),
                         );
                     }
-
-                    lint
                 },
             );
         }
@@ -307,8 +303,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     format!("<{} as {}>::{}", self_ty_name, trait_name, method_name.name,),
                     Applicability::MachineApplicable,
                 );
-
-                lint
             },
         );
     }
diff --git a/compiler/rustc_hir_typeck/src/method/probe.rs b/compiler/rustc_hir_typeck/src/method/probe.rs
index 74f469cb3..fe2d43a3c 100644
--- a/compiler/rustc_hir_typeck/src/method/probe.rs
+++ b/compiler/rustc_hir_typeck/src/method/probe.rs
@@ -445,7 +445,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         scope_expr_id,
                         span,
                         "type annotations needed",
-                        |lint| lint,
+                        |_| {},
                     );
                 }
             } else {
@@ -619,7 +619,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
     fn push_candidate(&mut self, candidate: Candidate<'tcx>, is_inherent: bool) {
         let is_accessible = if let Some(name) = self.method_name {
             let item = candidate.item;
-            let hir_id = self.tcx.hir().local_def_id_to_hir_id(self.body_id);
+            let hir_id = self.tcx.local_def_id_to_hir_id(self.body_id);
             let def_scope =
                 self.tcx.adjust_ident_and_get_scope(name, item.container_id(self.tcx), hir_id).1;
             item.visibility(self.tcx).is_accessible_from(def_scope, self.tcx)
@@ -801,15 +801,15 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
         // a `&self` method will wind up with an argument type like `&dyn Trait`.
         let trait_ref = principal.with_self_ty(self.tcx, self_ty);
         self.elaborate_bounds(iter::once(trait_ref), |this, new_trait_ref, item| {
-            if new_trait_ref.has_non_region_late_bound() {
-                this.tcx.sess.delay_span_bug(
+            if new_trait_ref.has_non_region_bound_vars() {
+                this.tcx.sess.span_delayed_bug(
                     this.span,
                     "tried to select method from HRTB with non-lifetime bound vars",
                 );
                 return;
             }
 
-            let new_trait_ref = this.erase_late_bound_regions(new_trait_ref);
+            let new_trait_ref = this.instantiate_bound_regions_with_erased(new_trait_ref);
 
             let (xform_self_ty, xform_ret_ty) =
                 this.xform_self_ty(item, new_trait_ref.self_ty(), new_trait_ref.args);
@@ -853,7 +853,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
         self.elaborate_bounds(bounds, |this, poly_trait_ref, item| {
             let trait_ref = this.instantiate_binder_with_fresh_vars(
                 this.span,
-                infer::LateBoundRegionConversionTime::FnCall,
+                infer::BoundRegionConversionTime::FnCall,
                 poly_trait_ref,
             );
 
@@ -971,7 +971,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
                     } else {
                         let new_trait_ref = self.instantiate_binder_with_fresh_vars(
                             self.span,
-                            infer::LateBoundRegionConversionTime::FnCall,
+                            infer::BoundRegionConversionTime::FnCall,
                             bound_trait_ref,
                         );
 
@@ -1135,7 +1135,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
                     .fcx
                     .probe_instantiate_query_response(
                         self.span,
-                        &self.orig_steps_var_values,
+                        self.orig_steps_var_values,
                         &step.self_ty,
                     )
                     .unwrap_or_else(|_| {
@@ -1427,8 +1427,6 @@ impl<'tcx> Pick<'tcx> {
                         ));
                     }
                 }
-
-                lint
             },
         );
     }
@@ -1509,7 +1507,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
             // match as well (or at least may match, sometimes we
             // don't have enough information to fully evaluate).
             match probe.kind {
-                InherentImplCandidate(ref args, ref ref_obligations) => {
+                InherentImplCandidate(args, ref ref_obligations) => {
                     // `xform_ret_ty` hasn't been normalized yet, only `xform_self_ty`,
                     // see the reasons mentioned in the comments in `assemble_inherent_impl_probe`
                     // for why this is necessary
@@ -1548,9 +1546,9 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
                     );
 
                     let candidate_obligations = impl_obligations
-                        .chain(norm_obligations.into_iter())
+                        .chain(norm_obligations)
                         .chain(ref_obligations.iter().cloned())
-                        .chain(normalization_obligations.into_iter());
+                        .chain(normalization_obligations);
 
                     // Evaluate those obligations to see if they might possibly hold.
                     for o in candidate_obligations {
@@ -1799,9 +1797,10 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
                         .iter()
                         .find(|cand| self.matches_by_doc_alias(cand.def_id))
                         .map(|cand| cand.name)
-                })
-                .unwrap();
-                Ok(applicable_close_candidates.into_iter().find(|method| method.name == best_name))
+                });
+                Ok(best_name.and_then(|best_name| {
+                    applicable_close_candidates.into_iter().find(|method| method.name == best_name)
+                }))
             }
         })
     }
@@ -1861,7 +1860,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
         // method yet. So create fresh variables here for those too,
         // if there are any.
         let generics = self.tcx.generics_of(method);
-        assert_eq!(args.len(), generics.parent_count as usize);
+        assert_eq!(args.len(), generics.parent_count);
 
         let xform_fn_sig = if generics.params.is_empty() {
             fn_sig.instantiate(self.tcx, args)
@@ -1885,7 +1884,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
             fn_sig.instantiate(self.tcx, args)
         };
 
-        self.erase_late_bound_regions(xform_fn_sig)
+        self.instantiate_bound_regions_with_erased(xform_fn_sig)
     }
 
     /// Gets the type of an impl and generate substitutions with inference vars.
@@ -1897,7 +1896,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
     }
 
     /// Replaces late-bound-regions bound by `value` with `'static` using
-    /// `ty::erase_late_bound_regions`.
+    /// `ty::instantiate_bound_regions_with_erased`.
     ///
     /// This is only a reasonable thing to do during the *probe* phase, not the *confirm* phase, of
     /// method matching. It is reasonable during the probe phase because we don't consider region
@@ -1914,11 +1913,11 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
     ///    region got replaced with the same variable, which requires a bit more coordination
     ///    and/or tracking the substitution and
     ///    so forth.
-    fn erase_late_bound_regions<T>(&self, value: ty::Binder<'tcx, T>) -> T
+    fn instantiate_bound_regions_with_erased<T>(&self, value: ty::Binder<'tcx, T>) -> T
     where
         T: TypeFoldable<TyCtxt<'tcx>>,
     {
-        self.tcx.erase_late_bound_regions(value)
+        self.tcx.instantiate_bound_regions_with_erased(value)
     }
 
     /// Determine if the given associated item type is relevant in the current context.
@@ -1939,7 +1938,7 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
         let Some(local_def_id) = def_id.as_local() else {
             return false;
         };
-        let hir_id = self.fcx.tcx.hir().local_def_id_to_hir_id(local_def_id);
+        let hir_id = self.fcx.tcx.local_def_id_to_hir_id(local_def_id);
         let attrs = self.fcx.tcx.hir().attrs(hir_id);
         for attr in attrs {
             let sym::doc = attr.name_or_empty() else {
diff --git a/compiler/rustc_hir_typeck/src/method/suggest.rs b/compiler/rustc_hir_typeck/src/method/suggest.rs
index bd40e6c10..7595f21d9 100644
--- a/compiler/rustc_hir_typeck/src/method/suggest.rs
+++ b/compiler/rustc_hir_typeck/src/method/suggest.rs
@@ -1,6 +1,8 @@
 //! Give useful errors and suggestions to users when an item can't be
 //! found or is otherwise invalid.
 
+// ignore-tidy-filelength
+
 use crate::errors;
 use crate::errors::{CandidateTraitNote, NoAssociatedItem};
 use crate::Expectation;
@@ -228,7 +230,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         }
                         if let hir::ExprKind::Path(hir::QPath::Resolved(None, path)) = kind
                             && let hir::def::Res::Local(hir_id) = path.res
-                            && let Some(hir::Node::Pat(b)) = self.tcx.hir().find(hir_id)
+                            && let Some(hir::Node::Pat(b)) = self.tcx.opt_hir_node(hir_id)
                             && let Some(hir::Node::Param(p)) = self.tcx.hir().find_parent(b.hir_id)
                             && let Some(node) = self.tcx.hir().find_parent(p.hir_id)
                             && let Some(decl) = node.fn_decl()
@@ -260,7 +262,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         rcvr_ty: Ty<'tcx>,
         rcvr_expr: &hir::Expr<'tcx>,
     ) -> DiagnosticBuilder<'_, ErrorGuaranteed> {
-        let (ty_str, _ty_file) = self.tcx.short_ty_string(rcvr_ty);
+        let mut file = None;
+        let ty_str = self.tcx.short_ty_string(rcvr_ty, &mut file);
         let mut err = struct_span_err!(
             self.tcx.sess,
             rcvr_expr.span,
@@ -278,6 +281,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 "a writer is needed before this format string",
             );
         };
+        if let Some(file) = file {
+            err.note(format!("the full type name has been written to '{}'", file.display()));
+        }
 
         err
     }
@@ -297,11 +303,15 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let mode = no_match_data.mode;
         let tcx = self.tcx;
         let rcvr_ty = self.resolve_vars_if_possible(rcvr_ty);
-        let ((mut ty_str, ty_file), short_ty_str) =
+        let mut ty_file = None;
+        let (mut ty_str, short_ty_str) =
             if trait_missing_method && let ty::Dynamic(predicates, _, _) = rcvr_ty.kind() {
-                ((predicates.to_string(), None), with_forced_trimmed_paths!(predicates.to_string()))
+                (predicates.to_string(), with_forced_trimmed_paths!(predicates.to_string()))
             } else {
-                (tcx.short_ty_string(rcvr_ty), with_forced_trimmed_paths!(rcvr_ty.to_string()))
+                (
+                    tcx.short_ty_string(rcvr_ty, &mut ty_file),
+                    with_forced_trimmed_paths!(rcvr_ty.to_string()),
+                )
             };
         let is_method = mode == Mode::MethodCall;
         let unsatisfied_predicates = &no_match_data.unsatisfied_predicates;
@@ -369,25 +379,23 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             tcx.is_diagnostic_item(sym::write_macro, def_id)
                 || tcx.is_diagnostic_item(sym::writeln_macro, def_id)
         }) && item_name.name == Symbol::intern("write_fmt");
-        let mut err =
-            if is_write && let SelfSource::MethodCall(rcvr_expr) = source
-            {
-                self.suggest_missing_writer(rcvr_ty, rcvr_expr)
-            } else {
-                tcx.sess.create_err(NoAssociatedItem {
-                    span,
-                    item_kind,
-                    item_name,
-                    ty_prefix: if trait_missing_method {
-                        // FIXME(mu001999) E0599 maybe not suitable here because it is for types
-                        Cow::from("trait")
-                    } else {
-                        rcvr_ty.prefix_string(self.tcx)
-                    },
-                    ty_str: ty_str_reported,
-                    trait_missing_method,
-                })
-            };
+        let mut err = if is_write && let SelfSource::MethodCall(rcvr_expr) = source {
+            self.suggest_missing_writer(rcvr_ty, rcvr_expr)
+        } else {
+            tcx.sess.create_err(NoAssociatedItem {
+                span,
+                item_kind,
+                item_name,
+                ty_prefix: if trait_missing_method {
+                    // FIXME(mu001999) E0599 maybe not suitable here because it is for types
+                    Cow::from("trait")
+                } else {
+                    rcvr_ty.prefix_string(self.tcx)
+                },
+                ty_str: ty_str_reported,
+                trait_missing_method,
+            })
+        };
         if tcx.sess.source_map().is_multiline(sugg_span) {
             err.span_label(sugg_span.with_hi(span.lo()), "");
         }
@@ -484,7 +492,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             self.note_internal_mutation_in_method(
                 &mut err,
                 rcvr_expr,
-                expected.to_option(&self),
+                expected.to_option(self),
                 rcvr_ty,
             );
         }
@@ -509,7 +517,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         if static_candidates.len() == 1 {
             self.suggest_associated_call_syntax(
                 &mut err,
-                &static_candidates,
+                static_candidates,
                 rcvr_ty,
                 source,
                 item_name,
@@ -605,16 +613,16 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     if let (ty::Param(_), ty::PredicateKind::Clause(ty::ClauseKind::Trait(p))) =
                         (self_ty.kind(), parent_pred.kind().skip_binder())
                     {
-                        let hir = self.tcx.hir();
                         let node = match p.trait_ref.self_ty().kind() {
                             ty::Param(_) => {
                                 // Account for `fn` items like in `issue-35677.rs` to
                                 // suggest restricting its type params.
-                                Some(hir.get_by_def_id(self.body_id))
-                            }
-                            ty::Adt(def, _) => {
-                                def.did().as_local().map(|def_id| hir.get_by_def_id(def_id))
+                                Some(self.tcx.hir_node_by_def_id(self.body_id))
                             }
+                            ty::Adt(def, _) => def
+                                .did()
+                                .as_local()
+                                .map(|def_id| self.tcx.hir_node_by_def_id(def_id)),
                             _ => None,
                         };
                         if let Some(hir::Node::Item(hir::Item { kind, .. })) = node
@@ -813,7 +821,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         span: item_span,
                         ..
                     })) => {
-                        tcx.sess.delay_span_bug(
+                        tcx.sess.span_delayed_bug(
                             *item_span,
                             "auto trait is invoked with no method error, but no error reported?",
                         );
@@ -969,7 +977,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         "the following trait bounds were not satisfied:\n{bound_list}"
                     ));
                 }
-                self.suggest_derive(&mut err, &unsatisfied_predicates);
+                self.suggest_derive(&mut err, unsatisfied_predicates);
 
                 unsatisfied_bounds = true;
             }
@@ -1170,8 +1178,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 args.map(|args| args.len() + 1),
                 source,
                 no_match_data.out_of_scope_traits.clone(),
-                &unsatisfied_predicates,
-                &static_candidates,
+                unsatisfied_predicates,
+                static_candidates,
                 unsatisfied_bounds,
                 expected.only_has_type(self),
                 trait_missing_method,
@@ -1240,20 +1248,27 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             }
         }
         // If an appropriate error source is not found, check method chain for possible candiates
-        if unsatisfied_predicates.is_empty() && let Mode::MethodCall = mode && let SelfSource::MethodCall(mut source_expr) = source {
+        if unsatisfied_predicates.is_empty()
+            && let Mode::MethodCall = mode
+            && let SelfSource::MethodCall(mut source_expr) = source
+        {
             let mut stack_methods = vec![];
             while let hir::ExprKind::MethodCall(_path_segment, rcvr_expr, _args, method_span) =
-            source_expr.kind
+                source_expr.kind
             {
-                 // Pop the matching receiver, to align on it's notional span
-                 if let Some(prev_match) = stack_methods.pop() {
-                    err.span_label(method_span, format!("{item_kind} `{item_name}` is available on `{prev_match}`"));
+                // Pop the matching receiver, to align on it's notional span
+                if let Some(prev_match) = stack_methods.pop() {
+                    err.span_label(
+                        method_span,
+                        format!("{item_kind} `{item_name}` is available on `{prev_match}`"),
+                    );
                 }
                 let rcvr_ty = self.resolve_vars_if_possible(
                     self.typeck_results
                         .borrow()
                         .expr_ty_adjusted_opt(rcvr_expr)
-                        .unwrap_or(Ty::new_misc_error(self.tcx)),);
+                        .unwrap_or(Ty::new_misc_error(self.tcx)),
+                );
 
                 for _matched_method in self.probe_for_name_many(
                     Mode::MethodCall,
@@ -1262,15 +1277,19 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     IsSuggestion(true),
                     rcvr_ty,
                     source_expr.hir_id,
-                    ProbeScope::TraitsInScope,) {
-                        // found a match, push to stack
-                        stack_methods.push(rcvr_ty);
+                    ProbeScope::TraitsInScope,
+                ) {
+                    // found a match, push to stack
+                    stack_methods.push(rcvr_ty);
                 }
                 source_expr = rcvr_expr;
             }
             // If there is a match at the start of the chain, add a label for it too!
             if let Some(prev_match) = stack_methods.pop() {
-                err.span_label(source_expr.span, format!("{item_kind} `{item_name}` is available on `{prev_match}`"));
+                err.span_label(
+                    source_expr.span,
+                    format!("{item_kind} `{item_name}` is available on `{prev_match}`"),
+                );
             }
         }
         self.note_derefed_ty_has_method(&mut err, source, rcvr_ty, item_name, expected);
@@ -1357,10 +1376,12 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             err,
                             self_source,
                             args,
-                            trait_ref.instantiate(
-                                self.tcx,
-                                self.fresh_args_for_item(sugg_span, impl_did)
-                            ).with_self_ty(self.tcx, rcvr_ty),
+                            trait_ref
+                                .instantiate(
+                                    self.tcx,
+                                    self.fresh_args_for_item(sugg_span, impl_did),
+                                )
+                                .with_self_ty(self.tcx, rcvr_ty),
                             idx,
                             sugg_span,
                             item,
@@ -1397,8 +1418,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             ty::TraitRef::new(
                                 self.tcx,
                                 trait_did,
-                                self.fresh_args_for_item(sugg_span, trait_did)
-                            ).with_self_ty(self.tcx, rcvr_ty),
+                                self.fresh_args_for_item(sugg_span, trait_did),
+                            )
+                            .with_self_ty(self.tcx, rcvr_ty),
                             idx,
                             sugg_span,
                             item,
@@ -1409,7 +1431,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 }
             }
         }
-        if !suggs.is_empty() && let Some(span) = sugg_span {
+        if !suggs.is_empty()
+            && let Some(span) = sugg_span
+        {
+            suggs.sort();
             err.span_suggestions(
                 span.with_hi(item_name.span.lo()),
                 "use fully-qualified syntax to disambiguate",
@@ -1438,7 +1463,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             .filter_map(|item| {
                 // Only assoc fns that return `Self`, `Option<Self>` or `Result<Self, _>`.
                 let ret_ty = self.tcx.fn_sig(item.def_id).skip_binder().output();
-                let ret_ty = self.tcx.erase_late_bound_regions(ret_ty);
+                let ret_ty = self.tcx.instantiate_bound_regions_with_erased(ret_ty);
                 let ty::Adt(def, args) = ret_ty.kind() else {
                     return None;
                 };
@@ -1574,10 +1599,12 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             {
                 let sig = self.tcx.fn_sig(assoc.def_id).instantiate_identity();
                 sig.inputs().skip_binder().get(0).and_then(|first| {
-                    if first.peel_refs() == rcvr_ty.peel_refs() {
-                        None
-                    } else {
+                    let impl_ty = self.tcx.type_of(*impl_did).instantiate_identity();
+                    // if the type of first arg is the same as the current impl type, we should take the first arg into assoc function
+                    if first.peel_refs() == impl_ty {
                         Some(first.ref_mutability().map_or("", |mutbl| mutbl.ref_prefix_str()))
+                    } else {
+                        None
                     }
                 })
             } else {
@@ -1585,39 +1612,39 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             };
             let mut applicability = Applicability::MachineApplicable;
             let args = if let SelfSource::MethodCall(receiver) = source
-                    && let Some(args) = args
-                {
-                    // The first arg is the same kind as the receiver
-                    let explicit_args = if first_arg.is_some() {
-                        std::iter::once(receiver).chain(args.iter()).collect::<Vec<_>>()
-                    } else {
-                        // There is no `Self` kind to infer the arguments from
-                        if has_unsuggestable_args {
-                            applicability = Applicability::HasPlaceholders;
-                        }
-                        args.iter().collect()
-                    };
-                    format!(
-                        "({}{})",
-                        first_arg.unwrap_or(""),
-                        explicit_args
-                            .iter()
-                            .map(|arg| self
-                                .tcx
-                                .sess
-                                .source_map()
-                                .span_to_snippet(arg.span)
-                                .unwrap_or_else(|_| {
-                                    applicability = Applicability::HasPlaceholders;
-                                    "_".to_owned()
-                                }))
-                            .collect::<Vec<_>>()
-                            .join(", "),
-                    )
+                && let Some(args) = args
+            {
+                // The first arg is the same kind as the receiver
+                let explicit_args = if first_arg.is_some() {
+                    std::iter::once(receiver).chain(args.iter()).collect::<Vec<_>>()
                 } else {
-                    applicability = Applicability::HasPlaceholders;
-                    "(...)".to_owned()
+                    // There is no `Self` kind to infer the arguments from
+                    if has_unsuggestable_args {
+                        applicability = Applicability::HasPlaceholders;
+                    }
+                    args.iter().collect()
                 };
+                format!(
+                    "({}{})",
+                    first_arg.unwrap_or(""),
+                    explicit_args
+                        .iter()
+                        .map(|arg| self
+                            .tcx
+                            .sess
+                            .source_map()
+                            .span_to_snippet(arg.span)
+                            .unwrap_or_else(|_| {
+                                applicability = Applicability::HasPlaceholders;
+                                "_".to_owned()
+                            }))
+                        .collect::<Vec<_>>()
+                        .join(", "),
+                )
+            } else {
+                applicability = Applicability::HasPlaceholders;
+                "(...)".to_owned()
+            };
             err.span_suggestion(
                 sugg_span,
                 "use associated function syntax instead",
@@ -1705,7 +1732,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             for (_, parent) in tcx.hir().parent_iter(expr.hir_id).take(5) {
                 if let Node::Expr(parent_expr) = parent {
                     let lang_item = match parent_expr.kind {
-                        ExprKind::Struct(ref qpath, _, _) => match **qpath {
+                        ExprKind::Struct(qpath, _, _) => match *qpath {
                             QPath::LangItem(LangItem::Range, ..) => Some(LangItem::Range),
                             QPath::LangItem(LangItem::RangeTo, ..) => Some(LangItem::RangeTo),
                             QPath::LangItem(LangItem::RangeToInclusive, ..) => {
@@ -1713,7 +1740,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             }
                             _ => None,
                         },
-                        ExprKind::Call(ref func, _) => match func.kind {
+                        ExprKind::Call(func, _) => match func.kind {
                             // `..=` desugars into `::std::ops::RangeInclusive::new(...)`.
                             ExprKind::Path(QPath::LangItem(LangItem::RangeInclusiveNew, ..)) => {
                                 Some(LangItem::RangeInclusiveStruct)
@@ -1732,7 +1759,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             eps.len() > 0 && eps.last().is_some_and(|ep| ep.span.contains(span))
                         }
                         // `..=` desugars into `::std::ops::RangeInclusive::new(...)`.
-                        hir::ExprKind::Call(ref func, ..) => func.span.contains(span),
+                        hir::ExprKind::Call(func, ..) => func.span.contains(span),
                         _ => false,
                     };
 
@@ -1826,7 +1853,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             );
             let concrete_type = if actual.is_integral() { "i32" } else { "f32" };
             match expr.kind {
-                ExprKind::Lit(ref lit) => {
+                ExprKind::Lit(lit) => {
                     // numeric literal
                     let snippet = tcx
                         .sess
@@ -1902,7 +1929,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             return;
         };
         let Some(mut diag) =
-            self.tcx.sess.diagnostic().steal_diagnostic(seg1.ident.span, StashKey::CallAssocMethod)
+            self.tcx.sess.dcx().steal_diagnostic(seg1.ident.span, StashKey::CallAssocMethod)
         else {
             return;
         };
@@ -1930,10 +1957,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         }
 
         let mut visitor = LetVisitor { result: None, ident_name: seg1.ident.name };
-        visitor.visit_body(&body);
+        visitor.visit_body(body);
 
         let parent = self.tcx.hir().parent_id(seg1.hir_id);
-        if let Some(Node::Expr(call_expr)) = self.tcx.hir().find(parent)
+        if let Some(Node::Expr(call_expr)) = self.tcx.opt_hir_node(parent)
             && let Some(expr) = visitor.result
             && let Some(self_ty) = self.node_ty_opt(expr.hir_id)
         {
@@ -1967,69 +1994,73 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         item_name: Ident,
         return_type: Option<Ty<'tcx>>,
     ) {
-        if let SelfSource::MethodCall(expr) = source
-            && let mod_id = self.tcx.parent_module(expr.hir_id).to_def_id()
-            && let Some((fields, args)) =
-                self.get_field_candidates_considering_privacy(span, actual, mod_id)
-        {
-            let call_expr = self.tcx.hir().expect_expr(self.tcx.hir().parent_id(expr.hir_id));
+        if let SelfSource::MethodCall(expr) = source {
+            let mod_id = self.tcx.parent_module(expr.hir_id).to_def_id();
+            for (fields, args) in
+                self.get_field_candidates_considering_privacy(span, actual, mod_id, expr.hir_id)
+            {
+                let call_expr = self.tcx.hir().expect_expr(self.tcx.hir().parent_id(expr.hir_id));
 
-            let lang_items = self.tcx.lang_items();
-            let never_mention_traits = [
-                lang_items.clone_trait(),
-                lang_items.deref_trait(),
-                lang_items.deref_mut_trait(),
-                self.tcx.get_diagnostic_item(sym::AsRef),
-                self.tcx.get_diagnostic_item(sym::AsMut),
-                self.tcx.get_diagnostic_item(sym::Borrow),
-                self.tcx.get_diagnostic_item(sym::BorrowMut),
-            ];
-            let candidate_fields: Vec<_> = fields
-                .filter_map(|candidate_field| {
-                    self.check_for_nested_field_satisfying(
-                        span,
-                        &|_, field_ty| {
-                            self.lookup_probe_for_diagnostic(
-                                item_name,
-                                field_ty,
-                                call_expr,
-                                ProbeScope::TraitsInScope,
-                                return_type,
-                            )
-                            .is_ok_and(|pick| {
-                                !never_mention_traits
-                                    .iter()
-                                    .flatten()
-                                    .any(|def_id| self.tcx.parent(pick.item.def_id) == *def_id)
-                            })
-                        },
-                        candidate_field,
-                        args,
-                        vec![],
-                        mod_id,
-                    )
-                })
-                .map(|field_path| {
-                    field_path
-                        .iter()
-                        .map(|id| id.name.to_ident_string())
-                        .collect::<Vec<String>>()
-                        .join(".")
-                })
-                .collect();
+                let lang_items = self.tcx.lang_items();
+                let never_mention_traits = [
+                    lang_items.clone_trait(),
+                    lang_items.deref_trait(),
+                    lang_items.deref_mut_trait(),
+                    self.tcx.get_diagnostic_item(sym::AsRef),
+                    self.tcx.get_diagnostic_item(sym::AsMut),
+                    self.tcx.get_diagnostic_item(sym::Borrow),
+                    self.tcx.get_diagnostic_item(sym::BorrowMut),
+                ];
+                let mut candidate_fields: Vec<_> = fields
+                    .into_iter()
+                    .filter_map(|candidate_field| {
+                        self.check_for_nested_field_satisfying(
+                            span,
+                            &|_, field_ty| {
+                                self.lookup_probe_for_diagnostic(
+                                    item_name,
+                                    field_ty,
+                                    call_expr,
+                                    ProbeScope::TraitsInScope,
+                                    return_type,
+                                )
+                                .is_ok_and(|pick| {
+                                    !never_mention_traits
+                                        .iter()
+                                        .flatten()
+                                        .any(|def_id| self.tcx.parent(pick.item.def_id) == *def_id)
+                                })
+                            },
+                            candidate_field,
+                            args,
+                            vec![],
+                            mod_id,
+                            expr.hir_id,
+                        )
+                    })
+                    .map(|field_path| {
+                        field_path
+                            .iter()
+                            .map(|id| id.name.to_ident_string())
+                            .collect::<Vec<String>>()
+                            .join(".")
+                    })
+                    .collect();
+                candidate_fields.sort();
 
-            let len = candidate_fields.len();
-            if len > 0 {
-                err.span_suggestions(
-                    item_name.span.shrink_to_lo(),
-                    format!(
-                        "{} of the expressions' fields {} a method of the same name",
-                        if len > 1 { "some" } else { "one" },
-                        if len > 1 { "have" } else { "has" },
-                    ),
-                    candidate_fields.iter().map(|path| format!("{path}.")),
-                    Applicability::MaybeIncorrect,
-                );
+                let len = candidate_fields.len();
+                if len > 0 {
+                    err.span_suggestions(
+                        item_name.span.shrink_to_lo(),
+                        format!(
+                            "{} of the expressions' fields {} a method of the same name",
+                            if len > 1 { "some" } else { "one" },
+                            if len > 1 { "have" } else { "has" },
+                        ),
+                        candidate_fields.iter().map(|path| format!("{path}.")),
+                        Applicability::MaybeIncorrect,
+                    );
+                }
             }
         }
     }
@@ -2267,7 +2298,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 ty::Adt(def, _) if def.did().is_local() => {
                     spans.push_span_label(
                         self.tcx.def_span(def.did()),
-                        format!("must implement `{}`", pred.trait_ref.print_only_trait_path()),
+                        format!("must implement `{}`", pred.trait_ref.print_trait_sugared()),
                     );
                 }
                 _ => {}
@@ -2278,7 +2309,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             let msg = if preds.len() == 1 {
                 format!(
                     "an implementation of `{}` might be missing for `{}`",
-                    preds[0].trait_ref.print_only_trait_path(),
+                    preds[0].trait_ref.print_trait_sugared(),
                     preds[0].self_ty()
                 )
             } else {
@@ -2548,13 +2579,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 self.tcx.item_name(*trait_did),
             )
         });
+        let mut sugg: Vec<_> = path_strings.chain(glob_path_strings).collect();
+        sugg.sort();
 
-        err.span_suggestions(
-            span,
-            msg,
-            path_strings.chain(glob_path_strings),
-            Applicability::MaybeIncorrect,
-        );
+        err.span_suggestions(span, msg, sugg, Applicability::MaybeIncorrect);
     }
 
     fn suggest_valid_traits(
@@ -2849,11 +2877,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                                 let id = item
                                     .def_id
                                     .as_local()
-                                    .map(|def_id| self.tcx.hir().local_def_id_to_hir_id(def_id));
+                                    .map(|def_id| self.tcx.hir_node_by_def_id(def_id));
                                 if let Some(hir::Node::TraitItem(hir::TraitItem {
                                     kind: hir::TraitItemKind::Fn(fn_sig, method),
                                     ..
-                                })) = id.map(|id| self.tcx.hir().get(id))
+                                })) = id
                                 {
                                     let self_first_arg = match method {
                                         hir::TraitFn::Required([ident, ..]) => {
@@ -2939,11 +2967,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 let type_param = generics.type_param(param, self.tcx);
                 let hir = self.tcx.hir();
                 if let Some(def_id) = type_param.def_id.as_local() {
-                    let id = hir.local_def_id_to_hir_id(def_id);
+                    let id = self.tcx.local_def_id_to_hir_id(def_id);
                     // Get the `hir::Param` to verify whether it already has any bounds.
                     // We do this to avoid suggesting code that ends up as `T: FooBar`,
                     // instead we suggest `T: Foo + Bar` in that case.
-                    match hir.get(id) {
+                    match self.tcx.hir_node(id) {
                         Node::GenericParam(param) => {
                             enum Introducer {
                                 Plus,
@@ -3163,7 +3191,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         }
 
         let parent = self.tcx.hir().parent_id(expr.hir_id);
-        if let Some(Node::Expr(call_expr)) = self.tcx.hir().find(parent)
+        if let Some(Node::Expr(call_expr)) = self.tcx.opt_hir_node(parent)
             && let hir::ExprKind::MethodCall(
                 hir::PathSegment { ident: method_name, .. },
                 self_expr,
@@ -3296,8 +3324,13 @@ fn print_disambiguation_help<'tcx>(
         {
             let def_kind_descr = tcx.def_kind_descr(item.kind.as_def_kind(), item.def_id);
             let item_name = item.ident(tcx);
-            let rcvr_ref = tcx.fn_sig(item.def_id).skip_binder().skip_binder().inputs()[0]
-                .ref_mutability()
+            let rcvr_ref = tcx
+                .fn_sig(item.def_id)
+                .skip_binder()
+                .skip_binder()
+                .inputs()
+                .get(0)
+                .and_then(|ty| ty.ref_mutability())
                 .map_or("", |mutbl| mutbl.ref_prefix_str());
             let args = format!(
                 "({}{})",
diff --git a/compiler/rustc_hir_typeck/src/op.rs b/compiler/rustc_hir_typeck/src/op.rs
index f40406c67..59b4b0032 100644
--- a/compiler/rustc_hir_typeck/src/op.rs
+++ b/compiler/rustc_hir_typeck/src/op.rs
@@ -4,7 +4,7 @@ use super::method::MethodCallee;
 use super::{has_expected_num_generic_args, FnCtxt};
 use crate::Expectation;
 use rustc_ast as ast;
-use rustc_errors::{self, struct_span_err, Applicability, Diagnostic, DiagnosticBuilder};
+use rustc_errors::{struct_span_err, Applicability, Diagnostic, DiagnosticBuilder};
 use rustc_hir as hir;
 use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
 use rustc_infer::traits::ObligationCauseCode;
@@ -291,7 +291,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             .push(autoref);
                     }
                 }
-                self.write_method_call(expr.hir_id, method);
+                self.write_method_call_and_enforce_effects(expr.hir_id, expr.span, method);
 
                 method.sig.output()
             }
@@ -781,7 +781,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         assert!(op.is_by_value());
         match self.lookup_op_method(operand_ty, None, Op::Unary(op, ex.span), expected) {
             Ok(method) => {
-                self.write_method_call(ex.hir_id, method);
+                self.write_method_call_and_enforce_effects(ex.hir_id, ex.span, method);
                 method.sig.output()
             }
             Err(errors) => {
@@ -900,7 +900,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         ) {
             self.tcx
                 .sess
-                .delay_span_bug(span, "operator didn't have the right number of generic args");
+                .span_delayed_bug(span, "operator didn't have the right number of generic args");
             return Err(vec![]);
         }
 
@@ -933,7 +933,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 // This path may do some inference, so make sure we've really
                 // doomed compilation so as to not accidentally stabilize new
                 // inference or something here...
-                self.tcx.sess.delay_span_bug(span, "this path really should be doomed...");
+                self.tcx.sess.span_delayed_bug(span, "this path really should be doomed...");
                 // Guide inference for the RHS expression if it's provided --
                 // this will allow us to better error reporting, at the expense
                 // of making some error messages a bit more specific.
diff --git a/compiler/rustc_hir_typeck/src/pat.rs b/compiler/rustc_hir_typeck/src/pat.rs
index b30f9b82f..56a420fab 100644
--- a/compiler/rustc_hir_typeck/src/pat.rs
+++ b/compiler/rustc_hir_typeck/src/pat.rs
@@ -178,6 +178,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         let ty = match pat.kind {
             PatKind::Wild => expected,
+            // FIXME(never_patterns): check the type is uninhabited. If that is not possible within
+            // typeck, do that in a later phase.
+            PatKind::Never => expected,
             PatKind::Lit(lt) => self.check_pat_lit(pat.span, lt, expected, ti),
             PatKind::Range(lhs, rhs, _) => self.check_pat_range(pat.span, lhs, rhs, expected, ti),
             PatKind::Binding(ba, var_id, _, sub) => {
@@ -287,9 +290,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             | PatKind::Box(_)
             | PatKind::Range(..)
             | PatKind::Slice(..) => AdjustMode::Peel,
+            // A never pattern behaves somewhat like a literal or unit variant.
+            PatKind::Never => AdjustMode::Peel,
             // String and byte-string literals result in types `&str` and `&[u8]` respectively.
             // All other literals result in non-reference types.
-            // As a result, we allow `if let 0 = &&0 {}` but not `if let "foo" = &&"foo {}`.
+            // As a result, we allow `if let 0 = &&0 {}` but not `if let "foo" = &&"foo" {}`.
             //
             // Call `resolve_vars_if_possible` here for inline const blocks.
             PatKind::Lit(lt) => match self.resolve_vars_if_possible(self.check_expr(lt)).kind() {
@@ -715,7 +720,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             && let PatKind::Binding(_, _, binding, ..) = inner.kind
         {
             let binding_parent_id = tcx.hir().parent_id(pat.hir_id);
-            let binding_parent = tcx.hir().get(binding_parent_id);
+            let binding_parent = tcx.hir_node(binding_parent_id);
             debug!(?inner, ?pat, ?binding_parent);
 
             let mutability = match mutbl {
@@ -743,6 +748,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         | PatKind::Slice(..) => "binding",
 
                         PatKind::Wild
+                        | PatKind::Never
                         | PatKind::Binding(..)
                         | PatKind::Path(..)
                         | PatKind::Box(..)
@@ -872,7 +878,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         self.demand_eqtype_pat(pat.span, expected, pat_ty, pat_info.top_info);
 
         // Type-check subpatterns.
-        if self.check_struct_pat_fields(pat_ty, &pat, variant, fields, has_rest_pat, pat_info) {
+        if self.check_struct_pat_fields(pat_ty, pat, variant, fields, has_rest_pat, pat_info) {
             pat_ty
         } else {
             Ty::new_misc_error(self.tcx)
@@ -893,7 +899,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let (res, opt_ty, segments) = path_resolution;
         match res {
             Res::Err => {
-                let e = tcx.sess.delay_span_bug(qpath.span(), "`Res::Err` but no error emitted");
+                let e = tcx.sess.span_delayed_bug(qpath.span(), "`Res::Err` but no error emitted");
                 self.set_tainted_by_errors(e);
                 return Ty::new_error(tcx, e);
             }
@@ -935,7 +941,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 Some(hir::Node::Item(hir::Item {
                     kind: hir::ItemKind::Const(_, _, body_id),
                     ..
-                })) => match self.tcx.hir().get(body_id.hir_id) {
+                })) => match self.tcx.hir_node(body_id.hir_id) {
                     hir::Node::Expr(expr) => {
                         if hir::is_range_literal(expr) {
                             let span = self.tcx.hir().span(body_id.hir_id);
@@ -1062,7 +1068,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let (res, opt_ty, segments) =
             self.resolve_ty_and_res_fully_qualified_call(qpath, pat.hir_id, pat.span, None);
         if res == Res::Err {
-            let e = tcx.sess.delay_span_bug(pat.span, "`Res::Err` but no error emitted");
+            let e = tcx.sess.span_delayed_bug(pat.span, "`Res::Err` but no error emitted");
             self.set_tainted_by_errors(e);
             on_error(e);
             return Ty::new_error(tcx, e);
@@ -1078,7 +1084,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         let variant = match res {
             Res::Err => {
-                let e = tcx.sess.delay_span_bug(pat.span, "`Res::Err` but no error emitted");
+                let e = tcx.sess.span_delayed_bug(pat.span, "`Res::Err` but no error emitted");
                 self.set_tainted_by_errors(e);
                 on_error(e);
                 return Ty::new_error(tcx, e);
@@ -1814,7 +1820,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         fn joined_uncovered_patterns(witnesses: &[&Ident]) -> String {
             const LIMIT: usize = 3;
             match witnesses {
-                [] => bug!(),
+                [] => {
+                    unreachable!(
+                        "expected an uncovered pattern, otherwise why are we emitting an error?"
+                    )
+                }
                 [witness] => format!("`{witness}`"),
                 [head @ .., tail] if head.len() < LIMIT => {
                     let head: Vec<_> = head.iter().map(<_>::to_string).collect();
@@ -1839,8 +1849,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         lint.note(format!(
             "the pattern is of type `{ty}` and the `non_exhaustive_omitted_patterns` attribute was found",
         ));
-
-        lint
     });
     }
 
diff --git a/compiler/rustc_hir_typeck/src/place_op.rs b/compiler/rustc_hir_typeck/src/place_op.rs
index 406434e09..79e41ef92 100644
--- a/compiler/rustc_hir_typeck/src/place_op.rs
+++ b/compiler/rustc_hir_typeck/src/place_op.rs
@@ -38,7 +38,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             span_bug!(expr.span, "input to deref is not a ref?");
         }
         let ty = self.make_overloaded_place_return_type(method).ty;
-        self.write_method_call(expr.hir_id, method);
+        self.write_method_call_and_enforce_effects(expr.hir_id, expr.span, method);
         Some(ty)
     }
 
@@ -179,7 +179,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 }
                 self.apply_adjustments(base_expr, adjustments);
 
-                self.write_method_call(expr.hir_id, method);
+                self.write_method_call_and_enforce_effects(expr.hir_id, expr.span, method);
 
                 return Some((input_ty, self.make_overloaded_place_return_type(method).ty));
             }
@@ -283,9 +283,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         // Gather up expressions we want to munge.
         let mut exprs = vec![expr];
 
-        while let hir::ExprKind::Field(ref expr, _)
-        | hir::ExprKind::Index(ref expr, _, _)
-        | hir::ExprKind::Unary(hir::UnOp::Deref, ref expr) = exprs.last().unwrap().kind
+        while let hir::ExprKind::Field(expr, _)
+        | hir::ExprKind::Index(expr, _, _)
+        | hir::ExprKind::Unary(hir::UnOp::Deref, expr) = exprs.last().unwrap().kind
         {
             exprs.push(expr);
         }
@@ -404,7 +404,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             None => return,
         };
         debug!("convert_place_op_to_mutable: method={:?}", method);
-        self.write_method_call(expr.hir_id, method);
+        self.write_method_call_and_enforce_effects(expr.hir_id, expr.span, method);
 
         let ty::Ref(region, _, hir::Mutability::Mut) = method.sig.inputs()[0].kind() else {
             span_bug!(expr.span, "input to mutable place op is not a mut ref?");
diff --git a/compiler/rustc_hir_typeck/src/rvalue_scopes.rs b/compiler/rustc_hir_typeck/src/rvalue_scopes.rs
index 04d841023..b9b3ed53d 100644
--- a/compiler/rustc_hir_typeck/src/rvalue_scopes.rs
+++ b/compiler/rustc_hir_typeck/src/rvalue_scopes.rs
@@ -69,12 +69,13 @@ pub fn resolve_rvalue_scopes<'a, 'tcx>(
     def_id: DefId,
 ) -> RvalueScopes {
     let tcx = &fcx.tcx;
-    let hir_map = tcx.hir();
     let mut rvalue_scopes = RvalueScopes::new();
     debug!("start resolving rvalue scopes, def_id={def_id:?}");
     debug!("rvalue_scope: rvalue_candidates={:?}", scope_tree.rvalue_candidates);
     for (&hir_id, candidate) in &scope_tree.rvalue_candidates {
-        let Some(Node::Expr(expr)) = hir_map.find(hir_id) else { bug!("hir node does not exist") };
+        let Some(Node::Expr(expr)) = tcx.opt_hir_node(hir_id) else {
+            bug!("hir node does not exist")
+        };
         record_rvalue_scope(&mut rvalue_scopes, expr, candidate);
     }
     rvalue_scopes
diff --git a/compiler/rustc_hir_typeck/src/upvar.rs b/compiler/rustc_hir_typeck/src/upvar.rs
index 17b81acd5..726ee02d7 100644
--- a/compiler/rustc_hir_typeck/src/upvar.rs
+++ b/compiler/rustc_hir_typeck/src/upvar.rs
@@ -221,7 +221,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         self.compute_min_captures(closure_def_id, capture_information, span);
 
-        let closure_hir_id = self.tcx.hir().local_def_id_to_hir_id(closure_def_id);
+        let closure_hir_id = self.tcx.local_def_id_to_hir_id(closure_def_id);
 
         if should_do_rust_2021_incompatible_closure_captures_analysis(self.tcx, closure_hir_id) {
             self.perform_2229_migration_analysis(closure_def_id, body_id, capture_clause, span);
@@ -261,7 +261,11 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             // Unify the (as yet unbound) type variable in the closure
             // args with the kind we inferred.
             let closure_kind_ty = closure_args.as_closure().kind_ty();
-            self.demand_eqtype(span, closure_kind.to_ty(self.tcx), closure_kind_ty);
+            self.demand_eqtype(
+                span,
+                Ty::from_closure_kind(self.tcx, closure_kind),
+                closure_kind_ty,
+            );
 
             // If we have an origin, store it.
             if let Some(mut origin) = origin {
@@ -315,11 +319,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         let final_tupled_upvars_type = Ty::new_tup(self.tcx, &final_upvar_tys);
         self.demand_suptype(span, args.tupled_upvars_ty(), final_tupled_upvars_type);
 
-        let fake_reads = delegate
-            .fake_reads
-            .into_iter()
-            .map(|(place, cause, hir_id)| (place, cause, hir_id))
-            .collect();
+        let fake_reads = delegate.fake_reads;
+
         self.typeck_results.borrow_mut().closure_fake_reads.insert(closure_def_id, fake_reads);
 
         if self.tcx.sess.opts.unstable_opts.profile_closures {
@@ -679,53 +680,41 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         // `tests/ui/closures/2229_closure_analysis/preserve_field_drop_order.rs`.
         for (_, captures) in &mut root_var_min_capture_list {
             captures.sort_by(|capture1, capture2| {
-                for (p1, p2) in capture1.place.projections.iter().zip(&capture2.place.projections) {
+                fn is_field<'a>(p: &&Projection<'a>) -> bool {
+                    match p.kind {
+                        ProjectionKind::Field(_, _) => true,
+                        ProjectionKind::Deref | ProjectionKind::OpaqueCast => false,
+                        p @ (ProjectionKind::Subslice | ProjectionKind::Index) => {
+                            bug!("ProjectionKind {:?} was unexpected", p)
+                        }
+                    }
+                }
+
+                // Need to sort only by Field projections, so filter away others.
+                // A previous implementation considered other projection types too
+                // but that caused ICE #118144
+                let capture1_field_projections = capture1.place.projections.iter().filter(is_field);
+                let capture2_field_projections = capture2.place.projections.iter().filter(is_field);
+
+                for (p1, p2) in capture1_field_projections.zip(capture2_field_projections) {
                     // We do not need to look at the `Projection.ty` fields here because at each
                     // step of the iteration, the projections will either be the same and therefore
                     // the types must be as well or the current projection will be different and
                     // we will return the result of comparing the field indexes.
                     match (p1.kind, p2.kind) {
-                        // Paths are the same, continue to next loop.
-                        (ProjectionKind::Deref, ProjectionKind::Deref) => {}
-                        (ProjectionKind::OpaqueCast, ProjectionKind::OpaqueCast) => {}
-                        (ProjectionKind::Field(i1, _), ProjectionKind::Field(i2, _))
-                            if i1 == i2 => {}
-
-                        // Fields are different, compare them.
                         (ProjectionKind::Field(i1, _), ProjectionKind::Field(i2, _)) => {
-                            return i1.cmp(&i2);
+                            // Compare only if paths are different.
+                            // Otherwise continue to the next iteration
+                            if i1 != i2 {
+                                return i1.cmp(&i2);
+                            }
                         }
-
-                        // We should have either a pair of `Deref`s or a pair of `Field`s.
-                        // Anything else is a bug.
-                        (
-                            l @ (ProjectionKind::Deref | ProjectionKind::Field(..)),
-                            r @ (ProjectionKind::Deref | ProjectionKind::Field(..)),
-                        ) => bug!(
-                            "ProjectionKinds Deref and Field were mismatched: ({:?}, {:?})",
-                            l,
-                            r
-                        ),
-                        (
-                            l @ (ProjectionKind::Index
-                            | ProjectionKind::Subslice
-                            | ProjectionKind::Deref
-                            | ProjectionKind::OpaqueCast
-                            | ProjectionKind::Field(..)),
-                            r @ (ProjectionKind::Index
-                            | ProjectionKind::Subslice
-                            | ProjectionKind::Deref
-                            | ProjectionKind::OpaqueCast
-                            | ProjectionKind::Field(..)),
-                        ) => bug!(
-                            "ProjectionKinds Index or Subslice were unexpected: ({:?}, {:?})",
-                            l,
-                            r
-                        ),
+                        // Given the filter above, this arm should never be hit
+                        (l, r) => bug!("ProjectionKinds {:?} or {:?} were unexpected", l, r),
                     }
                 }
 
-                self.tcx.sess.delay_span_bug(
+                self.tcx.sess.span_delayed_bug(
                     closure_span,
                     format!(
                         "two identical projections: ({:?}, {:?})",
@@ -763,7 +752,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             let (migration_string, migrated_variables_concat) =
                 migration_suggestion_for_2229(self.tcx, &need_migrations);
 
-            let closure_hir_id = self.tcx.hir().local_def_id_to_hir_id(closure_def_id);
+            let closure_hir_id = self.tcx.local_def_id_to_hir_id(closure_def_id);
             let closure_head_span = self.tcx.def_span(closure_def_id);
             self.tcx.struct_span_lint_hir(
                 lint::builtin::RUST_2021_INCOMPATIBLE_CLOSURE_CAPTURES,
@@ -853,7 +842,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             // Looks like a macro fragment. Try to find the real block.
                             if let Some(hir::Node::Expr(&hir::Expr {
                                 kind: hir::ExprKind::Block(block, ..), ..
-                            })) = self.tcx.hir().find(body_id.hir_id) {
+                            })) = self.tcx.opt_hir_node(body_id.hir_id) {
                                 // If the body is a block (with `{..}`), we use the span of that block.
                                 // E.g. with a `|| $body` expanded from a `m!({ .. })`, we use `{ .. }`, and not `$body`.
                                 // Since we know it's a block, we know we can insert the `let _ = ..` without
@@ -911,8 +900,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             Applicability::HasPlaceholders
                         );
                     }
-
-                    lint
                 },
             );
         }
@@ -1673,7 +1660,7 @@ fn apply_capture_kind_on_capture_ty<'tcx>(
 fn drop_location_span(tcx: TyCtxt<'_>, hir_id: hir::HirId) -> Span {
     let owner_id = tcx.hir().get_enclosing_scope(hir_id).unwrap();
 
-    let owner_node = tcx.hir().get(owner_id);
+    let owner_node = tcx.hir_node(owner_id);
     let owner_span = match owner_node {
         hir::Node::Item(item) => match item.kind {
             hir::ItemKind::Fn(_, _, owner_id) => tcx.hir().span(owner_id.hir_id),
diff --git a/compiler/rustc_hir_typeck/src/writeback.rs b/compiler/rustc_hir_typeck/src/writeback.rs
index 896aacc69..d0cf4575c 100644
--- a/compiler/rustc_hir_typeck/src/writeback.rs
+++ b/compiler/rustc_hir_typeck/src/writeback.rs
@@ -8,12 +8,16 @@ use rustc_errors::{ErrorGuaranteed, StashKey};
 use rustc_hir as hir;
 use rustc_hir::intravisit::{self, Visitor};
 use rustc_infer::infer::error_reporting::TypeAnnotationNeeded::E0282;
+use rustc_middle::traits::ObligationCause;
 use rustc_middle::ty::adjustment::{Adjust, Adjustment, PointerCoercion};
-use rustc_middle::ty::fold::{TypeFoldable, TypeFolder, TypeSuperFoldable};
+use rustc_middle::ty::fold::{TypeFoldable, TypeFolder};
 use rustc_middle::ty::visit::TypeVisitableExt;
+use rustc_middle::ty::TypeSuperFoldable;
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_span::symbol::sym;
 use rustc_span::Span;
+use rustc_trait_selection::solve;
+use rustc_trait_selection::traits::error_reporting::TypeErrCtxtExt;
 
 use std::mem;
 
@@ -47,7 +51,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         // Type only exists for constants and statics, not functions.
         match self.tcx.hir().body_owner_kind(item_def_id) {
             hir::BodyOwnerKind::Const { .. } | hir::BodyOwnerKind::Static(_) => {
-                let item_hir_id = self.tcx.hir().local_def_id_to_hir_id(item_def_id);
+                let item_hir_id = self.tcx.local_def_id_to_hir_id(item_def_id);
                 wbcx.visit_node_id(body.value.span, item_hir_id);
             }
             hir::BodyOwnerKind::Closure | hir::BodyOwnerKind::Fn => (),
@@ -218,7 +222,7 @@ impl<'cx, 'tcx> WritebackCx<'cx, 'tcx> {
                 // When encountering `return [0][0]` outside of a `fn` body we can encounter a base
                 // that isn't in the type table. We assume more relevant errors have already been
                 // emitted, so we delay an ICE if none have. (#64638)
-                self.tcx().sess.delay_span_bug(e.span, format!("bad base: `{base:?}`"));
+                self.tcx().sess.span_delayed_bug(e.span, format!("bad base: `{base:?}`"));
             }
             if let Some(base_ty) = base_ty
                 && let ty::Ref(_, base_ty_inner, _) = *base_ty.kind()
@@ -311,7 +315,9 @@ impl<'cx, 'tcx> Visitor<'tcx> for WritebackCx<'cx, 'tcx> {
                 // Nothing to write back here
             }
             hir::GenericParamKind::Type { .. } | hir::GenericParamKind::Const { .. } => {
-                self.tcx().sess.delay_span_bug(p.span, format!("unexpected generic param: {p:?}"));
+                self.tcx()
+                    .sess
+                    .span_delayed_bug(p.span, format!("unexpected generic param: {p:?}"));
             }
         }
     }
@@ -382,7 +388,7 @@ impl<'cx, 'tcx> WritebackCx<'cx, 'tcx> {
                 .to_sorted(hcx, false)
                 .into_iter()
                 .map(|(&closure_def_id, data)| {
-                    let closure_hir_id = self.tcx().hir().local_def_id_to_hir_id(closure_def_id);
+                    let closure_hir_id = self.tcx().local_def_id_to_hir_id(closure_def_id);
                     let data = self.resolve(*data, &closure_hir_id);
                     (closure_def_id, data)
                 })
@@ -407,7 +413,7 @@ impl<'cx, 'tcx> WritebackCx<'cx, 'tcx> {
                                 .map(|captured_place| {
                                     let locatable =
                                         captured_place.info.path_expr_id.unwrap_or_else(|| {
-                                            self.tcx().hir().local_def_id_to_hir_id(closure_def_id)
+                                            self.tcx().local_def_id_to_hir_id(closure_def_id)
                                         });
                                     self.resolve(captured_place.clone(), &locatable)
                                 })
@@ -433,7 +439,7 @@ impl<'cx, 'tcx> WritebackCx<'cx, 'tcx> {
                     let resolved_fake_reads = fake_reads
                         .iter()
                         .map(|(place, cause, hir_id)| {
-                            let locatable = self.tcx().hir().local_def_id_to_hir_id(closure_def_id);
+                            let locatable = self.tcx().local_def_id_to_hir_id(closure_def_id);
                             let resolved_fake_read = self.resolve(place.clone(), &locatable);
                             (resolved_fake_read, *cause, *hir_id)
                         })
@@ -498,8 +504,8 @@ impl<'cx, 'tcx> WritebackCx<'cx, 'tcx> {
 
             if !errors_buffer.is_empty() {
                 errors_buffer.sort_by_key(|diag| diag.span.primary_span());
-                for mut diag in errors_buffer {
-                    self.tcx().sess.diagnostic().emit_diagnostic(&mut diag);
+                for diag in errors_buffer {
+                    self.tcx().sess.dcx().emit_diagnostic(diag);
                 }
             }
         }
@@ -693,24 +699,22 @@ impl<'cx, 'tcx> WritebackCx<'cx, 'tcx> {
         }
     }
 
-    fn resolve<T>(&mut self, x: T, span: &dyn Locatable) -> T
+    fn resolve<T>(&mut self, value: T, span: &dyn Locatable) -> T
     where
         T: TypeFoldable<TyCtxt<'tcx>>,
     {
-        let mut resolver = Resolver::new(self.fcx, span, self.body);
-        let x = x.fold_with(&mut resolver);
-        if cfg!(debug_assertions) && x.has_infer() {
-            span_bug!(span.to_span(self.fcx.tcx), "writeback: `{:?}` has inference variables", x);
-        }
+        let value = self.fcx.resolve_vars_if_possible(value);
+        let value = value.fold_with(&mut Resolver::new(self.fcx, span, self.body));
+        assert!(!value.has_infer());
 
         // We may have introduced e.g. `ty::Error`, if inference failed, make sure
         // to mark the `TypeckResults` as tainted in that case, so that downstream
         // users of the typeck results don't produce extra errors, or worse, ICEs.
-        if let Some(e) = resolver.replaced_with_error {
-            self.typeck_results.tainted_by_errors = Some(e);
+        if let Err(guar) = value.error_reported() {
+            self.typeck_results.tainted_by_errors = Some(guar);
         }
 
-        x
+        value
     }
 }
 
@@ -730,15 +734,13 @@ impl Locatable for hir::HirId {
     }
 }
 
-/// The Resolver. This is the type folding engine that detects
-/// unresolved types and so forth.
 struct Resolver<'cx, 'tcx> {
     fcx: &'cx FnCtxt<'cx, 'tcx>,
     span: &'cx dyn Locatable,
     body: &'tcx hir::Body<'tcx>,
-
-    /// Set to `Some` if any `Ty` or `ty::Const` had to be replaced with an `Error`.
-    replaced_with_error: Option<ErrorGuaranteed>,
+    /// Whether we should normalize using the new solver, disabled
+    /// both when using the old solver and when resolving predicates.
+    should_normalize: bool,
 }
 
 impl<'cx, 'tcx> Resolver<'cx, 'tcx> {
@@ -747,7 +749,7 @@ impl<'cx, 'tcx> Resolver<'cx, 'tcx> {
         span: &'cx dyn Locatable,
         body: &'tcx hir::Body<'tcx>,
     ) -> Resolver<'cx, 'tcx> {
-        Resolver { fcx, span, body, replaced_with_error: None }
+        Resolver { fcx, span, body, should_normalize: fcx.next_trait_solver() }
     }
 
     fn report_error(&self, p: impl Into<ty::GenericArg<'tcx>>) -> ErrorGuaranteed {
@@ -766,26 +768,42 @@ impl<'cx, 'tcx> Resolver<'cx, 'tcx> {
                 .emit(),
         }
     }
-}
 
-struct EraseEarlyRegions<'tcx> {
-    tcx: TyCtxt<'tcx>,
-}
+    fn handle_term<T>(
+        &mut self,
+        value: T,
+        outer_exclusive_binder: impl FnOnce(T) -> ty::DebruijnIndex,
+        new_err: impl Fn(TyCtxt<'tcx>, ErrorGuaranteed) -> T,
+    ) -> T
+    where
+        T: Into<ty::GenericArg<'tcx>> + TypeSuperFoldable<TyCtxt<'tcx>> + Copy,
+    {
+        let tcx = self.fcx.tcx;
+        // We must deeply normalize in the new solver, since later lints
+        // expect that types that show up in the typeck are fully
+        // normalized.
+        let value = if self.should_normalize {
+            let body_id = tcx.hir().body_owner_def_id(self.body.id());
+            let cause = ObligationCause::misc(self.span.to_span(tcx), body_id);
+            let at = self.fcx.at(&cause, self.fcx.param_env);
+            let universes = vec![None; outer_exclusive_binder(value).as_usize()];
+            solve::deeply_normalize_with_skipped_universes(at, value, universes).unwrap_or_else(
+                |errors| {
+                    let guar = self.fcx.err_ctxt().report_fulfillment_errors(errors);
+                    new_err(tcx, guar)
+                },
+            )
+        } else {
+            value
+        };
 
-impl<'tcx> TypeFolder<TyCtxt<'tcx>> for EraseEarlyRegions<'tcx> {
-    fn interner(&self) -> TyCtxt<'tcx> {
-        self.tcx
-    }
-    fn fold_ty(&mut self, ty: Ty<'tcx>) -> Ty<'tcx> {
-        if ty.has_type_flags(ty::TypeFlags::HAS_FREE_REGIONS) {
-            ty.super_fold_with(self)
+        if value.has_non_region_infer() {
+            let guar = self.report_error(value);
+            new_err(tcx, guar)
         } else {
-            ty
+            tcx.fold_regions(value, |_, _| tcx.lifetimes.re_erased)
         }
     }
-    fn fold_region(&mut self, r: ty::Region<'tcx>) -> ty::Region<'tcx> {
-        if r.is_late_bound() { r } else { self.tcx.lifetimes.re_erased }
-    }
 }
 
 impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Resolver<'cx, 'tcx> {
@@ -793,56 +811,28 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Resolver<'cx, 'tcx> {
         self.fcx.tcx
     }
 
-    fn fold_ty(&mut self, t: Ty<'tcx>) -> Ty<'tcx> {
-        match self.fcx.fully_resolve(t) {
-            Ok(t) if self.fcx.next_trait_solver() => {
-                // We must normalize erasing regions here, since later lints
-                // expect that types that show up in the typeck are fully
-                // normalized.
-                if let Ok(t) = self.fcx.tcx.try_normalize_erasing_regions(self.fcx.param_env, t) {
-                    t
-                } else {
-                    EraseEarlyRegions { tcx: self.fcx.tcx }.fold_ty(t)
-                }
-            }
-            Ok(t) => {
-                // Do not anonymize late-bound regions
-                // (e.g. keep `for<'a>` named `for<'a>`).
-                // This allows NLL to generate error messages that
-                // refer to the higher-ranked lifetime names written by the user.
-                EraseEarlyRegions { tcx: self.fcx.tcx }.fold_ty(t)
-            }
-            Err(_) => {
-                debug!("Resolver::fold_ty: input type `{:?}` not fully resolvable", t);
-                let e = self.report_error(t);
-                self.replaced_with_error = Some(e);
-                Ty::new_error(self.fcx.tcx, e)
-            }
-        }
-    }
-
     fn fold_region(&mut self, r: ty::Region<'tcx>) -> ty::Region<'tcx> {
-        debug_assert!(!r.is_late_bound(), "Should not be resolving bound region.");
+        debug_assert!(!r.is_bound(), "Should not be resolving bound region.");
         self.fcx.tcx.lifetimes.re_erased
     }
 
+    fn fold_ty(&mut self, ty: Ty<'tcx>) -> Ty<'tcx> {
+        self.handle_term(ty, Ty::outer_exclusive_binder, Ty::new_error)
+    }
+
     fn fold_const(&mut self, ct: ty::Const<'tcx>) -> ty::Const<'tcx> {
-        match self.fcx.fully_resolve(ct) {
-            Ok(ct) => self.fcx.tcx.erase_regions(ct),
-            Err(_) => {
-                debug!("Resolver::fold_const: input const `{:?}` not fully resolvable", ct);
-                let e = self.report_error(ct);
-                self.replaced_with_error = Some(e);
-                ty::Const::new_error(self.fcx.tcx, e, ct.ty())
-            }
-        }
+        self.handle_term(ct, ty::Const::outer_exclusive_binder, |tcx, guar| {
+            ty::Const::new_error(tcx, guar, ct.ty())
+        })
     }
-}
 
-///////////////////////////////////////////////////////////////////////////
-// During type check, we store promises with the result of trait
-// lookup rather than the actual results (because the results are not
-// necessarily available immediately). These routines unwind the
-// promises. It is expected that we will have already reported any
-// errors that may be encountered, so if the promises store an error,
-// a dummy result is returned.
+    fn fold_predicate(&mut self, predicate: ty::Predicate<'tcx>) -> ty::Predicate<'tcx> {
+        // Do not normalize predicates in the new solver. The new solver is
+        // supposed to handle unnormalized predicates and incorrectly normalizing
+        // them can be unsound, e.g. for `WellFormed` predicates.
+        let prev = mem::replace(&mut self.should_normalize, false);
+        let predicate = predicate.super_fold_with(self);
+        self.should_normalize = prev;
+        predicate
+    }
+}
diff --git a/compiler/rustc_incremental/src/assert_dep_graph.rs b/compiler/rustc_incremental/src/assert_dep_graph.rs
index 1b160eca9..a5e0654c8 100644
--- a/compiler/rustc_incremental/src/assert_dep_graph.rs
+++ b/compiler/rustc_incremental/src/assert_dep_graph.rs
@@ -122,7 +122,7 @@ impl<'tcx> IfThisChanged<'tcx> {
 
     fn process_attrs(&mut self, def_id: LocalDefId) {
         let def_path_hash = self.tcx.def_path_hash(def_id.to_def_id());
-        let hir_id = self.tcx.hir().local_def_id_to_hir_id(def_id);
+        let hir_id = self.tcx.local_def_id_to_hir_id(def_id);
         let attrs = self.tcx.hir().attrs(hir_id);
         for attr in attrs {
             if attr.has_name(sym::rustc_if_this_changed) {
@@ -231,13 +231,13 @@ fn dump_graph(query: &DepGraphQuery) {
             // Expect one of: "-> target", "source -> target", or "source ->".
             let edge_filter =
                 EdgeFilter::new(&string).unwrap_or_else(|e| bug!("invalid filter: {}", e));
-            let sources = node_set(&query, &edge_filter.source);
-            let targets = node_set(&query, &edge_filter.target);
-            filter_nodes(&query, &sources, &targets)
+            let sources = node_set(query, &edge_filter.source);
+            let targets = node_set(query, &edge_filter.target);
+            filter_nodes(query, &sources, &targets)
         }
         Err(_) => query.nodes().into_iter().map(|n| n.kind).collect(),
     };
-    let edges = filter_edges(&query, &nodes);
+    let edges = filter_edges(query, &nodes);
 
     {
         // dump a .txt file with just the edges:
diff --git a/compiler/rustc_incremental/src/lib.rs b/compiler/rustc_incremental/src/lib.rs
index dde138973..367a15e14 100644
--- a/compiler/rustc_incremental/src/lib.rs
+++ b/compiler/rustc_incremental/src/lib.rs
@@ -2,9 +2,9 @@
 
 #![deny(missing_docs)]
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
-#![cfg_attr(not(bootstrap), allow(internal_features))]
+#![doc(rust_logo)]
+#![feature(rustdoc_internals)]
+#![allow(internal_features)]
 #![recursion_limit = "256"]
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
@@ -28,7 +28,4 @@ pub use persist::save_work_product_index;
 pub use persist::setup_dep_graph;
 pub use persist::LoadResult;
 
-use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
-use rustc_fluent_macro::fluent_messages;
-
-fluent_messages! { "../messages.ftl" }
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
diff --git a/compiler/rustc_incremental/src/persist/dirty_clean.rs b/compiler/rustc_incremental/src/persist/dirty_clean.rs
index 5dd06c6ca..8a7b97139 100644
--- a/compiler/rustc_incremental/src/persist/dirty_clean.rs
+++ b/compiler/rustc_incremental/src/persist/dirty_clean.rs
@@ -32,7 +32,6 @@ use rustc_middle::hir::nested_filter;
 use rustc_middle::ty::TyCtxt;
 use rustc_span::symbol::{sym, Symbol};
 use rustc_span::Span;
-use std::iter::FromIterator;
 use thin_vec::ThinVec;
 
 const LOADED_FROM_DISK: Symbol = sym::loaded_from_disk;
@@ -233,7 +232,7 @@ impl<'tcx> DirtyCleanVisitor<'tcx> {
     /// Return all DepNode labels that should be asserted for this item.
     /// index=0 is the "name" used for error messages
     fn auto_labels(&mut self, item_id: LocalDefId, attr: &Attribute) -> (&'static str, Labels) {
-        let node = self.tcx.hir().get_by_def_id(item_id);
+        let node = self.tcx.hir_node_by_def_id(item_id);
         let (name, labels) = match node {
             HirNode::Item(item) => {
                 match item.kind {
@@ -378,15 +377,15 @@ impl<'tcx> DirtyCleanVisitor<'tcx> {
             };
             self.checked_attrs.insert(attr.id);
             for label in assertion.clean.items().into_sorted_stable_ord() {
-                let dep_node = DepNode::from_label_string(self.tcx, &label, def_path_hash).unwrap();
+                let dep_node = DepNode::from_label_string(self.tcx, label, def_path_hash).unwrap();
                 self.assert_clean(item_span, dep_node);
             }
             for label in assertion.dirty.items().into_sorted_stable_ord() {
-                let dep_node = DepNode::from_label_string(self.tcx, &label, def_path_hash).unwrap();
+                let dep_node = DepNode::from_label_string(self.tcx, label, def_path_hash).unwrap();
                 self.assert_dirty(item_span, dep_node);
             }
             for label in assertion.loaded_from_disk.items().into_sorted_stable_ord() {
-                let dep_node = DepNode::from_label_string(self.tcx, &label, def_path_hash).unwrap();
+                let dep_node = DepNode::from_label_string(self.tcx, label, def_path_hash).unwrap();
                 self.assert_loaded_from_disk(item_span, dep_node);
             }
         }
diff --git a/compiler/rustc_incremental/src/persist/file_format.rs b/compiler/rustc_incremental/src/persist/file_format.rs
index 25bf83f64..b5742b97d 100644
--- a/compiler/rustc_incremental/src/persist/file_format.rs
+++ b/compiler/rustc_incremental/src/persist/file_format.rs
@@ -80,8 +80,8 @@ where
             );
             debug!("save: data written to disk successfully");
         }
-        Err(err) => {
-            sess.emit_err(errors::WriteNew { name, path: path_buf, err });
+        Err((path, err)) => {
+            sess.emit_err(errors::WriteNew { name, path, err });
         }
     }
 }
diff --git a/compiler/rustc_incremental/src/persist/fs.rs b/compiler/rustc_incremental/src/persist/fs.rs
index f56fb0d05..92297a27a 100644
--- a/compiler/rustc_incremental/src/persist/fs.rs
+++ b/compiler/rustc_incremental/src/persist/fs.rs
@@ -262,7 +262,7 @@ pub(crate) fn prepare_session_directory(
                     directory."
             );
 
-            sess.init_incr_comp_session(session_dir, directory_lock, false);
+            sess.init_incr_comp_session(session_dir, directory_lock);
             return Ok(());
         };
 
@@ -276,7 +276,7 @@ pub(crate) fn prepare_session_directory(
                 sess.emit_warning(errors::HardLinkFailed { path: &session_dir });
             }
 
-            sess.init_incr_comp_session(session_dir, directory_lock, true);
+            sess.init_incr_comp_session(session_dir, directory_lock);
             return Ok(());
         } else {
             debug!("copying failed - trying next directory");
@@ -312,7 +312,7 @@ pub fn finalize_session_directory(sess: &Session, svh: Option<Svh>) {
 
     let incr_comp_session_dir: PathBuf = sess.incr_comp_session_dir().clone();
 
-    if let Some(_) = sess.has_errors_or_delayed_span_bugs() {
+    if let Some(_) = sess.has_errors_or_span_delayed_bugs() {
         // If there have been any errors during compilation, we don't want to
         // publish this session directory. Rather, we'll just delete it.
 
@@ -499,7 +499,7 @@ fn lock_directory(
 }
 
 fn delete_session_dir_lock_file(sess: &Session, lock_file_path: &Path) {
-    if let Err(err) = safe_remove_file(&lock_file_path) {
+    if let Err(err) = safe_remove_file(lock_file_path) {
         sess.emit_warning(errors::DeleteLock { path: lock_file_path, err });
     }
 }
@@ -847,10 +847,10 @@ pub(crate) fn garbage_collect_session_directories(sess: &Session) -> io::Result<
 fn delete_old(sess: &Session, path: &Path) {
     debug!("garbage_collect_session_directories() - deleting `{}`", path.display());
 
-    if let Err(err) = safe_remove_dir_all(&path) {
-        sess.emit_warning(errors::SessionGcFailed { path: &path, err });
+    if let Err(err) = safe_remove_dir_all(path) {
+        sess.emit_warning(errors::SessionGcFailed { path: path, err });
     } else {
-        delete_session_dir_lock_file(sess, &lock_file_path(&path));
+        delete_session_dir_lock_file(sess, &lock_file_path(path));
     }
 }
 
diff --git a/compiler/rustc_incremental/src/persist/load.rs b/compiler/rustc_incremental/src/persist/load.rs
index 6dfc40969..f2c1d0b83 100644
--- a/compiler/rustc_incremental/src/persist/load.rs
+++ b/compiler/rustc_incremental/src/persist/load.rs
@@ -99,7 +99,7 @@ fn load_dep_graph(sess: &Session) -> LoadResult<(SerializedDepGraph, WorkProduct
 
     // Calling `sess.incr_comp_session_dir()` will panic if `sess.opts.incremental.is_none()`.
     // Fortunately, we just checked that this isn't the case.
-    let path = dep_graph_path(&sess);
+    let path = dep_graph_path(sess);
     let expected_hash = sess.opts.dep_tracking_hash(false);
 
     let mut prev_work_products = UnordMap::default();
diff --git a/compiler/rustc_incremental/src/persist/save.rs b/compiler/rustc_incremental/src/persist/save.rs
index fa21320be..ff0953e9f 100644
--- a/compiler/rustc_incremental/src/persist/save.rs
+++ b/compiler/rustc_incremental/src/persist/save.rs
@@ -32,7 +32,7 @@ pub fn save_dep_graph(tcx: TyCtxt<'_>) {
             return;
         }
         // This is going to be deleted in finalize_session_directory, so let's not create it
-        if let Some(_) = sess.has_errors_or_delayed_span_bugs() {
+        if let Some(_) = sess.has_errors_or_span_delayed_bugs() {
             return;
         }
 
@@ -50,9 +50,6 @@ pub fn save_dep_graph(tcx: TyCtxt<'_>) {
         join(
             move || {
                 sess.time("incr_comp_persist_dep_graph", || {
-                    if let Err(err) = tcx.dep_graph.encode(&tcx.sess.prof) {
-                        sess.emit_err(errors::WriteDepGraph { path: &staging_dep_graph_path, err });
-                    }
                     if let Err(err) = fs::rename(&staging_dep_graph_path, &dep_graph_path) {
                         sess.emit_err(errors::MoveDepGraph {
                             from: &staging_dep_graph_path,
@@ -90,7 +87,7 @@ pub fn save_work_product_index(
         return;
     }
     // This is going to be deleted in finalize_session_directory, so let's not create it
-    if let Some(_) = sess.has_errors_or_delayed_span_bugs() {
+    if let Some(_) = sess.has_errors_or_span_delayed_bugs() {
         return;
     }
 
diff --git a/compiler/rustc_incremental/src/persist/work_product.rs b/compiler/rustc_incremental/src/persist/work_product.rs
index fb96bed5a..1ee3d1392 100644
--- a/compiler/rustc_incremental/src/persist/work_product.rs
+++ b/compiler/rustc_incremental/src/persist/work_product.rs
@@ -31,7 +31,7 @@ pub fn copy_cgu_workproduct_to_incr_comp_cache_dir(
             }
             Err(err) => {
                 sess.emit_warning(errors::CopyWorkProductToCache {
-                    from: &path,
+                    from: path,
                     to: &path_in_incr_dir,
                     err,
                 });
diff --git a/compiler/rustc_index/Cargo.toml b/compiler/rustc_index/Cargo.toml
index 856f8a67d..3a4c813b5 100644
--- a/compiler/rustc_index/Cargo.toml
+++ b/compiler/rustc_index/Cargo.toml
@@ -6,6 +6,7 @@ edition = "2021"
 [dependencies]
 # tidy-alphabetical-start
 arrayvec = { version = "0.7", default-features = false }
+rustc_index_macros = { path = "../rustc_index_macros", default-features = false }
 rustc_macros = { path = "../rustc_macros", optional = true }
 rustc_serialize = { path = "../rustc_serialize", optional = true }
 smallvec = "1.8.1"
@@ -14,5 +15,5 @@ smallvec = "1.8.1"
 [features]
 # tidy-alphabetical-start
 default = ["nightly"]
-nightly = ["rustc_serialize", "rustc_macros"]
+nightly = ["rustc_serialize", "rustc_macros", "rustc_index_macros/nightly"]
 # tidy-alphabetical-end
diff --git a/compiler/rustc_index/src/bit_set.rs b/compiler/rustc_index/src/bit_set.rs
index ece61ff12..3ea1a52ae 100644
--- a/compiler/rustc_index/src/bit_set.rs
+++ b/compiler/rustc_index/src/bit_set.rs
@@ -9,6 +9,7 @@ use std::slice;
 use arrayvec::ArrayVec;
 use smallvec::{smallvec, SmallVec};
 
+#[cfg(feature = "nightly")]
 use rustc_macros::{Decodable, Encodable};
 
 use crate::{Idx, IndexVec};
@@ -111,7 +112,8 @@ macro_rules! bit_relations_inherent_impls {
 /// to or greater than the domain size. All operations that involve two bitsets
 /// will panic if the bitsets have differing domain sizes.
 ///
-#[derive(Eq, PartialEq, Hash, Decodable, Encodable)]
+#[cfg_attr(feature = "nightly", derive(Decodable, Encodable))]
+#[derive(Eq, PartialEq, Hash)]
 pub struct BitSet<T> {
     domain_size: usize,
     words: SmallVec<[Word; 2]>,
@@ -237,23 +239,12 @@ impl<T: Idx> BitSet<T> {
         new_word != word
     }
 
-    /// Gets a slice of the underlying words.
-    pub fn words(&self) -> &[Word] {
-        &self.words
-    }
-
     /// Iterates over the indices of set bits in a sorted order.
     #[inline]
     pub fn iter(&self) -> BitIter<'_, T> {
         BitIter::new(&self.words)
     }
 
-    /// Duplicates the set as a hybrid set.
-    pub fn to_hybrid(&self) -> HybridBitSet<T> {
-        // Note: we currently don't bother trying to make a Sparse set.
-        HybridBitSet::Dense(self.to_owned())
-    }
-
     /// Set `self = self | other`. In contrast to `union` returns `true` if the set contains at
     /// least one bit that is not in `other` (i.e. `other` is not a superset of `self`).
     ///
@@ -502,10 +493,21 @@ impl<T: Idx> ChunkedBitSet<T> {
         match *chunk {
             Zeros(chunk_domain_size) => {
                 if chunk_domain_size > 1 {
-                    // We take some effort to avoid copying the words.
-                    let words = Rc::<[Word; CHUNK_WORDS]>::new_zeroed();
-                    // SAFETY: `words` can safely be all zeroes.
-                    let mut words = unsafe { words.assume_init() };
+                    #[cfg(feature = "nightly")]
+                    let mut words = {
+                        // We take some effort to avoid copying the words.
+                        let words = Rc::<[Word; CHUNK_WORDS]>::new_zeroed();
+                        // SAFETY: `words` can safely be all zeroes.
+                        unsafe { words.assume_init() }
+                    };
+                    #[cfg(not(feature = "nightly"))]
+                    let mut words = {
+                        let words = mem::MaybeUninit::<[Word; CHUNK_WORDS]>::zeroed();
+                        // SAFETY: `words` can safely be all zeroes.
+                        let words = unsafe { words.assume_init() };
+                        // Unfortunate possibly-large copy
+                        Rc::new(words)
+                    };
                     let words_ref = Rc::get_mut(&mut words).unwrap();
 
                     let (word_index, mask) = chunk_word_index_and_mask(elem);
@@ -556,10 +558,21 @@ impl<T: Idx> ChunkedBitSet<T> {
             Zeros(_) => false,
             Ones(chunk_domain_size) => {
                 if chunk_domain_size > 1 {
-                    // We take some effort to avoid copying the words.
-                    let words = Rc::<[Word; CHUNK_WORDS]>::new_zeroed();
-                    // SAFETY: `words` can safely be all zeroes.
-                    let mut words = unsafe { words.assume_init() };
+                    #[cfg(feature = "nightly")]
+                    let mut words = {
+                        // We take some effort to avoid copying the words.
+                        let words = Rc::<[Word; CHUNK_WORDS]>::new_zeroed();
+                        // SAFETY: `words` can safely be all zeroes.
+                        unsafe { words.assume_init() }
+                    };
+                    #[cfg(not(feature = "nightly"))]
+                    let mut words = {
+                        let words = mem::MaybeUninit::<[Word; CHUNK_WORDS]>::zeroed();
+                        // SAFETY: `words` can safely be all zeroes.
+                        let words = unsafe { words.assume_init() };
+                        // Unfortunate possibly-large copy
+                        Rc::new(words)
+                    };
                     let words_ref = Rc::get_mut(&mut words).unwrap();
 
                     // Set only the bits in use.
@@ -1575,7 +1588,8 @@ impl<T: Idx> From<BitSet<T>> for GrowableBitSet<T> {
 ///
 /// All operations that involve a row and/or column index will panic if the
 /// index exceeds the relevant bound.
-#[derive(Clone, Eq, PartialEq, Hash, Decodable, Encodable)]
+#[cfg_attr(feature = "nightly", derive(Decodable, Encodable))]
+#[derive(Clone, Eq, PartialEq, Hash)]
 pub struct BitMatrix<R: Idx, C: Idx> {
     num_rows: usize,
     num_columns: usize,
@@ -1601,11 +1615,11 @@ impl<R: Idx, C: Idx> BitMatrix<R, C> {
     pub fn from_row_n(row: &BitSet<C>, num_rows: usize) -> BitMatrix<R, C> {
         let num_columns = row.domain_size();
         let words_per_row = num_words(num_columns);
-        assert_eq!(words_per_row, row.words().len());
+        assert_eq!(words_per_row, row.words.len());
         BitMatrix {
             num_rows,
             num_columns,
-            words: iter::repeat(row.words()).take(num_rows).flatten().cloned().collect(),
+            words: iter::repeat(&row.words).take(num_rows).flatten().cloned().collect(),
             marker: PhantomData,
         }
     }
@@ -1700,9 +1714,9 @@ impl<R: Idx, C: Idx> BitMatrix<R, C> {
         assert_eq!(with.domain_size(), self.num_columns);
         let (write_start, write_end) = self.range(write);
         let mut changed = false;
-        for (read_index, write_index) in iter::zip(0..with.words().len(), write_start..write_end) {
+        for (read_index, write_index) in iter::zip(0..with.words.len(), write_start..write_end) {
             let word = self.words[write_index];
-            let new_word = word | with.words()[read_index];
+            let new_word = word | with.words[read_index];
             self.words[write_index] = new_word;
             changed |= word != new_word;
         }
@@ -2002,57 +2016,10 @@ impl std::fmt::Debug for FiniteBitSet<u32> {
     }
 }
 
-impl FiniteBitSetTy for u64 {
-    const DOMAIN_SIZE: u32 = 64;
-
-    const FILLED: Self = Self::MAX;
-    const EMPTY: Self = Self::MIN;
-
-    const ONE: Self = 1u64;
-    const ZERO: Self = 0u64;
-
-    fn checked_shl(self, rhs: u32) -> Option<Self> {
-        self.checked_shl(rhs)
-    }
-
-    fn checked_shr(self, rhs: u32) -> Option<Self> {
-        self.checked_shr(rhs)
-    }
-}
-
-impl std::fmt::Debug for FiniteBitSet<u64> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{:064b}", self.0)
-    }
-}
-
-impl FiniteBitSetTy for u128 {
-    const DOMAIN_SIZE: u32 = 128;
-
-    const FILLED: Self = Self::MAX;
-    const EMPTY: Self = Self::MIN;
-
-    const ONE: Self = 1u128;
-    const ZERO: Self = 0u128;
-
-    fn checked_shl(self, rhs: u32) -> Option<Self> {
-        self.checked_shl(rhs)
-    }
-
-    fn checked_shr(self, rhs: u32) -> Option<Self> {
-        self.checked_shr(rhs)
-    }
-}
-
-impl std::fmt::Debug for FiniteBitSet<u128> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{:0128b}", self.0)
-    }
-}
-
 /// A fixed-sized bitset type represented by an integer type. Indices outwith than the range
 /// representable by `T` are considered set.
-#[derive(Copy, Clone, Eq, PartialEq, Decodable, Encodable)]
+#[cfg_attr(feature = "nightly", derive(Decodable, Encodable))]
+#[derive(Copy, Clone, Eq, PartialEq)]
 pub struct FiniteBitSet<T: FiniteBitSetTy>(pub T);
 
 impl<T: FiniteBitSetTy> FiniteBitSet<T> {
diff --git a/compiler/rustc_index/src/lib.rs b/compiler/rustc_index/src/lib.rs
index 061c55c01..185e0c7d6 100644
--- a/compiler/rustc_index/src/lib.rs
+++ b/compiler/rustc_index/src/lib.rs
@@ -14,7 +14,6 @@
 )]
 #![cfg_attr(feature = "nightly", allow(internal_features))]
 
-#[cfg(feature = "nightly")]
 pub mod bit_set;
 #[cfg(feature = "nightly")]
 pub mod interval;
@@ -25,8 +24,7 @@ mod vec;
 
 pub use {idx::Idx, slice::IndexSlice, vec::IndexVec};
 
-#[cfg(feature = "rustc_macros")]
-pub use rustc_macros::newtype_index;
+pub use rustc_index_macros::newtype_index;
 
 /// Type size assertion. The first argument is a type and the second argument is its expected size.
 ///
diff --git a/compiler/rustc_index/src/vec.rs b/compiler/rustc_index/src/vec.rs
index 99e72e49f..66c5cc774 100644
--- a/compiler/rustc_index/src/vec.rs
+++ b/compiler/rustc_index/src/vec.rs
@@ -137,10 +137,6 @@ impl<I: Idx, T> IndexVec<I, T> {
         self.raw.truncate(a)
     }
 
-    pub fn convert_index_type<Ix: Idx>(self) -> IndexVec<Ix, T> {
-        IndexVec::from_raw(self.raw)
-    }
-
     /// Grows the index vector so that it contains an entry for
     /// `elem`; if that is already true, then has no
     /// effect. Otherwise, inserts new values as needed by invoking
diff --git a/compiler/rustc_index/src/vec/tests.rs b/compiler/rustc_index/src/vec/tests.rs
index cb0f0db22..381d79c24 100644
--- a/compiler/rustc_index/src/vec/tests.rs
+++ b/compiler/rustc_index/src/vec/tests.rs
@@ -1,9 +1,8 @@
-#![allow(dead_code)]
-
 // Allows the macro invocation below to work
 use crate as rustc_index;
 
-rustc_macros::newtype_index! {
+crate::newtype_index! {
+    #[orderable]
     #[max = 0xFFFF_FFFA]
     struct MyIdx {}
 }
diff --git a/compiler/rustc_index_macros/Cargo.toml b/compiler/rustc_index_macros/Cargo.toml
new file mode 100644
index 000000000..c4ca29db3
--- /dev/null
+++ b/compiler/rustc_index_macros/Cargo.toml
@@ -0,0 +1,17 @@
+[package]
+name = "rustc_index_macros"
+version = "0.0.0"
+edition = "2021"
+
+[lib]
+proc-macro = true
+
+[dependencies]
+synstructure = "0.13.0"
+syn = { version = "2.0.9", features = ["full"] }
+proc-macro2 = "1"
+quote = "1"
+
+[features]
+default = ["nightly"]
+nightly = []
\ No newline at end of file
diff --git a/compiler/rustc_index_macros/src/lib.rs b/compiler/rustc_index_macros/src/lib.rs
new file mode 100644
index 000000000..ac374a41e
--- /dev/null
+++ b/compiler/rustc_index_macros/src/lib.rs
@@ -0,0 +1,41 @@
+#![cfg_attr(feature = "nightly", feature(allow_internal_unstable))]
+#![cfg_attr(feature = "nightly", allow(internal_features))]
+
+use proc_macro::TokenStream;
+
+mod newtype;
+
+/// Creates a struct type `S` that can be used as an index with
+/// `IndexVec` and so on.
+///
+/// There are two ways of interacting with these indices:
+///
+/// - The `From` impls are the preferred way. So you can do
+///   `S::from(v)` with a `usize` or `u32`. And you can convert back
+///   to an integer with `u32::from(s)`.
+///
+/// - Alternatively, you can use the methods `S::new(v)` and `s.index()`
+///   to create/return a value.
+///
+/// Internally, the index uses a u32, so the index must not exceed
+/// `u32::MAX`.
+///
+/// The impls provided by default are Clone, Copy, PartialEq, Eq, and Hash.
+///
+/// Accepted attributes for customization:
+/// - `#[derive(HashStable_Generic)]`/`#[derive(HashStable)]`: derives
+///   `HashStable`, as normal.
+/// - `#[encodable]`: derives `Encodable`/`Decodable`.
+/// - `#[orderable]`: derives `PartialOrd`/`Ord`, plus step-related methods.
+/// - `#[debug_format = "Foo({})"]`: derives `Debug` with particular output.
+/// - `#[max = 0xFFFF_FFFD]`: specifies the max value, which allows niche
+///   optimizations. The default max value is 0xFFFF_FF00.
+/// - `#[gate_rustc_only]`: makes parts of the generated code nightly-only.
+#[proc_macro]
+#[cfg_attr(
+    feature = "nightly",
+    allow_internal_unstable(step_trait, rustc_attrs, trusted_step, spec_option_partial_eq)
+)]
+pub fn newtype_index(input: TokenStream) -> TokenStream {
+    newtype::newtype(input)
+}
diff --git a/compiler/rustc_macros/src/newtype.rs b/compiler/rustc_index_macros/src/newtype.rs
index 72b47de1a..df1318c83 100644
--- a/compiler/rustc_macros/src/newtype.rs
+++ b/compiler/rustc_index_macros/src/newtype.rs
@@ -22,17 +22,24 @@ impl Parse for Newtype {
         let mut debug_format: Option<Lit> = None;
         let mut max = None;
         let mut consts = Vec::new();
-        let mut encodable = true;
-        let mut ord = true;
+        let mut encodable = false;
+        let mut ord = false;
+        let mut gate_rustc_only = quote! {};
+        let mut gate_rustc_only_cfg = quote! { all() };
 
         attrs.retain(|attr| match attr.path().get_ident() {
             Some(ident) => match &*ident.to_string() {
-                "custom_encodable" => {
-                    encodable = false;
+                "gate_rustc_only" => {
+                    gate_rustc_only = quote! { #[cfg(feature = "nightly")] };
+                    gate_rustc_only_cfg = quote! { feature = "nightly" };
                     false
                 }
-                "no_ord_impl" => {
-                    ord = false;
+                "encodable" => {
+                    encodable = true;
+                    false
+                }
+                "orderable" => {
+                    ord = true;
                     false
                 }
                 "max" => {
@@ -88,11 +95,13 @@ impl Parse for Newtype {
 
         let encodable_impls = if encodable {
             quote! {
+                #gate_rustc_only
                 impl<D: ::rustc_serialize::Decoder> ::rustc_serialize::Decodable<D> for #name {
                     fn decode(d: &mut D) -> Self {
                         Self::from_u32(d.read_u32())
                     }
                 }
+                #gate_rustc_only
                 impl<E: ::rustc_serialize::Encoder> ::rustc_serialize::Encodable<E> for #name {
                     fn encode(&self, e: &mut E) {
                         e.emit_u32(self.private);
@@ -110,6 +119,7 @@ impl Parse for Newtype {
 
         let step = if ord {
             quote! {
+                #gate_rustc_only
                 impl ::std::iter::Step for #name {
                     #[inline]
                     fn steps_between(start: &Self, end: &Self) -> Option<usize> {
@@ -131,6 +141,7 @@ impl Parse for Newtype {
                 }
 
                 // Safety: The implementation of `Step` upholds all invariants.
+                #gate_rustc_only
                 unsafe impl ::std::iter::TrustedStep for #name {}
             }
         } else {
@@ -148,6 +159,7 @@ impl Parse for Newtype {
         let spec_partial_eq_impl = if let Lit::Int(max) = &max {
             if let Ok(max_val) = max.base10_parse::<u32>() {
                 quote! {
+                    #gate_rustc_only
                     impl core::option::SpecOptionPartialEq for #name {
                         #[inline]
                         fn eq(l: &Option<Self>, r: &Option<Self>) -> bool {
@@ -173,8 +185,8 @@ impl Parse for Newtype {
         Ok(Self(quote! {
             #(#attrs)*
             #[derive(Clone, Copy, PartialEq, Eq, Hash, #(#derive_paths),*)]
-            #[rustc_layout_scalar_valid_range_end(#max)]
-            #[rustc_pass_by_value]
+            #[cfg_attr(#gate_rustc_only_cfg, rustc_layout_scalar_valid_range_end(#max))]
+            #[cfg_attr(#gate_rustc_only_cfg, rustc_pass_by_value)]
             #vis struct #name {
                 private: u32,
             }
diff --git a/compiler/rustc_infer/Cargo.toml b/compiler/rustc_infer/Cargo.toml
index 00251a192..73a02a431 100644
--- a/compiler/rustc_infer/Cargo.toml
+++ b/compiler/rustc_infer/Cargo.toml
@@ -15,7 +15,6 @@ rustc_hir = { path = "../rustc_hir" }
 rustc_index = { path = "../rustc_index" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_middle = { path = "../rustc_middle" }
-rustc_serialize = { path = "../rustc_serialize" }
 rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
diff --git a/compiler/rustc_infer/src/errors/mod.rs b/compiler/rustc_infer/src/errors/mod.rs
index 3ff1a5c0c..a0768fc71 100644
--- a/compiler/rustc_infer/src/errors/mod.rs
+++ b/compiler/rustc_infer/src/errors/mod.rs
@@ -363,9 +363,7 @@ impl AddToDiagnostic for AddLifetimeParamsSuggestion<'_> {
                 return false;
             };
 
-            let hir_id = self.tcx.hir().local_def_id_to_hir_id(anon_reg.def_id);
-
-            let node = self.tcx.hir().get(hir_id);
+            let node = self.tcx.hir_node_by_def_id(anon_reg.def_id);
             let is_impl = matches!(&node, hir::Node::ImplItem(_));
             let generics = match node {
                 hir::Node::Item(&hir::Item {
diff --git a/compiler/rustc_infer/src/errors/note_and_explain.rs b/compiler/rustc_infer/src/errors/note_and_explain.rs
index 57bc14ebc..68bf36a16 100644
--- a/compiler/rustc_infer/src/errors/note_and_explain.rs
+++ b/compiler/rustc_infer/src/errors/note_and_explain.rs
@@ -1,6 +1,6 @@
 use crate::fluent_generated as fluent;
 use crate::infer::error_reporting::nice_region_error::find_anon_type;
-use rustc_errors::{self, AddToDiagnostic, Diagnostic, IntoDiagnosticArg, SubdiagnosticMessage};
+use rustc_errors::{AddToDiagnostic, Diagnostic, IntoDiagnosticArg, SubdiagnosticMessage};
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_span::{symbol::kw, Span};
 
@@ -17,7 +17,7 @@ impl<'a> DescriptionCtx<'a> {
         alt_span: Option<Span>,
     ) -> Option<Self> {
         let (span, kind, arg) = match *region {
-            ty::ReEarlyBound(ref br) => {
+            ty::ReEarlyParam(ref br) => {
                 let scope = region.free_region_binding_scope(tcx).expect_local();
                 let span = if let Some(param) =
                     tcx.hir().get_generics(scope).and_then(|generics| generics.get_named(br.name))
@@ -32,7 +32,7 @@ impl<'a> DescriptionCtx<'a> {
                     (Some(span), "as_defined_anon", String::new())
                 }
             }
-            ty::ReFree(ref fr) => {
+            ty::ReLateParam(ref fr) => {
                 if !fr.bound_region.is_named()
                     && let Some((ty, _)) = find_anon_type(tcx, region, &fr.bound_region)
                 {
@@ -70,11 +70,14 @@ impl<'a> DescriptionCtx<'a> {
             ty::RePlaceholder(_) | ty::ReError(_) => return None,
 
             // FIXME(#13998) RePlaceholder should probably print like
-            // ReFree rather than dumping Debug output on the user.
+            // ReLateParam rather than dumping Debug output on the user.
             //
             // We shouldn't really be having unification failures with ReVar
-            // and ReLateBound though.
-            ty::ReVar(_) | ty::ReLateBound(..) | ty::ReErased => {
+            // and ReBound though.
+            //
+            // FIXME(@lcnr): figure out why we have to handle `ReBound`
+            // here, this feels somewhat off.
+            ty::ReVar(_) | ty::ReBound(..) | ty::ReErased => {
                 (alt_span, "revar", format!("{region:?}"))
             }
         };
diff --git a/compiler/rustc_infer/src/infer/at.rs b/compiler/rustc_infer/src/infer/at.rs
index 2797d0797..09313cd97 100644
--- a/compiler/rustc_infer/src/infer/at.rs
+++ b/compiler/rustc_infer/src/infer/at.rs
@@ -65,8 +65,15 @@ impl<'tcx> InferCtxt<'tcx> {
 
     /// Forks the inference context, creating a new inference context with the same inference
     /// variables in the same state. This can be used to "branch off" many tests from the same
-    /// common state. Used in coherence.
+    /// common state.
     pub fn fork(&self) -> Self {
+        self.fork_with_intercrate(self.intercrate)
+    }
+
+    /// Forks the inference context, creating a new inference context with the same inference
+    /// variables in the same state, except possibly changing the intercrate mode. This can be
+    /// used to "branch off" many tests from the same common state. Used in negative coherence.
+    pub fn fork_with_intercrate(&self, intercrate: bool) -> Self {
         Self {
             tcx: self.tcx,
             defining_use_anchor: self.defining_use_anchor,
@@ -81,7 +88,7 @@ impl<'tcx> InferCtxt<'tcx> {
             tainted_by_errors: self.tainted_by_errors.clone(),
             err_count_on_creation: self.err_count_on_creation,
             universe: self.universe.clone(),
-            intercrate: self.intercrate,
+            intercrate,
             next_trait_solver: self.next_trait_solver,
         }
     }
@@ -441,7 +448,11 @@ impl<'tcx> ToTrace<'tcx> for ty::TraitRef<'tcx> {
     ) -> TypeTrace<'tcx> {
         TypeTrace {
             cause: cause.clone(),
-            values: TraitRefs(ExpectedFound::new(a_is_expected, a, b)),
+            values: PolyTraitRefs(ExpectedFound::new(
+                a_is_expected,
+                ty::Binder::dummy(a),
+                ty::Binder::dummy(b),
+            )),
         }
     }
 }
diff --git a/compiler/rustc_infer/src/infer/canonical/canonicalizer.rs b/compiler/rustc_infer/src/infer/canonical/canonicalizer.rs
index 0e2f9ba70..5b00ef422 100644
--- a/compiler/rustc_infer/src/infer/canonical/canonicalizer.rs
+++ b/compiler/rustc_infer/src/infer/canonical/canonicalizer.rs
@@ -9,11 +9,9 @@ use crate::infer::canonical::{
     Canonical, CanonicalTyVarKind, CanonicalVarInfo, CanonicalVarKind, OriginalQueryValues,
 };
 use crate::infer::InferCtxt;
-use rustc_middle::ty::flags::FlagComputation;
 use rustc_middle::ty::fold::{TypeFoldable, TypeFolder, TypeSuperFoldable};
 use rustc_middle::ty::GenericArg;
 use rustc_middle::ty::{self, BoundVar, InferConst, List, Ty, TyCtxt, TypeFlags, TypeVisitableExt};
-use std::sync::atomic::Ordering;
 
 use rustc_data_structures::fx::FxHashMap;
 use rustc_index::Idx;
@@ -37,40 +35,13 @@ impl<'tcx> InferCtxt<'tcx> {
     /// [c]: https://rust-lang.github.io/chalk/book/canonical_queries/canonicalization.html#canonicalizing-the-query
     pub fn canonicalize_query<V>(
         &self,
-        value: V,
-        query_state: &mut OriginalQueryValues<'tcx>,
-    ) -> Canonical<'tcx, V>
-    where
-        V: TypeFoldable<TyCtxt<'tcx>>,
-    {
-        self.tcx.sess.perf_stats.queries_canonicalized.fetch_add(1, Ordering::Relaxed);
-
-        Canonicalizer::canonicalize(value, self, self.tcx, &CanonicalizeAllFreeRegions, query_state)
-    }
-
-    /// Like [Self::canonicalize_query], but preserves distinct universes. For
-    /// example, canonicalizing `&'?0: Trait<'?1>`, where `'?0` is in `U1` and
-    /// `'?1` is in `U3` would be canonicalized to have `?0` in `U1` and `'?1`
-    /// in `U2`.
-    ///
-    /// This is used for Chalk integration.
-    pub fn canonicalize_query_preserving_universes<V>(
-        &self,
-        value: V,
+        value: ty::ParamEnvAnd<'tcx, V>,
         query_state: &mut OriginalQueryValues<'tcx>,
-    ) -> Canonical<'tcx, V>
+    ) -> Canonical<'tcx, ty::ParamEnvAnd<'tcx, V>>
     where
         V: TypeFoldable<TyCtxt<'tcx>>,
     {
-        self.tcx.sess.perf_stats.queries_canonicalized.fetch_add(1, Ordering::Relaxed);
-
-        Canonicalizer::canonicalize(
-            value,
-            self,
-            self.tcx,
-            &CanonicalizeAllFreeRegionsPreservingUniverses,
-            query_state,
-        )
+        self.canonicalize_query_with_mode(value, query_state, &CanonicalizeAllFreeRegions)
     }
 
     /// Canonicalizes a query *response* `V`. When we canonicalize a
@@ -105,7 +76,7 @@ impl<'tcx> InferCtxt<'tcx> {
         let mut query_state = OriginalQueryValues::default();
         Canonicalizer::canonicalize(
             value,
-            self,
+            Some(self),
             self.tcx,
             &CanonicalizeQueryResponse,
             &mut query_state,
@@ -119,7 +90,7 @@ impl<'tcx> InferCtxt<'tcx> {
         let mut query_state = OriginalQueryValues::default();
         Canonicalizer::canonicalize(
             value,
-            self,
+            Some(self),
             self.tcx,
             &CanonicalizeUserTypeAnnotation,
             &mut query_state,
@@ -132,21 +103,53 @@ impl<'tcx> InferCtxt<'tcx> {
     /// handling of `'static` regions (e.g. trait evaluation).
     pub fn canonicalize_query_keep_static<V>(
         &self,
-        value: V,
+        value: ty::ParamEnvAnd<'tcx, V>,
         query_state: &mut OriginalQueryValues<'tcx>,
-    ) -> Canonical<'tcx, V>
+    ) -> Canonical<'tcx, ty::ParamEnvAnd<'tcx, V>>
+    where
+        V: TypeFoldable<TyCtxt<'tcx>>,
+    {
+        self.canonicalize_query_with_mode(
+            value,
+            query_state,
+            &CanonicalizeFreeRegionsOtherThanStatic,
+        )
+    }
+
+    fn canonicalize_query_with_mode<V>(
+        &self,
+        value: ty::ParamEnvAnd<'tcx, V>,
+        query_state: &mut OriginalQueryValues<'tcx>,
+        canonicalize_region_mode: &dyn CanonicalizeMode,
+    ) -> Canonical<'tcx, ty::ParamEnvAnd<'tcx, V>>
     where
         V: TypeFoldable<TyCtxt<'tcx>>,
     {
-        self.tcx.sess.perf_stats.queries_canonicalized.fetch_add(1, Ordering::Relaxed);
+        let (param_env, value) = value.into_parts();
+        let base = self.tcx.canonical_param_env_cache.get_or_insert(
+            self.tcx,
+            param_env,
+            query_state,
+            |tcx, param_env, query_state| {
+                Canonicalizer::canonicalize(
+                    param_env,
+                    None,
+                    tcx,
+                    &CanonicalizeFreeRegionsOtherThanStatic,
+                    query_state,
+                )
+            },
+        );
 
-        Canonicalizer::canonicalize(
+        Canonicalizer::canonicalize_with_base(
+            base,
             value,
-            self,
+            Some(self),
             self.tcx,
-            &CanonicalizeFreeRegionsOtherThanStatic,
+            canonicalize_region_mode,
             query_state,
         )
+        .unchecked_map(|(param_env, value)| param_env.and(value))
     }
 }
 
@@ -176,10 +179,24 @@ impl CanonicalizeMode for CanonicalizeQueryResponse {
     fn canonicalize_free_region<'tcx>(
         &self,
         canonicalizer: &mut Canonicalizer<'_, 'tcx>,
-        r: ty::Region<'tcx>,
+        mut r: ty::Region<'tcx>,
     ) -> ty::Region<'tcx> {
+        let infcx = canonicalizer.infcx.unwrap();
+
+        if let ty::ReVar(vid) = *r {
+            r = infcx
+                .inner
+                .borrow_mut()
+                .unwrap_region_constraints()
+                .opportunistic_resolve_var(canonicalizer.tcx, vid);
+            debug!(
+                "canonical: region var found with vid {vid:?}, \
+                     opportunistically resolved to {r:?}",
+            );
+        };
+
         match *r {
-            ty::ReFree(_) | ty::ReErased | ty::ReStatic | ty::ReEarlyBound(..) => r,
+            ty::ReLateParam(_) | ty::ReErased | ty::ReStatic | ty::ReEarlyParam(..) => r,
 
             ty::RePlaceholder(placeholder) => canonicalizer.canonical_var_for_region(
                 CanonicalVarInfo { kind: CanonicalVarKind::PlaceholderRegion(placeholder) },
@@ -187,7 +204,8 @@ impl CanonicalizeMode for CanonicalizeQueryResponse {
             ),
 
             ty::ReVar(vid) => {
-                let universe = canonicalizer.region_var_universe(vid);
+                let universe =
+                    infcx.inner.borrow_mut().unwrap_region_constraints().var_universe(vid);
                 canonicalizer.canonical_var_for_region(
                     CanonicalVarInfo { kind: CanonicalVarKind::Region(universe) },
                     r,
@@ -202,8 +220,8 @@ impl CanonicalizeMode for CanonicalizeQueryResponse {
                 //
                 // rust-lang/rust#57464: `impl Trait` can leak local
                 // scopes (in manner violating typeck). Therefore, use
-                // `delay_span_bug` to allow type error over an ICE.
-                canonicalizer.tcx.sess.delay_span_bug(
+                // `span_delayed_bug` to allow type error over an ICE.
+                canonicalizer.tcx.sess.span_delayed_bug(
                     rustc_span::DUMMY_SP,
                     format!("unexpected region in query response: `{r:?}`"),
                 );
@@ -230,9 +248,13 @@ impl CanonicalizeMode for CanonicalizeUserTypeAnnotation {
         r: ty::Region<'tcx>,
     ) -> ty::Region<'tcx> {
         match *r {
-            ty::ReEarlyBound(_) | ty::ReFree(_) | ty::ReErased | ty::ReStatic | ty::ReError(_) => r,
+            ty::ReEarlyParam(_)
+            | ty::ReLateParam(_)
+            | ty::ReErased
+            | ty::ReStatic
+            | ty::ReError(_) => r,
             ty::ReVar(_) => canonicalizer.canonical_var_for_region_in_root_universe(r),
-            ty::RePlaceholder(..) | ty::ReLateBound(..) => {
+            ty::RePlaceholder(..) | ty::ReBound(..) => {
                 // We only expect region names that the user can type.
                 bug!("unexpected region in query response: `{:?}`", r)
             }
@@ -268,30 +290,6 @@ impl CanonicalizeMode for CanonicalizeAllFreeRegions {
     }
 }
 
-struct CanonicalizeAllFreeRegionsPreservingUniverses;
-
-impl CanonicalizeMode for CanonicalizeAllFreeRegionsPreservingUniverses {
-    fn canonicalize_free_region<'tcx>(
-        &self,
-        canonicalizer: &mut Canonicalizer<'_, 'tcx>,
-        r: ty::Region<'tcx>,
-    ) -> ty::Region<'tcx> {
-        let universe = canonicalizer.infcx.universe_of_region(r);
-        canonicalizer.canonical_var_for_region(
-            CanonicalVarInfo { kind: CanonicalVarKind::Region(universe) },
-            r,
-        )
-    }
-
-    fn any(&self) -> bool {
-        true
-    }
-
-    fn preserve_universes(&self) -> bool {
-        true
-    }
-}
-
 struct CanonicalizeFreeRegionsOtherThanStatic;
 
 impl CanonicalizeMode for CanonicalizeFreeRegionsOtherThanStatic {
@@ -313,7 +311,8 @@ impl CanonicalizeMode for CanonicalizeFreeRegionsOtherThanStatic {
 }
 
 struct Canonicalizer<'cx, 'tcx> {
-    infcx: &'cx InferCtxt<'tcx>,
+    /// Set to `None` to disable the resolution of inference variables.
+    infcx: Option<&'cx InferCtxt<'tcx>>,
     tcx: TyCtxt<'tcx>,
     variables: SmallVec<[CanonicalVarInfo<'tcx>; 8]>,
     query_state: &'cx mut OriginalQueryValues<'tcx>,
@@ -343,7 +342,7 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> {
 
     fn fold_region(&mut self, r: ty::Region<'tcx>) -> ty::Region<'tcx> {
         match *r {
-            ty::ReLateBound(index, ..) => {
+            ty::ReBound(index, ..) => {
                 if index >= self.binder_index {
                     bug!("escaping late-bound region during canonicalization");
                 } else {
@@ -351,25 +350,12 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> {
                 }
             }
 
-            ty::ReVar(vid) => {
-                let resolved = self
-                    .infcx
-                    .inner
-                    .borrow_mut()
-                    .unwrap_region_constraints()
-                    .opportunistic_resolve_var(self.tcx, vid);
-                debug!(
-                    "canonical: region var found with vid {vid:?}, \
-                     opportunistically resolved to {resolved:?}",
-                );
-                self.canonicalize_mode.canonicalize_free_region(self, resolved)
-            }
-
             ty::ReStatic
-            | ty::ReEarlyBound(..)
+            | ty::ReEarlyParam(..)
             | ty::ReError(_)
-            | ty::ReFree(_)
+            | ty::ReLateParam(_)
             | ty::RePlaceholder(..)
+            | ty::ReVar(_)
             | ty::ReErased => self.canonicalize_mode.canonicalize_free_region(self, r),
         }
     }
@@ -380,14 +366,14 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> {
                 // We need to canonicalize the *root* of our ty var.
                 // This is so that our canonical response correctly reflects
                 // any equated inference vars correctly!
-                let root_vid = self.infcx.root_var(vid);
+                let root_vid = self.infcx.unwrap().root_var(vid);
                 if root_vid != vid {
-                    t = Ty::new_var(self.infcx.tcx, root_vid);
+                    t = Ty::new_var(self.tcx, root_vid);
                     vid = root_vid;
                 }
 
                 debug!("canonical: type var found with vid {:?}", vid);
-                match self.infcx.probe_ty_var(vid) {
+                match self.infcx.unwrap().probe_ty_var(vid) {
                     // `t` could be a float / int variable; canonicalize that instead.
                     Ok(t) => {
                         debug!("(resolved to {:?})", t);
@@ -412,7 +398,7 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> {
             }
 
             ty::Infer(ty::IntVar(vid)) => {
-                let nt = self.infcx.opportunistic_resolve_int_var(vid);
+                let nt = self.infcx.unwrap().opportunistic_resolve_int_var(vid);
                 if nt != t {
                     return self.fold_ty(nt);
                 } else {
@@ -423,7 +409,7 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> {
                 }
             }
             ty::Infer(ty::FloatVar(vid)) => {
-                let nt = self.infcx.opportunistic_resolve_float_var(vid);
+                let nt = self.infcx.unwrap().opportunistic_resolve_float_var(vid);
                 if nt != t {
                     return self.fold_ty(nt);
                 } else {
@@ -494,14 +480,14 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> {
                 // We need to canonicalize the *root* of our const var.
                 // This is so that our canonical response correctly reflects
                 // any equated inference vars correctly!
-                let root_vid = self.infcx.root_const_var(vid);
+                let root_vid = self.infcx.unwrap().root_const_var(vid);
                 if root_vid != vid {
-                    ct = ty::Const::new_var(self.infcx.tcx, root_vid, ct.ty());
+                    ct = ty::Const::new_var(self.tcx, root_vid, ct.ty());
                     vid = root_vid;
                 }
 
                 debug!("canonical: const var found with vid {:?}", vid);
-                match self.infcx.probe_const_var(vid) {
+                match self.infcx.unwrap().probe_const_var(vid) {
                     Ok(c) => {
                         debug!("(resolved to {:?})", c);
                         return self.fold_const(c);
@@ -522,8 +508,8 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> {
                 }
             }
             ty::ConstKind::Infer(InferConst::EffectVar(vid)) => {
-                match self.infcx.probe_effect_var(vid) {
-                    Some(value) => return self.fold_const(value.as_const(self.infcx.tcx)),
+                match self.infcx.unwrap().probe_effect_var(vid) {
+                    Some(value) => return self.fold_const(value.as_const(self.tcx)),
                     None => {
                         return self.canonicalize_const_var(
                             CanonicalVarInfo { kind: CanonicalVarKind::Effect },
@@ -553,8 +539,11 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> {
             _ => {}
         }
 
-        let flags = FlagComputation::for_const(ct);
-        if flags.intersects(self.needs_canonical_flags) { ct.super_fold_with(self) } else { ct }
+        if ct.flags().intersects(self.needs_canonical_flags) {
+            ct.super_fold_with(self)
+        } else {
+            ct
+        }
     }
 }
 
@@ -563,7 +552,7 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
     /// `canonicalize_query` and `canonicalize_response`.
     fn canonicalize<V>(
         value: V,
-        infcx: &InferCtxt<'tcx>,
+        infcx: Option<&InferCtxt<'tcx>>,
         tcx: TyCtxt<'tcx>,
         canonicalize_region_mode: &dyn CanonicalizeMode,
         query_state: &mut OriginalQueryValues<'tcx>,
@@ -571,6 +560,33 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
     where
         V: TypeFoldable<TyCtxt<'tcx>>,
     {
+        let base = Canonical {
+            max_universe: ty::UniverseIndex::ROOT,
+            variables: List::empty(),
+            value: (),
+        };
+        Canonicalizer::canonicalize_with_base(
+            base,
+            value,
+            infcx,
+            tcx,
+            canonicalize_region_mode,
+            query_state,
+        )
+        .unchecked_map(|((), val)| val)
+    }
+
+    fn canonicalize_with_base<U, V>(
+        base: Canonical<'tcx, U>,
+        value: V,
+        infcx: Option<&InferCtxt<'tcx>>,
+        tcx: TyCtxt<'tcx>,
+        canonicalize_region_mode: &dyn CanonicalizeMode,
+        query_state: &mut OriginalQueryValues<'tcx>,
+    ) -> Canonical<'tcx, (U, V)>
+    where
+        V: TypeFoldable<TyCtxt<'tcx>>,
+    {
         let needs_canonical_flags = if canonicalize_region_mode.any() {
             TypeFlags::HAS_INFER | TypeFlags::HAS_PLACEHOLDER | TypeFlags::HAS_FREE_REGIONS
         } else {
@@ -579,12 +595,7 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
 
         // Fast path: nothing that needs to be canonicalized.
         if !value.has_type_flags(needs_canonical_flags) {
-            let canon_value = Canonical {
-                max_universe: ty::UniverseIndex::ROOT,
-                variables: List::empty(),
-                value,
-            };
-            return canon_value;
+            return base.unchecked_map(|b| (b, value));
         }
 
         let mut canonicalizer = Canonicalizer {
@@ -592,11 +603,20 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
             tcx,
             canonicalize_mode: canonicalize_region_mode,
             needs_canonical_flags,
-            variables: SmallVec::new(),
+            variables: SmallVec::from_slice(base.variables),
             query_state,
             indices: FxHashMap::default(),
             binder_index: ty::INNERMOST,
         };
+        if canonicalizer.query_state.var_values.spilled() {
+            canonicalizer.indices = canonicalizer
+                .query_state
+                .var_values
+                .iter()
+                .enumerate()
+                .map(|(i, &kind)| (kind, BoundVar::new(i)))
+                .collect();
+        }
         let out_value = value.fold_with(&mut canonicalizer);
 
         // Once we have canonicalized `out_value`, it should not
@@ -613,7 +633,7 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
             .max()
             .unwrap_or(ty::UniverseIndex::ROOT);
 
-        Canonical { max_universe, variables: canonical_variables, value: out_value }
+        Canonical { max_universe, variables: canonical_variables, value: (base.value, out_value) }
     }
 
     /// Creates a canonical variable replacing `kind` from the input,
@@ -762,11 +782,6 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
         )
     }
 
-    /// Returns the universe in which `vid` is defined.
-    fn region_var_universe(&self, vid: ty::RegionVid) -> ty::UniverseIndex {
-        self.infcx.inner.borrow_mut().unwrap_region_constraints().var_universe(vid)
-    }
-
     /// Creates a canonical variable (with the given `info`)
     /// representing the region `r`; return a region referencing it.
     fn canonical_var_for_region(
@@ -776,7 +791,7 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
     ) -> ty::Region<'tcx> {
         let var = self.canonical_var(info, r.into());
         let br = ty::BoundRegion { var, kind: ty::BrAnon };
-        ty::Region::new_late_bound(self.interner(), self.binder_index, br)
+        ty::Region::new_bound(self.interner(), self.binder_index, br)
     }
 
     /// Given a type variable `ty_var` of the given kind, first check
@@ -784,14 +799,9 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
     /// *that*. Otherwise, create a new canonical variable for
     /// `ty_var`.
     fn canonicalize_ty_var(&mut self, info: CanonicalVarInfo<'tcx>, ty_var: Ty<'tcx>) -> Ty<'tcx> {
-        let infcx = self.infcx;
-        let bound_to = infcx.shallow_resolve(ty_var);
-        if bound_to != ty_var {
-            self.fold_ty(bound_to)
-        } else {
-            let var = self.canonical_var(info, ty_var.into());
-            Ty::new_bound(self.tcx, self.binder_index, var.into())
-        }
+        debug_assert!(!self.infcx.is_some_and(|infcx| ty_var != infcx.shallow_resolve(ty_var)));
+        let var = self.canonical_var(info, ty_var.into());
+        Ty::new_bound(self.tcx, self.binder_index, var.into())
     }
 
     /// Given a type variable `const_var` of the given kind, first check
@@ -803,13 +813,10 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {
         info: CanonicalVarInfo<'tcx>,
         const_var: ty::Const<'tcx>,
     ) -> ty::Const<'tcx> {
-        let infcx = self.infcx;
-        let bound_to = infcx.shallow_resolve(const_var);
-        if bound_to != const_var {
-            self.fold_const(bound_to)
-        } else {
-            let var = self.canonical_var(info, const_var.into());
-            ty::Const::new_bound(self.tcx, self.binder_index, var, self.fold_ty(const_var.ty()))
-        }
+        debug_assert!(
+            !self.infcx.is_some_and(|infcx| const_var != infcx.shallow_resolve(const_var))
+        );
+        let var = self.canonical_var(info, const_var.into());
+        ty::Const::new_bound(self.tcx, self.binder_index, var, self.fold_ty(const_var.ty()))
     }
 }
diff --git a/compiler/rustc_infer/src/infer/canonical/mod.rs b/compiler/rustc_infer/src/infer/canonical/mod.rs
index 3c4c4644f..386fdb09b 100644
--- a/compiler/rustc_infer/src/infer/canonical/mod.rs
+++ b/compiler/rustc_infer/src/infer/canonical/mod.rs
@@ -101,7 +101,7 @@ impl<'tcx> InferCtxt<'tcx> {
     /// variable, then you'll get a new inference variable; if it is a
     /// universally quantified variable, you get a placeholder.
     ///
-    /// FIXME(-Ztrait-solver=next): This is public because it's used by the
+    /// FIXME(-Znext-solver): This is public because it's used by the
     /// new trait solver which has a different canonicalization routine.
     /// We should somehow deduplicate all of this.
     pub fn instantiate_canonical_var(
diff --git a/compiler/rustc_infer/src/infer/canonical/query_response.rs b/compiler/rustc_infer/src/infer/canonical/query_response.rs
index ed1010821..8cca4c623 100644
--- a/compiler/rustc_infer/src/infer/canonical/query_response.rs
+++ b/compiler/rustc_infer/src/infer/canonical/query_response.rs
@@ -166,10 +166,7 @@ impl<'tcx> InferCtxt<'tcx> {
     }
 
     fn take_opaque_types_for_query_response(&self) -> Vec<(ty::OpaqueTypeKey<'tcx>, Ty<'tcx>)> {
-        std::mem::take(&mut self.inner.borrow_mut().opaque_type_storage.opaque_types)
-            .into_iter()
-            .map(|(k, v)| (k, v.hidden_type.ty))
-            .collect()
+        self.take_opaque_types().into_iter().map(|(k, v)| (k, v.hidden_type.ty)).collect()
     }
 
     /// Given the (canonicalized) result to a canonical query,
@@ -460,7 +457,7 @@ impl<'tcx> InferCtxt<'tcx> {
                 }
                 GenericArgKind::Lifetime(result_value) => {
                     // e.g., here `result_value` might be `'?1` in the example above...
-                    if let ty::ReLateBound(debruijn, br) = *result_value {
+                    if let ty::ReBound(debruijn, br) = *result_value {
                         // ... in which case we would set `canonical_vars[0]` to `Some('static)`.
 
                         // We only allow a `ty::INNERMOST` index in substitutions.
@@ -552,7 +549,7 @@ impl<'tcx> InferCtxt<'tcx> {
         // `query_response.var_values` after applying the substitution
         // `result_subst`.
         let substituted_query_response = |index: BoundVar| -> GenericArg<'tcx> {
-            query_response.substitute_projected(self.tcx, &result_subst, |v| v.var_values[index])
+            query_response.substitute_projected(self.tcx, result_subst, |v| v.var_values[index])
         };
 
         // Unify the original value for each variable with the value
diff --git a/compiler/rustc_infer/src/infer/canonical/substitute.rs b/compiler/rustc_infer/src/infer/canonical/substitute.rs
index f368b30fb..e0b97bb16 100644
--- a/compiler/rustc_infer/src/infer/canonical/substitute.rs
+++ b/compiler/rustc_infer/src/infer/canonical/substitute.rs
@@ -11,7 +11,7 @@ use rustc_middle::ty::fold::{FnMutDelegate, TypeFoldable};
 use rustc_middle::ty::GenericArgKind;
 use rustc_middle::ty::{self, TyCtxt};
 
-/// FIXME(-Ztrait-solver=next): This or public because it is shared with the
+/// FIXME(-Znext-solver): This or public because it is shared with the
 /// new trait solver implementation. We should deduplicate canonicalization.
 pub trait CanonicalExt<'tcx, V> {
     /// Instantiate the wrapped value, replacing each canonical value
diff --git a/compiler/rustc_infer/src/infer/error_reporting/mod.rs b/compiler/rustc_infer/src/infer/error_reporting/mod.rs
index 26d071a01..d396c4100 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/mod.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/mod.rs
@@ -131,13 +131,13 @@ pub struct TypeErrCtxt<'a, 'tcx> {
 
 impl Drop for TypeErrCtxt<'_, '_> {
     fn drop(&mut self) {
-        if let Some(_) = self.infcx.tcx.sess.has_errors_or_delayed_span_bugs() {
+        if let Some(_) = self.infcx.tcx.sess.has_errors_or_span_delayed_bugs() {
             // ok, emitted an error.
         } else {
             self.infcx
                 .tcx
                 .sess
-                .delay_good_path_bug("used a `TypeErrCtxt` without raising an error or lint");
+                .good_path_delayed_bug("used a `TypeErrCtxt` without raising an error or lint");
         }
     }
 }
@@ -155,7 +155,7 @@ impl TypeErrCtxt<'_, '_> {
 impl<'tcx> Deref for TypeErrCtxt<'_, 'tcx> {
     type Target = InferCtxt<'tcx>;
     fn deref(&self) -> &InferCtxt<'tcx> {
-        &self.infcx
+        self.infcx
     }
 }
 
@@ -168,17 +168,17 @@ pub(super) fn note_and_explain_region<'tcx>(
     alt_span: Option<Span>,
 ) {
     let (description, span) = match *region {
-        ty::ReEarlyBound(_) | ty::ReFree(_) | ty::RePlaceholder(_) | ty::ReStatic => {
+        ty::ReEarlyParam(_) | ty::ReLateParam(_) | ty::RePlaceholder(_) | ty::ReStatic => {
             msg_span_from_named_region(tcx, region, alt_span)
         }
 
         ty::ReError(_) => return,
 
         // We shouldn't really be having unification failures with ReVar
-        // and ReLateBound though.
-        ty::ReVar(_) | ty::ReLateBound(..) | ty::ReErased => {
-            (format!("lifetime `{region}`"), alt_span)
-        }
+        // and ReBound though.
+        //
+        // FIXME(@lcnr): Figure out whether this is reachable and if so, why.
+        ty::ReVar(_) | ty::ReBound(..) | ty::ReErased => (format!("lifetime `{region}`"), alt_span),
     };
 
     emit_msg_span(err, prefix, description, span, suffix);
@@ -202,7 +202,7 @@ fn msg_span_from_named_region<'tcx>(
     alt_span: Option<Span>,
 ) -> (String, Option<Span>) {
     match *region {
-        ty::ReEarlyBound(ref br) => {
+        ty::ReEarlyParam(ref br) => {
             let scope = region.free_region_binding_scope(tcx).expect_local();
             let span = if let Some(param) =
                 tcx.hir().get_generics(scope).and_then(|generics| generics.get_named(br.name))
@@ -218,7 +218,7 @@ fn msg_span_from_named_region<'tcx>(
             };
             (text, Some(span))
         }
-        ty::ReFree(ref fr) => {
+        ty::ReLateParam(ref fr) => {
             if !fr.bound_region.is_named()
                 && let Some((ty, _)) = find_anon_type(tcx, region, &fr.bound_region)
             {
@@ -315,7 +315,7 @@ pub fn unexpected_hidden_region_diagnostic<'tcx>(
 
     // Explain the region we are capturing.
     match *hidden_region {
-        ty::ReEarlyBound(_) | ty::ReFree(_) | ty::ReStatic => {
+        ty::ReEarlyParam(_) | ty::ReLateParam(_) | ty::ReStatic => {
             // Assuming regionck succeeded (*), we ought to always be
             // capturing *some* region from the fn header, and hence it
             // ought to be free. So under normal circumstances, we will go
@@ -517,7 +517,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
 
         self.tcx
             .sess
-            .delay_span_bug(self.tcx.def_span(generic_param_scope), "expected region errors")
+            .span_delayed_bug(self.tcx.def_span(generic_param_scope), "expected region errors")
     }
 
     // This method goes through all the errors and try to group certain types
@@ -860,7 +860,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                         self.suggest_boxing_for_return_impl_trait(
                             err,
                             ret_sp,
-                            prior_arms.iter().chain(std::iter::once(&arm_span)).map(|s| *s),
+                            prior_arms.iter().chain(std::iter::once(&arm_span)).copied(),
                         );
                     }
                 }
@@ -891,7 +891,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                 }
                 // don't suggest wrapping either blocks in `if .. {} else {}`
                 let is_empty_arm = |id| {
-                    let hir::Node::Block(blk) = self.tcx.hir().get(id) else {
+                    let hir::Node::Block(blk) = self.tcx.hir_node(id) else {
                         return false;
                     };
                     if blk.expr.is_some() || !blk.stmts.is_empty() {
@@ -1019,8 +1019,8 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
     /// ```
     fn cmp_type_arg(
         &self,
-        mut t1_out: &mut DiagnosticStyledString,
-        mut t2_out: &mut DiagnosticStyledString,
+        t1_out: &mut DiagnosticStyledString,
+        t2_out: &mut DiagnosticStyledString,
         path: String,
         sub: &'tcx [ty::GenericArg<'tcx>],
         other_path: String,
@@ -1031,13 +1031,13 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
         let sub = self.tcx.mk_args(sub);
         for (i, ta) in sub.types().enumerate() {
             if ta == other_ty {
-                self.highlight_outer(&mut t1_out, &mut t2_out, path, sub, i, other_ty);
+                self.highlight_outer(t1_out, t2_out, path, sub, i, other_ty);
                 return Some(());
             }
             if let ty::Adt(def, _) = ta.kind() {
                 let path_ = self.tcx.def_path_str(def.did());
                 if path_ == other_path {
-                    self.highlight_outer(&mut t1_out, &mut t2_out, path, sub, i, other_ty);
+                    self.highlight_outer(t1_out, t2_out, path, sub, i, other_ty);
                     return Some(());
                 }
             }
@@ -1181,37 +1181,54 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
         debug!("cmp(t1={}, t1.kind={:?}, t2={}, t2.kind={:?})", t1, t1.kind(), t2, t2.kind());
 
         // helper functions
-        fn equals<'tcx>(a: Ty<'tcx>, b: Ty<'tcx>) -> bool {
-            match (a.kind(), b.kind()) {
-                (a, b) if *a == *b => true,
-                (&ty::Int(_), &ty::Infer(ty::InferTy::IntVar(_)))
-                | (
-                    &ty::Infer(ty::InferTy::IntVar(_)),
-                    &ty::Int(_) | &ty::Infer(ty::InferTy::IntVar(_)),
-                )
-                | (&ty::Float(_), &ty::Infer(ty::InferTy::FloatVar(_)))
-                | (
-                    &ty::Infer(ty::InferTy::FloatVar(_)),
-                    &ty::Float(_) | &ty::Infer(ty::InferTy::FloatVar(_)),
-                ) => true,
-                _ => false,
+        let recurse = |t1, t2, values: &mut (DiagnosticStyledString, DiagnosticStyledString)| {
+            let (x1, x2) = self.cmp(t1, t2);
+            (values.0).0.extend(x1.0);
+            (values.1).0.extend(x2.0);
+        };
+
+        fn fmt_region<'tcx>(region: ty::Region<'tcx>) -> String {
+            let mut r = region.to_string();
+            if r == "'_" {
+                r.clear();
+            } else {
+                r.push(' ');
             }
+            format!("&{r}")
         }
 
-        fn push_ty_ref<'tcx>(
+        fn push_ref<'tcx>(
             region: ty::Region<'tcx>,
-            ty: Ty<'tcx>,
             mutbl: hir::Mutability,
             s: &mut DiagnosticStyledString,
         ) {
-            let mut r = region.to_string();
-            if r == "'_" {
-                r.clear();
+            s.push_highlighted(fmt_region(region));
+            s.push_highlighted(mutbl.prefix_str());
+        }
+
+        fn cmp_ty_refs<'tcx>(
+            r1: ty::Region<'tcx>,
+            mut1: hir::Mutability,
+            r2: ty::Region<'tcx>,
+            mut2: hir::Mutability,
+            ss: &mut (DiagnosticStyledString, DiagnosticStyledString),
+        ) {
+            let (r1, r2) = (fmt_region(r1), fmt_region(r2));
+            if r1 != r2 {
+                ss.0.push_highlighted(r1);
+                ss.1.push_highlighted(r2);
             } else {
-                r.push(' ');
+                ss.0.push_normal(r1);
+                ss.1.push_normal(r2);
+            }
+
+            if mut1 != mut2 {
+                ss.0.push_highlighted(mut1.prefix_str());
+                ss.1.push_highlighted(mut2.prefix_str());
+            } else {
+                ss.0.push_normal(mut1.prefix_str());
+                ss.1.push_normal(mut2.prefix_str());
             }
-            s.push_highlighted(format!("&{}{}", r, mutbl.prefix_str()));
-            s.push_normal(ty.to_string());
         }
 
         // process starts here
@@ -1285,7 +1302,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                         if lifetimes.0 != lifetimes.1 {
                             values.0.push_highlighted(l1);
                             values.1.push_highlighted(l2);
-                        } else if lifetimes.0.is_late_bound() {
+                        } else if lifetimes.0.is_bound() {
                             values.0.push_normal(l1);
                             values.1.push_normal(l2);
                         } else {
@@ -1310,9 +1327,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                             values.0.push_normal("_");
                             values.1.push_normal("_");
                         } else {
-                            let (x1, x2) = self.cmp(ta1, ta2);
-                            (values.0).0.extend(x1.0);
-                            (values.1).0.extend(x2.0);
+                            recurse(ta1, ta2, &mut values);
                         }
                         self.push_comma(&mut values.0, &mut values.1, len, i);
                     }
@@ -1418,27 +1433,24 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                 }
             }
 
-            // When finding T != &T, highlight only the borrow
-            (&ty::Ref(r1, ref_ty1, mutbl1), _) if equals(ref_ty1, t2) => {
+            // When finding `&T != &T`, compare the references, then recurse into pointee type
+            (&ty::Ref(r1, ref_ty1, mutbl1), &ty::Ref(r2, ref_ty2, mutbl2)) => {
                 let mut values = (DiagnosticStyledString::new(), DiagnosticStyledString::new());
-                push_ty_ref(r1, ref_ty1, mutbl1, &mut values.0);
-                values.1.push_normal(t2.to_string());
+                cmp_ty_refs(r1, mutbl1, r2, mutbl2, &mut values);
+                recurse(ref_ty1, ref_ty2, &mut values);
                 values
             }
-            (_, &ty::Ref(r2, ref_ty2, mutbl2)) if equals(t1, ref_ty2) => {
+            // When finding T != &T, highlight the borrow
+            (&ty::Ref(r1, ref_ty1, mutbl1), _) => {
                 let mut values = (DiagnosticStyledString::new(), DiagnosticStyledString::new());
-                values.0.push_normal(t1.to_string());
-                push_ty_ref(r2, ref_ty2, mutbl2, &mut values.1);
+                push_ref(r1, mutbl1, &mut values.0);
+                recurse(ref_ty1, t2, &mut values);
                 values
             }
-
-            // When encountering &T != &mut T, highlight only the borrow
-            (&ty::Ref(r1, ref_ty1, mutbl1), &ty::Ref(r2, ref_ty2, mutbl2))
-                if equals(ref_ty1, ref_ty2) =>
-            {
+            (_, &ty::Ref(r2, ref_ty2, mutbl2)) => {
                 let mut values = (DiagnosticStyledString::new(), DiagnosticStyledString::new());
-                push_ty_ref(r1, ref_ty1, mutbl1, &mut values.0);
-                push_ty_ref(r2, ref_ty2, mutbl2, &mut values.1);
+                push_ref(r2, mutbl2, &mut values.1);
+                recurse(t1, ref_ty2, &mut values);
                 values
             }
 
@@ -1448,9 +1460,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                     (DiagnosticStyledString::normal("("), DiagnosticStyledString::normal("("));
                 let len = args1.len();
                 for (i, (left, right)) in args1.iter().zip(args2).enumerate() {
-                    let (x1, x2) = self.cmp(left, right);
-                    (values.0).0.extend(x1.0);
-                    (values.1).0.extend(x2.0);
+                    recurse(left, right, &mut values);
                     self.push_comma(&mut values.0, &mut values.1, len, i);
                 }
                 if len == 1 {
@@ -1667,9 +1677,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                             .report(diag);
                         (false, Mismatch::Fixed("signature"))
                     }
-                    ValuePairs::TraitRefs(_) | ValuePairs::PolyTraitRefs(_) => {
-                        (false, Mismatch::Fixed("trait"))
-                    }
+                    ValuePairs::PolyTraitRefs(_) => (false, Mismatch::Fixed("trait")),
                     ValuePairs::Aliases(infer::ExpectedFound { expected, .. }) => {
                         (false, Mismatch::Fixed(self.tcx.def_descr(expected.def_id)))
                     }
@@ -1735,7 +1743,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
             }
         }
 
-        if let Some((expected, found, exp_p, found_p)) = expected_found {
+        if let Some((expected, found, path)) = expected_found {
             let (expected_label, found_label, exp_found) = match exp_found {
                 Mismatch::Variable(ef) => (
                     ef.expected.prefix_string(self.tcx),
@@ -1861,40 +1869,31 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                 }
                 TypeError::Sorts(values) => {
                     let extra = expected == found;
-                    let sort_string = |ty: Ty<'tcx>, path: Option<PathBuf>| {
-                        let mut s = match (extra, ty.kind()) {
-                            (true, ty::Alias(ty::Opaque, ty::AliasTy { def_id, .. })) => {
-                                let sm = self.tcx.sess.source_map();
-                                let pos = sm.lookup_char_pos(self.tcx.def_span(*def_id).lo());
-                                format!(
-                                    " (opaque type at <{}:{}:{}>)",
-                                    sm.filename_for_diagnostics(&pos.file.name),
-                                    pos.line,
-                                    pos.col.to_usize() + 1,
-                                )
-                            }
-                            (true, ty::Alias(ty::Projection, proj))
-                                if self.tcx.is_impl_trait_in_trait(proj.def_id) =>
-                            {
-                                let sm = self.tcx.sess.source_map();
-                                let pos = sm.lookup_char_pos(self.tcx.def_span(proj.def_id).lo());
-                                format!(
-                                    " (trait associated opaque type at <{}:{}:{}>)",
-                                    sm.filename_for_diagnostics(&pos.file.name),
-                                    pos.line,
-                                    pos.col.to_usize() + 1,
-                                )
-                            }
-                            (true, _) => format!(" ({})", ty.sort_string(self.tcx)),
-                            (false, _) => "".to_string(),
-                        };
-                        if let Some(path) = path {
-                            s.push_str(&format!(
-                                "\nthe full type name has been written to '{}'",
-                                path.display(),
-                            ));
+                    let sort_string = |ty: Ty<'tcx>| match (extra, ty.kind()) {
+                        (true, ty::Alias(ty::Opaque, ty::AliasTy { def_id, .. })) => {
+                            let sm = self.tcx.sess.source_map();
+                            let pos = sm.lookup_char_pos(self.tcx.def_span(*def_id).lo());
+                            format!(
+                                " (opaque type at <{}:{}:{}>)",
+                                sm.filename_for_diagnostics(&pos.file.name),
+                                pos.line,
+                                pos.col.to_usize() + 1,
+                            )
                         }
-                        s
+                        (true, ty::Alias(ty::Projection, proj))
+                            if self.tcx.is_impl_trait_in_trait(proj.def_id) =>
+                        {
+                            let sm = self.tcx.sess.source_map();
+                            let pos = sm.lookup_char_pos(self.tcx.def_span(proj.def_id).lo());
+                            format!(
+                                " (trait associated opaque type at <{}:{}:{}>)",
+                                sm.filename_for_diagnostics(&pos.file.name),
+                                pos.line,
+                                pos.col.to_usize() + 1,
+                            )
+                        }
+                        (true, _) => format!(" ({})", ty.sort_string(self.tcx)),
+                        (false, _) => "".to_string(),
                     };
                     if !(values.expected.is_simple_text(self.tcx)
                         && values.found.is_simple_text(self.tcx))
@@ -1925,9 +1924,15 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                                     expected,
                                     &found_label,
                                     found,
-                                    &sort_string(values.expected, exp_p),
-                                    &sort_string(values.found, found_p),
+                                    &sort_string(values.expected),
+                                    &sort_string(values.found),
                                 );
+                                if let Some(path) = path {
+                                    diag.note(format!(
+                                        "the full type name has been written to '{}'",
+                                        path.display(),
+                                    ));
+                                }
                             }
                         }
                     }
@@ -1999,7 +2004,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
             diag.span_note(span, "this closure does not fulfill the lifetime requirements");
             self.suggest_for_all_lifetime_closure(
                 span,
-                self.tcx.hir().get_by_def_id(def_id),
+                self.tcx.hir_node_by_def_id(def_id),
                 &exp_found,
                 diag,
             );
@@ -2093,7 +2098,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
         if let &(MatchExpressionArm(box MatchExpressionArmCause { source, .. })
         | BlockTailExpression(.., source)) = code
             && let hir::MatchSource::TryDesugar(_) = source
-            && let Some((expected_ty, found_ty, _, _)) = self.values_str(trace.values)
+            && let Some((expected_ty, found_ty, _)) = self.values_str(trace.values)
         {
             suggestions.push(TypeErrorAdditionalDiags::TryCannotConvert {
                 found: found_ty.content(),
@@ -2113,7 +2118,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
         let TypeError::FixedArraySize(sz) = terr else {
             return None;
         };
-        let tykind = match hir.find_by_def_id(trace.cause.body_id) {
+        let tykind = match self.tcx.opt_hir_node_by_def_id(trace.cause.body_id) {
             Some(hir::Node::Item(hir::Item { kind: hir::ItemKind::Fn(_, _, body_id), .. })) => {
                 let body = hir.body(*body_id);
                 struct LetVisitor<'v> {
@@ -2211,33 +2216,20 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
     fn values_str(
         &self,
         values: ValuePairs<'tcx>,
-    ) -> Option<(DiagnosticStyledString, DiagnosticStyledString, Option<PathBuf>, Option<PathBuf>)>
-    {
+    ) -> Option<(DiagnosticStyledString, DiagnosticStyledString, Option<PathBuf>)> {
         match values {
             infer::Regions(exp_found) => self.expected_found_str(exp_found),
             infer::Terms(exp_found) => self.expected_found_str_term(exp_found),
             infer::Aliases(exp_found) => self.expected_found_str(exp_found),
             infer::ExistentialTraitRef(exp_found) => self.expected_found_str(exp_found),
             infer::ExistentialProjection(exp_found) => self.expected_found_str(exp_found),
-            infer::TraitRefs(exp_found) => {
-                let pretty_exp_found = ty::error::ExpectedFound {
-                    expected: exp_found.expected.print_only_trait_path(),
-                    found: exp_found.found.print_only_trait_path(),
-                };
-                match self.expected_found_str(pretty_exp_found) {
-                    Some((expected, found, _, _)) if expected == found => {
-                        self.expected_found_str(exp_found)
-                    }
-                    ret => ret,
-                }
-            }
             infer::PolyTraitRefs(exp_found) => {
                 let pretty_exp_found = ty::error::ExpectedFound {
-                    expected: exp_found.expected.print_only_trait_path(),
-                    found: exp_found.found.print_only_trait_path(),
+                    expected: exp_found.expected.print_trait_sugared(),
+                    found: exp_found.found.print_trait_sugared(),
                 };
                 match self.expected_found_str(pretty_exp_found) {
-                    Some((expected, found, _, _)) if expected == found => {
+                    Some((expected, found, _)) if expected == found => {
                         self.expected_found_str(exp_found)
                     }
                     ret => ret,
@@ -2249,7 +2241,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                     return None;
                 }
                 let (exp, fnd) = self.cmp_fn_sig(&exp_found.expected, &exp_found.found);
-                Some((exp, fnd, None, None))
+                Some((exp, fnd, None))
             }
         }
     }
@@ -2257,8 +2249,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
     fn expected_found_str_term(
         &self,
         exp_found: ty::error::ExpectedFound<ty::Term<'tcx>>,
-    ) -> Option<(DiagnosticStyledString, DiagnosticStyledString, Option<PathBuf>, Option<PathBuf>)>
-    {
+    ) -> Option<(DiagnosticStyledString, DiagnosticStyledString, Option<PathBuf>)> {
         let exp_found = self.resolve_vars_if_possible(exp_found);
         if exp_found.references_error() {
             return None;
@@ -2273,25 +2264,21 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                 let len = self.tcx.sess().diagnostic_width() + 40;
                 let exp_s = exp.content();
                 let fnd_s = fnd.content();
-                let mut exp_p = None;
-                let mut fnd_p = None;
+                let mut path = None;
                 if exp_s.len() > len {
-                    let (exp_s, exp_path) = self.tcx.short_ty_string(expected);
+                    let exp_s = self.tcx.short_ty_string(expected, &mut path);
                     exp = DiagnosticStyledString::highlighted(exp_s);
-                    exp_p = exp_path;
                 }
                 if fnd_s.len() > len {
-                    let (fnd_s, fnd_path) = self.tcx.short_ty_string(found);
+                    let fnd_s = self.tcx.short_ty_string(found, &mut path);
                     fnd = DiagnosticStyledString::highlighted(fnd_s);
-                    fnd_p = fnd_path;
                 }
-                (exp, fnd, exp_p, fnd_p)
+                (exp, fnd, path)
             }
             _ => (
                 DiagnosticStyledString::highlighted(exp_found.expected.to_string()),
                 DiagnosticStyledString::highlighted(exp_found.found.to_string()),
                 None,
-                None,
             ),
         })
     }
@@ -2300,8 +2287,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
     fn expected_found_str<T: fmt::Display + TypeFoldable<TyCtxt<'tcx>>>(
         &self,
         exp_found: ty::error::ExpectedFound<T>,
-    ) -> Option<(DiagnosticStyledString, DiagnosticStyledString, Option<PathBuf>, Option<PathBuf>)>
-    {
+    ) -> Option<(DiagnosticStyledString, DiagnosticStyledString, Option<PathBuf>)> {
         let exp_found = self.resolve_vars_if_possible(exp_found);
         if exp_found.references_error() {
             return None;
@@ -2311,7 +2297,6 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
             DiagnosticStyledString::highlighted(exp_found.expected.to_string()),
             DiagnosticStyledString::highlighted(exp_found.found.to_string()),
             None,
-            None,
         ))
     }
 
@@ -2351,6 +2336,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
 
         let labeled_user_string = match bound_kind {
             GenericKind::Param(ref p) => format!("the parameter type `{p}`"),
+            GenericKind::Placeholder(ref p) => format!("the placeholder type `{p:?}`"),
             GenericKind::Alias(ref p) => match p.kind(self.tcx) {
                 ty::AliasKind::Projection | ty::AliasKind::Inherent => {
                     format!("the associated type `{p}`")
@@ -2364,7 +2350,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
             span,
             format!("{labeled_user_string} may not live long enough"),
             match sub.kind() {
-                ty::ReEarlyBound(_) | ty::ReFree(_) if sub.has_name() => error_code!(E0309),
+                ty::ReEarlyParam(_) | ty::ReLateParam(_) if sub.has_name() => error_code!(E0309),
                 ty::ReStatic => error_code!(E0310),
                 _ => error_code!(E0311),
             },
@@ -2372,7 +2358,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
 
         '_explain: {
             let (description, span) = match sub.kind() {
-                ty::ReEarlyBound(_) | ty::ReFree(_) | ty::ReStatic => {
+                ty::ReEarlyParam(_) | ty::ReLateParam(_) | ty::ReStatic => {
                     msg_span_from_named_region(self.tcx, sub, Some(span))
                 }
                 _ => (format!("lifetime `{sub}`"), Some(span)),
@@ -2455,7 +2441,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
 
             if !suggs.is_empty() {
                 err.multipart_suggestion_verbose(
-                    format!("{msg}"),
+                    msg,
                     suggs,
                     Applicability::MaybeIncorrect, // Issue #41966
                 );
@@ -2515,7 +2501,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
 
         let (lifetime_def_id, lifetime_scope) = match self.tcx.is_suitable_region(lifetime) {
             Some(info) if !lifetime.has_name() => {
-                (info.boundregion.get_id().unwrap().expect_local(), info.def_id)
+                (info.bound_region.get_id().unwrap().expect_local(), info.def_id)
             }
             _ => return lifetime.get_name_or_anon().to_string(),
         };
@@ -2594,8 +2580,8 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
 
         if let infer::Subtype(ref sup_trace) = sup_origin
             && let infer::Subtype(ref sub_trace) = sub_origin
-            && let Some((sup_expected, sup_found, _, _)) = self.values_str(sup_trace.values)
-            && let Some((sub_expected, sub_found, _, _)) = self.values_str(sub_trace.values)
+            && let Some((sup_expected, sup_found, _)) = self.values_str(sup_trace.values)
+            && let Some((sub_expected, sub_found, _)) = self.values_str(sub_trace.values)
             && sub_expected == sup_expected
             && sub_found == sup_found
         {
@@ -2667,11 +2653,6 @@ impl<'tcx> TypeRelation<'tcx> for SameTypeModuloInfer<'_, 'tcx> {
         self.0.tcx
     }
 
-    fn param_env(&self) -> ty::ParamEnv<'tcx> {
-        // Unused, only for consts which we treat as always equal
-        ty::ParamEnv::empty()
-    }
-
     fn tag(&self) -> &'static str {
         "SameTypeModuloInfer"
     }
@@ -2714,8 +2695,8 @@ impl<'tcx> TypeRelation<'tcx> for SameTypeModuloInfer<'_, 'tcx> {
         a: ty::Region<'tcx>,
         b: ty::Region<'tcx>,
     ) -> RelateResult<'tcx, ty::Region<'tcx>> {
-        if (a.is_var() && b.is_free_or_static())
-            || (b.is_var() && a.is_free_or_static())
+        if (a.is_var() && b.is_free())
+            || (b.is_var() && a.is_free())
             || (a.is_var() && b.is_var())
             || a == b
         {
@@ -2768,18 +2749,20 @@ impl<'tcx> InferCtxt<'tcx> {
             infer::AddrOfRegion(_) => " for borrow expression".to_string(),
             infer::Autoref(_) => " for autoref".to_string(),
             infer::Coercion(_) => " for automatic coercion".to_string(),
-            infer::LateBoundRegion(_, br, infer::FnCall) => {
+            infer::BoundRegion(_, br, infer::FnCall) => {
                 format!(" for lifetime parameter {}in function call", br_string(br))
             }
-            infer::LateBoundRegion(_, br, infer::HigherRankedType) => {
+            infer::BoundRegion(_, br, infer::HigherRankedType) => {
                 format!(" for lifetime parameter {}in generic type", br_string(br))
             }
-            infer::LateBoundRegion(_, br, infer::AssocTypeProjection(def_id)) => format!(
+            infer::BoundRegion(_, br, infer::AssocTypeProjection(def_id)) => format!(
                 " for lifetime parameter {}in trait containing associated type `{}`",
                 br_string(br),
                 self.tcx.associated_item(def_id).name
             ),
-            infer::EarlyBoundRegion(_, name) => format!(" for lifetime parameter `{name}`"),
+            infer::RegionParameterDefinition(_, name) => {
+                format!(" for lifetime parameter `{name}`")
+            }
             infer::UpvarRegion(ref upvar_id, _) => {
                 let var_name = self.tcx.hir().name(upvar_id.var_path.hir_id);
                 format!(" for capture of `{var_name}` by closure")
@@ -2906,9 +2889,6 @@ impl<'tcx> ObligationCauseExt<'tcx> for ObligationCause<'tcx> {
             CompareImplItemObligation { kind: ty::AssocKind::Const, .. } => {
                 "const is compatible with trait"
             }
-            ExprAssignable => "expression is assignable",
-            IfExpression { .. } => "`if` and `else` have incompatible types",
-            IfExpressionWithNoElse => "`if` missing an `else` returns `()`",
             MainFunctionType => "`main` function has the correct type",
             StartFunctionType => "`#[start]` function has the correct type",
             LangFunctionType(_) => "lang item function has the correct type",
@@ -2929,9 +2909,6 @@ impl IntoDiagnosticArg for ObligationCauseAsDiagArg<'_> {
             CompareImplItemObligation { kind: ty::AssocKind::Fn, .. } => "method_compat",
             CompareImplItemObligation { kind: ty::AssocKind::Type, .. } => "type_compat",
             CompareImplItemObligation { kind: ty::AssocKind::Const, .. } => "const_compat",
-            ExprAssignable => "expr_assignable",
-            IfExpression { .. } => "if_else_different",
-            IfExpressionWithNoElse => "no_else",
             MainFunctionType => "fn_main_correct_type",
             StartFunctionType => "fn_start_correct_type",
             LangFunctionType(_) => "fn_lang_correct_type",
@@ -3004,7 +2981,7 @@ impl<'tcx> InferCtxt<'tcx> {
     /// Given a [`hir::HirId`] for a block, get the span of its last expression
     /// or statement, peeling off any inner blocks.
     pub fn find_block_span_from_hir_id(&self, hir_id: hir::HirId) -> Span {
-        match self.tcx.hir().get(hir_id) {
+        match self.tcx.hir_node(hir_id) {
             hir::Node::Block(blk) => self.find_block_span(blk),
             // The parser was in a weird state if either of these happen, but
             // it's better not to panic.
diff --git a/compiler/rustc_infer/src/infer/error_reporting/need_type_info.rs b/compiler/rustc_infer/src/infer/error_reporting/need_type_info.rs
index 4beb51da7..817a4451d 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/need_type_info.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/need_type_info.rs
@@ -232,7 +232,9 @@ fn ty_to_string<'tcx>(
 /// something users are familiar with. Directly printing the `fn_sig` of closures also
 /// doesn't work as they actually use the "rust-call" API.
 fn closure_as_fn_str<'tcx>(infcx: &InferCtxt<'tcx>, ty: Ty<'tcx>) -> String {
-    let ty::Closure(_, args) = ty.kind() else { unreachable!() };
+    let ty::Closure(_, args) = ty.kind() else {
+        bug!("cannot convert non-closure to fn str in `closure_as_fn_str`")
+    };
     let fn_sig = args.as_closure().sig();
     let args = fn_sig
         .inputs()
@@ -374,7 +376,7 @@ impl<'tcx> InferCtxt<'tcx> {
                 multi_suggestions,
                 bad_label,
             }
-            .into_diagnostic(&self.tcx.sess.parse_sess.span_diagnostic),
+            .into_diagnostic(self.tcx.sess.dcx()),
             TypeAnnotationNeeded::E0283 => AmbiguousImpl {
                 span,
                 source_kind,
@@ -384,7 +386,7 @@ impl<'tcx> InferCtxt<'tcx> {
                 multi_suggestions,
                 bad_label,
             }
-            .into_diagnostic(&self.tcx.sess.parse_sess.span_diagnostic),
+            .into_diagnostic(self.tcx.sess.dcx()),
             TypeAnnotationNeeded::E0284 => AmbiguousReturn {
                 span,
                 source_kind,
@@ -394,7 +396,7 @@ impl<'tcx> InferCtxt<'tcx> {
                 multi_suggestions,
                 bad_label,
             }
-            .into_diagnostic(&self.tcx.sess.parse_sess.span_diagnostic),
+            .into_diagnostic(self.tcx.sess.dcx()),
         }
     }
 }
@@ -419,7 +421,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
             return self.bad_inference_failure_err(failure_span, arg_data, error_code);
         };
 
-        let mut local_visitor = FindInferSourceVisitor::new(&self, typeck_results, arg);
+        let mut local_visitor = FindInferSourceVisitor::new(self, typeck_results, arg);
         if let Some(body_id) = self.tcx.hir().maybe_body_owned_by(
             self.tcx.typeck_root_def_id(body_def_id.to_def_id()).expect_local(),
         ) {
@@ -581,7 +583,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                 multi_suggestions,
                 bad_label: None,
             }
-            .into_diagnostic(&self.tcx.sess.parse_sess.span_diagnostic),
+            .into_diagnostic(self.tcx.sess.dcx()),
             TypeAnnotationNeeded::E0283 => AmbiguousImpl {
                 span,
                 source_kind,
@@ -591,7 +593,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                 multi_suggestions,
                 bad_label: None,
             }
-            .into_diagnostic(&self.tcx.sess.parse_sess.span_diagnostic),
+            .into_diagnostic(self.tcx.sess.dcx()),
             TypeAnnotationNeeded::E0284 => AmbiguousReturn {
                 span,
                 source_kind,
@@ -601,7 +603,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                 multi_suggestions,
                 bad_label: None,
             }
-            .into_diagnostic(&self.tcx.sess.parse_sess.span_diagnostic),
+            .into_diagnostic(self.tcx.sess.dcx()),
         }
     }
 }
@@ -1087,7 +1089,7 @@ impl<'a, 'tcx> FindInferSourceVisitor<'a, 'tcx> {
 
                 Box::new(segment.into_iter())
             }
-            hir::QPath::LangItem(_, _, _) => Box::new(iter::empty()),
+            hir::QPath::LangItem(_, _) => Box::new(iter::empty()),
         }
     }
 }
diff --git a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/different_lifetimes.rs b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/different_lifetimes.rs
index 1a60bab18..f56d5d7d3 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/different_lifetimes.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/different_lifetimes.rs
@@ -70,9 +70,9 @@ impl<'a, 'tcx> NiceRegionError<'a, 'tcx> {
 
         let anon_reg_sub = self.tcx().is_suitable_region(sub)?;
         let scope_def_id_sup = anon_reg_sup.def_id;
-        let bregion_sup = anon_reg_sup.boundregion;
+        let bregion_sup = anon_reg_sup.bound_region;
         let scope_def_id_sub = anon_reg_sub.def_id;
-        let bregion_sub = anon_reg_sub.boundregion;
+        let bregion_sub = anon_reg_sub.bound_region;
 
         let ty_sup = find_anon_type(self.tcx(), sup, &bregion_sup)?;
 
diff --git a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/find_anon_type.rs b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/find_anon_type.rs
index 0df417d09..4f74365d0 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/find_anon_type.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/find_anon_type.rs
@@ -26,8 +26,7 @@ pub fn find_anon_type<'tcx>(
     br: &ty::BoundRegionKind,
 ) -> Option<(&'tcx hir::Ty<'tcx>, &'tcx hir::FnSig<'tcx>)> {
     let anon_reg = tcx.is_suitable_region(region)?;
-    let hir_id = tcx.hir().local_def_id_to_hir_id(anon_reg.def_id);
-    let fn_sig = tcx.hir().get(hir_id).fn_sig()?;
+    let fn_sig = tcx.hir_node_by_def_id(anon_reg.def_id).fn_sig()?;
 
     fn_sig
         .decl
@@ -96,7 +95,7 @@ impl<'tcx> Visitor<'tcx> for FindNestedTypeVisitor<'tcx> {
                 }
             }
 
-            hir::TyKind::Ref(ref lifetime, _) => {
+            hir::TyKind::Ref(lifetime, _) => {
                 // the lifetime of the Ref
                 let hir_id = lifetime.hir_id;
                 match (self.tcx.named_bound_var(hir_id), self.bound_region) {
diff --git a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/mismatched_static_lifetime.rs b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/mismatched_static_lifetime.rs
index 6901955af..df7907e1f 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/mismatched_static_lifetime.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/mismatched_static_lifetime.rs
@@ -81,7 +81,7 @@ impl<'a, 'tcx> NiceRegionError<'a, 'tcx> {
             for matching_def_id in v.0 {
                 let mut hir_v =
                     super::static_impl_trait::HirTraitObjectVisitor(&mut traits, matching_def_id);
-                hir_v.visit_ty(&impl_self_ty);
+                hir_v.visit_ty(impl_self_ty);
             }
 
             if traits.is_empty() {
diff --git a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/placeholder_error.rs b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/placeholder_error.rs
index c38e5b8cd..d98ca995d 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/placeholder_error.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/placeholder_error.rs
@@ -197,11 +197,6 @@ impl<'tcx> NiceRegionError<'_, 'tcx> {
         value_pairs: &ValuePairs<'tcx>,
     ) -> Option<DiagnosticBuilder<'tcx, ErrorGuaranteed>> {
         let (expected_args, found_args, trait_def_id) = match value_pairs {
-            ValuePairs::TraitRefs(ExpectedFound { expected, found })
-                if expected.def_id == found.def_id =>
-            {
-                (expected.args, found.args, expected.def_id)
-            }
             ValuePairs::PolyTraitRefs(ExpectedFound { expected, found })
                 if expected.def_id() == found.def_id() =>
             {
diff --git a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/static_impl_trait.rs b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/static_impl_trait.rs
index e2be6cf42..01b43f719 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/static_impl_trait.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/static_impl_trait.rs
@@ -78,7 +78,7 @@ impl<'a, 'tcx> NiceRegionError<'a, 'tcx> {
                         has_impl_path,
                         impl_path,
                     });
-                    if self.find_impl_on_dyn_trait(&mut err, param.param_ty, &ctxt) {
+                    if self.find_impl_on_dyn_trait(&mut err, param.param_ty, ctxt) {
                         let reported = err.emit();
                         return Some(reported);
                     } else {
@@ -204,7 +204,7 @@ impl<'a, 'tcx> NiceRegionError<'a, 'tcx> {
             && let ObligationCauseCode::UnifyReceiver(ctxt) = cause.code()
             // Handle case of `impl Foo for dyn Bar { fn qux(&self) {} }` introducing a
             // `'static` lifetime when called as a method on a binding: `bar.qux()`.
-            && self.find_impl_on_dyn_trait(&mut err, param.param_ty, &ctxt)
+            && self.find_impl_on_dyn_trait(&mut err, param.param_ty, ctxt)
         {
             override_error_code = Some(ctxt.assoc_item.name);
         }
@@ -460,7 +460,7 @@ impl<'a, 'tcx> NiceRegionError<'a, 'tcx> {
                 tcx.hir().trait_impls(trait_did).iter().find_map(|&impl_did| {
                     if let Node::Item(Item {
                         kind: ItemKind::Impl(hir::Impl { self_ty, .. }), ..
-                    }) = tcx.hir().find_by_def_id(impl_did)?
+                    }) = tcx.opt_hir_node_by_def_id(impl_did)?
                         && trait_objects.iter().all(|did| {
                             // FIXME: we should check `self_ty` against the receiver
                             // type in the `UnifyReceiver` context, but for now, use
@@ -530,7 +530,7 @@ impl<'a, 'tcx> NiceRegionError<'a, 'tcx> {
         for found_did in found_dids {
             let mut traits = vec![];
             let mut hir_v = HirTraitObjectVisitor(&mut traits, *found_did);
-            hir_v.visit_ty(&self_ty);
+            hir_v.visit_ty(self_ty);
             for &span in &traits {
                 let subdiag = DynTraitConstraintSuggestion { span, ident };
                 subdiag.add_to_diagnostic(err);
diff --git a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/trait_impl_difference.rs b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/trait_impl_difference.rs
index cb51254a1..b16d5c509 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/trait_impl_difference.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/trait_impl_difference.rs
@@ -101,7 +101,7 @@ impl<'a, 'tcx> NiceRegionError<'a, 'tcx> {
             ty::AssocKind::Fn => {
                 let hir = self.tcx().hir();
                 if let Some(hir_id) =
-                    assoc_item.def_id.as_local().map(|id| hir.local_def_id_to_hir_id(id))
+                    assoc_item.def_id.as_local().map(|id| self.tcx().local_def_id_to_hir_id(id))
                 {
                     if let Some(decl) = hir.fn_decl_by_hir_id(hir_id) {
                         visitor.visit_fn_decl(decl);
diff --git a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/util.rs b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/util.rs
index be6d1a375..bfff00b94 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/util.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/nice_region_error/util.rs
@@ -41,8 +41,8 @@ pub fn find_param_with_region<'tcx>(
     replace_region: Region<'tcx>,
 ) -> Option<AnonymousParamInfo<'tcx>> {
     let (id, bound_region) = match *anon_region {
-        ty::ReFree(ref free_region) => (free_region.scope, free_region.bound_region),
-        ty::ReEarlyBound(ebr) => {
+        ty::ReLateParam(late_param) => (late_param.scope, late_param.bound_region),
+        ty::ReEarlyParam(ebr) => {
             (tcx.parent(ebr.def_id), ty::BoundRegionKind::BrNamed(ebr.def_id, ebr.name))
         }
         _ => return None, // not a free region
@@ -50,11 +50,10 @@ pub fn find_param_with_region<'tcx>(
 
     let hir = &tcx.hir();
     let def_id = id.as_local()?;
-    let hir_id = hir.local_def_id_to_hir_id(def_id);
 
     // FIXME: use def_kind
     // Don't perform this on closures
-    match hir.get(hir_id) {
+    match tcx.hir_node_by_def_id(def_id) {
         hir::Node::Expr(&hir::Expr { kind: hir::ExprKind::Closure { .. }, .. }) => {
             return None;
         }
@@ -161,7 +160,6 @@ impl<'a, 'tcx> NiceRegionError<'a, 'tcx> {
             && self
                 .tcx()
                 .opt_associated_item(scope_def_id.to_def_id())
-                .map(|i| i.fn_has_self_parameter)
-                == Some(true)
+                .is_some_and(|i| i.fn_has_self_parameter)
     }
 }
diff --git a/compiler/rustc_infer/src/infer/error_reporting/note.rs b/compiler/rustc_infer/src/infer/error_reporting/note.rs
index 8d3cd23b7..859c10ef1 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/note.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/note.rs
@@ -22,7 +22,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
             infer::Subtype(ref trace) => RegionOriginNote::WithRequirement {
                 span: trace.cause.span,
                 requirement: ObligationCauseAsDiagArg(trace.cause.clone()),
-                expected_found: self.values_str(trace.values).map(|(e, f, _, _)| (e, f)),
+                expected_found: self.values_str(trace.values).map(|(e, f, _)| (e, f)),
             }
             .add_to_diagnostic(err),
             infer::Reborrow(span) => {
@@ -63,7 +63,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                     .add_to_diagnostic(err);
             }
             infer::CheckAssociatedTypeBounds { ref parent, .. } => {
-                self.note_region_origin(err, &parent);
+                self.note_region_origin(err, parent);
             }
             infer::AscribeUserTypeProvePredicate(span) => {
                 RegionOriginNote::Plain {
@@ -140,7 +140,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                     span,
                     notes: reference_valid.into_iter().chain(content_valid).collect(),
                 }
-                .into_diagnostic(&self.tcx.sess.parse_sess.span_diagnostic)
+                .into_diagnostic(self.tcx.sess.dcx())
             }
             infer::RelateObjectBound(span) => {
                 let object_valid = note_and_explain::RegionExplanation::new(
@@ -161,7 +161,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                     span,
                     notes: object_valid.into_iter().chain(pointer_valid).collect(),
                 }
-                .into_diagnostic(&self.tcx.sess.parse_sess.span_diagnostic)
+                .into_diagnostic(self.tcx.sess.dcx())
             }
             infer::RelateParamBound(span, ty, opt_span) => {
                 let prefix = match *sub {
@@ -177,7 +177,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                     self.tcx, sub, opt_span, prefix, suffix,
                 );
                 FulfillReqLifetime { span, ty: self.resolve_vars_if_possible(ty), note }
-                    .into_diagnostic(&self.tcx.sess.parse_sess.span_diagnostic)
+                    .into_diagnostic(self.tcx.sess.dcx())
             }
             infer::RelateRegionParamBound(span) => {
                 let param_instantiated = note_and_explain::RegionExplanation::new(
@@ -198,7 +198,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                     span,
                     notes: param_instantiated.into_iter().chain(param_must_outlive).collect(),
                 }
-                .into_diagnostic(&self.tcx.sess.parse_sess.span_diagnostic)
+                .into_diagnostic(self.tcx.sess.dcx())
             }
             infer::ReferenceOutlivesReferent(ty, span) => {
                 let pointer_valid = note_and_explain::RegionExplanation::new(
@@ -220,7 +220,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                     ty: self.resolve_vars_if_possible(ty),
                     notes: pointer_valid.into_iter().chain(data_valid).collect(),
                 }
-                .into_diagnostic(&self.tcx.sess.parse_sess.span_diagnostic)
+                .into_diagnostic(self.tcx.sess.dcx())
             }
             infer::CompareImplItemObligation { span, impl_item_def_id, trait_item_def_id } => {
                 let mut err = self.report_extra_impl_obligation(
@@ -281,7 +281,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                     span,
                     notes: instantiated.into_iter().chain(must_outlive).collect(),
                 }
-                .into_diagnostic(&self.tcx.sess.parse_sess.span_diagnostic)
+                .into_diagnostic(self.tcx.sess.dcx())
             }
         };
         if sub.is_error() || sup.is_error() {
@@ -375,7 +375,9 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                     err.span_note(span, "the lifetime requirement is introduced here");
                     err
                 } else {
-                    unreachable!()
+                    unreachable!(
+                        "control flow ensures we have a `BindingObligation` or `ExprBindingObligation` here..."
+                    )
                 }
             }
             infer::Subtype(box trace) => {
diff --git a/compiler/rustc_infer/src/infer/error_reporting/note_and_explain.rs b/compiler/rustc_infer/src/infer/error_reporting/note_and_explain.rs
index a8ae43d52..afb3c5c1e 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/note_and_explain.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/note_and_explain.rs
@@ -87,7 +87,8 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                     | (ty::Alias(ty::Projection, proj), ty::Param(p))
                         if !tcx.is_impl_trait_in_trait(proj.def_id) =>
                     {
-                        let parent = tcx.generics_of(body_owner_def_id)
+                        let parent = tcx
+                            .generics_of(body_owner_def_id)
                             .opt_type_param(p, tcx)
                             .and_then(|param| {
                                 let p_def_id = param.def_id;
@@ -104,7 +105,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                                     );
                                 }
                                 p_def_id.as_local().and_then(|id| {
-                                    let local_id = tcx.hir().local_def_id_to_hir_id(id);
+                                    let local_id = tcx.local_def_id_to_hir_id(id);
                                     let generics = tcx.hir().find_parent(local_id)?.generics()?;
                                     Some((id, generics))
                                 })
@@ -654,7 +655,7 @@ fn foo(&self) -> Self::T { String::new() }
         // When `body_owner` is an `impl` or `trait` item, look in its associated types for
         // `expected` and point at it.
         let parent_id = tcx.hir().get_parent_item(hir_id);
-        let item = tcx.hir().find_by_def_id(parent_id.def_id);
+        let item = tcx.opt_hir_node_by_def_id(parent_id.def_id);
 
         debug!("expected_projection parent item {:?}", item);
 
diff --git a/compiler/rustc_infer/src/infer/error_reporting/suggest.rs b/compiler/rustc_infer/src/infer/error_reporting/suggest.rs
index fe18d0029..bbe07b8ed 100644
--- a/compiler/rustc_infer/src/infer/error_reporting/suggest.rs
+++ b/compiler/rustc_infer/src/infer/error_reporting/suggest.rs
@@ -42,7 +42,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
         ]
         .into_iter()
         .find_map(|(id, ty)| {
-            let hir::Node::Block(blk) = self.tcx.hir().get(id?) else { return None };
+            let hir::Node::Block(blk) = self.tcx.hir_node(id?) else { return None };
             self.could_remove_semicolon(blk, ty)
         });
         match remove_semicolon {
@@ -62,7 +62,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                 let mut ret = None;
                 for (id, ty) in [(first_id, second_ty), (second_id, first_ty)] {
                     if let Some(id) = id
-                        && let hir::Node::Block(blk) = self.tcx.hir().get(id)
+                        && let hir::Node::Block(blk) = self.tcx.hir_node(id)
                         && let Some(diag) = self.consider_returning_binding_diag(blk, ty)
                     {
                         ret = Some(diag);
@@ -511,7 +511,7 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
             }
 
             let mut visitor = IfVisitor { err_span: span, found_if: false, result: false };
-            visitor.visit_body(&body);
+            visitor.visit_body(body);
             if visitor.result {
                 return Some(TypeErrorAdditionalDiags::AddLetForLetChains {
                     span: span.shrink_to_lo(),
@@ -572,8 +572,8 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
 
                 if let ty::Ref(expected_region, _, _) = expected.kind()
                     && let ty::Ref(found_region, _, _) = found.kind()
-                    && expected_region.is_late_bound()
-                    && !found_region.is_late_bound()
+                    && expected_region.is_bound()
+                    && !found_region.is_bound()
                     && let hir::TyKind::Infer = arg_hir.kind
                 {
                     // If the expected region is late bound, the found region is not, and users are asking compiler
@@ -636,10 +636,10 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
             return None;
         }
         let last_stmt = blk.stmts.last()?;
-        let hir::StmtKind::Semi(ref last_expr) = last_stmt.kind else {
+        let hir::StmtKind::Semi(last_expr) = last_stmt.kind else {
             return None;
         };
-        let last_expr_ty = self.typeck_results.as_ref()?.expr_ty_opt(*last_expr)?;
+        let last_expr_ty = self.typeck_results.as_ref()?.expr_ty_opt(last_expr)?;
         let needs_box = match (last_expr_ty.kind(), expected_ty.kind()) {
             _ if last_expr_ty.references_error() => return None,
             _ if self.same_type_modulo_infer(last_expr_ty, expected_ty) => {
@@ -668,26 +668,16 @@ impl<'tcx> TypeErrCtxt<'_, 'tcx> {
                     (
                         hir::ItemKind::OpaqueTy(hir::OpaqueTy { bounds: last_bounds, .. }),
                         hir::ItemKind::OpaqueTy(hir::OpaqueTy { bounds: exp_bounds, .. }),
-                    ) if std::iter::zip(*last_bounds, *exp_bounds).all(|(left, right)| {
-                        match (left, right) {
-                            (
-                                hir::GenericBound::Trait(tl, ml),
-                                hir::GenericBound::Trait(tr, mr),
-                            ) if tl.trait_ref.trait_def_id() == tr.trait_ref.trait_def_id()
+                    ) if std::iter::zip(*last_bounds, *exp_bounds).all(|(left, right)| match (
+                        left, right,
+                    ) {
+                        (hir::GenericBound::Trait(tl, ml), hir::GenericBound::Trait(tr, mr))
+                            if tl.trait_ref.trait_def_id() == tr.trait_ref.trait_def_id()
                                 && ml == mr =>
-                            {
-                                true
-                            }
-                            (
-                                hir::GenericBound::LangItemTrait(langl, _, _, argsl),
-                                hir::GenericBound::LangItemTrait(langr, _, _, argsr),
-                            ) if langl == langr => {
-                                // FIXME: consider the bounds!
-                                debug!("{:?} {:?}", argsl, argsr);
-                                true
-                            }
-                            _ => false,
+                        {
+                            true
                         }
+                        _ => false,
                     }) =>
                     {
                         StatementAsExpression::NeedsBoxing
diff --git a/compiler/rustc_infer/src/infer/free_regions.rs b/compiler/rustc_infer/src/infer/free_regions.rs
index ed1a2a117..0dde3082d 100644
--- a/compiler/rustc_infer/src/infer/free_regions.rs
+++ b/compiler/rustc_infer/src/infer/free_regions.rs
@@ -22,8 +22,8 @@ impl<'a, 'tcx> RegionRelations<'a, 'tcx> {
         Self { tcx, free_regions }
     }
 
-    pub fn lub_free_regions(&self, r_a: Region<'tcx>, r_b: Region<'tcx>) -> Region<'tcx> {
-        self.free_regions.lub_free_regions(self.tcx, r_a, r_b)
+    pub fn lub_param_regions(&self, r_a: Region<'tcx>, r_b: Region<'tcx>) -> Region<'tcx> {
+        self.free_regions.lub_param_regions(self.tcx, r_a, r_b)
     }
 }
 
@@ -59,7 +59,7 @@ impl<'tcx> FreeRegionMap<'tcx> {
         r_a: Region<'tcx>,
         r_b: Region<'tcx>,
     ) -> bool {
-        assert!(r_a.is_free_or_static() && r_b.is_free_or_static());
+        assert!(r_a.is_free() && r_b.is_free());
         let re_static = tcx.lifetimes.re_static;
         if self.check_relation(re_static, r_b) {
             // `'a <= 'static` is always true, and not stored in the
@@ -80,15 +80,15 @@ impl<'tcx> FreeRegionMap<'tcx> {
     /// cases, this is more conservative than necessary, in order to
     /// avoid making arbitrary choices. See
     /// `TransitiveRelation::postdom_upper_bound` for more details.
-    pub fn lub_free_regions(
+    pub fn lub_param_regions(
         &self,
         tcx: TyCtxt<'tcx>,
         r_a: Region<'tcx>,
         r_b: Region<'tcx>,
     ) -> Region<'tcx> {
-        debug!("lub_free_regions(r_a={:?}, r_b={:?})", r_a, r_b);
-        assert!(r_a.is_free());
-        assert!(r_b.is_free());
+        debug!("lub_param_regions(r_a={:?}, r_b={:?})", r_a, r_b);
+        assert!(r_a.is_param());
+        assert!(r_b.is_param());
         let result = if r_a == r_b {
             r_a
         } else {
@@ -97,7 +97,7 @@ impl<'tcx> FreeRegionMap<'tcx> {
                 Some(r) => r,
             }
         };
-        debug!("lub_free_regions(r_a={:?}, r_b={:?}) = {:?}", r_a, r_b, result);
+        debug!("lub_param_regions(r_a={:?}, r_b={:?}) = {:?}", r_a, r_b, result);
         result
     }
 }
diff --git a/compiler/rustc_infer/src/infer/freshen.rs b/compiler/rustc_infer/src/infer/freshen.rs
index 35204478c..11ab86277 100644
--- a/compiler/rustc_infer/src/infer/freshen.rs
+++ b/compiler/rustc_infer/src/infer/freshen.rs
@@ -110,13 +110,13 @@ impl<'a, 'tcx> TypeFolder<TyCtxt<'tcx>> for TypeFreshener<'a, 'tcx> {
 
     fn fold_region(&mut self, r: ty::Region<'tcx>) -> ty::Region<'tcx> {
         match *r {
-            ty::ReLateBound(..) => {
+            ty::ReBound(..) => {
                 // leave bound regions alone
                 r
             }
 
-            ty::ReEarlyBound(..)
-            | ty::ReFree(_)
+            ty::ReEarlyParam(..)
+            | ty::ReLateParam(_)
             | ty::ReVar(_)
             | ty::RePlaceholder(..)
             | ty::ReStatic
diff --git a/compiler/rustc_infer/src/infer/fudge.rs b/compiler/rustc_infer/src/infer/fudge.rs
index 7e878ac06..8ca97ae1b 100644
--- a/compiler/rustc_infer/src/infer/fudge.rs
+++ b/compiler/rustc_infer/src/infer/fudge.rs
@@ -187,7 +187,7 @@ impl<'a, 'tcx> TypeFolder<TyCtxt<'tcx>> for InferenceFudger<'a, 'tcx> {
                 if self.type_vars.0.contains(&vid) {
                     // This variable was created during the fudging.
                     // Recreate it with a fresh variable here.
-                    let idx = (vid.as_usize() - self.type_vars.0.start.as_usize()) as usize;
+                    let idx = vid.as_usize() - self.type_vars.0.start.as_usize();
                     let origin = self.type_vars.1[idx];
                     self.infcx.next_ty_var(origin)
                 } else {
@@ -236,7 +236,7 @@ impl<'a, 'tcx> TypeFolder<TyCtxt<'tcx>> for InferenceFudger<'a, 'tcx> {
             if self.const_vars.0.contains(&vid) {
                 // This variable was created during the fudging.
                 // Recreate it with a fresh variable here.
-                let idx = (vid.index() - self.const_vars.0.start.index()) as usize;
+                let idx = vid.index() - self.const_vars.0.start.index();
                 let origin = self.const_vars.1[idx];
                 self.infcx.next_const_var(ct.ty(), origin)
             } else {
diff --git a/compiler/rustc_infer/src/infer/lexical_region_resolve/mod.rs b/compiler/rustc_infer/src/infer/lexical_region_resolve/mod.rs
index bee0a978a..0fbc4a0ce 100644
--- a/compiler/rustc_infer/src/infer/lexical_region_resolve/mod.rs
+++ b/compiler/rustc_infer/src/infer/lexical_region_resolve/mod.rs
@@ -16,8 +16,8 @@ use rustc_data_structures::intern::Interned;
 use rustc_index::{IndexSlice, IndexVec};
 use rustc_middle::ty::fold::TypeFoldable;
 use rustc_middle::ty::{self, Ty, TyCtxt};
-use rustc_middle::ty::{ReEarlyBound, ReErased, ReError, ReFree, ReStatic};
-use rustc_middle::ty::{ReLateBound, RePlaceholder, ReVar};
+use rustc_middle::ty::{ReBound, RePlaceholder, ReVar};
+use rustc_middle::ty::{ReEarlyParam, ReErased, ReError, ReLateParam, ReStatic};
 use rustc_middle::ty::{Region, RegionVid};
 use rustc_span::Span;
 use std::fmt;
@@ -31,13 +31,12 @@ use super::outlives::test_type_match;
 /// all the variables as well as a set of errors that must be reported.
 #[instrument(level = "debug", skip(region_rels, var_infos, data))]
 pub(crate) fn resolve<'tcx>(
-    param_env: ty::ParamEnv<'tcx>,
     region_rels: &RegionRelations<'_, 'tcx>,
     var_infos: VarInfos,
     data: RegionConstraintData<'tcx>,
 ) -> (LexicalRegionResolutions<'tcx>, Vec<RegionResolutionError<'tcx>>) {
     let mut errors = vec![];
-    let mut resolver = LexicalResolver { param_env, region_rels, var_infos, data };
+    let mut resolver = LexicalResolver { region_rels, var_infos, data };
     let values = resolver.infer_variable_values(&mut errors);
     (values, errors)
 }
@@ -120,7 +119,6 @@ struct RegionAndOrigin<'tcx> {
 type RegionGraph<'tcx> = Graph<(), Constraint<'tcx>>;
 
 struct LexicalResolver<'cx, 'tcx> {
-    param_env: ty::ParamEnv<'tcx>,
     region_rels: &'cx RegionRelations<'cx, 'tcx>,
     var_infos: VarInfos,
     data: RegionConstraintData<'tcx>,
@@ -378,7 +376,7 @@ impl<'cx, 'tcx> LexicalResolver<'cx, 'tcx> {
                     // so it doesn't really matter if it's shorter or longer than an empty region
                     ReError(_) => false,
 
-                    ReLateBound(..) | ReErased => {
+                    ReBound(..) | ReErased => {
                         bug!("cannot relate region: {:?}", a);
                     }
 
@@ -390,7 +388,7 @@ impl<'cx, 'tcx> LexicalResolver<'cx, 'tcx> {
                         );
                     }
 
-                    ReStatic | ReEarlyBound(_) | ReFree(_) => {
+                    ReStatic | ReEarlyParam(_) | ReLateParam(_) => {
                         // nothing lives longer than `'static`
 
                         // All empty regions are less than early-bound, free,
@@ -411,7 +409,7 @@ impl<'cx, 'tcx> LexicalResolver<'cx, 'tcx> {
                     // so it doesn't really matter if it's shorter or longer than an empty region
                     ReError(_) => false,
 
-                    ReLateBound(..) | ReErased => {
+                    ReBound(..) | ReErased => {
                         bug!("cannot relate region: {:?}", b);
                     }
 
@@ -423,9 +421,9 @@ impl<'cx, 'tcx> LexicalResolver<'cx, 'tcx> {
                         );
                     }
 
-                    ReStatic | ReEarlyBound(_) | ReFree(_) => {
+                    ReStatic | ReEarlyParam(_) | ReLateParam(_) => {
                         // nothing lives longer than `'static`
-                        // All empty regions are less than early-bound, free,
+                        // All empty regions are less than early-bound, late-bound,
                         // and scope regions.
                         true
                     }
@@ -450,8 +448,7 @@ impl<'cx, 'tcx> LexicalResolver<'cx, 'tcx> {
 
         // Check for the case where we know that `'b: 'static` -- in that case,
         // `a <= b` for all `a`.
-        let b_free_or_static = b.is_free_or_static();
-        if b_free_or_static && sub_free_regions(tcx.lifetimes.re_static, b) {
+        if b.is_free() && sub_free_regions(tcx.lifetimes.re_static, b) {
             return true;
         }
 
@@ -460,8 +457,7 @@ impl<'cx, 'tcx> LexicalResolver<'cx, 'tcx> {
         // `lub` relationship defined below, since sometimes the "lub"
         // is actually the `postdom_upper_bound` (see
         // `TransitiveRelation` for more details).
-        let a_free_or_static = a.is_free_or_static();
-        if a_free_or_static && b_free_or_static {
+        if a.is_free() && b.is_free() {
             return sub_free_regions(a, b);
         }
 
@@ -478,7 +474,7 @@ impl<'cx, 'tcx> LexicalResolver<'cx, 'tcx> {
     #[instrument(level = "trace", skip(self), ret)]
     fn lub_concrete_regions(&self, a: Region<'tcx>, b: Region<'tcx>) -> Region<'tcx> {
         match (*a, *b) {
-            (ReLateBound(..), _) | (_, ReLateBound(..)) | (ReErased, _) | (_, ReErased) => {
+            (ReBound(..), _) | (_, ReBound(..)) | (ReErased, _) | (_, ReErased) => {
                 bug!("cannot relate region: LUB({:?}, {:?})", a, b);
             }
 
@@ -501,8 +497,8 @@ impl<'cx, 'tcx> LexicalResolver<'cx, 'tcx> {
                 self.tcx().lifetimes.re_static
             }
 
-            (ReEarlyBound(_) | ReFree(_), ReEarlyBound(_) | ReFree(_)) => {
-                self.region_rels.lub_free_regions(a, b)
+            (ReEarlyParam(_) | ReLateParam(_), ReEarlyParam(_) | ReLateParam(_)) => {
+                self.region_rels.lub_param_regions(a, b)
             }
 
             // For these types, we cannot define any additional
@@ -683,17 +679,13 @@ impl<'cx, 'tcx> LexicalResolver<'cx, 'tcx> {
         for constraint in self.data.constraints.keys() {
             match *constraint {
                 Constraint::VarSubVar(a_id, b_id) => {
-                    graph.add_edge(
-                        NodeIndex(a_id.index() as usize),
-                        NodeIndex(b_id.index() as usize),
-                        *constraint,
-                    );
+                    graph.add_edge(NodeIndex(a_id.index()), NodeIndex(b_id.index()), *constraint);
                 }
                 Constraint::RegSubVar(_, b_id) => {
-                    graph.add_edge(dummy_source, NodeIndex(b_id.index() as usize), *constraint);
+                    graph.add_edge(dummy_source, NodeIndex(b_id.index()), *constraint);
                 }
                 Constraint::VarSubReg(a_id, _) => {
-                    graph.add_edge(NodeIndex(a_id.index() as usize), dummy_sink, *constraint);
+                    graph.add_edge(NodeIndex(a_id.index()), dummy_sink, *constraint);
                 }
                 Constraint::RegSubReg(..) => {
                     // this would be an edge from `dummy_source` to
@@ -723,13 +715,13 @@ impl<'cx, 'tcx> LexicalResolver<'cx, 'tcx> {
             return;
         }
 
-        // We place free regions first because we are special casing
-        // SubSupConflict(ReFree, ReFree) when reporting error, and so
+        // We place late-bound regions first because we are special casing
+        // SubSupConflict(ReLateParam, ReLateParam) when reporting error, and so
         // the user will more likely get a specific suggestion.
         fn region_order_key(x: &RegionAndOrigin<'_>) -> u8 {
             match *x.region {
-                ReEarlyBound(_) => 0,
-                ReFree(_) => 1,
+                ReEarlyParam(_) => 0,
+                ReLateParam(_) => 1,
                 _ => 2,
             }
         }
@@ -804,7 +796,7 @@ impl<'cx, 'tcx> LexicalResolver<'cx, 'tcx> {
 
         // Errors in earlier passes can yield error variables without
         // resolution errors here; delay ICE in favor of those errors.
-        self.tcx().sess.delay_span_bug(
+        self.tcx().sess.span_delayed_bug(
             self.var_infos[node_idx].origin.span(),
             format!(
                 "collect_error_for_expanding_node() could not find \
@@ -880,7 +872,7 @@ impl<'cx, 'tcx> LexicalResolver<'cx, 'tcx> {
         ) {
             debug!("process_edges(source_vid={:?}, dir={:?})", source_vid, dir);
 
-            let source_node_index = NodeIndex(source_vid.index() as usize);
+            let source_node_index = NodeIndex(source_vid.index());
             for (_, edge) in graph.adjacent_edges(source_node_index, dir) {
                 match edge.data {
                     Constraint::VarSubVar(from_vid, to_vid) => {
@@ -920,12 +912,8 @@ impl<'cx, 'tcx> LexicalResolver<'cx, 'tcx> {
         match bound {
             VerifyBound::IfEq(verify_if_eq_b) => {
                 let verify_if_eq_b = var_values.normalize(self.region_rels.tcx, *verify_if_eq_b);
-                match test_type_match::extract_verify_if_eq(
-                    self.tcx(),
-                    self.param_env,
-                    &verify_if_eq_b,
-                    generic_ty,
-                ) {
+                match test_type_match::extract_verify_if_eq(self.tcx(), &verify_if_eq_b, generic_ty)
+                {
                     Some(r) => {
                         self.bound_is_met(&VerifyBound::OutlivedBy(r), var_values, generic_ty, min)
                     }
diff --git a/compiler/rustc_infer/src/infer/mod.rs b/compiler/rustc_infer/src/infer/mod.rs
index 4ee897ffe..e092bbbfd 100644
--- a/compiler/rustc_infer/src/infer/mod.rs
+++ b/compiler/rustc_infer/src/infer/mod.rs
@@ -1,11 +1,11 @@
 pub use self::at::DefineOpaqueTypes;
 pub use self::freshen::TypeFreshener;
 pub use self::lexical_region_resolve::RegionResolutionError;
-pub use self::LateBoundRegionConversionTime::*;
+pub use self::BoundRegionConversionTime::*;
 pub use self::RegionVariableOrigin::*;
 pub use self::SubregionOrigin::*;
 pub use self::ValuePairs::*;
-pub use combine::ObligationEmittingRelation;
+pub use relate::combine::ObligationEmittingRelation;
 use rustc_data_structures::captures::Captures;
 use rustc_data_structures::undo_log::UndoLogs;
 use rustc_middle::infer::unify_key::{ConstVidKey, EffectVidKey};
@@ -32,7 +32,7 @@ use rustc_middle::ty::error::{ExpectedFound, TypeError};
 use rustc_middle::ty::fold::BoundVarReplacerDelegate;
 use rustc_middle::ty::fold::{TypeFoldable, TypeFolder, TypeSuperFoldable};
 use rustc_middle::ty::relate::RelateResult;
-use rustc_middle::ty::visit::{TypeVisitable, TypeVisitableExt};
+use rustc_middle::ty::visit::TypeVisitableExt;
 pub use rustc_middle::ty::IntVarValue;
 use rustc_middle::ty::{self, GenericParamDefKind, InferConst, InferTy, Ty, TyCtxt};
 use rustc_middle::ty::{ConstVid, EffectVid, FloatVid, IntVid, TyVid};
@@ -43,7 +43,6 @@ use rustc_span::{Span, DUMMY_SP};
 use std::cell::{Cell, RefCell};
 use std::fmt;
 
-use self::combine::CombineFields;
 use self::error_reporting::TypeErrCtxt;
 use self::free_regions::RegionRelations;
 use self::lexical_region_resolve::LexicalRegionResolutions;
@@ -51,29 +50,23 @@ use self::region_constraints::{GenericKind, VarInfos, VerifyBound};
 use self::region_constraints::{
     RegionConstraintCollector, RegionConstraintStorage, RegionSnapshot,
 };
+pub use self::relate::combine::CombineFields;
+pub use self::relate::nll as nll_relate;
 use self::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
 
 pub mod at;
 pub mod canonical;
-mod combine;
-mod equate;
 pub mod error_reporting;
 pub mod free_regions;
 mod freshen;
 mod fudge;
-mod generalize;
-mod glb;
-mod higher_ranked;
-pub mod lattice;
 mod lexical_region_resolve;
-mod lub;
-pub mod nll_relate;
 pub mod opaque_types;
 pub mod outlives;
 mod projection;
 pub mod region_constraints;
+mod relate;
 pub mod resolve;
-mod sub;
 pub mod type_variable;
 mod undo_log;
 
@@ -98,6 +91,8 @@ pub(crate) type UnificationTable<'a, 'tcx, T> = ut::UnificationTable<
 /// call to `start_snapshot` and `rollback_to`.
 #[derive(Clone)]
 pub struct InferCtxtInner<'tcx> {
+    undo_log: InferCtxtUndoLogs<'tcx>,
+
     /// Cache for projections.
     ///
     /// This cache is snapshotted along with the infcx.
@@ -162,8 +157,6 @@ pub struct InferCtxtInner<'tcx> {
     /// that all type inference variables have been bound and so forth.
     region_obligations: Vec<RegionObligation<'tcx>>,
 
-    undo_log: InferCtxtUndoLogs<'tcx>,
-
     /// Caches for opaque type inference.
     opaque_type_storage: OpaqueTypeStorage<'tcx>,
 }
@@ -171,9 +164,10 @@ pub struct InferCtxtInner<'tcx> {
 impl<'tcx> InferCtxtInner<'tcx> {
     fn new() -> InferCtxtInner<'tcx> {
         InferCtxtInner {
+            undo_log: InferCtxtUndoLogs::default(),
+
             projection_cache: Default::default(),
             type_variable_storage: type_variable::TypeVariableStorage::new(),
-            undo_log: InferCtxtUndoLogs::default(),
             const_unification_storage: ut::UnificationTableStorage::new(),
             int_unification_storage: ut::UnificationTableStorage::new(),
             float_unification_storage: ut::UnificationTableStorage::new(),
@@ -345,37 +339,57 @@ pub struct InferCtxt<'tcx> {
 impl<'tcx> ty::InferCtxtLike for InferCtxt<'tcx> {
     type Interner = TyCtxt<'tcx>;
 
-    fn universe_of_ty(&self, ty: ty::InferTy) -> Option<ty::UniverseIndex> {
-        use InferTy::*;
-        match ty {
-            // FIXME(BoxyUwU): this is kind of jank and means that printing unresolved
-            // ty infers will give you the universe of the var it resolved to not the universe
-            // it actually had. It also means that if you have a `?0.1` and infer it to `u8` then
-            // try to print out `?0.1` it will just print `?0`.
-            TyVar(ty_vid) => match self.probe_ty_var(ty_vid) {
-                Err(universe) => Some(universe),
-                Ok(_) => None,
-            },
-            IntVar(_) | FloatVar(_) | FreshTy(_) | FreshIntTy(_) | FreshFloatTy(_) => None,
+    fn interner(&self) -> TyCtxt<'tcx> {
+        self.tcx
+    }
+
+    fn universe_of_ty(&self, vid: TyVid) -> Option<ty::UniverseIndex> {
+        // FIXME(BoxyUwU): this is kind of jank and means that printing unresolved
+        // ty infers will give you the universe of the var it resolved to not the universe
+        // it actually had. It also means that if you have a `?0.1` and infer it to `u8` then
+        // try to print out `?0.1` it will just print `?0`.
+        match self.probe_ty_var(vid) {
+            Err(universe) => Some(universe),
+            Ok(_) => None,
         }
     }
 
-    fn universe_of_ct(&self, ct: ty::InferConst) -> Option<ty::UniverseIndex> {
-        use ty::InferConst::*;
-        match ct {
-            // Same issue as with `universe_of_ty`
-            Var(ct_vid) => match self.probe_const_var(ct_vid) {
-                Err(universe) => Some(universe),
-                Ok(_) => None,
-            },
-            EffectVar(_) => None,
-            Fresh(_) => None,
+    fn universe_of_ct(&self, ct: ConstVid) -> Option<ty::UniverseIndex> {
+        // Same issue as with `universe_of_ty`
+        match self.probe_const_var(ct) {
+            Err(universe) => Some(universe),
+            Ok(_) => None,
         }
     }
 
     fn universe_of_lt(&self, lt: ty::RegionVid) -> Option<ty::UniverseIndex> {
         Some(self.universe_of_region_vid(lt))
     }
+
+    fn root_ty_var(&self, vid: TyVid) -> TyVid {
+        self.root_var(vid)
+    }
+
+    fn probe_ty_var(&self, vid: TyVid) -> Option<Ty<'tcx>> {
+        self.probe_ty_var(vid).ok()
+    }
+
+    fn opportunistic_resolve_lt_var(&self, vid: ty::RegionVid) -> Option<ty::Region<'tcx>> {
+        let re = self
+            .inner
+            .borrow_mut()
+            .unwrap_region_constraints()
+            .opportunistic_resolve_var(self.tcx, vid);
+        if *re == ty::ReVar(vid) { None } else { Some(re) }
+    }
+
+    fn root_ct_var(&self, vid: ConstVid) -> ConstVid {
+        self.root_const_var(vid)
+    }
+
+    fn probe_ct_var(&self, vid: ConstVid) -> Option<ty::Const<'tcx>> {
+        self.probe_const_var(vid).ok()
+    }
 }
 
 /// See the `error_reporting` module for more details.
@@ -384,7 +398,6 @@ pub enum ValuePairs<'tcx> {
     Regions(ExpectedFound<ty::Region<'tcx>>),
     Terms(ExpectedFound<ty::Term<'tcx>>),
     Aliases(ExpectedFound<ty::AliasTy<'tcx>>),
-    TraitRefs(ExpectedFound<ty::TraitRef<'tcx>>),
     PolyTraitRefs(ExpectedFound<ty::PolyTraitRef<'tcx>>),
     PolySigs(ExpectedFound<ty::PolyFnSig<'tcx>>),
     ExistentialTraitRef(ExpectedFound<ty::PolyExistentialTraitRef<'tcx>>),
@@ -472,9 +485,9 @@ impl<'tcx> SubregionOrigin<'tcx> {
     }
 }
 
-/// Times when we replace late-bound regions with variables:
+/// Times when we replace bound regions with existentials:
 #[derive(Clone, Copy, Debug)]
-pub enum LateBoundRegionConversionTime {
+pub enum BoundRegionConversionTime {
     /// when a fn is called
     FnCall,
 
@@ -508,11 +521,14 @@ pub enum RegionVariableOrigin {
     Coercion(Span),
 
     /// Region variables created as the values for early-bound regions.
-    EarlyBoundRegion(Span, Symbol),
+    ///
+    /// FIXME(@lcnr): This can also store a `DefId`, similar to
+    /// `TypeVariableOriginKind::TypeParameterDefinition`.
+    RegionParameterDefinition(Span, Symbol),
 
-    /// Region variables created for bound regions
-    /// in a function or method that is called.
-    LateBoundRegion(Span, ty::BoundRegionKind, LateBoundRegionConversionTime),
+    /// Region variables created when instantiating a binder with
+    /// existential variables, e.g. when calling a function or method.
+    BoundRegion(Span, ty::BoundRegionKind, BoundRegionConversionTime),
 
     UpvarRegion(ty::UpvarId, Span),
 
@@ -697,10 +713,6 @@ impl<'tcx> InferCtxtBuilder<'tcx> {
 }
 
 impl<'tcx, T> InferOk<'tcx, T> {
-    pub fn unit(self) -> InferOk<'tcx, ()> {
-        InferOk { value: (), obligations: self.obligations }
-    }
-
     /// Extracts `value`, registering any obligations into `fulfill_cx`.
     pub fn into_value_registering_obligations(
         self,
@@ -757,7 +769,7 @@ impl<'tcx> InferCtxt<'tcx> {
     pub fn type_var_origin(&self, ty: Ty<'tcx>) -> Option<TypeVariableOrigin> {
         match *ty.kind() {
             ty::Infer(ty::TyVar(vid)) => {
-                Some(*self.inner.borrow_mut().type_variables().var_origin(vid))
+                Some(self.inner.borrow_mut().type_variables().var_origin(vid))
             }
             _ => None,
         }
@@ -767,11 +779,11 @@ impl<'tcx> InferCtxt<'tcx> {
         freshen::TypeFreshener::new(self)
     }
 
-    pub fn unsolved_variables(&self) -> Vec<Ty<'tcx>> {
+    pub fn unresolved_variables(&self) -> Vec<Ty<'tcx>> {
         let mut inner = self.inner.borrow_mut();
         let mut vars: Vec<Ty<'_>> = inner
             .type_variables()
-            .unsolved_variables()
+            .unresolved_variables()
             .into_iter()
             .map(|t| Ty::new_var(self.tcx, t))
             .collect();
@@ -903,12 +915,14 @@ impl<'tcx> InferCtxt<'tcx> {
         self.inner.borrow().undo_log.opaque_types_in_snapshot(&snapshot.undo_snapshot)
     }
 
-    pub fn can_sub<T>(&self, param_env: ty::ParamEnv<'tcx>, a: T, b: T) -> bool
+    pub fn can_sub<T>(&self, param_env: ty::ParamEnv<'tcx>, expected: T, actual: T) -> bool
     where
         T: at::ToTrace<'tcx>,
     {
         let origin = &ObligationCause::dummy();
-        self.probe(|_| self.at(origin, param_env).sub(DefineOpaqueTypes::No, a, b).is_ok())
+        self.probe(|_| {
+            self.at(origin, param_env).sub(DefineOpaqueTypes::No, expected, actual).is_ok()
+        })
     }
 
     pub fn can_eq<T>(&self, param_env: ty::ParamEnv<'tcx>, a: T, b: T) -> bool
@@ -1007,15 +1021,10 @@ impl<'tcx> InferCtxt<'tcx> {
             _ => {}
         }
 
-        Ok(self.commit_if_ok(|_snapshot| {
-            let ty::SubtypePredicate { a_is_expected, a, b } =
-                self.instantiate_binder_with_placeholders(predicate);
-
-            let ok =
-                self.at(cause, param_env).sub_exp(DefineOpaqueTypes::No, a_is_expected, a, b)?;
+        let ty::SubtypePredicate { a_is_expected, a, b } =
+            self.instantiate_binder_with_placeholders(predicate);
 
-            Ok(ok.unit())
-        }))
+        Ok(self.at(cause, param_env).sub_exp(DefineOpaqueTypes::No, a_is_expected, a, b))
     }
 
     pub fn region_outlives_predicate(
@@ -1165,7 +1174,7 @@ impl<'tcx> InferCtxt<'tcx> {
             GenericParamDefKind::Lifetime => {
                 // Create a region inference variable for the given
                 // region parameter definition.
-                self.next_region_var(EarlyBoundRegion(span, param.name)).into()
+                self.next_region_var(RegionParameterDefinition(span, param.name)).into()
             }
             GenericParamDefKind::Type { .. } => {
                 // Create a type inference variable for the given
@@ -1278,12 +1287,7 @@ impl<'tcx> InferCtxt<'tcx> {
     pub fn region_var_origin(&self, vid: ty::RegionVid) -> RegionVariableOrigin {
         let mut inner = self.inner.borrow_mut();
         let inner = &mut *inner;
-        inner
-            .region_constraint_storage
-            .as_mut()
-            .expect("regions already resolved")
-            .with_log(&mut inner.undo_log)
-            .var_origin(vid)
+        inner.unwrap_region_constraints().var_origin(vid)
     }
 
     /// Clone the list of variable regions. This is used only during NLL processing
@@ -1401,17 +1405,6 @@ impl<'tcx> InferCtxt<'tcx> {
         value.fold_with(&mut r)
     }
 
-    /// Returns the first unresolved type or const variable contained in `T`.
-    pub fn first_unresolved_const_or_ty_var<T>(
-        &self,
-        value: &T,
-    ) -> Option<(ty::Term<'tcx>, Option<Span>)>
-    where
-        T: TypeVisitable<TyCtxt<'tcx>>,
-    {
-        value.visit_with(&mut resolve::UnresolvedTypeOrConstFinder::new(self)).break_value()
-    }
-
     pub fn probe_const_var(&self, vid: ty::ConstVid) -> Result<ty::Const<'tcx>, ty::UniverseIndex> {
         match self.inner.borrow_mut().const_unification_table().probe_value(vid).val {
             ConstVariableValue::Known { value } => Ok(value),
@@ -1440,7 +1433,7 @@ impl<'tcx> InferCtxt<'tcx> {
                     let guar = self
                         .tcx
                         .sess
-                        .delay_span_bug(DUMMY_SP, format!("`{value:?}` is not fully resolved"));
+                        .span_delayed_bug(DUMMY_SP, format!("`{value:?}` is not fully resolved"));
                     Ok(self.tcx.fold_regions(value, |re, _| {
                         if re.is_var() { ty::Region::new_error(self.tcx, guar) } else { re }
                     }))
@@ -1456,13 +1449,13 @@ impl<'tcx> InferCtxt<'tcx> {
     // variables in the current universe.
     //
     // Use this method if you'd like to find some substitution of the binder's
-    // variables (e.g. during a method call). If there isn't a [`LateBoundRegionConversionTime`]
+    // variables (e.g. during a method call). If there isn't a [`BoundRegionConversionTime`]
     // that corresponds to your use case, consider whether or not you should
     // use [`InferCtxt::instantiate_binder_with_placeholders`] instead.
     pub fn instantiate_binder_with_fresh_vars<T>(
         &self,
         span: Span,
-        lbrct: LateBoundRegionConversionTime,
+        lbrct: BoundRegionConversionTime,
         value: ty::Binder<'tcx, T>,
     ) -> T
     where
@@ -1475,7 +1468,7 @@ impl<'tcx> InferCtxt<'tcx> {
         struct ToFreshVars<'a, 'tcx> {
             infcx: &'a InferCtxt<'tcx>,
             span: Span,
-            lbrct: LateBoundRegionConversionTime,
+            lbrct: BoundRegionConversionTime,
             map: FxHashMap<ty::BoundVar, ty::GenericArg<'tcx>>,
         }
 
@@ -1485,7 +1478,7 @@ impl<'tcx> InferCtxt<'tcx> {
                     .entry(br.var)
                     .or_insert_with(|| {
                         self.infcx
-                            .next_region_var(LateBoundRegion(self.span, br.kind, self.lbrct))
+                            .next_region_var(BoundRegion(self.span, br.kind, self.lbrct))
                             .into()
                     })
                     .expect_region()
@@ -2041,8 +2034,8 @@ impl RegionVariableOrigin {
             | AddrOfRegion(a)
             | Autoref(a)
             | Coercion(a)
-            | EarlyBoundRegion(a, ..)
-            | LateBoundRegion(a, ..)
+            | RegionParameterDefinition(a, ..)
+            | BoundRegion(a, ..)
             | UpvarRegion(_, a) => a,
             Nll(..) => bug!("NLL variable used with `span`"),
         }
diff --git a/compiler/rustc_infer/src/infer/opaque_types.rs b/compiler/rustc_infer/src/infer/opaque_types.rs
index 7a5dec22f..a492c6bf9 100644
--- a/compiler/rustc_infer/src/infer/opaque_types.rs
+++ b/compiler/rustc_infer/src/infer/opaque_types.rs
@@ -374,7 +374,7 @@ impl<'tcx> InferCtxt<'tcx> {
     /// in its defining scope.
     #[instrument(skip(self), level = "trace", ret)]
     pub fn opaque_type_origin(&self, def_id: LocalDefId) -> Option<OpaqueTyOrigin> {
-        let opaque_hir_id = self.tcx.hir().local_def_id_to_hir_id(def_id);
+        let opaque_hir_id = self.tcx.local_def_id_to_hir_id(def_id);
         let parent_def_id = match self.defining_use_anchor {
             DefiningAnchor::Bubble | DefiningAnchor::Error => return None,
             DefiningAnchor::Bind(bind) => bind,
@@ -432,7 +432,7 @@ where
     fn visit_region(&mut self, r: ty::Region<'tcx>) -> ControlFlow<Self::BreakTy> {
         match *r {
             // ignore bound regions, keep visiting
-            ty::ReLateBound(_, _) => ControlFlow::Continue(()),
+            ty::ReBound(_, _) => ControlFlow::Continue(()),
             _ => {
                 (self.op)(r);
                 ControlFlow::Continue(())
@@ -447,7 +447,7 @@ where
         }
 
         match ty.kind() {
-            ty::Closure(_, ref args) => {
+            ty::Closure(_, args) => {
                 // Skip lifetime parameters of the enclosing item(s)
 
                 for upvar in args.as_closure().upvar_tys() {
@@ -456,7 +456,7 @@ where
                 args.as_closure().sig_as_fn_ptr_ty().visit_with(self);
             }
 
-            ty::Coroutine(_, ref args, _) => {
+            ty::Coroutine(_, args, _) => {
                 // Skip lifetime parameters of the enclosing item(s)
                 // Also skip the witness type, because that has no free regions.
 
@@ -468,7 +468,7 @@ where
                 args.as_coroutine().resume_ty().visit_with(self);
             }
 
-            ty::Alias(ty::Opaque, ty::AliasTy { def_id, ref args, .. }) => {
+            ty::Alias(ty::Opaque, ty::AliasTy { def_id, args, .. }) => {
                 // Skip lifetime parameters that are not captures.
                 let variances = self.tcx.variances_of(*def_id);
 
@@ -575,7 +575,7 @@ impl<'tcx> InferCtxt<'tcx> {
                 .register(opaque_type_key, OpaqueHiddenType { ty: hidden_ty, span });
             if let Some(prev) = prev {
                 obligations.extend(
-                    self.at(&cause, param_env)
+                    self.at(cause, param_env)
                         .eq_exp(DefineOpaqueTypes::Yes, a_is_expected, prev, hidden_ty)?
                         .obligations,
                 );
@@ -671,7 +671,7 @@ impl<'tcx> InferCtxt<'tcx> {
 /// and `opaque_hir_id` is the `HirId` of the definition of the opaque type `Baz`.
 /// For the above example, this function returns `true` for `f1` and `false` for `f2`.
 fn may_define_opaque_type(tcx: TyCtxt<'_>, def_id: LocalDefId, opaque_hir_id: hir::HirId) -> bool {
-    let mut hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
+    let mut hir_id = tcx.local_def_id_to_hir_id(def_id);
 
     // Named opaque types can be defined by any siblings or children of siblings.
     let scope = tcx.hir().get_defining_scope(opaque_hir_id);
@@ -683,8 +683,8 @@ fn may_define_opaque_type(tcx: TyCtxt<'_>, def_id: LocalDefId, opaque_hir_id: hi
     let res = hir_id == scope;
     trace!(
         "may_define_opaque_type(def={:?}, opaque_node={:?}) = {}",
-        tcx.hir().find(hir_id),
-        tcx.hir().get(opaque_hir_id),
+        tcx.opt_hir_node(hir_id),
+        tcx.hir_node(opaque_hir_id),
         res
     );
     res
diff --git a/compiler/rustc_infer/src/infer/opaque_types/table.rs b/compiler/rustc_infer/src/infer/opaque_types/table.rs
index a737761ba..715006a50 100644
--- a/compiler/rustc_infer/src/infer/opaque_types/table.rs
+++ b/compiler/rustc_infer/src/infer/opaque_types/table.rs
@@ -41,7 +41,7 @@ impl<'tcx> Drop for OpaqueTypeStorage<'tcx> {
     fn drop(&mut self) {
         if !self.opaque_types.is_empty() {
             ty::tls::with(|tcx| {
-                tcx.sess.delay_span_bug(DUMMY_SP, format!("{:?}", self.opaque_types))
+                tcx.sess.span_delayed_bug(DUMMY_SP, format!("{:?}", self.opaque_types))
             });
         }
     }
diff --git a/compiler/rustc_infer/src/infer/outlives/components.rs b/compiler/rustc_infer/src/infer/outlives/components.rs
index 38819e8ad..47038cfd4 100644
--- a/compiler/rustc_infer/src/infer/outlives/components.rs
+++ b/compiler/rustc_infer/src/infer/outlives/components.rs
@@ -11,6 +11,7 @@ use smallvec::{smallvec, SmallVec};
 pub enum Component<'tcx> {
     Region(ty::Region<'tcx>),
     Param(ty::ParamTy),
+    Placeholder(ty::PlaceholderType),
     UnresolvedInferenceVariable(ty::InferTy),
 
     // Projections like `T::Foo` are tricky because a constraint like
@@ -97,12 +98,12 @@ fn compute_components<'tcx>(
                 compute_components(tcx, element, out, visited);
             }
 
-            ty::Closure(_, ref args) => {
+            ty::Closure(_, args) => {
                 let tupled_ty = args.as_closure().tupled_upvars_ty();
                 compute_components(tcx, tupled_ty, out, visited);
             }
 
-            ty::Coroutine(_, ref args, _) => {
+            ty::Coroutine(_, args, _) => {
                 // Same as the closure case
                 let tupled_ty = args.as_coroutine().tupled_upvars_ty();
                 compute_components(tcx, tupled_ty, out, visited);
@@ -120,6 +121,10 @@ fn compute_components<'tcx>(
                 out.push(Component::Param(p));
             }
 
+            ty::Placeholder(p) => {
+                out.push(Component::Placeholder(p));
+            }
+
             // For projections, we prefer to generate an obligation like
             // `<P0 as Trait<P1...Pn>>::Foo: 'a`, because this gives the
             // regionck more ways to prove that it holds. However,
@@ -176,7 +181,6 @@ fn compute_components<'tcx>(
             ty::Tuple(..) |       // ...
             ty::FnPtr(_) |        // OutlivesFunction (*)
             ty::Dynamic(..) |     // OutlivesObject, OutlivesFragment (*)
-            ty::Placeholder(..) |
             ty::Bound(..) |
             ty::Error(_) => {
                 // (*) Function pointers and trait objects are both binders.
@@ -199,7 +203,9 @@ pub(super) fn compute_alias_components_recursive<'tcx>(
     out: &mut SmallVec<[Component<'tcx>; 4]>,
     visited: &mut SsoHashSet<GenericArg<'tcx>>,
 ) {
-    let ty::Alias(kind, alias_ty) = alias_ty.kind() else { bug!() };
+    let ty::Alias(kind, alias_ty) = alias_ty.kind() else {
+        unreachable!("can only call `compute_alias_components_recursive` on an alias type")
+    };
     let opt_variances = if *kind == ty::Opaque { tcx.variances_of(alias_ty.def_id) } else { &[] };
     for (index, child) in alias_ty.args.iter().enumerate() {
         if opt_variances.get(index) == Some(&ty::Bivariant) {
@@ -213,8 +219,8 @@ pub(super) fn compute_alias_components_recursive<'tcx>(
                 compute_components(tcx, ty, out, visited);
             }
             GenericArgKind::Lifetime(lt) => {
-                // Ignore late-bound regions.
-                if !lt.is_late_bound() {
+                // Ignore higher ranked regions.
+                if !lt.is_bound() {
                     out.push(Component::Region(lt));
                 }
             }
@@ -241,8 +247,8 @@ fn compute_components_recursive<'tcx>(
                 compute_components(tcx, ty, out, visited);
             }
             GenericArgKind::Lifetime(lt) => {
-                // Ignore late-bound regions.
-                if !lt.is_late_bound() {
+                // Ignore higher ranked regions.
+                if !lt.is_bound() {
                     out.push(Component::Region(lt));
                 }
             }
diff --git a/compiler/rustc_infer/src/infer/outlives/env.rs b/compiler/rustc_infer/src/infer/outlives/env.rs
index 47e3dd762..f8dbfdde3 100644
--- a/compiler/rustc_infer/src/infer/outlives/env.rs
+++ b/compiler/rustc_infer/src/infer/outlives/env.rs
@@ -138,8 +138,8 @@ impl<'tcx> OutlivesEnvironmentBuilder<'tcx> {
                 }
                 OutlivesBound::RegionSubRegion(r_a, r_b) => match (*r_a, *r_b) {
                     (
-                        ty::ReStatic | ty::ReEarlyBound(_) | ty::ReFree(_),
-                        ty::ReStatic | ty::ReEarlyBound(_) | ty::ReFree(_),
+                        ty::ReStatic | ty::ReEarlyParam(_) | ty::ReLateParam(_),
+                        ty::ReStatic | ty::ReEarlyParam(_) | ty::ReLateParam(_),
                     ) => self.region_relation.add(r_a, r_b),
                     (ty::ReError(_), _) | (_, ty::ReError(_)) => {}
                     // FIXME(#109628): We shouldn't have existential variables in implied bounds.
diff --git a/compiler/rustc_infer/src/infer/outlives/for_liveness.rs b/compiler/rustc_infer/src/infer/outlives/for_liveness.rs
index 398ac94ee..42e3d6cad 100644
--- a/compiler/rustc_infer/src/infer/outlives/for_liveness.rs
+++ b/compiler/rustc_infer/src/infer/outlives/for_liveness.rs
@@ -37,7 +37,7 @@ where
     fn visit_region(&mut self, r: ty::Region<'tcx>) -> ControlFlow<Self::BreakTy> {
         match *r {
             // ignore bound regions, keep visiting
-            ty::ReLateBound(_, _) => ControlFlow::Continue(()),
+            ty::ReBound(_, _) => ControlFlow::Continue(()),
             _ => {
                 (self.op)(r);
                 ControlFlow::Continue(())
@@ -84,7 +84,6 @@ where
                         } else {
                             test_type_match::extract_verify_if_eq(
                                 tcx,
-                                param_env,
                                 &outlives.map_bound(|ty::OutlivesPredicate(ty, bound)| {
                                     VerifyIfEq { ty, bound }
                                 }),
@@ -112,7 +111,9 @@ where
                     };
 
                     for (idx, s) in args.iter().enumerate() {
-                        if variances.map(|variances| variances[idx]) != Some(ty::Variance::Bivariant) {
+                        if variances.map(|variances| variances[idx])
+                            != Some(ty::Variance::Bivariant)
+                        {
                             s.visit_with(self)?;
                         }
                     }
diff --git a/compiler/rustc_infer/src/infer/outlives/mod.rs b/compiler/rustc_infer/src/infer/outlives/mod.rs
index 0987915f4..6379f84aa 100644
--- a/compiler/rustc_infer/src/infer/outlives/mod.rs
+++ b/compiler/rustc_infer/src/infer/outlives/mod.rs
@@ -3,7 +3,7 @@ use self::env::OutlivesEnvironment;
 use super::region_constraints::RegionConstraintData;
 use super::{InferCtxt, RegionResolutionError};
 use crate::infer::free_regions::RegionRelations;
-use crate::infer::lexical_region_resolve::{self, LexicalRegionResolutions};
+use crate::infer::lexical_region_resolve;
 use rustc_middle::traits::query::OutlivesBound;
 use rustc_middle::ty;
 
@@ -37,32 +37,6 @@ pub fn explicit_outlives_bounds<'tcx>(
 }
 
 impl<'tcx> InferCtxt<'tcx> {
-    pub fn skip_region_resolution(&self) {
-        let (var_infos, _) = {
-            let mut inner = self.inner.borrow_mut();
-            let inner = &mut *inner;
-            // Note: `inner.region_obligations` may not be empty, because we
-            // didn't necessarily call `process_registered_region_obligations`.
-            // This is okay, because that doesn't introduce new vars.
-            inner
-                .region_constraint_storage
-                .take()
-                .expect("regions already resolved")
-                .with_log(&mut inner.undo_log)
-                .into_infos_and_data()
-        };
-
-        let lexical_region_resolutions = LexicalRegionResolutions {
-            values: rustc_index::IndexVec::from_elem_n(
-                crate::infer::lexical_region_resolve::VarValue::Value(self.tcx.lifetimes.re_erased),
-                var_infos.len(),
-            ),
-        };
-
-        let old_value = self.lexical_region_resolutions.replace(Some(lexical_region_resolutions));
-        assert!(old_value.is_none());
-    }
-
     /// Process the region constraints and return any errors that
     /// result. After this, no more unification operations should be
     /// done -- or the compiler will panic -- but it is legal to use
@@ -93,7 +67,7 @@ impl<'tcx> InferCtxt<'tcx> {
         let region_rels = &RegionRelations::new(self.tcx, outlives_env.free_region_map());
 
         let (lexical_region_resolutions, errors) =
-            lexical_region_resolve::resolve(outlives_env.param_env, region_rels, var_infos, data);
+            lexical_region_resolve::resolve(region_rels, var_infos, data);
 
         let old_value = self.lexical_region_resolutions.replace(Some(lexical_region_resolutions));
         assert!(old_value.is_none());
diff --git a/compiler/rustc_infer/src/infer/outlives/obligations.rs b/compiler/rustc_infer/src/infer/outlives/obligations.rs
index f36802e12..395830d93 100644
--- a/compiler/rustc_infer/src/infer/outlives/obligations.rs
+++ b/compiler/rustc_infer/src/infer/outlives/obligations.rs
@@ -136,7 +136,7 @@ impl<'tcx> InferCtxt<'tcx> {
             let outlives = &mut TypeOutlives::new(
                 self,
                 self.tcx,
-                &outlives_env.region_bound_pairs(),
+                outlives_env.region_bound_pairs(),
                 None,
                 outlives_env.param_env,
             );
@@ -243,15 +243,18 @@ where
                 Component::Param(param_ty) => {
                     self.param_ty_must_outlive(origin, region, *param_ty);
                 }
+                Component::Placeholder(placeholder_ty) => {
+                    self.placeholder_ty_must_outlive(origin, region, *placeholder_ty);
+                }
                 Component::Alias(alias_ty) => self.alias_ty_must_outlive(origin, region, *alias_ty),
                 Component::EscapingAlias(subcomponents) => {
-                    self.components_must_outlive(origin, &subcomponents, region, category);
+                    self.components_must_outlive(origin, subcomponents, region, category);
                 }
                 Component::UnresolvedInferenceVariable(v) => {
                     // ignore this, we presume it will yield an error
                     // later, since if a type variable is not resolved by
                     // this point it never will be
-                    self.tcx.sess.delay_span_bug(
+                    self.tcx.sess.span_delayed_bug(
                         origin.span(),
                         format!("unresolved inference variable in outlives: {v:?}"),
                     );
@@ -267,11 +270,29 @@ where
         region: ty::Region<'tcx>,
         param_ty: ty::ParamTy,
     ) {
-        let verify_bound = self.verify_bound.param_bound(param_ty);
+        let verify_bound = self.verify_bound.param_or_placeholder_bound(param_ty.to_ty(self.tcx));
         self.delegate.push_verify(origin, GenericKind::Param(param_ty), region, verify_bound);
     }
 
     #[instrument(level = "debug", skip(self))]
+    fn placeholder_ty_must_outlive(
+        &mut self,
+        origin: infer::SubregionOrigin<'tcx>,
+        region: ty::Region<'tcx>,
+        placeholder_ty: ty::PlaceholderType,
+    ) {
+        let verify_bound = self
+            .verify_bound
+            .param_or_placeholder_bound(Ty::new_placeholder(self.tcx, placeholder_ty));
+        self.delegate.push_verify(
+            origin,
+            GenericKind::Placeholder(placeholder_ty),
+            region,
+            verify_bound,
+        );
+    }
+
+    #[instrument(level = "debug", skip(self))]
     fn alias_ty_must_outlive(
         &mut self,
         origin: infer::SubregionOrigin<'tcx>,
diff --git a/compiler/rustc_infer/src/infer/outlives/test_type_match.rs b/compiler/rustc_infer/src/infer/outlives/test_type_match.rs
index 6f973ee37..236dc4ec3 100644
--- a/compiler/rustc_infer/src/infer/outlives/test_type_match.rs
+++ b/compiler/rustc_infer/src/infer/outlives/test_type_match.rs
@@ -36,19 +36,18 @@ use crate::infer::region_constraints::VerifyIfEq;
 /// like are used. This is a particular challenge since this function is invoked
 /// very late in inference and hence cannot make use of the normal inference
 /// machinery.
-#[instrument(level = "debug", skip(tcx, param_env))]
+#[instrument(level = "debug", skip(tcx))]
 pub fn extract_verify_if_eq<'tcx>(
     tcx: TyCtxt<'tcx>,
-    param_env: ty::ParamEnv<'tcx>,
     verify_if_eq_b: &ty::Binder<'tcx, VerifyIfEq<'tcx>>,
     test_ty: Ty<'tcx>,
 ) -> Option<ty::Region<'tcx>> {
     assert!(!verify_if_eq_b.has_escaping_bound_vars());
-    let mut m = MatchAgainstHigherRankedOutlives::new(tcx, param_env);
+    let mut m = MatchAgainstHigherRankedOutlives::new(tcx);
     let verify_if_eq = verify_if_eq_b.skip_binder();
     m.relate(verify_if_eq.ty, test_ty).ok()?;
 
-    if let ty::RegionKind::ReLateBound(depth, br) = verify_if_eq.bound.kind() {
+    if let ty::RegionKind::ReBound(depth, br) = verify_if_eq.bound.kind() {
         assert!(depth == ty::INNERMOST);
         match m.map.get(&br) {
             Some(&r) => Some(r),
@@ -73,10 +72,9 @@ pub fn extract_verify_if_eq<'tcx>(
 }
 
 /// True if a (potentially higher-ranked) outlives
-#[instrument(level = "debug", skip(tcx, param_env))]
+#[instrument(level = "debug", skip(tcx))]
 pub(super) fn can_match_erased_ty<'tcx>(
     tcx: TyCtxt<'tcx>,
-    param_env: ty::ParamEnv<'tcx>,
     outlives_predicate: ty::Binder<'tcx, ty::TypeOutlivesPredicate<'tcx>>,
     erased_ty: Ty<'tcx>,
 ) -> bool {
@@ -87,25 +85,20 @@ pub(super) fn can_match_erased_ty<'tcx>(
         // pointless micro-optimization
         true
     } else {
-        MatchAgainstHigherRankedOutlives::new(tcx, param_env).relate(outlives_ty, erased_ty).is_ok()
+        MatchAgainstHigherRankedOutlives::new(tcx).relate(outlives_ty, erased_ty).is_ok()
     }
 }
 
 struct MatchAgainstHigherRankedOutlives<'tcx> {
     tcx: TyCtxt<'tcx>,
-    param_env: ty::ParamEnv<'tcx>,
     pattern_depth: ty::DebruijnIndex,
     map: FxHashMap<ty::BoundRegion, ty::Region<'tcx>>,
 }
 
 impl<'tcx> MatchAgainstHigherRankedOutlives<'tcx> {
-    fn new(
-        tcx: TyCtxt<'tcx>,
-        param_env: ty::ParamEnv<'tcx>,
-    ) -> MatchAgainstHigherRankedOutlives<'tcx> {
+    fn new(tcx: TyCtxt<'tcx>) -> MatchAgainstHigherRankedOutlives<'tcx> {
         MatchAgainstHigherRankedOutlives {
             tcx,
-            param_env,
             pattern_depth: ty::INNERMOST,
             map: FxHashMap::default(),
         }
@@ -144,15 +137,13 @@ impl<'tcx> MatchAgainstHigherRankedOutlives<'tcx> {
 
 impl<'tcx> TypeRelation<'tcx> for MatchAgainstHigherRankedOutlives<'tcx> {
     fn tag(&self) -> &'static str {
-        "Match"
+        "MatchAgainstHigherRankedOutlives"
     }
 
     fn tcx(&self) -> TyCtxt<'tcx> {
         self.tcx
     }
-    fn param_env(&self) -> ty::ParamEnv<'tcx> {
-        self.param_env
-    }
+
     fn a_is_expected(&self) -> bool {
         true
     } // irrelevant
@@ -177,7 +168,7 @@ impl<'tcx> TypeRelation<'tcx> for MatchAgainstHigherRankedOutlives<'tcx> {
         value: ty::Region<'tcx>,
     ) -> RelateResult<'tcx, ty::Region<'tcx>> {
         debug!("self.pattern_depth = {:?}", self.pattern_depth);
-        if let ty::RegionKind::ReLateBound(depth, br) = pattern.kind()
+        if let ty::RegionKind::ReBound(depth, br) = pattern.kind()
             && depth == self.pattern_depth
         {
             self.bind(br, value)
diff --git a/compiler/rustc_infer/src/infer/outlives/verify.rs b/compiler/rustc_infer/src/infer/outlives/verify.rs
index 7f0a4717d..90282f58e 100644
--- a/compiler/rustc_infer/src/infer/outlives/verify.rs
+++ b/compiler/rustc_infer/src/infer/outlives/verify.rs
@@ -1,7 +1,7 @@
 use crate::infer::outlives::components::{compute_alias_components_recursive, Component};
 use crate::infer::outlives::env::RegionBoundPairs;
 use crate::infer::region_constraints::VerifyIfEq;
-use crate::infer::VerifyBound;
+use crate::infer::{GenericKind, VerifyBound};
 use rustc_data_structures::sso::SsoHashSet;
 use rustc_middle::ty::GenericArg;
 use rustc_middle::ty::{self, OutlivesPredicate, Ty, TyCtxt};
@@ -37,11 +37,11 @@ impl<'cx, 'tcx> VerifyBoundCx<'cx, 'tcx> {
     }
 
     #[instrument(level = "debug", skip(self))]
-    pub fn param_bound(&self, param_ty: ty::ParamTy) -> VerifyBound<'tcx> {
+    pub fn param_or_placeholder_bound(&self, ty: Ty<'tcx>) -> VerifyBound<'tcx> {
         // Start with anything like `T: 'a` we can scrape from the
         // environment. If the environment contains something like
         // `for<'a> T: 'a`, then we know that `T` outlives everything.
-        let declared_bounds_from_env = self.declared_generic_bounds_from_env(param_ty);
+        let declared_bounds_from_env = self.declared_generic_bounds_from_env(ty);
         debug!(?declared_bounds_from_env);
         let mut param_bounds = vec![];
         for declared_bound in declared_bounds_from_env {
@@ -51,7 +51,7 @@ impl<'cx, 'tcx> VerifyBoundCx<'cx, 'tcx> {
                 param_bounds.push(VerifyBound::OutlivedBy(region));
             } else {
                 // This is `for<'a> T: 'a`. This means that `T` outlives everything! All done here.
-                debug!("found that {param_ty:?} outlives any lifetime, returning empty vector");
+                debug!("found that {ty:?} outlives any lifetime, returning empty vector");
                 return VerifyBound::AllBounds(vec![]);
             }
         }
@@ -130,12 +130,7 @@ impl<'cx, 'tcx> VerifyBoundCx<'cx, 'tcx> {
         // see the extensive comment in projection_must_outlive
         let recursive_bound = {
             let mut components = smallvec![];
-            compute_alias_components_recursive(
-                self.tcx,
-                alias_ty_as_ty.into(),
-                &mut components,
-                visited,
-            );
+            compute_alias_components_recursive(self.tcx, alias_ty_as_ty, &mut components, visited);
             self.bound_from_components(&components, visited)
         };
 
@@ -168,7 +163,10 @@ impl<'cx, 'tcx> VerifyBoundCx<'cx, 'tcx> {
     ) -> VerifyBound<'tcx> {
         match *component {
             Component::Region(lt) => VerifyBound::OutlivedBy(lt),
-            Component::Param(param_ty) => self.param_bound(param_ty),
+            Component::Param(param_ty) => self.param_or_placeholder_bound(param_ty.to_ty(self.tcx)),
+            Component::Placeholder(placeholder_ty) => {
+                self.param_or_placeholder_bound(Ty::new_placeholder(self.tcx, placeholder_ty))
+            }
             Component::Alias(alias_ty) => self.alias_bound(alias_ty, visited),
             Component::EscapingAlias(ref components) => {
                 self.bound_from_components(components, visited)
@@ -177,7 +175,7 @@ impl<'cx, 'tcx> VerifyBoundCx<'cx, 'tcx> {
                 // ignore this, we presume it will yield an error
                 // later, since if a type variable is not resolved by
                 // this point it never will be
-                self.tcx.sess.delay_span_bug(
+                self.tcx.sess.span_delayed_bug(
                     rustc_span::DUMMY_SP,
                     format!("unresolved inference variable in outlives: {v:?}"),
                 );
@@ -195,9 +193,9 @@ impl<'cx, 'tcx> VerifyBoundCx<'cx, 'tcx> {
     /// bounds, but all the bounds it returns can be relied upon.
     fn declared_generic_bounds_from_env(
         &self,
-        param_ty: ty::ParamTy,
+        generic_ty: Ty<'tcx>,
     ) -> Vec<ty::Binder<'tcx, ty::OutlivesPredicate<Ty<'tcx>, ty::Region<'tcx>>>> {
-        let generic_ty = param_ty.to_ty(self.tcx);
+        assert!(matches!(generic_ty.kind(), ty::Param(_) | ty::Placeholder(_)));
         self.declared_generic_bounds_from_env_for_erased_ty(generic_ty)
     }
 
@@ -242,10 +240,20 @@ impl<'cx, 'tcx> VerifyBoundCx<'cx, 'tcx> {
                     "declared_generic_bounds_from_env_for_erased_ty: region_bound_pair = {:?}",
                     (r, p)
                 );
+                // Fast path for the common case.
+                match (&p, erased_ty.kind()) {
+                    // In outlive routines, all types are expected to be fully normalized.
+                    // And therefore we can safely use structural equality for alias types.
+                    (GenericKind::Param(p1), ty::Param(p2)) if p1 == p2 => {}
+                    (GenericKind::Placeholder(p1), ty::Placeholder(p2)) if p1 == p2 => {}
+                    (GenericKind::Alias(a1), ty::Alias(_, a2)) if a1.def_id == a2.def_id => {}
+                    _ => return None,
+                }
+
                 let p_ty = p.to_ty(tcx);
                 let erased_p_ty = self.tcx.erase_regions(p_ty);
                 (erased_p_ty == erased_ty)
-                    .then_some(ty::Binder::dummy(ty::OutlivesPredicate(p.to_ty(tcx), r)))
+                    .then_some(ty::Binder::dummy(ty::OutlivesPredicate(p_ty, r)))
             });
 
         param_bounds
@@ -314,14 +322,8 @@ impl<'cx, 'tcx> VerifyBoundCx<'cx, 'tcx> {
     ) -> impl Iterator<Item = ty::Binder<'tcx, ty::OutlivesPredicate<Ty<'tcx>, ty::Region<'tcx>>>>
     {
         let tcx = self.tcx;
-        let param_env = self.param_env;
         clauses.filter_map(|p| p.as_type_outlives_clause()).filter(move |outlives_predicate| {
-            super::test_type_match::can_match_erased_ty(
-                tcx,
-                param_env,
-                *outlives_predicate,
-                erased_ty,
-            )
+            super::test_type_match::can_match_erased_ty(tcx, *outlives_predicate, erased_ty)
         })
     }
 }
diff --git a/compiler/rustc_infer/src/infer/region_constraints/leak_check.rs b/compiler/rustc_infer/src/infer/region_constraints/leak_check.rs
index b6ff8f2f5..e06596df7 100644
--- a/compiler/rustc_infer/src/infer/region_constraints/leak_check.rs
+++ b/compiler/rustc_infer/src/infer/region_constraints/leak_check.rs
@@ -81,7 +81,7 @@ impl<'tcx> RegionConstraintCollector<'_, 'tcx> {
             return Ok(());
         }
 
-        let mini_graph = &MiniGraph::new(tcx, &self, only_consider_snapshot);
+        let mini_graph = &MiniGraph::new(tcx, self, only_consider_snapshot);
 
         let mut leak_check = LeakCheck::new(tcx, outer_universe, max_universe, mini_graph, self);
         leak_check.assign_placeholder_values()?;
@@ -341,11 +341,13 @@ impl<'tcx> SccUniverse<'tcx> {
 }
 
 rustc_index::newtype_index! {
+    #[orderable]
     #[debug_format = "LeakCheckNode({})"]
     struct LeakCheckNode {}
 }
 
 rustc_index::newtype_index! {
+    #[orderable]
     #[debug_format = "LeakCheckScc({})"]
     struct LeakCheckScc {}
 }
diff --git a/compiler/rustc_infer/src/infer/region_constraints/mod.rs b/compiler/rustc_infer/src/infer/region_constraints/mod.rs
index 3fa9a7333..cbd8040c9 100644
--- a/compiler/rustc_infer/src/infer/region_constraints/mod.rs
+++ b/compiler/rustc_infer/src/infer/region_constraints/mod.rs
@@ -16,7 +16,7 @@ use rustc_index::IndexVec;
 use rustc_middle::infer::unify_key::{RegionVidKey, UnifiedRegion};
 use rustc_middle::ty::ReStatic;
 use rustc_middle::ty::{self, Ty, TyCtxt};
-use rustc_middle::ty::{ReLateBound, ReVar};
+use rustc_middle::ty::{ReBound, ReVar};
 use rustc_middle::ty::{Region, RegionVid};
 use rustc_span::Span;
 
@@ -147,6 +147,7 @@ pub struct Verify<'tcx> {
 #[derive(Copy, Clone, PartialEq, Eq, Hash, TypeFoldable, TypeVisitable)]
 pub enum GenericKind<'tcx> {
     Param(ty::ParamTy),
+    Placeholder(ty::PlaceholderType),
     Alias(ty::AliasTy<'tcx>),
 }
 
@@ -316,7 +317,7 @@ impl<'tcx> RegionConstraintStorage<'tcx> {
         match undo_entry {
             AddVar(vid) => {
                 self.var_infos.pop().unwrap();
-                assert_eq!(self.var_infos.len(), vid.index() as usize);
+                assert_eq!(self.var_infos.len(), vid.index());
             }
             AddConstraint(ref constraint) => {
                 self.data.constraints.remove(constraint);
@@ -531,7 +532,7 @@ impl<'tcx> RegionConstraintCollector<'_, 'tcx> {
         debug!("origin = {:#?}", origin);
 
         match (*sub, *sup) {
-            (ReLateBound(..), _) | (_, ReLateBound(..)) => {
+            (ReBound(..), _) | (_, ReBound(..)) => {
                 span_bug!(origin.span(), "cannot relate bound region: {:?} <= {:?}", sub, sup);
             }
             (_, ReStatic) => {
@@ -662,12 +663,12 @@ impl<'tcx> RegionConstraintCollector<'_, 'tcx> {
         match *region {
             ty::ReStatic
             | ty::ReErased
-            | ty::ReFree(..)
-            | ty::ReEarlyBound(..)
+            | ty::ReLateParam(..)
+            | ty::ReEarlyParam(..)
             | ty::ReError(_) => ty::UniverseIndex::ROOT,
             ty::RePlaceholder(placeholder) => placeholder.universe,
             ty::ReVar(vid) => self.var_universe(vid),
-            ty::ReLateBound(..) => bug!("universe(): encountered bound region {:?}", region),
+            ty::ReBound(..) => bug!("universe(): encountered bound region {:?}", region),
         }
     }
 
@@ -707,6 +708,7 @@ impl<'tcx> fmt::Debug for GenericKind<'tcx> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match *self {
             GenericKind::Param(ref p) => write!(f, "{p:?}"),
+            GenericKind::Placeholder(ref p) => write!(f, "{p:?}"),
             GenericKind::Alias(ref p) => write!(f, "{p:?}"),
         }
     }
@@ -716,6 +718,7 @@ impl<'tcx> fmt::Display for GenericKind<'tcx> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match *self {
             GenericKind::Param(ref p) => write!(f, "{p}"),
+            GenericKind::Placeholder(ref p) => write!(f, "{p:?}"),
             GenericKind::Alias(ref p) => write!(f, "{p}"),
         }
     }
@@ -725,6 +728,7 @@ impl<'tcx> GenericKind<'tcx> {
     pub fn to_ty(&self, tcx: TyCtxt<'tcx>) -> Ty<'tcx> {
         match *self {
             GenericKind::Param(ref p) => p.to_ty(tcx),
+            GenericKind::Placeholder(ref p) => Ty::new_placeholder(tcx, *p),
             GenericKind::Alias(ref p) => p.to_ty(tcx),
         }
     }
diff --git a/compiler/rustc_infer/src/infer/combine.rs b/compiler/rustc_infer/src/infer/relate/combine.rs
index 2a9e20b9f..ee911c432 100644
--- a/compiler/rustc_infer/src/infer/combine.rs
+++ b/compiler/rustc_infer/src/infer/relate/combine.rs
@@ -3,7 +3,7 @@
 //! combining two instances of various things and yielding a new instance.
 //! These combiner methods always yield a `Result<T>`. To relate two
 //! types, you can use `infcx.at(cause, param_env)` which then allows
-//! you to use the relevant methods of [At](super::at::At).
+//! you to use the relevant methods of [At](crate::infer::at::At).
 //!
 //! Combiners mostly do their specific behavior and then hand off the
 //! bulk of the work to [InferCtxt::super_combine_tys] and
@@ -23,11 +23,11 @@
 //! this should be correctly updated.
 
 use super::equate::Equate;
+use super::generalize::{self, CombineDelegate, Generalization};
 use super::glb::Glb;
 use super::lub::Lub;
 use super::sub::Sub;
-use super::{DefineOpaqueTypes, InferCtxt, TypeTrace};
-use crate::infer::generalize::{self, CombineDelegate, Generalization};
+use crate::infer::{DefineOpaqueTypes, InferCtxt, TypeTrace};
 use crate::traits::{Obligation, PredicateObligations};
 use rustc_middle::infer::canonical::OriginalQueryValues;
 use rustc_middle::infer::unify_key::{ConstVarValue, ConstVariableValue, EffectVarValue};
@@ -35,6 +35,7 @@ use rustc_middle::infer::unify_key::{ConstVariableOrigin, ConstVariableOriginKin
 use rustc_middle::ty::error::{ExpectedFound, TypeError};
 use rustc_middle::ty::relate::{RelateResult, TypeRelation};
 use rustc_middle::ty::{self, InferConst, ToPredicate, Ty, TyCtxt, TypeVisitableExt};
+use rustc_middle::ty::{AliasRelationDirection, TyVar};
 use rustc_middle::ty::{IntType, UintType};
 use rustc_span::DUMMY_SP;
 
@@ -102,15 +103,19 @@ impl<'tcx> InferCtxt<'tcx> {
             }
 
             // We don't expect `TyVar` or `Fresh*` vars at this point with lazy norm.
-            (
-                ty::Alias(..),
-                ty::Infer(ty::TyVar(_) | ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_)),
-            )
-            | (
-                ty::Infer(ty::TyVar(_) | ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_)),
-                ty::Alias(..),
-            ) if self.next_trait_solver() => {
-                bug!()
+            (ty::Alias(..), ty::Infer(ty::TyVar(_))) | (ty::Infer(ty::TyVar(_)), ty::Alias(..))
+                if self.next_trait_solver() =>
+            {
+                bug!(
+                    "We do not expect to encounter `TyVar` this late in combine \
+                    -- they should have been handled earlier"
+                )
+            }
+            (_, ty::Infer(ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_)))
+            | (ty::Infer(ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_)), _)
+                if self.next_trait_solver() =>
+            {
+                bug!("We do not expect to encounter `Fresh` variables in the new solver")
             }
 
             (_, ty::Alias(..)) | (ty::Alias(..), _) if self.next_trait_solver() => {
@@ -171,11 +176,11 @@ impl<'tcx> InferCtxt<'tcx> {
             // two const param's types are able to be equal has to go through a canonical query with the actual logic
             // in `rustc_trait_selection`.
             let canonical = self.canonicalize_query(
-                (relation.param_env(), a.ty(), b.ty()),
+                relation.param_env().and((a.ty(), b.ty())),
                 &mut OriginalQueryValues::default(),
             );
             self.tcx.check_tys_might_be_eq(canonical).map_err(|_| {
-                self.tcx.sess.delay_span_bug(
+                self.tcx.sess.span_delayed_bug(
                     DUMMY_SP,
                     format!("cannot relate consts of different types (a={a:?}, b={b:?})",),
                 )
@@ -326,7 +331,9 @@ impl<'tcx> InferCtxt<'tcx> {
     ) -> RelateResult<'tcx, ty::Const<'tcx>> {
         let span =
             self.inner.borrow_mut().const_unification_table().probe_value(target_vid).origin.span;
-        let Generalization { value, needs_wf: _ } = generalize::generalize(
+        // FIXME(generic_const_exprs): Occurs check failures for unevaluated
+        // constants and generic expressions are not yet handled correctly.
+        let Generalization { value_may_be_infer: value, needs_wf: _ } = generalize::generalize(
             self,
             &mut CombineDelegate { infcx: self, span, param_env },
             ct,
@@ -445,7 +452,7 @@ impl<'infcx, 'tcx> CombineFields<'infcx, 'tcx> {
         // `'?2` and `?3` are fresh region/type inference
         // variables. (Down below, we will relate `a_ty <: b_ty`,
         // adding constraints like `'x: '?2` and `?1 <: ?3`.)
-        let Generalization { value: b_ty, needs_wf } = generalize::generalize(
+        let Generalization { value_may_be_infer: b_ty, needs_wf } = generalize::generalize(
             self.infcx,
             &mut CombineDelegate {
                 infcx: self.infcx,
@@ -457,8 +464,12 @@ impl<'infcx, 'tcx> CombineFields<'infcx, 'tcx> {
             ambient_variance,
         )?;
 
-        debug!(?b_ty);
-        self.infcx.inner.borrow_mut().type_variables().instantiate(b_vid, b_ty);
+        // Constrain `b_vid` to the generalized type `b_ty`.
+        if let &ty::Infer(TyVar(b_ty_vid)) = b_ty.kind() {
+            self.infcx.inner.borrow_mut().type_variables().equate(b_vid, b_ty_vid);
+        } else {
+            self.infcx.inner.borrow_mut().type_variables().instantiate(b_vid, b_ty);
+        }
 
         if needs_wf {
             self.obligations.push(Obligation::new(
@@ -477,25 +488,71 @@ impl<'infcx, 'tcx> CombineFields<'infcx, 'tcx> {
         // relations wind up attributed to the same spans. We need
         // to associate causes/spans with each of the relations in
         // the stack to get this right.
-        match ambient_variance {
-            ty::Variance::Invariant => self.equate(a_is_expected).relate(a_ty, b_ty),
-            ty::Variance::Covariant => self.sub(a_is_expected).relate(a_ty, b_ty),
-            ty::Variance::Contravariant => self.sub(a_is_expected).relate_with_variance(
-                ty::Contravariant,
-                ty::VarianceDiagInfo::default(),
-                a_ty,
-                b_ty,
-            ),
-            ty::Variance::Bivariant => {
-                unreachable!("no code should be generalizing bivariantly (currently)")
+        if b_ty.is_ty_var() {
+            // This happens for cases like `<?0 as Trait>::Assoc == ?0`.
+            // We can't instantiate `?0` here as that would result in a
+            // cyclic type. We instead delay the unification in case
+            // the alias can be normalized to something which does not
+            // mention `?0`.
+            if self.infcx.next_trait_solver() {
+                let (lhs, rhs, direction) = match ambient_variance {
+                    ty::Variance::Invariant => {
+                        (a_ty.into(), b_ty.into(), AliasRelationDirection::Equate)
+                    }
+                    ty::Variance::Covariant => {
+                        (a_ty.into(), b_ty.into(), AliasRelationDirection::Subtype)
+                    }
+                    ty::Variance::Contravariant => {
+                        (b_ty.into(), a_ty.into(), AliasRelationDirection::Subtype)
+                    }
+                    ty::Variance::Bivariant => unreachable!("bivariant generalization"),
+                };
+                self.obligations.push(Obligation::new(
+                    self.tcx(),
+                    self.trace.cause.clone(),
+                    self.param_env,
+                    ty::PredicateKind::AliasRelate(lhs, rhs, direction),
+                ));
+            } else {
+                match a_ty.kind() {
+                    &ty::Alias(ty::AliasKind::Projection, data) => {
+                        // FIXME: This does not handle subtyping correctly, we could
+                        // instead create a new inference variable for `a_ty`, emitting
+                        // `Projection(a_ty, a_infer)` and `a_infer <: b_ty`.
+                        self.obligations.push(Obligation::new(
+                            self.tcx(),
+                            self.trace.cause.clone(),
+                            self.param_env,
+                            ty::ProjectionPredicate { projection_ty: data, term: b_ty.into() },
+                        ))
+                    }
+                    // The old solver only accepts projection predicates for associated types.
+                    ty::Alias(
+                        ty::AliasKind::Inherent | ty::AliasKind::Weak | ty::AliasKind::Opaque,
+                        _,
+                    ) => return Err(TypeError::CyclicTy(a_ty)),
+                    _ => bug!("generalizated `{a_ty:?} to infer, not an alias"),
+                }
             }
-        }?;
+        } else {
+            match ambient_variance {
+                ty::Variance::Invariant => self.equate(a_is_expected).relate(a_ty, b_ty),
+                ty::Variance::Covariant => self.sub(a_is_expected).relate(a_ty, b_ty),
+                ty::Variance::Contravariant => self.sub(a_is_expected).relate_with_variance(
+                    ty::Contravariant,
+                    ty::VarianceDiagInfo::default(),
+                    a_ty,
+                    b_ty,
+                ),
+                ty::Variance::Bivariant => unreachable!("bivariant generalization"),
+            }?;
+        }
 
         Ok(())
     }
 
     pub fn register_obligations(&mut self, obligations: PredicateObligations<'tcx>) {
-        self.obligations.extend(obligations.into_iter());
+        self.obligations.extend(obligations);
     }
 
     pub fn register_predicates(&mut self, obligations: impl IntoIterator<Item: ToPredicate<'tcx>>) {
@@ -506,6 +563,8 @@ impl<'infcx, 'tcx> CombineFields<'infcx, 'tcx> {
 }
 
 pub trait ObligationEmittingRelation<'tcx>: TypeRelation<'tcx> {
+    fn param_env(&self) -> ty::ParamEnv<'tcx>;
+
     /// Register obligations that must hold in order for this relation to hold
     fn register_obligations(&mut self, obligations: PredicateObligations<'tcx>);
 
diff --git a/compiler/rustc_infer/src/infer/equate.rs b/compiler/rustc_infer/src/infer/relate/equate.rs
index 5d929394e..cb62f2583 100644
--- a/compiler/rustc_infer/src/infer/equate.rs
+++ b/compiler/rustc_infer/src/infer/relate/equate.rs
@@ -1,8 +1,6 @@
-use crate::infer::DefineOpaqueTypes;
-use crate::traits::PredicateObligations;
-
 use super::combine::{CombineFields, ObligationEmittingRelation};
-use super::Subtype;
+use crate::infer::{DefineOpaqueTypes, SubregionOrigin};
+use crate::traits::PredicateObligations;
 
 use rustc_middle::ty::relate::{self, Relate, RelateResult, TypeRelation};
 use rustc_middle::ty::GenericArgsRef;
@@ -35,10 +33,6 @@ impl<'tcx> TypeRelation<'tcx> for Equate<'_, '_, 'tcx> {
         self.fields.tcx()
     }
 
-    fn param_env(&self) -> ty::ParamEnv<'tcx> {
-        self.fields.param_env
-    }
-
     fn a_is_expected(&self) -> bool {
         self.a_is_expected
     }
@@ -133,7 +127,7 @@ impl<'tcx> TypeRelation<'tcx> for Equate<'_, '_, 'tcx> {
         b: ty::Region<'tcx>,
     ) -> RelateResult<'tcx, ty::Region<'tcx>> {
         debug!("{}.regions({:?}, {:?})", self.tag(), a, b);
-        let origin = Subtype(Box::new(self.fields.trace.clone()));
+        let origin = SubregionOrigin::Subtype(Box::new(self.fields.trace.clone()));
         self.fields
             .infcx
             .inner
@@ -176,6 +170,10 @@ impl<'tcx> TypeRelation<'tcx> for Equate<'_, '_, 'tcx> {
 }
 
 impl<'tcx> ObligationEmittingRelation<'tcx> for Equate<'_, '_, 'tcx> {
+    fn param_env(&self) -> ty::ParamEnv<'tcx> {
+        self.fields.param_env
+    }
+
     fn register_predicates(&mut self, obligations: impl IntoIterator<Item: ty::ToPredicate<'tcx>>) {
         self.fields.register_predicates(obligations);
     }
diff --git a/compiler/rustc_infer/src/infer/generalize.rs b/compiler/rustc_infer/src/infer/relate/generalize.rs
index 17fe3aa7b..665af7381 100644
--- a/compiler/rustc_infer/src/infer/generalize.rs
+++ b/compiler/rustc_infer/src/infer/relate/generalize.rs
@@ -1,19 +1,22 @@
+use std::mem;
+
 use rustc_data_structures::sso::SsoHashMap;
 use rustc_hir::def_id::DefId;
 use rustc_middle::infer::unify_key::{ConstVarValue, ConstVariableValue};
 use rustc_middle::ty::error::TypeError;
 use rustc_middle::ty::relate::{self, Relate, RelateResult, TypeRelation};
-use rustc_middle::ty::{self, InferConst, Term, Ty, TyCtxt, TypeVisitableExt};
+use rustc_middle::ty::visit::MaxUniverse;
+use rustc_middle::ty::{self, InferConst, Term, Ty, TyCtxt, TypeVisitable, TypeVisitableExt};
 use rustc_span::Span;
 
 use crate::infer::nll_relate::TypeRelatingDelegate;
-use crate::infer::type_variable::TypeVariableValue;
+use crate::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind, TypeVariableValue};
 use crate::infer::{InferCtxt, RegionVariableOrigin};
 
 /// Attempts to generalize `term` for the type variable `for_vid`.
 /// This checks for cycles -- that is, whether the type `term`
 /// references `for_vid`.
-pub(super) fn generalize<'tcx, D: GeneralizerDelegate<'tcx>, T: Into<Term<'tcx>> + Relate<'tcx>>(
+pub fn generalize<'tcx, D: GeneralizerDelegate<'tcx>, T: Into<Term<'tcx>> + Relate<'tcx>>(
     infcx: &InferCtxt<'tcx>,
     delegate: &mut D,
     term: T,
@@ -38,14 +41,15 @@ pub(super) fn generalize<'tcx, D: GeneralizerDelegate<'tcx>, T: Into<Term<'tcx>>
         root_vid,
         for_universe,
         root_term: term.into(),
+        in_alias: false,
         needs_wf: false,
         cache: Default::default(),
     };
 
     assert!(!term.has_escaping_bound_vars());
-    let value = generalizer.relate(term, term)?;
+    let value_may_be_infer = generalizer.relate(term, term)?;
     let needs_wf = generalizer.needs_wf;
-    Ok(Generalization { value, needs_wf })
+    Ok(Generalization { value_may_be_infer, needs_wf })
 }
 
 /// Abstracts the handling of region vars between HIR and MIR/NLL typechecking
@@ -55,6 +59,8 @@ pub trait GeneralizerDelegate<'tcx> {
 
     fn forbid_inference_vars() -> bool;
 
+    fn span(&self) -> Span;
+
     fn generalize_region(&mut self, universe: ty::UniverseIndex) -> ty::Region<'tcx>;
 }
 
@@ -73,6 +79,10 @@ impl<'tcx> GeneralizerDelegate<'tcx> for CombineDelegate<'_, 'tcx> {
         false
     }
 
+    fn span(&self) -> Span {
+        self.span
+    }
+
     fn generalize_region(&mut self, universe: ty::UniverseIndex) -> ty::Region<'tcx> {
         // FIXME: This is non-ideal because we don't give a
         // very descriptive origin for this region variable.
@@ -93,6 +103,10 @@ where
         <Self as TypeRelatingDelegate<'tcx>>::forbid_inference_vars()
     }
 
+    fn span(&self) -> Span {
+        <Self as TypeRelatingDelegate<'tcx>>::span(&self)
+    }
+
     fn generalize_region(&mut self, universe: ty::UniverseIndex) -> ty::Region<'tcx> {
         <Self as TypeRelatingDelegate<'tcx>>::generalize_existential(self, universe)
     }
@@ -139,6 +153,13 @@ struct Generalizer<'me, 'tcx, D> {
 
     cache: SsoHashMap<Ty<'tcx>, Ty<'tcx>>,
 
+    /// This is set once we're generalizing the arguments of an alias.
+    ///
+    /// This is necessary to correctly handle
+    /// `<T as Bar<<?0 as Foo>::Assoc>::Assoc == ?0`. This equality can
+    /// hold by either normalizing the outer or the inner associated type.
+    in_alias: bool,
+
     /// See the field `needs_wf` in `Generalization`.
     needs_wf: bool,
 }
@@ -161,10 +182,6 @@ where
         self.infcx.tcx
     }
 
-    fn param_env(&self) -> ty::ParamEnv<'tcx> {
-        self.delegate.param_env()
-    }
-
     fn tag(&self) -> &'static str {
         "Generalizer"
     }
@@ -193,7 +210,7 @@ where
                 opt_variances,
                 a_subst,
                 b_subst,
-                true,
+                false,
             )
         }
     }
@@ -274,7 +291,7 @@ where
                                 ty::Covariant | ty::Contravariant => (),
                             }
 
-                            let origin = *inner.type_variables().var_origin(vid);
+                            let origin = inner.type_variables().var_origin(vid);
                             let new_var_id =
                                 inner.type_variables().new_var(self.for_universe, origin);
                             let u = Ty::new_var(self.tcx(), new_var_id);
@@ -309,6 +326,44 @@ where
                 }
             }
 
+            ty::Alias(kind, data) => {
+                // An occurs check failure inside of an alias does not mean
+                // that the types definitely don't unify. We may be able
+                // to normalize the alias after all.
+                //
+                // We handle this by lazily equating the alias and generalizing
+                // it to an inference variable.
+                let is_nested_alias = mem::replace(&mut self.in_alias, true);
+                let result = match self.relate(data, data) {
+                    Ok(data) => Ok(Ty::new_alias(self.tcx(), kind, data)),
+                    Err(e) => {
+                        if is_nested_alias {
+                            return Err(e);
+                        } else {
+                            let mut visitor = MaxUniverse::new();
+                            t.visit_with(&mut visitor);
+                            let infer_replacement_is_complete =
+                                self.for_universe.can_name(visitor.max_universe())
+                                    && !t.has_escaping_bound_vars();
+                            if !infer_replacement_is_complete {
+                                warn!("may incompletely handle alias type: {t:?}");
+                            }
+
+                            debug!("generalization failure in alias");
+                            Ok(self.infcx.next_ty_var_in_universe(
+                                TypeVariableOrigin {
+                                    kind: TypeVariableOriginKind::MiscVariable,
+                                    span: self.delegate.span(),
+                                },
+                                self.for_universe,
+                            ))
+                        }
+                    }
+                };
+                self.in_alias = is_nested_alias;
+                result
+            }
+
             _ => relate::structurally_relate_tys(self, t, t),
         }?;
 
@@ -327,7 +382,7 @@ where
         match *r {
             // Never make variables for regions bound within the type itself,
             // nor for erased regions.
-            ty::ReLateBound(..) | ty::ReErased => {
+            ty::ReBound(..) | ty::ReErased => {
                 return Ok(r);
             }
 
@@ -340,8 +395,8 @@ where
             ty::RePlaceholder(..)
             | ty::ReVar(..)
             | ty::ReStatic
-            | ty::ReEarlyBound(..)
-            | ty::ReFree(..) => {
+            | ty::ReEarlyParam(..)
+            | ty::ReLateParam(..) => {
                 // see common code below
             }
         }
@@ -457,7 +512,15 @@ where
 /// indicating whether further WF checks are needed.
 #[derive(Debug)]
 pub struct Generalization<T> {
-    pub value: T,
+    /// When generalizing `<?0 as Trait>::Assoc` or
+    /// `<T as Bar<<?0 as Foo>::Assoc>>::Assoc`
+    /// for `?0` generalization returns an inference
+    /// variable.
+    ///
+    /// This has to be handled wotj care as it can
+    /// otherwise very easily result in infinite
+    /// recursion.
+    pub value_may_be_infer: T,
 
     /// If true, then the generalized type may not be well-formed,
     /// even if the source type is well-formed, so we should add an
diff --git a/compiler/rustc_infer/src/infer/glb.rs b/compiler/rustc_infer/src/infer/relate/glb.rs
index 2f659d9a6..aa8912430 100644
--- a/compiler/rustc_infer/src/infer/glb.rs
+++ b/compiler/rustc_infer/src/infer/relate/glb.rs
@@ -1,13 +1,12 @@
 //! Greatest lower bound. See [`lattice`].
 
+use rustc_middle::ty::relate::{Relate, RelateResult, TypeRelation};
+use rustc_middle::ty::{self, Ty, TyCtxt, TypeVisitableExt};
+
 use super::combine::{CombineFields, ObligationEmittingRelation};
 use super::lattice::{self, LatticeDir};
-use super::Subtype;
-use super::{DefineOpaqueTypes, InferCtxt};
-
+use crate::infer::{DefineOpaqueTypes, InferCtxt, SubregionOrigin};
 use crate::traits::{ObligationCause, PredicateObligations};
-use rustc_middle::ty::relate::{Relate, RelateResult, TypeRelation};
-use rustc_middle::ty::{self, Ty, TyCtxt, TypeVisitableExt};
 
 /// "Greatest lower bound" (common subtype)
 pub struct Glb<'combine, 'infcx, 'tcx> {
@@ -33,10 +32,6 @@ impl<'tcx> TypeRelation<'tcx> for Glb<'_, '_, 'tcx> {
         self.fields.tcx()
     }
 
-    fn param_env(&self) -> ty::ParamEnv<'tcx> {
-        self.fields.param_env
-    }
-
     fn a_is_expected(&self) -> bool {
         self.a_is_expected
     }
@@ -68,7 +63,7 @@ impl<'tcx> TypeRelation<'tcx> for Glb<'_, '_, 'tcx> {
     ) -> RelateResult<'tcx, ty::Region<'tcx>> {
         debug!("{}.regions({:?}, {:?})", self.tag(), a, b);
 
-        let origin = Subtype(Box::new(self.fields.trace.clone()));
+        let origin = SubregionOrigin::Subtype(Box::new(self.fields.trace.clone()));
         // GLB(&'static u8, &'a u8) == &RegionLUB('static, 'a) u8 == &'static u8
         Ok(self.fields.infcx.inner.borrow_mut().unwrap_region_constraints().lub_regions(
             self.tcx(),
@@ -139,6 +134,10 @@ impl<'combine, 'infcx, 'tcx> LatticeDir<'infcx, 'tcx> for Glb<'combine, 'infcx,
 }
 
 impl<'tcx> ObligationEmittingRelation<'tcx> for Glb<'_, '_, 'tcx> {
+    fn param_env(&self) -> ty::ParamEnv<'tcx> {
+        self.fields.param_env
+    }
+
     fn register_predicates(&mut self, obligations: impl IntoIterator<Item: ty::ToPredicate<'tcx>>) {
         self.fields.register_predicates(obligations);
     }
diff --git a/compiler/rustc_infer/src/infer/higher_ranked/mod.rs b/compiler/rustc_infer/src/infer/relate/higher_ranked.rs
index 510b1797d..440df8c89 100644
--- a/compiler/rustc_infer/src/infer/higher_ranked/mod.rs
+++ b/compiler/rustc_infer/src/infer/relate/higher_ranked.rs
@@ -2,8 +2,8 @@
 //! the end of the file for details.
 
 use super::combine::CombineFields;
-use super::{HigherRankedType, InferCtxt};
 use crate::infer::CombinedSnapshot;
+use crate::infer::{HigherRankedType, InferCtxt};
 use rustc_middle::ty::fold::FnMutDelegate;
 use rustc_middle::ty::relate::{Relate, RelateResult, TypeRelation};
 use rustc_middle::ty::{self, Binder, Ty, TyCtxt, TypeFoldable};
diff --git a/compiler/rustc_infer/src/infer/lattice.rs b/compiler/rustc_infer/src/infer/relate/lattice.rs
index 9ef35429f..744e2dfa3 100644
--- a/compiler/rustc_infer/src/infer/lattice.rs
+++ b/compiler/rustc_infer/src/infer/relate/lattice.rs
@@ -18,10 +18,10 @@
 //! [lattices]: https://en.wikipedia.org/wiki/Lattice_(order)
 
 use super::combine::ObligationEmittingRelation;
-use super::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
-use super::{DefineOpaqueTypes, InferCtxt};
-
+use crate::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
+use crate::infer::{DefineOpaqueTypes, InferCtxt};
 use crate::traits::ObligationCause;
+
 use rustc_middle::ty::relate::RelateResult;
 use rustc_middle::ty::TyVar;
 use rustc_middle::ty::{self, Ty};
diff --git a/compiler/rustc_infer/src/infer/lub.rs b/compiler/rustc_infer/src/infer/relate/lub.rs
index e41ec7e6c..87d777530 100644
--- a/compiler/rustc_infer/src/infer/lub.rs
+++ b/compiler/rustc_infer/src/infer/relate/lub.rs
@@ -2,10 +2,9 @@
 
 use super::combine::{CombineFields, ObligationEmittingRelation};
 use super::lattice::{self, LatticeDir};
-use super::Subtype;
-use super::{DefineOpaqueTypes, InferCtxt};
-
+use crate::infer::{DefineOpaqueTypes, InferCtxt, SubregionOrigin};
 use crate::traits::{ObligationCause, PredicateObligations};
+
 use rustc_middle::ty::relate::{Relate, RelateResult, TypeRelation};
 use rustc_middle::ty::{self, Ty, TyCtxt, TypeVisitableExt};
 
@@ -33,10 +32,6 @@ impl<'tcx> TypeRelation<'tcx> for Lub<'_, '_, 'tcx> {
         self.fields.tcx()
     }
 
-    fn param_env(&self) -> ty::ParamEnv<'tcx> {
-        self.fields.param_env
-    }
-
     fn a_is_expected(&self) -> bool {
         self.a_is_expected
     }
@@ -68,7 +63,7 @@ impl<'tcx> TypeRelation<'tcx> for Lub<'_, '_, 'tcx> {
     ) -> RelateResult<'tcx, ty::Region<'tcx>> {
         debug!("{}.regions({:?}, {:?})", self.tag(), a, b);
 
-        let origin = Subtype(Box::new(self.fields.trace.clone()));
+        let origin = SubregionOrigin::Subtype(Box::new(self.fields.trace.clone()));
         // LUB(&'static u8, &'a u8) == &RegionGLB('static, 'a) u8 == &'a u8
         Ok(self.fields.infcx.inner.borrow_mut().unwrap_region_constraints().glb_regions(
             self.tcx(),
@@ -139,6 +134,10 @@ impl<'combine, 'infcx, 'tcx> LatticeDir<'infcx, 'tcx> for Lub<'combine, 'infcx,
 }
 
 impl<'tcx> ObligationEmittingRelation<'tcx> for Lub<'_, '_, 'tcx> {
+    fn param_env(&self) -> ty::ParamEnv<'tcx> {
+        self.fields.param_env
+    }
+
     fn register_predicates(&mut self, obligations: impl IntoIterator<Item: ty::ToPredicate<'tcx>>) {
         self.fields.register_predicates(obligations);
     }
diff --git a/compiler/rustc_infer/src/infer/relate/mod.rs b/compiler/rustc_infer/src/infer/relate/mod.rs
new file mode 100644
index 000000000..f688c2b74
--- /dev/null
+++ b/compiler/rustc_infer/src/infer/relate/mod.rs
@@ -0,0 +1,12 @@
+//! This module contains the definitions of most `TypeRelation`s in the type system
+//! (except for some relations used for diagnostics and heuristics in the compiler).
+
+pub(super) mod combine;
+mod equate;
+pub(super) mod generalize;
+mod glb;
+mod higher_ranked;
+mod lattice;
+mod lub;
+pub mod nll;
+mod sub;
diff --git a/compiler/rustc_infer/src/infer/nll_relate/mod.rs b/compiler/rustc_infer/src/infer/relate/nll.rs
index c80491643..1ef865cfc 100644
--- a/compiler/rustc_infer/src/infer/nll_relate/mod.rs
+++ b/compiler/rustc_infer/src/infer/relate/nll.rs
@@ -30,8 +30,8 @@ use rustc_middle::ty::{self, InferConst, Ty, TyCtxt};
 use rustc_span::{Span, Symbol};
 use std::fmt::Debug;
 
-use crate::infer::combine::ObligationEmittingRelation;
-use crate::infer::generalize::{self, Generalization};
+use super::combine::ObligationEmittingRelation;
+use super::generalize::{self, Generalization};
 use crate::infer::InferCtxt;
 use crate::infer::{TypeVariableOrigin, TypeVariableOriginKind};
 use crate::traits::{Obligation, PredicateObligations};
@@ -113,9 +113,6 @@ pub trait TypeRelatingDelegate<'tcx> {
     fn forbid_inference_vars() -> bool;
 }
 
-#[derive(Copy, Clone)]
-struct UniversallyQuantified(bool);
-
 impl<'me, 'tcx, D> TypeRelating<'me, 'tcx, D>
 where
     D: TypeRelatingDelegate<'tcx>,
@@ -217,13 +214,17 @@ where
     }
 
     fn generalize(&mut self, ty: Ty<'tcx>, for_vid: ty::TyVid) -> RelateResult<'tcx, Ty<'tcx>> {
-        let Generalization { value: ty, needs_wf: _ } = generalize::generalize(
+        let Generalization { value_may_be_infer: ty, needs_wf: _ } = generalize::generalize(
             self.infcx,
             &mut self.delegate,
             ty,
             for_vid,
             self.ambient_variance,
         )?;
+
+        if ty.is_ty_var() {
+            span_bug!(self.delegate.span(), "occurs check failure in MIR typeck");
+        }
         Ok(ty)
     }
 
@@ -246,7 +247,9 @@ where
         let (a, b) = match (a.kind(), b.kind()) {
             (&ty::Alias(ty::Opaque, ..), _) => (a, generalize(b, false)?),
             (_, &ty::Alias(ty::Opaque, ..)) => (generalize(a, true)?, b),
-            _ => unreachable!(),
+            _ => unreachable!(
+                "expected at least one opaque type in `relate_opaques`, got {a} and {b}."
+            ),
         };
         let cause = ObligationCause::dummy_with_span(self.delegate.span());
         let obligations = self
@@ -428,10 +431,6 @@ where
         self.infcx.tcx
     }
 
-    fn param_env(&self) -> ty::ParamEnv<'tcx> {
-        self.delegate.param_env()
-    }
-
     fn tag(&self) -> &'static str {
         "nll::subtype"
     }
@@ -497,7 +496,7 @@ where
                     // shouldn't ever fail. Instead, it unconditionally emits an
                     // alias-relate goal.
                     assert!(!self.infcx.next_trait_solver());
-                    self.tcx().sess.delay_span_bug(
+                    self.tcx().sess.span_delayed_bug(
                         self.delegate.span(),
                         "failure to relate an opaque to itself should result in an error later on",
                     );
@@ -555,7 +554,7 @@ where
         match b.kind() {
             ty::ConstKind::Infer(InferConst::Var(_)) if D::forbid_inference_vars() => {
                 // Forbid inference variables in the RHS.
-                self.infcx.tcx.sess.delay_span_bug(
+                self.infcx.tcx.sess.span_delayed_bug(
                     self.delegate.span(),
                     format!("unexpected inference var {b:?}",),
                 );
@@ -667,6 +666,10 @@ impl<'tcx, D> ObligationEmittingRelation<'tcx> for TypeRelating<'_, 'tcx, D>
 where
     D: TypeRelatingDelegate<'tcx>,
 {
+    fn param_env(&self) -> ty::ParamEnv<'tcx> {
+        self.delegate.param_env()
+    }
+
     fn register_predicates(&mut self, obligations: impl IntoIterator<Item: ty::ToPredicate<'tcx>>) {
         self.delegate.register_obligations(
             obligations
@@ -706,7 +709,9 @@ where
             ),
             // FIXME(deferred_projection_equality): Implement this when we trigger it.
             // Probably just need to do nothing here.
-            ty::Variance::Bivariant => unreachable!(),
+            ty::Variance::Bivariant => {
+                unreachable!("cannot defer an alias-relate goal with Bivariant variance (yet?)")
+            }
         })]);
     }
 }
diff --git a/compiler/rustc_infer/src/infer/sub.rs b/compiler/rustc_infer/src/infer/relate/sub.rs
index 0c3bb633b..36876acd7 100644
--- a/compiler/rustc_infer/src/infer/sub.rs
+++ b/compiler/rustc_infer/src/infer/relate/sub.rs
@@ -1,7 +1,7 @@
 use super::combine::CombineFields;
-use super::{DefineOpaqueTypes, ObligationEmittingRelation, SubregionOrigin};
-
+use crate::infer::{DefineOpaqueTypes, ObligationEmittingRelation, SubregionOrigin};
 use crate::traits::{Obligation, PredicateObligations};
+
 use rustc_middle::ty::relate::{Cause, Relate, RelateResult, TypeRelation};
 use rustc_middle::ty::visit::TypeVisitableExt;
 use rustc_middle::ty::TyVar;
@@ -39,10 +39,6 @@ impl<'tcx> TypeRelation<'tcx> for Sub<'_, '_, 'tcx> {
         self.fields.infcx.tcx
     }
 
-    fn param_env(&self) -> ty::ParamEnv<'tcx> {
-        self.fields.param_env
-    }
-
     fn a_is_expected(&self) -> bool {
         self.a_is_expected
     }
@@ -203,6 +199,10 @@ impl<'tcx> TypeRelation<'tcx> for Sub<'_, '_, 'tcx> {
 }
 
 impl<'tcx> ObligationEmittingRelation<'tcx> for Sub<'_, '_, 'tcx> {
+    fn param_env(&self) -> ty::ParamEnv<'tcx> {
+        self.fields.param_env
+    }
+
     fn register_predicates(&mut self, obligations: impl IntoIterator<Item: ty::ToPredicate<'tcx>>) {
         self.fields.register_predicates(obligations);
     }
diff --git a/compiler/rustc_infer/src/infer/resolve.rs b/compiler/rustc_infer/src/infer/resolve.rs
index 18a9cb6b4..f317ccee6 100644
--- a/compiler/rustc_infer/src/infer/resolve.rs
+++ b/compiler/rustc_infer/src/infer/resolve.rs
@@ -134,7 +134,7 @@ impl<'a, 'tcx> TypeVisitor<TyCtxt<'tcx>> for UnresolvedTypeOrConstFinder<'a, 'tc
                 if let TypeVariableOrigin {
                     kind: TypeVariableOriginKind::TypeParameterDefinition(_, _),
                     span,
-                } = *ty_vars.var_origin(ty_vid)
+                } = ty_vars.var_origin(ty_vid)
                 {
                     Some(span)
                 } else {
@@ -260,3 +260,85 @@ impl<'a, 'tcx> FallibleTypeFolder<TyCtxt<'tcx>> for FullTypeResolver<'a, 'tcx> {
         }
     }
 }
+
+///////////////////////////////////////////////////////////////////////////
+// EAGER RESOLUTION
+
+/// Resolves ty, region, and const vars to their inferred values or their root vars.
+pub struct EagerResolver<'a, 'tcx> {
+    infcx: &'a InferCtxt<'tcx>,
+}
+
+impl<'a, 'tcx> EagerResolver<'a, 'tcx> {
+    pub fn new(infcx: &'a InferCtxt<'tcx>) -> Self {
+        EagerResolver { infcx }
+    }
+}
+
+impl<'tcx> TypeFolder<TyCtxt<'tcx>> for EagerResolver<'_, 'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
+        self.infcx.tcx
+    }
+
+    fn fold_ty(&mut self, t: Ty<'tcx>) -> Ty<'tcx> {
+        match *t.kind() {
+            ty::Infer(ty::TyVar(vid)) => match self.infcx.probe_ty_var(vid) {
+                Ok(t) => t.fold_with(self),
+                Err(_) => Ty::new_var(self.infcx.tcx, self.infcx.root_var(vid)),
+            },
+            ty::Infer(ty::IntVar(vid)) => self.infcx.opportunistic_resolve_int_var(vid),
+            ty::Infer(ty::FloatVar(vid)) => self.infcx.opportunistic_resolve_float_var(vid),
+            _ => {
+                if t.has_infer() {
+                    t.super_fold_with(self)
+                } else {
+                    t
+                }
+            }
+        }
+    }
+
+    fn fold_region(&mut self, r: ty::Region<'tcx>) -> ty::Region<'tcx> {
+        match *r {
+            ty::ReVar(vid) => self
+                .infcx
+                .inner
+                .borrow_mut()
+                .unwrap_region_constraints()
+                .opportunistic_resolve_var(self.infcx.tcx, vid),
+            _ => r,
+        }
+    }
+
+    fn fold_const(&mut self, c: ty::Const<'tcx>) -> ty::Const<'tcx> {
+        match c.kind() {
+            ty::ConstKind::Infer(ty::InferConst::Var(vid)) => {
+                // FIXME: we need to fold the ty too, I think.
+                match self.infcx.probe_const_var(vid) {
+                    Ok(c) => c.fold_with(self),
+                    Err(_) => {
+                        ty::Const::new_var(self.infcx.tcx, self.infcx.root_const_var(vid), c.ty())
+                    }
+                }
+            }
+            ty::ConstKind::Infer(ty::InferConst::EffectVar(vid)) => {
+                debug_assert_eq!(c.ty(), self.infcx.tcx.types.bool);
+                match self.infcx.probe_effect_var(vid) {
+                    Some(c) => c.as_const(self.infcx.tcx),
+                    None => ty::Const::new_infer(
+                        self.infcx.tcx,
+                        ty::InferConst::EffectVar(self.infcx.root_effect_var(vid)),
+                        self.infcx.tcx.types.bool,
+                    ),
+                }
+            }
+            _ => {
+                if c.has_infer() {
+                    c.super_fold_with(self)
+                } else {
+                    c
+                }
+            }
+        }
+    }
+}
diff --git a/compiler/rustc_infer/src/infer/type_variable.rs b/compiler/rustc_infer/src/infer/type_variable.rs
index bc83f8d3f..58b811015 100644
--- a/compiler/rustc_infer/src/infer/type_variable.rs
+++ b/compiler/rustc_infer/src/infer/type_variable.rs
@@ -1,4 +1,5 @@
 use rustc_hir::def_id::DefId;
+use rustc_index::IndexVec;
 use rustc_middle::ty::{self, Ty, TyVid};
 use rustc_span::symbol::Symbol;
 use rustc_span::Span;
@@ -11,14 +12,13 @@ use std::cmp;
 use std::marker::PhantomData;
 use std::ops::Range;
 
-use rustc_data_structures::undo_log::{Rollback, UndoLogs};
+use rustc_data_structures::undo_log::Rollback;
 
 /// Represents a single undo-able action that affects a type inference variable.
 #[derive(Clone)]
 pub(crate) enum UndoLog<'tcx> {
     EqRelation(sv::UndoLog<ut::Delegate<TyVidEqKey<'tcx>>>),
     SubRelation(sv::UndoLog<ut::Delegate<ty::TyVid>>),
-    Values(sv::UndoLog<Delegate>),
 }
 
 /// Convert from a specific kind of undo to the more general UndoLog
@@ -35,34 +35,19 @@ impl<'tcx> From<sv::UndoLog<ut::Delegate<ty::TyVid>>> for UndoLog<'tcx> {
     }
 }
 
-/// Convert from a specific kind of undo to the more general UndoLog
-impl<'tcx> From<sv::UndoLog<Delegate>> for UndoLog<'tcx> {
-    fn from(l: sv::UndoLog<Delegate>) -> Self {
-        UndoLog::Values(l)
-    }
-}
-
-/// Convert from a specific kind of undo to the more general UndoLog
-impl<'tcx> From<Instantiate> for UndoLog<'tcx> {
-    fn from(l: Instantiate) -> Self {
-        UndoLog::Values(sv::UndoLog::Other(l))
-    }
-}
-
 impl<'tcx> Rollback<UndoLog<'tcx>> for TypeVariableStorage<'tcx> {
     fn reverse(&mut self, undo: UndoLog<'tcx>) {
         match undo {
             UndoLog::EqRelation(undo) => self.eq_relations.reverse(undo),
             UndoLog::SubRelation(undo) => self.sub_relations.reverse(undo),
-            UndoLog::Values(undo) => self.values.reverse(undo),
         }
     }
 }
 
 #[derive(Clone)]
 pub struct TypeVariableStorage<'tcx> {
-    values: sv::SnapshotVecStorage<Delegate>,
-
+    /// The origins of each type variable.
+    values: IndexVec<TyVid, TypeVariableData>,
     /// Two variables are unified in `eq_relations` when we have a
     /// constraint `?X == ?Y`. This table also stores, for each key,
     /// the known value.
@@ -168,15 +153,10 @@ impl<'tcx> TypeVariableValue<'tcx> {
     }
 }
 
-#[derive(Clone)]
-pub(crate) struct Instantiate;
-
-pub(crate) struct Delegate;
-
 impl<'tcx> TypeVariableStorage<'tcx> {
     pub fn new() -> TypeVariableStorage<'tcx> {
         TypeVariableStorage {
-            values: sv::SnapshotVecStorage::new(),
+            values: Default::default(),
             eq_relations: ut::UnificationTableStorage::new(),
             sub_relations: ut::UnificationTableStorage::new(),
         }
@@ -194,6 +174,11 @@ impl<'tcx> TypeVariableStorage<'tcx> {
     pub(crate) fn eq_relations_ref(&self) -> &ut::UnificationTableStorage<TyVidEqKey<'tcx>> {
         &self.eq_relations
     }
+
+    pub(super) fn finalize_rollback(&mut self) {
+        debug_assert!(self.values.len() >= self.eq_relations.len());
+        self.values.truncate(self.eq_relations.len());
+    }
 }
 
 impl<'tcx> TypeVariableTable<'_, 'tcx> {
@@ -201,8 +186,8 @@ impl<'tcx> TypeVariableTable<'_, 'tcx> {
     ///
     /// Note that this function does not return care whether
     /// `vid` has been unified with something else or not.
-    pub fn var_origin(&self, vid: ty::TyVid) -> &TypeVariableOrigin {
-        &self.storage.values.get(vid.as_usize()).origin
+    pub fn var_origin(&self, vid: ty::TyVid) -> TypeVariableOrigin {
+        self.storage.values[vid].origin
     }
 
     /// Records that `a == b`, depending on `dir`.
@@ -229,20 +214,14 @@ impl<'tcx> TypeVariableTable<'_, 'tcx> {
     /// Precondition: `vid` must not have been previously instantiated.
     pub fn instantiate(&mut self, vid: ty::TyVid, ty: Ty<'tcx>) {
         let vid = self.root_var(vid);
+        debug_assert!(!ty.is_ty_var(), "instantiating ty var with var: {vid:?} {ty:?}");
         debug_assert!(self.probe(vid).is_unknown());
         debug_assert!(
             self.eq_relations().probe_value(vid).is_unknown(),
-            "instantiating type variable `{:?}` twice: new-value = {:?}, old-value={:?}",
-            vid,
-            ty,
+            "instantiating type variable `{vid:?}` twice: new-value = {ty:?}, old-value={:?}",
             self.eq_relations().probe_value(vid)
         );
         self.eq_relations().union_value(vid, TypeVariableValue::Known { value: ty });
-
-        // Hack: we only need this so that `types_escaping_snapshot`
-        // can see what has been unified; see the Delegate impl for
-        // more details.
-        self.undo_log.push(Instantiate);
     }
 
     /// Creates a new type variable.
@@ -263,14 +242,14 @@ impl<'tcx> TypeVariableTable<'_, 'tcx> {
         let eq_key = self.eq_relations().new_key(TypeVariableValue::Unknown { universe });
 
         let sub_key = self.sub_relations().new_key(());
-        assert_eq!(eq_key.vid, sub_key);
+        debug_assert_eq!(eq_key.vid, sub_key);
 
-        let index = self.values().push(TypeVariableData { origin });
-        assert_eq!(eq_key.vid.as_u32(), index as u32);
+        let index = self.storage.values.push(TypeVariableData { origin });
+        debug_assert_eq!(eq_key.vid, index);
 
         debug!("new_var(index={:?}, universe={:?}, origin={:?})", eq_key.vid, universe, origin);
 
-        eq_key.vid
+        index
     }
 
     /// Returns the number of type variables created thus far.
@@ -331,13 +310,6 @@ impl<'tcx> TypeVariableTable<'_, 'tcx> {
     }
 
     #[inline]
-    fn values(
-        &mut self,
-    ) -> sv::SnapshotVec<Delegate, &mut Vec<TypeVariableData>, &mut InferCtxtUndoLogs<'tcx>> {
-        self.storage.values.with_log(self.undo_log)
-    }
-
-    #[inline]
     fn eq_relations(&mut self) -> super::UnificationTable<'_, 'tcx, TyVidEqKey<'tcx>> {
         self.storage.eq_relations.with_log(self.undo_log)
     }
@@ -355,16 +327,14 @@ impl<'tcx> TypeVariableTable<'_, 'tcx> {
         let range = TyVid::from_usize(value_count)..TyVid::from_usize(self.num_vars());
         (
             range.start..range.end,
-            (range.start.as_usize()..range.end.as_usize())
-                .map(|index| self.storage.values.get(index).origin)
-                .collect(),
+            (range.start..range.end).map(|index| self.var_origin(index)).collect(),
         )
     }
 
     /// Returns indices of all variables that are not yet
     /// instantiated.
-    pub fn unsolved_variables(&mut self) -> Vec<ty::TyVid> {
-        (0..self.storage.values.len())
+    pub fn unresolved_variables(&mut self) -> Vec<ty::TyVid> {
+        (0..self.num_vars())
             .filter_map(|i| {
                 let vid = ty::TyVid::from_usize(i);
                 match self.probe(vid) {
@@ -376,26 +346,6 @@ impl<'tcx> TypeVariableTable<'_, 'tcx> {
     }
 }
 
-impl sv::SnapshotVecDelegate for Delegate {
-    type Value = TypeVariableData;
-    type Undo = Instantiate;
-
-    fn reverse(_values: &mut Vec<TypeVariableData>, _action: Instantiate) {
-        // We don't actually have to *do* anything to reverse an
-        // instantiation; the value for a variable is stored in the
-        // `eq_relations` and hence its rollback code will handle
-        // it. In fact, we could *almost* just remove the
-        // `SnapshotVec` entirely, except that we would have to
-        // reproduce *some* of its logic, since we want to know which
-        // type variables have been instantiated since the snapshot
-        // was started, so we can implement `types_escaping_snapshot`.
-        //
-        // (If we extended the `UnificationTable` to let us see which
-        // values have been unified and so forth, that might also
-        // suffice.)
-    }
-}
-
 ///////////////////////////////////////////////////////////////////////////
 
 /// These structs (a newtyped TyVid) are used as the unification key
diff --git a/compiler/rustc_infer/src/infer/undo_log.rs b/compiler/rustc_infer/src/infer/undo_log.rs
index 565573051..be02452d8 100644
--- a/compiler/rustc_infer/src/infer/undo_log.rs
+++ b/compiler/rustc_infer/src/infer/undo_log.rs
@@ -32,7 +32,7 @@ pub(crate) enum UndoLog<'tcx> {
 }
 
 macro_rules! impl_from {
-    ($($ctor: ident ($ty: ty),)*) => {
+    ($($ctor:ident ($ty:ty),)*) => {
         $(
         impl<'tcx> From<$ty> for UndoLog<'tcx> {
             fn from(x: $ty) -> Self {
@@ -50,8 +50,6 @@ impl_from! {
 
     TypeVariables(sv::UndoLog<ut::Delegate<type_variable::TyVidEqKey<'tcx>>>),
     TypeVariables(sv::UndoLog<ut::Delegate<ty::TyVid>>),
-    TypeVariables(sv::UndoLog<type_variable::Delegate>),
-    TypeVariables(type_variable::Instantiate),
 
     IntUnificationTable(sv::UndoLog<ut::Delegate<ty::IntVid>>),
 
@@ -140,6 +138,8 @@ impl<'tcx> InferCtxtInner<'tcx> {
             self.reverse(undo);
         }
 
+        self.type_variable_storage.finalize_rollback();
+
         if self.undo_log.num_open_snapshots == 1 {
             // After the root snapshot the undo log should be empty.
             assert!(snapshot.undo_len == 0);
diff --git a/compiler/rustc_infer/src/lib.rs b/compiler/rustc_infer/src/lib.rs
index 4a6d1bc68..221b78048 100644
--- a/compiler/rustc_infer/src/lib.rs
+++ b/compiler/rustc_infer/src/lib.rs
@@ -13,9 +13,9 @@
 //! This API is completely unstable and subject to change.
 
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
-#![cfg_attr(not(bootstrap), allow(internal_features))]
+#![doc(rust_logo)]
+#![feature(rustdoc_internals)]
+#![allow(internal_features)]
 #![feature(associated_type_bounds)]
 #![feature(box_patterns)]
 #![feature(control_flow_enum)]
@@ -37,11 +37,8 @@ extern crate tracing;
 #[macro_use]
 extern crate rustc_middle;
 
-use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
-use rustc_fluent_macro::fluent_messages;
-
 mod errors;
 pub mod infer;
 pub mod traits;
 
-fluent_messages! { "../messages.ftl" }
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
diff --git a/compiler/rustc_infer/src/traits/error_reporting/mod.rs b/compiler/rustc_infer/src/traits/error_reporting/mod.rs
index 329660119..b3cfd843a 100644
--- a/compiler/rustc_infer/src/traits/error_reporting/mod.rs
+++ b/compiler/rustc_infer/src/traits/error_reporting/mod.rs
@@ -101,12 +101,19 @@ pub fn report_object_safety_error<'tcx>(
          to be resolvable dynamically; for more information visit \
          <https://doc.rust-lang.org/reference/items/traits.html#object-safety>",
     );
+
+    // Only provide the help if its a local trait, otherwise it's not actionable.
     if trait_span.is_some() {
         let mut reported_violations: Vec<_> = reported_violations.into_iter().collect();
         reported_violations.sort();
-        for violation in reported_violations {
-            // Only provide the help if its a local trait, otherwise it's not actionable.
-            violation.solution(&mut err);
+
+        let mut potential_solutions: Vec<_> =
+            reported_violations.into_iter().map(|violation| violation.solution()).collect();
+        potential_solutions.sort();
+        // Allows us to skip suggesting that the same item should be moved to another trait multiple times.
+        potential_solutions.dedup();
+        for solution in potential_solutions {
+            solution.add_to(&mut err);
         }
     }
 
diff --git a/compiler/rustc_infer/src/traits/mod.rs b/compiler/rustc_infer/src/traits/mod.rs
index a26e676c5..b9be17891 100644
--- a/compiler/rustc_infer/src/traits/mod.rs
+++ b/compiler/rustc_infer/src/traits/mod.rs
@@ -135,7 +135,7 @@ pub enum FulfillmentErrorCode<'tcx> {
     CodeSubtypeError(ExpectedFound<Ty<'tcx>>, TypeError<'tcx>), // always comes from a SubtypePredicate
     CodeConstEquateError(ExpectedFound<Const<'tcx>>, TypeError<'tcx>),
     CodeAmbiguity {
-        /// Overflow reported from the new solver `-Ztrait-solver=next`, which will
+        /// Overflow reported from the new solver `-Znext-solver`, which will
         /// be reported as an regular error as opposed to a fatal error.
         overflow: bool,
     },
diff --git a/compiler/rustc_infer/src/traits/util.rs b/compiler/rustc_infer/src/traits/util.rs
index 3c566e0dd..50190058a 100644
--- a/compiler/rustc_infer/src/traits/util.rs
+++ b/compiler/rustc_infer/src/traits/util.rs
@@ -261,9 +261,14 @@ impl<'tcx, O: Elaboratable<'tcx>> Elaborator<'tcx, O> {
     fn elaborate(&mut self, elaboratable: &O) {
         let tcx = self.visited.tcx;
 
-        let bound_predicate = elaboratable.predicate().kind();
-        match bound_predicate.skip_binder() {
-            ty::PredicateKind::Clause(ty::ClauseKind::Trait(data)) => {
+        // We only elaborate clauses.
+        let Some(clause) = elaboratable.predicate().as_clause() else {
+            return;
+        };
+
+        let bound_clause = clause.kind();
+        match bound_clause.skip_binder() {
+            ty::ClauseKind::Trait(data) => {
                 // Negative trait bounds do not imply any supertrait bounds
                 if data.polarity == ty::ImplPolarity::Negative {
                     return;
@@ -280,52 +285,16 @@ impl<'tcx, O: Elaboratable<'tcx>> Elaborator<'tcx, O> {
                 let obligations =
                     predicates.predicates.iter().enumerate().map(|(index, &(clause, span))| {
                         elaboratable.child_with_derived_cause(
-                            clause.subst_supertrait(tcx, &bound_predicate.rebind(data.trait_ref)),
+                            clause.subst_supertrait(tcx, &bound_clause.rebind(data.trait_ref)),
                             span,
-                            bound_predicate.rebind(data),
+                            bound_clause.rebind(data),
                             index,
                         )
                     });
                 debug!(?data, ?obligations, "super_predicates");
                 self.extend_deduped(obligations);
             }
-            ty::PredicateKind::Clause(ty::ClauseKind::WellFormed(..)) => {
-                // Currently, we do not elaborate WF predicates,
-                // although we easily could.
-            }
-            ty::PredicateKind::ObjectSafe(..) => {
-                // Currently, we do not elaborate object-safe
-                // predicates.
-            }
-            ty::PredicateKind::Subtype(..) => {
-                // Currently, we do not "elaborate" predicates like `X <: Y`,
-                // though conceivably we might.
-            }
-            ty::PredicateKind::Coerce(..) => {
-                // Currently, we do not "elaborate" predicates like `X -> Y`,
-                // though conceivably we might.
-            }
-            ty::PredicateKind::Clause(ty::ClauseKind::Projection(..)) => {
-                // Nothing to elaborate in a projection predicate.
-            }
-            ty::PredicateKind::ClosureKind(..) => {
-                // Nothing to elaborate when waiting for a closure's kind to be inferred.
-            }
-            ty::PredicateKind::Clause(ty::ClauseKind::ConstEvaluatable(..)) => {
-                // Currently, we do not elaborate const-evaluatable
-                // predicates.
-            }
-            ty::PredicateKind::ConstEquate(..) => {
-                // Currently, we do not elaborate const-equate
-                // predicates.
-            }
-            ty::PredicateKind::Clause(ty::ClauseKind::RegionOutlives(..)) => {
-                // Nothing to elaborate from `'a: 'b`.
-            }
-            ty::PredicateKind::Clause(ty::ClauseKind::TypeOutlives(ty::OutlivesPredicate(
-                ty_max,
-                r_min,
-            ))) => {
+            ty::ClauseKind::TypeOutlives(ty::OutlivesPredicate(ty_max, r_min)) => {
                 // We know that `T: 'a` for some type `T`. We can
                 // often elaborate this. For example, if we know that
                 // `[U]: 'a`, that implies that `U: 'a`. Similarly, if
@@ -340,7 +309,7 @@ impl<'tcx, O: Elaboratable<'tcx>> Elaborator<'tcx, O> {
                 // consider this as evidence that `T: 'static`, but
                 // I'm a bit wary of such constructions and so for now
                 // I want to be conservative. --nmatsakis
-                if r_min.is_late_bound() {
+                if r_min.is_bound() {
                     return;
                 }
 
@@ -351,7 +320,7 @@ impl<'tcx, O: Elaboratable<'tcx>> Elaborator<'tcx, O> {
                         .into_iter()
                         .filter_map(|component| match component {
                             Component::Region(r) => {
-                                if r.is_late_bound() {
+                                if r.is_bound() {
                                     None
                                 } else {
                                     Some(ty::ClauseKind::RegionOutlives(ty::OutlivesPredicate(
@@ -365,6 +334,11 @@ impl<'tcx, O: Elaboratable<'tcx>> Elaborator<'tcx, O> {
                                 Some(ty::ClauseKind::TypeOutlives(ty::OutlivesPredicate(ty, r_min)))
                             }
 
+                            Component::Placeholder(p) => {
+                                let ty = Ty::new_placeholder(tcx, p);
+                                Some(ty::ClauseKind::TypeOutlives(ty::OutlivesPredicate(ty, r_min)))
+                            }
+
                             Component::UnresolvedInferenceVariable(_) => None,
 
                             Component::Alias(alias_ty) => {
@@ -383,15 +357,25 @@ impl<'tcx, O: Elaboratable<'tcx>> Elaborator<'tcx, O> {
                             }
                         })
                         .map(|clause| {
-                            elaboratable.child(bound_predicate.rebind(clause).to_predicate(tcx))
+                            elaboratable.child(bound_clause.rebind(clause).to_predicate(tcx))
                         }),
                 );
             }
-            ty::PredicateKind::Ambiguous => {}
-            ty::PredicateKind::AliasRelate(..) => {
-                // No
+            ty::ClauseKind::RegionOutlives(..) => {
+                // Nothing to elaborate from `'a: 'b`.
+            }
+            ty::ClauseKind::WellFormed(..) => {
+                // Currently, we do not elaborate WF predicates,
+                // although we easily could.
+            }
+            ty::ClauseKind::Projection(..) => {
+                // Nothing to elaborate in a projection predicate.
+            }
+            ty::ClauseKind::ConstEvaluatable(..) => {
+                // Currently, we do not elaborate const-evaluatable
+                // predicates.
             }
-            ty::PredicateKind::Clause(ty::ClauseKind::ConstArgHasType(..)) => {
+            ty::ClauseKind::ConstArgHasType(..) => {
                 // Nothing to elaborate
             }
         }
diff --git a/compiler/rustc_interface/Cargo.toml b/compiler/rustc_interface/Cargo.toml
index fd587e53f..319e81758 100644
--- a/compiler/rustc_interface/Cargo.toml
+++ b/compiler/rustc_interface/Cargo.toml
@@ -40,6 +40,7 @@ rustc_privacy = { path = "../rustc_privacy" }
 rustc_query_impl = { path = "../rustc_query_impl" }
 rustc_query_system = { path = "../rustc_query_system" }
 rustc_resolve = { path = "../rustc_resolve" }
+rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
 rustc_symbol_mangling = { path = "../rustc_symbol_mangling" }
diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs
index c4962707f..d58d60fc8 100644
--- a/compiler/rustc_interface/src/interface.rs
+++ b/compiler/rustc_interface/src/interface.rs
@@ -1,26 +1,24 @@
 use crate::util;
 
 use rustc_ast::token;
-use rustc_ast::{self as ast, LitKind, MetaItemKind};
+use rustc_ast::{LitKind, MetaItemKind};
 use rustc_codegen_ssa::traits::CodegenBackend;
 use rustc_data_structures::defer;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::stable_hasher::StableHasher;
 use rustc_data_structures::sync::Lrc;
 use rustc_errors::registry::Registry;
-use rustc_errors::{ErrorGuaranteed, Handler};
+use rustc_errors::{DiagCtxt, ErrorGuaranteed};
 use rustc_lint::LintStore;
+use rustc_middle::ty;
 use rustc_middle::util::Providers;
-use rustc_middle::{bug, ty};
 use rustc_parse::maybe_new_parser_from_source_str;
 use rustc_query_impl::QueryCtxt;
 use rustc_query_system::query::print_query_stack;
-use rustc_session::config::{
-    self, Cfg, CheckCfg, ExpectedValues, Input, OutFileName, OutputFilenames,
-};
+use rustc_session::config::{self, Cfg, CheckCfg, ExpectedValues, Input, OutFileName};
 use rustc_session::filesearch::sysroot_candidates;
 use rustc_session::parse::ParseSess;
-use rustc_session::{lint, CompilerIO, EarlyErrorHandler, Session};
+use rustc_session::{lint, CompilerIO, EarlyDiagCtxt, Session};
 use rustc_span::source_map::FileLoader;
 use rustc_span::symbol::sym;
 use rustc_span::FileName;
@@ -38,33 +36,13 @@ pub type Result<T> = result::Result<T, ErrorGuaranteed>;
 /// Can be used to run `rustc_interface` queries.
 /// Created by passing [`Config`] to [`run_compiler`].
 pub struct Compiler {
-    pub(crate) sess: Lrc<Session>,
-    codegen_backend: Lrc<dyn CodegenBackend>,
-    pub(crate) register_lints: Option<Box<dyn Fn(&Session, &mut LintStore) + Send + Sync>>,
+    pub sess: Session,
+    pub codegen_backend: Box<dyn CodegenBackend>,
     pub(crate) override_queries: Option<fn(&Session, &mut Providers)>,
 }
 
-impl Compiler {
-    pub fn session(&self) -> &Lrc<Session> {
-        &self.sess
-    }
-    pub fn codegen_backend(&self) -> &Lrc<dyn CodegenBackend> {
-        &self.codegen_backend
-    }
-    pub fn register_lints(&self) -> &Option<Box<dyn Fn(&Session, &mut LintStore) + Send + Sync>> {
-        &self.register_lints
-    }
-    pub fn build_output_filenames(
-        &self,
-        sess: &Session,
-        attrs: &[ast::Attribute],
-    ) -> OutputFilenames {
-        util::build_output_filenames(attrs, sess)
-    }
-}
-
 /// Converts strings provided as `--cfg [cfgspec]` into a `Cfg`.
-pub(crate) fn parse_cfg(handler: &EarlyErrorHandler, cfgs: Vec<String>) -> Cfg {
+pub(crate) fn parse_cfg(dcx: &DiagCtxt, cfgs: Vec<String>) -> Cfg {
     cfgs.into_iter()
         .map(|s| {
             let sess = ParseSess::with_silent_emitter(Some(format!(
@@ -74,10 +52,13 @@ pub(crate) fn parse_cfg(handler: &EarlyErrorHandler, cfgs: Vec<String>) -> Cfg {
 
             macro_rules! error {
                 ($reason: expr) => {
-                    handler.early_error(format!(
+                    #[allow(rustc::untranslatable_diagnostic)]
+                    #[allow(rustc::diagnostic_outside_of_impl)]
+                    dcx.struct_fatal(format!(
                         concat!("invalid `--cfg` argument: `{}` (", $reason, ")"),
                         s
-                    ));
+                    ))
+                    .emit();
                 };
             }
 
@@ -119,14 +100,13 @@ pub(crate) fn parse_cfg(handler: &EarlyErrorHandler, cfgs: Vec<String>) -> Cfg {
 }
 
 /// Converts strings provided as `--check-cfg [specs]` into a `CheckCfg`.
-pub(crate) fn parse_check_cfg(handler: &EarlyErrorHandler, specs: Vec<String>) -> CheckCfg {
+pub(crate) fn parse_check_cfg(dcx: &DiagCtxt, specs: Vec<String>) -> CheckCfg {
     // If any --check-cfg is passed then exhaustive_values and exhaustive_names
     // are enabled by default.
     let exhaustive_names = !specs.is_empty();
     let exhaustive_values = !specs.is_empty();
     let mut check_cfg = CheckCfg { exhaustive_names, exhaustive_values, ..CheckCfg::default() };
 
-    let mut old_syntax = None;
     for s in specs {
         let sess = ParseSess::with_silent_emitter(Some(format!(
             "this error occurred on the command line: `--check-cfg={s}`"
@@ -135,10 +115,13 @@ pub(crate) fn parse_check_cfg(handler: &EarlyErrorHandler, specs: Vec<String>) -
 
         macro_rules! error {
             ($reason:expr) => {
-                handler.early_error(format!(
+                #[allow(rustc::untranslatable_diagnostic)]
+                #[allow(rustc::diagnostic_outside_of_impl)]
+                dcx.struct_fatal(format!(
                     concat!("invalid `--check-cfg` argument: `{}` (", $reason, ")"),
                     s
                 ))
+                .emit()
             };
         }
 
@@ -164,162 +147,101 @@ pub(crate) fn parse_check_cfg(handler: &EarlyErrorHandler, specs: Vec<String>) -
             expected_error();
         };
 
-        let mut set_old_syntax = || {
-            // defaults are flipped for the old syntax
-            if old_syntax == None {
-                check_cfg.exhaustive_names = false;
-                check_cfg.exhaustive_values = false;
-            }
-            old_syntax = Some(true);
-        };
-
-        if meta_item.has_name(sym::names) {
-            set_old_syntax();
-
-            check_cfg.exhaustive_names = true;
-            for arg in args {
-                if arg.is_word() && let Some(ident) = arg.ident() {
-                    check_cfg.expecteds.entry(ident.name).or_insert(ExpectedValues::Any);
-                } else {
-                    error!("`names()` arguments must be simple identifiers");
-                }
-            }
-        } else if meta_item.has_name(sym::values) {
-            set_old_syntax();
-
-            if let Some((name, values)) = args.split_first() {
-                if name.is_word() && let Some(ident) = name.ident() {
-                    let expected_values = check_cfg
-                        .expecteds
-                        .entry(ident.name)
-                        .and_modify(|expected_values| match expected_values {
-                            ExpectedValues::Some(_) => {}
-                            ExpectedValues::Any => {
-                                // handle the case where names(...) was done
-                                // before values by changing to a list
-                                *expected_values = ExpectedValues::Some(FxHashSet::default());
-                            }
-                        })
-                        .or_insert_with(|| ExpectedValues::Some(FxHashSet::default()));
+        if !meta_item.has_name(sym::cfg) {
+            expected_error();
+        }
 
-                    let ExpectedValues::Some(expected_values) = expected_values else {
-                        bug!("`expected_values` should be a list a values")
-                    };
+        let mut names = Vec::new();
+        let mut values: FxHashSet<_> = Default::default();
 
-                    for val in values {
-                        if let Some(LitKind::Str(s, _)) = val.lit().map(|lit| &lit.kind) {
-                            expected_values.insert(Some(*s));
-                        } else {
-                            error!("`values()` arguments must be string literals");
-                        }
-                    }
+        let mut any_specified = false;
+        let mut values_specified = false;
+        let mut values_any_specified = false;
 
-                    if values.is_empty() {
-                        expected_values.insert(None);
-                    }
-                } else {
-                    error!("`values()` first argument must be a simple identifier");
+        for arg in args {
+            if arg.is_word()
+                && let Some(ident) = arg.ident()
+            {
+                if values_specified {
+                    error!("`cfg()` names cannot be after values");
                 }
-            } else if args.is_empty() {
-                check_cfg.exhaustive_values = true;
-            } else {
-                expected_error();
-            }
-        } else if meta_item.has_name(sym::cfg) {
-            old_syntax = Some(false);
-
-            let mut names = Vec::new();
-            let mut values: FxHashSet<_> = Default::default();
-
-            let mut any_specified = false;
-            let mut values_specified = false;
-            let mut values_any_specified = false;
-
-            for arg in args {
-                if arg.is_word() && let Some(ident) = arg.ident() {
-                    if values_specified {
-                        error!("`cfg()` names cannot be after values");
-                    }
-                    names.push(ident);
-                } else if arg.has_name(sym::any) && let Some(args) = arg.meta_item_list() {
-                    if any_specified {
-                        error!("`any()` cannot be specified multiple times");
-                    }
-                    any_specified = true;
-                    if !args.is_empty() {
-                        error!("`any()` must be empty");
-                    }
-                } else if arg.has_name(sym::values) && let Some(args) = arg.meta_item_list() {
-                    if names.is_empty() {
-                        error!("`values()` cannot be specified before the names");
-                    } else if values_specified {
-                        error!("`values()` cannot be specified multiple times");
-                    }
-                    values_specified = true;
-
-                    for arg in args {
-                        if let Some(LitKind::Str(s, _)) = arg.lit().map(|lit| &lit.kind) {
-                            values.insert(Some(*s));
-                        } else if arg.has_name(sym::any) && let Some(args) = arg.meta_item_list() {
-                            if values_any_specified {
-                                error!("`any()` in `values()` cannot be specified multiple times");
-                            }
-                            values_any_specified = true;
-                            if !args.is_empty() {
-                                error!("`any()` must be empty");
-                            }
-                        } else {
-                            error!("`values()` arguments must be string literals or `any()`");
+                names.push(ident);
+            } else if arg.has_name(sym::any)
+                && let Some(args) = arg.meta_item_list()
+            {
+                if any_specified {
+                    error!("`any()` cannot be specified multiple times");
+                }
+                any_specified = true;
+                if !args.is_empty() {
+                    error!("`any()` must be empty");
+                }
+            } else if arg.has_name(sym::values)
+                && let Some(args) = arg.meta_item_list()
+            {
+                if names.is_empty() {
+                    error!("`values()` cannot be specified before the names");
+                } else if values_specified {
+                    error!("`values()` cannot be specified multiple times");
+                }
+                values_specified = true;
+
+                for arg in args {
+                    if let Some(LitKind::Str(s, _)) = arg.lit().map(|lit| &lit.kind) {
+                        values.insert(Some(*s));
+                    } else if arg.has_name(sym::any)
+                        && let Some(args) = arg.meta_item_list()
+                    {
+                        if values_any_specified {
+                            error!("`any()` in `values()` cannot be specified multiple times");
                         }
+                        values_any_specified = true;
+                        if !args.is_empty() {
+                            error!("`any()` must be empty");
+                        }
+                    } else {
+                        error!("`values()` arguments must be string literals or `any()`");
                     }
-                } else {
-                    error!(
-                        "`cfg()` arguments must be simple identifiers, `any()` or `values(...)`"
-                    );
                 }
+            } else {
+                error!("`cfg()` arguments must be simple identifiers, `any()` or `values(...)`");
             }
+        }
 
-            if values.is_empty() && !values_any_specified && !any_specified {
-                values.insert(None);
-            } else if !values.is_empty() && values_any_specified {
-                error!(
-                    "`values()` arguments cannot specify string literals and `any()` at the same time"
-                );
-            }
+        if values.is_empty() && !values_any_specified && !any_specified {
+            values.insert(None);
+        } else if !values.is_empty() && values_any_specified {
+            error!(
+                "`values()` arguments cannot specify string literals and `any()` at the same time"
+            );
+        }
 
-            if any_specified {
-                if names.is_empty()
-                    && values.is_empty()
-                    && !values_specified
-                    && !values_any_specified
-                {
-                    check_cfg.exhaustive_names = false;
-                } else {
-                    error!("`cfg(any())` can only be provided in isolation");
-                }
+        if any_specified {
+            if names.is_empty() && values.is_empty() && !values_specified && !values_any_specified {
+                check_cfg.exhaustive_names = false;
             } else {
-                for name in names {
-                    check_cfg
-                        .expecteds
-                        .entry(name.name)
-                        .and_modify(|v| match v {
-                            ExpectedValues::Some(v) if !values_any_specified => {
-                                v.extend(values.clone())
-                            }
-                            ExpectedValues::Some(_) => *v = ExpectedValues::Any,
-                            ExpectedValues::Any => {}
-                        })
-                        .or_insert_with(|| {
-                            if values_any_specified {
-                                ExpectedValues::Any
-                            } else {
-                                ExpectedValues::Some(values.clone())
-                            }
-                        });
-                }
+                error!("`cfg(any())` can only be provided in isolation");
             }
         } else {
-            expected_error();
+            for name in names {
+                check_cfg
+                    .expecteds
+                    .entry(name.name)
+                    .and_modify(|v| match v {
+                        ExpectedValues::Some(v) if !values_any_specified => {
+                            v.extend(values.clone())
+                        }
+                        ExpectedValues::Some(_) => *v = ExpectedValues::Any,
+                        ExpectedValues::Any => {}
+                    })
+                    .or_insert_with(|| {
+                        if values_any_specified {
+                            ExpectedValues::Any
+                        } else {
+                            ExpectedValues::Some(values.clone())
+                        }
+                    });
+            }
         }
     }
 
@@ -392,19 +314,23 @@ pub fn run_compiler<R: Send>(config: Config, f: impl FnOnce(&Compiler) -> R + Se
     // Set parallel mode before thread pool creation, which will create `Lock`s.
     rustc_data_structures::sync::set_dyn_thread_safe_mode(config.opts.unstable_opts.threads > 1);
 
+    // Check jobserver before run_in_thread_pool_with_globals, which call jobserver::acquire_thread
+    let early_dcx = EarlyDiagCtxt::new(config.opts.error_format);
+    early_dcx.initialize_checked_jobserver();
+
     util::run_in_thread_pool_with_globals(
         config.opts.edition,
         config.opts.unstable_opts.threads,
         || {
             crate::callbacks::setup_callbacks();
 
-            let handler = EarlyErrorHandler::new(config.opts.error_format);
+            let early_dcx = EarlyDiagCtxt::new(config.opts.error_format);
 
             let codegen_backend = if let Some(make_codegen_backend) = config.make_codegen_backend {
                 make_codegen_backend(&config.opts)
             } else {
                 util::get_codegen_backend(
-                    &handler,
+                    &early_dcx,
                     &config.opts.maybe_sysroot,
                     config.opts.unstable_opts.codegen_backend.as_deref(),
                 )
@@ -421,7 +347,7 @@ pub fn run_compiler<R: Send>(config: Config, f: impl FnOnce(&Compiler) -> R + Se
             ) {
                 Ok(bundle) => bundle,
                 Err(e) => {
-                    handler.early_error(format!("failed to load fluent bundle: {e}"));
+                    early_dcx.early_error(format!("failed to load fluent bundle: {e}"));
                 }
             };
 
@@ -432,7 +358,7 @@ pub fn run_compiler<R: Send>(config: Config, f: impl FnOnce(&Compiler) -> R + Se
             let target_override = codegen_backend.target_override(&config.opts);
 
             let mut sess = rustc_session::build_session(
-                &handler,
+                early_dcx,
                 config.opts,
                 CompilerIO {
                     input: config.input,
@@ -454,12 +380,12 @@ pub fn run_compiler<R: Send>(config: Config, f: impl FnOnce(&Compiler) -> R + Se
 
             codegen_backend.init(&sess);
 
-            let cfg = parse_cfg(&handler, config.crate_cfg);
+            let cfg = parse_cfg(&sess.dcx(), config.crate_cfg);
             let mut cfg = config::build_configuration(&sess, cfg);
             util::add_configuration(&mut cfg, &mut sess, &*codegen_backend);
             sess.parse_sess.config = cfg;
 
-            let mut check_cfg = parse_check_cfg(&handler, config.crate_check_cfg);
+            let mut check_cfg = parse_check_cfg(&sess.dcx(), config.crate_check_cfg);
             check_cfg.fill_well_known(&sess.target);
             sess.parse_sess.check_config = check_cfg;
 
@@ -473,12 +399,18 @@ pub fn run_compiler<R: Send>(config: Config, f: impl FnOnce(&Compiler) -> R + Se
                 sess.opts.untracked_state_hash = hasher.finish()
             }
 
-            let compiler = Compiler {
-                sess: Lrc::new(sess),
-                codegen_backend: Lrc::from(codegen_backend),
-                register_lints: config.register_lints,
-                override_queries: config.override_queries,
-            };
+            // Even though the session holds the lint store, we can't build the
+            // lint store until after the session exists. And we wait until now
+            // so that `register_lints` sees the fully initialized session.
+            let mut lint_store = rustc_lint::new_lint_store(sess.enable_internal_lints());
+            if let Some(register_lints) = config.register_lints.as_deref() {
+                register_lints(&sess, &mut lint_store);
+                sess.registered_lints = true;
+            }
+            sess.lint_store = Some(Lrc::new(lint_store));
+
+            let compiler =
+                Compiler { sess, codegen_backend, override_queries: config.override_queries };
 
             rustc_span::set_source_map(compiler.sess.parse_sess.clone_source_map(), move || {
                 let r = {
@@ -499,21 +431,21 @@ pub fn run_compiler<R: Send>(config: Config, f: impl FnOnce(&Compiler) -> R + Se
 }
 
 pub fn try_print_query_stack(
-    handler: &Handler,
+    dcx: &DiagCtxt,
     num_frames: Option<usize>,
     file: Option<std::fs::File>,
 ) {
     eprintln!("query stack during panic:");
 
     // Be careful relying on global state here: this code is called from
-    // a panic hook, which means that the global `Handler` may be in a weird
+    // a panic hook, which means that the global `DiagCtxt` may be in a weird
     // state if it was responsible for triggering the panic.
     let i = ty::tls::with_context_opt(|icx| {
         if let Some(icx) = icx {
             ty::print::with_no_queries!(print_query_stack(
                 QueryCtxt::new(icx.tcx),
                 icx.query,
-                handler,
+                dcx,
                 num_frames,
                 file,
             ))
diff --git a/compiler/rustc_interface/src/lib.rs b/compiler/rustc_interface/src/lib.rs
index ffa2667a3..cfa464478 100644
--- a/compiler/rustc_interface/src/lib.rs
+++ b/compiler/rustc_interface/src/lib.rs
@@ -6,16 +6,12 @@
 #![feature(let_chains)]
 #![feature(try_blocks)]
 #![recursion_limit = "256"]
-#![allow(rustc::potential_query_instability)]
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
 
 #[macro_use]
 extern crate tracing;
 
-use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
-use rustc_fluent_macro::fluent_messages;
-
 mod callbacks;
 mod errors;
 pub mod interface;
@@ -32,4 +28,4 @@ pub use queries::Queries;
 #[cfg(test)]
 mod tests;
 
-fluent_messages! { "../messages.ftl" }
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs
index 7d14d088e..21beb90d7 100644
--- a/compiler/rustc_interface/src/passes.rs
+++ b/compiler/rustc_interface/src/passes.rs
@@ -22,7 +22,7 @@ use rustc_middle::ty::{self, GlobalCtxt, RegisteredTools, TyCtxt};
 use rustc_middle::util::Providers;
 use rustc_mir_build as mir_build;
 use rustc_parse::{parse_crate_from_file, parse_crate_from_source_str, validate_attr};
-use rustc_passes::{self, abi_test, hir_stats, layout_test};
+use rustc_passes::{abi_test, hir_stats, layout_test};
 use rustc_resolve::Resolver;
 use rustc_session::code_stats::VTableSizeInfo;
 use rustc_session::config::{CrateType, Input, OutFileName, OutputFilenames, OutputType};
@@ -39,7 +39,7 @@ use std::any::Any;
 use std::ffi::OsString;
 use std::io::{self, BufWriter, Write};
 use std::path::{Path, PathBuf};
-use std::sync::{Arc, LazyLock};
+use std::sync::LazyLock;
 use std::{env, fs, iter};
 
 pub fn parse<'a>(sess: &'a Session) -> PResult<'a, ast::Crate> {
@@ -56,7 +56,7 @@ pub fn parse<'a>(sess: &'a Session) -> PResult<'a, ast::Crate> {
     }
 
     if let Some(ref s) = sess.opts.unstable_opts.show_span {
-        rustc_ast_passes::show_span::run(sess.diagnostic(), s, &krate);
+        rustc_ast_passes::show_span::run(sess.dcx(), s, &krate);
     }
 
     if sess.opts.unstable_opts.hir_stats {
@@ -72,17 +72,6 @@ fn count_nodes(krate: &ast::Crate) -> usize {
     counter.count
 }
 
-pub(crate) fn create_lint_store(
-    sess: &Session,
-    register_lints: Option<impl Fn(&Session, &mut LintStore)>,
-) -> LintStore {
-    let mut lint_store = rustc_lint::new_lint_store(sess.enable_internal_lints());
-    if let Some(register_lints) = register_lints {
-        register_lints(sess, &mut lint_store);
-    }
-    lint_store
-}
-
 fn pre_expansion_lint<'a>(
     sess: &Session,
     features: &Features,
@@ -125,7 +114,7 @@ impl LintStoreExpand for LintStoreExpandImpl<'_> {
     }
 }
 
-/// Runs the "early phases" of the compiler: initial `cfg` processing, loading compiler plugins,
+/// Runs the "early phases" of the compiler: initial `cfg` processing,
 /// syntax expansion, secondary `cfg` expansion, synthesis of a test
 /// harness if one is to be provided, injection of a dependency on the
 /// standard library and prelude, and name resolution.
@@ -138,7 +127,7 @@ fn configure_and_expand(
     let tcx = resolver.tcx();
     let sess = tcx.sess;
     let features = tcx.features();
-    let lint_store = unerased_lint_store(tcx);
+    let lint_store = unerased_lint_store(tcx.sess);
     let crate_name = tcx.crate_name(LOCAL_CRATE);
     let lint_check_node = (&krate, pre_configured_attrs);
     pre_expansion_lint(
@@ -161,7 +150,7 @@ fn configure_and_expand(
         )
     });
 
-    util::check_attr_crate_type(sess, pre_configured_attrs, &mut resolver.lint_buffer());
+    util::check_attr_crate_type(sess, pre_configured_attrs, resolver.lint_buffer());
 
     // Expand all macros
     krate = sess.time("macro_expand_crate", || {
@@ -278,7 +267,7 @@ fn configure_and_expand(
             is_proc_macro_crate,
             has_proc_macro_decls,
             is_test_crate,
-            sess.diagnostic(),
+            sess.dcx(),
         )
     });
 
@@ -295,16 +284,16 @@ fn early_lint_checks(tcx: TyCtxt<'_>, (): ()) {
     let mut lint_buffer = resolver.lint_buffer.steal();
 
     if sess.opts.unstable_opts.input_stats {
-        eprintln!("Post-expansion node count: {}", count_nodes(&krate));
+        eprintln!("Post-expansion node count: {}", count_nodes(krate));
     }
 
     if sess.opts.unstable_opts.hir_stats {
-        hir_stats::print_ast_stats(&krate, "POST EXPANSION AST STATS", "ast-stats-2");
+        hir_stats::print_ast_stats(krate, "POST EXPANSION AST STATS", "ast-stats-2");
     }
 
     // Needs to go *after* expansion to be able to check the results of macro expansion.
     sess.time("complete_gated_feature_checking", || {
-        rustc_ast_passes::feature_gate::check_crate(&krate, sess, tcx.features());
+        rustc_ast_passes::feature_gate::check_crate(krate, sess, tcx.features());
     });
 
     // Add all buffered lints from the `ParseSess` to the `Session`.
@@ -317,6 +306,8 @@ fn early_lint_checks(tcx: TyCtxt<'_>, (): ()) {
 
     // Gate identifiers containing invalid Unicode codepoints that were recovered during lexing.
     sess.parse_sess.bad_unicode_identifiers.with_lock(|identifiers| {
+        // We will soon sort, so the initial order does not matter.
+        #[allow(rustc::potential_query_instability)]
         let mut identifiers: Vec<_> = identifiers.drain().collect();
         identifiers.sort_by_key(|&(key, _)| key);
         for (ident, mut spans) in identifiers.into_iter() {
@@ -330,7 +321,7 @@ fn early_lint_checks(tcx: TyCtxt<'_>, (): ()) {
         }
     });
 
-    let lint_store = unerased_lint_store(tcx);
+    let lint_store = unerased_lint_store(tcx.sess);
     rustc_lint::check_ast_node(
         sess,
         tcx.features(),
@@ -442,6 +433,9 @@ fn write_out_deps(tcx: TyCtxt<'_>, outputs: &OutputFilenames, out_filenames: &[P
             escape_dep_filename(&file.prefer_local().to_string())
         };
 
+        // The entries will be used to declare dependencies beween files in a
+        // Makefile-like output, so the iteration order does not matter.
+        #[allow(rustc::potential_query_instability)]
         let extra_tracked_files =
             file_depinfo.iter().map(|path_sym| normalize_path(PathBuf::from(path_sym.as_str())));
         files.extend(extra_tracked_files);
@@ -497,6 +491,8 @@ fn write_out_deps(tcx: TyCtxt<'_>, outputs: &OutputFilenames, out_filenames: &[P
             // Emit special comments with information about accessed environment variables.
             let env_depinfo = sess.parse_sess.env_depinfo.borrow();
             if !env_depinfo.is_empty() {
+                // We will soon sort, so the initial order does not matter.
+                #[allow(rustc::potential_query_instability)]
                 let mut envs: Vec<_> = env_depinfo
                     .iter()
                     .map(|(k, v)| (escape_dep_env(*k), v.map(escape_dep_env)))
@@ -530,13 +526,11 @@ fn write_out_deps(tcx: TyCtxt<'_>, outputs: &OutputFilenames, out_filenames: &[P
     match result {
         Ok(_) => {
             if sess.opts.json_artifact_notifications {
-                sess.parse_sess
-                    .span_diagnostic
-                    .emit_artifact_notification(&deps_filename, "dep-info");
+                sess.dcx().emit_artifact_notification(deps_filename, "dep-info");
             }
         }
         Err(error) => {
-            sess.emit_fatal(errors::ErrorWritingDependencies { path: &deps_filename, error });
+            sess.emit_fatal(errors::ErrorWritingDependencies { path: deps_filename, error });
         }
     }
 }
@@ -564,23 +558,27 @@ fn resolver_for_lowering<'tcx>(
     tcx.arena.alloc(Steal::new((untracked_resolver_for_lowering, Lrc::new(krate))))
 }
 
-fn output_filenames(tcx: TyCtxt<'_>, (): ()) -> Arc<OutputFilenames> {
+pub(crate) fn write_dep_info(tcx: TyCtxt<'_>) {
+    // Make sure name resolution and macro expansion is run for
+    // the side-effect of providing a complete set of all
+    // accessed files and env vars.
+    let _ = tcx.resolver_for_lowering(());
+
     let sess = tcx.sess;
-    let _timer = sess.timer("prepare_outputs");
-    let (_, krate) = &*tcx.resolver_for_lowering(()).borrow();
+    let _timer = sess.timer("write_dep_info");
     let crate_name = tcx.crate_name(LOCAL_CRATE);
 
-    let outputs = util::build_output_filenames(&krate.attrs, sess);
+    let outputs = tcx.output_filenames(());
     let output_paths =
         generated_output_paths(tcx, &outputs, sess.io.output_file.is_some(), crate_name);
 
     // Ensure the source file isn't accidentally overwritten during compilation.
-    if let Some(ref input_path) = sess.io.input.opt_path() {
+    if let Some(input_path) = sess.io.input.opt_path() {
         if sess.opts.will_create_output_file() {
             if output_contains_path(&output_paths, input_path) {
                 sess.emit_fatal(errors::InputFileWouldBeOverWritten { path: input_path });
             }
-            if let Some(ref dir_path) = output_conflicts_with_dir(&output_paths) {
+            if let Some(dir_path) = output_conflicts_with_dir(&output_paths) {
                 sess.emit_fatal(errors::GeneratedFileConflictsWithDirectory {
                     input_path,
                     dir_path,
@@ -607,15 +605,12 @@ fn output_filenames(tcx: TyCtxt<'_>, (): ()) -> Arc<OutputFilenames> {
             }
         }
     }
-
-    outputs.into()
 }
 
 pub static DEFAULT_QUERY_PROVIDERS: LazyLock<Providers> = LazyLock::new(|| {
     let providers = &mut Providers::default();
     providers.analysis = analysis;
     providers.hir_crate = rustc_ast_lowering::lower_to_hir;
-    providers.output_filenames = output_filenames;
     providers.resolver_for_lowering = resolver_for_lowering;
     providers.early_lint_checks = early_lint_checks;
     proc_macro_decls::provide(providers);
@@ -645,7 +640,6 @@ pub fn create_global_ctxt<'tcx>(
     compiler: &'tcx Compiler,
     crate_types: Vec<CrateType>,
     stable_crate_id: StableCrateId,
-    lint_store: Lrc<LintStore>,
     dep_graph: DepGraph,
     untracked: Untracked,
     gcx_cell: &'tcx OnceLock<GlobalCtxt<'tcx>>,
@@ -657,10 +651,10 @@ pub fn create_global_ctxt<'tcx>(
     // incr. comp. yet.
     dep_graph.assert_ignored();
 
-    let sess = &compiler.session();
+    let sess = &compiler.sess;
     let query_result_on_disk_cache = rustc_incremental::load_query_result_cache(sess);
 
-    let codegen_backend = compiler.codegen_backend();
+    let codegen_backend = &compiler.codegen_backend;
     let mut providers = *DEFAULT_QUERY_PROVIDERS;
     codegen_backend.provide(&mut providers);
 
@@ -676,7 +670,6 @@ pub fn create_global_ctxt<'tcx>(
                 sess,
                 crate_types,
                 stable_crate_id,
-                lint_store,
                 arena,
                 hir_arena,
                 untracked,
@@ -769,7 +762,7 @@ fn analysis(tcx: TyCtxt<'_>, (): ()) -> Result<()> {
     });
 
     tcx.hir().par_body_owners(|def_id| {
-        if let rustc_hir::def::DefKind::Coroutine = tcx.def_kind(def_id) {
+        if tcx.is_coroutine(def_id.to_def_id()) {
             tcx.ensure().mir_coroutine_witnesses(def_id);
             tcx.ensure().check_coroutine_obligations(def_id);
         }
diff --git a/compiler/rustc_interface/src/queries.rs b/compiler/rustc_interface/src/queries.rs
index 3a5f788e8..8a553b95e 100644
--- a/compiler/rustc_interface/src/queries.rs
+++ b/compiler/rustc_interface/src/queries.rs
@@ -1,13 +1,14 @@
 use crate::errors::{FailedWritingFile, RustcErrorFatal, RustcErrorUnexpectedAnnotation};
 use crate::interface::{Compiler, Result};
-use crate::{passes, util};
+use crate::{errors, passes, util};
 
 use rustc_ast as ast;
 use rustc_codegen_ssa::traits::CodegenBackend;
 use rustc_codegen_ssa::CodegenResults;
 use rustc_data_structures::steal::Steal;
 use rustc_data_structures::svh::Svh;
-use rustc_data_structures::sync::{AppendOnlyIndexVec, FreezeLock, Lrc, OnceLock, WorkerLocal};
+use rustc_data_structures::sync::{AppendOnlyIndexVec, FreezeLock, OnceLock, WorkerLocal};
+use rustc_hir::def::DefKind;
 use rustc_hir::def_id::{StableCrateId, CRATE_DEF_ID, LOCAL_CRATE};
 use rustc_hir::definitions::Definitions;
 use rustc_incremental::setup_dep_graph;
@@ -15,9 +16,11 @@ use rustc_metadata::creader::CStore;
 use rustc_middle::arena::Arena;
 use rustc_middle::dep_graph::DepGraph;
 use rustc_middle::ty::{GlobalCtxt, TyCtxt};
+use rustc_serialize::opaque::FileEncodeResult;
 use rustc_session::config::{self, CrateType, OutputFilenames, OutputType};
 use rustc_session::cstore::Untracked;
-use rustc_session::{output::find_crate_name, Session};
+use rustc_session::output::find_crate_name;
+use rustc_session::Session;
 use rustc_span::symbol::sym;
 use std::any::Any;
 use std::cell::{RefCell, RefMut};
@@ -83,7 +86,6 @@ pub struct Queries<'tcx> {
     hir_arena: WorkerLocal<rustc_hir::Arena<'tcx>>,
 
     parse: Query<ast::Crate>,
-    pre_configure: Query<(ast::Crate, ast::AttrVec)>,
     // This just points to what's in `gcx_cell`.
     gcx: Query<&'tcx GlobalCtxt<'tcx>>,
 }
@@ -96,29 +98,26 @@ impl<'tcx> Queries<'tcx> {
             arena: WorkerLocal::new(|_| Arena::default()),
             hir_arena: WorkerLocal::new(|_| rustc_hir::Arena::default()),
             parse: Default::default(),
-            pre_configure: Default::default(),
             gcx: Default::default(),
         }
     }
 
-    fn session(&self) -> &Lrc<Session> {
-        &self.compiler.sess
-    }
-    fn codegen_backend(&self) -> &Lrc<dyn CodegenBackend> {
-        self.compiler.codegen_backend()
+    pub fn finish(&self) -> FileEncodeResult {
+        if let Some(gcx) = self.gcx_cell.get() { gcx.finish() } else { Ok(0) }
     }
 
     pub fn parse(&self) -> Result<QueryResult<'_, ast::Crate>> {
-        self.parse
-            .compute(|| passes::parse(self.session()).map_err(|mut parse_error| parse_error.emit()))
+        self.parse.compute(|| {
+            passes::parse(&self.compiler.sess).map_err(|mut parse_error| parse_error.emit())
+        })
     }
 
-    #[deprecated = "pre_configure may be made private in the future. If you need it please open an issue with your use case."]
-    pub fn pre_configure(&self) -> Result<QueryResult<'_, (ast::Crate, ast::AttrVec)>> {
-        self.pre_configure.compute(|| {
+    pub fn global_ctxt(&'tcx self) -> Result<QueryResult<'_, &'tcx GlobalCtxt<'tcx>>> {
+        self.gcx.compute(|| {
+            let sess = &self.compiler.sess;
+
             let mut krate = self.parse()?.steal();
 
-            let sess = self.session();
             rustc_builtin_macros::cmdline_attrs::inject(
                 &mut krate,
                 &sess.parse_sess,
@@ -127,15 +126,6 @@ impl<'tcx> Queries<'tcx> {
 
             let pre_configured_attrs =
                 rustc_expand::config::pre_configure_attrs(sess, &krate.attrs);
-            Ok((krate, pre_configured_attrs))
-        })
-    }
-
-    pub fn global_ctxt(&'tcx self) -> Result<QueryResult<'_, &'tcx GlobalCtxt<'tcx>>> {
-        self.gcx.compute(|| {
-            let sess = self.session();
-            #[allow(deprecated)]
-            let (krate, pre_configured_attrs) = self.pre_configure()?.steal();
 
             // parse `#[crate_name]` even if `--crate-name` was passed, to make sure it matches.
             let crate_name = find_crate_name(sess, &pre_configured_attrs);
@@ -146,12 +136,11 @@ impl<'tcx> Queries<'tcx> {
                 sess.opts.cg.metadata.clone(),
                 sess.cfg_version,
             );
+            let outputs = util::build_output_filenames(&pre_configured_attrs, sess);
             let dep_graph = setup_dep_graph(sess, crate_name, stable_crate_id)?;
 
-            let lint_store =
-                Lrc::new(passes::create_lint_store(sess, self.compiler.register_lints.as_deref()));
             let cstore = FreezeLock::new(Box::new(CStore::new(
-                self.codegen_backend().metadata_loader(),
+                self.compiler.codegen_backend.metadata_loader(),
                 stable_crate_id,
             )) as _);
             let definitions = FreezeLock::new(Definitions::new(stable_crate_id));
@@ -164,7 +153,6 @@ impl<'tcx> Queries<'tcx> {
                 self.compiler,
                 crate_types,
                 stable_crate_id,
-                lint_store,
                 dep_graph,
                 untracked,
                 &self.gcx_cell,
@@ -183,25 +171,20 @@ impl<'tcx> Queries<'tcx> {
                     crate_name,
                 )));
                 feed.crate_for_resolver(tcx.arena.alloc(Steal::new((krate, pre_configured_attrs))));
+                feed.output_filenames(Arc::new(outputs));
+
+                let feed = tcx.feed_local_def_id(CRATE_DEF_ID);
+                feed.def_kind(DefKind::Mod);
             });
             Ok(qcx)
         })
     }
 
-    pub fn ongoing_codegen(&'tcx self) -> Result<Box<dyn Any>> {
+    pub fn write_dep_info(&'tcx self) -> Result<()> {
         self.global_ctxt()?.enter(|tcx| {
-            // Don't do code generation if there were any errors
-            self.session().compile_status()?;
-
-            // If we have any delayed bugs, for example because we created TyKind::Error earlier,
-            // it's likely that codegen will only cause more ICEs, obscuring the original problem
-            self.session().diagnostic().flush_delayed();
-
-            // Hook for UI tests.
-            Self::check_for_rustc_errors_attr(tcx);
-
-            Ok(passes::start_codegen(&**self.codegen_backend(), tcx))
-        })
+            passes::write_dep_info(tcx);
+        });
+        Ok(())
     }
 
     /// Check for the `#[rustc_error]` annotation, which forces an error in codegen. This is used
@@ -211,16 +194,16 @@ impl<'tcx> Queries<'tcx> {
         let Some((def_id, _)) = tcx.entry_fn(()) else { return };
         for attr in tcx.get_attrs(def_id, sym::rustc_error) {
             match attr.meta_item_list() {
-                // Check if there is a `#[rustc_error(delay_span_bug_from_inside_query)]`.
+                // Check if there is a `#[rustc_error(span_delayed_bug_from_inside_query)]`.
                 Some(list)
                     if list.iter().any(|list_item| {
                         matches!(
                             list_item.ident().map(|i| i.name),
-                            Some(sym::delay_span_bug_from_inside_query)
+                            Some(sym::span_delayed_bug_from_inside_query)
                         )
                     }) =>
                 {
-                    tcx.ensure().trigger_delay_span_bug(def_id);
+                    tcx.ensure().trigger_span_delayed_bug(def_id);
                 }
 
                 // Bare `#[rustc_error]`.
@@ -238,68 +221,61 @@ impl<'tcx> Queries<'tcx> {
         }
     }
 
-    pub fn linker(&'tcx self, ongoing_codegen: Box<dyn Any>) -> Result<Linker> {
-        let sess = self.session().clone();
-        let codegen_backend = self.codegen_backend().clone();
+    pub fn codegen_and_build_linker(&'tcx self) -> Result<Linker> {
+        self.global_ctxt()?.enter(|tcx| {
+            // Don't do code generation if there were any errors
+            self.compiler.sess.compile_status()?;
 
-        let (crate_hash, prepare_outputs, dep_graph) = self.global_ctxt()?.enter(|tcx| {
-            (
-                if tcx.needs_crate_hash() { Some(tcx.crate_hash(LOCAL_CRATE)) } else { None },
-                tcx.output_filenames(()).clone(),
-                tcx.dep_graph.clone(),
-            )
-        });
+            // If we have any delayed bugs, for example because we created TyKind::Error earlier,
+            // it's likely that codegen will only cause more ICEs, obscuring the original problem
+            self.compiler.sess.dcx().flush_delayed();
 
-        Ok(Linker {
-            sess,
-            codegen_backend,
+            // Hook for UI tests.
+            Self::check_for_rustc_errors_attr(tcx);
 
-            dep_graph,
-            prepare_outputs,
-            crate_hash,
-            ongoing_codegen,
+            let ongoing_codegen = passes::start_codegen(&*self.compiler.codegen_backend, tcx);
+
+            Ok(Linker {
+                dep_graph: tcx.dep_graph.clone(),
+                output_filenames: tcx.output_filenames(()).clone(),
+                crate_hash: if tcx.needs_crate_hash() {
+                    Some(tcx.crate_hash(LOCAL_CRATE))
+                } else {
+                    None
+                },
+                ongoing_codegen,
+            })
         })
     }
 }
 
 pub struct Linker {
-    // compilation inputs
-    sess: Lrc<Session>,
-    codegen_backend: Lrc<dyn CodegenBackend>,
-
-    // compilation outputs
     dep_graph: DepGraph,
-    prepare_outputs: Arc<OutputFilenames>,
+    output_filenames: Arc<OutputFilenames>,
     // Only present when incr. comp. is enabled.
     crate_hash: Option<Svh>,
     ongoing_codegen: Box<dyn Any>,
 }
 
 impl Linker {
-    pub fn link(self) -> Result<()> {
-        let (codegen_results, work_products) = self.codegen_backend.join_codegen(
-            self.ongoing_codegen,
-            &self.sess,
-            &self.prepare_outputs,
-        )?;
+    pub fn link(self, sess: &Session, codegen_backend: &dyn CodegenBackend) -> Result<()> {
+        let (codegen_results, work_products) =
+            codegen_backend.join_codegen(self.ongoing_codegen, sess, &self.output_filenames)?;
 
-        self.sess.compile_status()?;
+        sess.compile_status()?;
 
-        let sess = &self.sess;
-        let dep_graph = self.dep_graph;
         sess.time("serialize_work_products", || {
-            rustc_incremental::save_work_product_index(sess, &dep_graph, work_products)
+            rustc_incremental::save_work_product_index(sess, &self.dep_graph, work_products)
         });
 
-        let prof = self.sess.prof.clone();
-        prof.generic_activity("drop_dep_graph").run(move || drop(dep_graph));
+        let prof = sess.prof.clone();
+        prof.generic_activity("drop_dep_graph").run(move || drop(self.dep_graph));
 
         // Now that we won't touch anything in the incremental compilation directory
         // any more, we can finalize it (which involves renaming it)
-        rustc_incremental::finalize_session_directory(&self.sess, self.crate_hash);
+        rustc_incremental::finalize_session_directory(sess, self.crate_hash);
 
-        if !self
-            .sess
+        if !sess
             .opts
             .output_types
             .keys()
@@ -309,14 +285,19 @@ impl Linker {
         }
 
         if sess.opts.unstable_opts.no_link {
-            let rlink_file = self.prepare_outputs.with_extension(config::RLINK_EXT);
-            CodegenResults::serialize_rlink(sess, &rlink_file, &codegen_results)
-                .map_err(|error| sess.emit_fatal(FailedWritingFile { path: &rlink_file, error }))?;
+            let rlink_file = self.output_filenames.with_extension(config::RLINK_EXT);
+            CodegenResults::serialize_rlink(
+                sess,
+                &rlink_file,
+                &codegen_results,
+                &*self.output_filenames,
+            )
+            .map_err(|error| sess.emit_fatal(FailedWritingFile { path: &rlink_file, error }))?;
             return Ok(());
         }
 
         let _timer = sess.prof.verbose_generic_activity("link_crate");
-        self.codegen_backend.link(&self.sess, codegen_results, &self.prepare_outputs)
+        codegen_backend.link(sess, codegen_results, &self.output_filenames)
     }
 }
 
@@ -325,6 +306,7 @@ impl Compiler {
     where
         F: for<'tcx> FnOnce(&'tcx Queries<'tcx>) -> T,
     {
+        // Must declare `_timer` first so that it is dropped after `queries`.
         let mut _timer = None;
         let queries = Queries::new(self);
         let ret = f(&queries);
@@ -337,15 +319,19 @@ impl Compiler {
             // after this point, they'll show up as "<unknown>" in self-profiling data.
             {
                 let _prof_timer =
-                    queries.session().prof.generic_activity("self_profile_alloc_query_strings");
+                    queries.compiler.sess.prof.generic_activity("self_profile_alloc_query_strings");
                 gcx.enter(rustc_query_impl::alloc_self_profile_query_strings);
             }
 
-            self.session()
-                .time("serialize_dep_graph", || gcx.enter(rustc_incremental::save_dep_graph));
+            self.sess.time("serialize_dep_graph", || gcx.enter(rustc_incremental::save_dep_graph));
         }
 
-        _timer = Some(self.session().timer("free_global_ctxt"));
+        // The timer's lifetime spans the dropping of `queries`, which contains
+        // the global context.
+        _timer = Some(self.sess.timer("free_global_ctxt"));
+        if let Err((path, error)) = queries.finish() {
+            self.sess.emit_err(errors::FailedWritingFile { path: &path, error });
+        }
 
         ret
     }
diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs
index d30816955..04a7714d4 100644
--- a/compiler/rustc_interface/src/tests.rs
+++ b/compiler/rustc_interface/src/tests.rs
@@ -5,15 +5,15 @@ use rustc_errors::{emitter::HumanReadableErrorType, registry, ColorConfig};
 use rustc_session::config::{
     build_configuration, build_session_options, rustc_optgroups, BranchProtection, CFGuard, Cfg,
     DebugInfo, DumpMonoStatsFormat, ErrorOutputType, ExternEntry, ExternLocation, Externs,
-    InliningThreshold, Input, InstrumentCoverage, InstrumentXRay, LinkSelfContained,
-    LinkerPluginLto, LocationDetail, LtoCli, MirSpanview, OomStrategy, Options, OutFileName,
-    OutputType, OutputTypes, PAuthKey, PacRet, Passes, Polonius, ProcMacroExecutionStrategy, Strip,
-    SwitchWithOptPath, SymbolManglingVersion, TraitSolver, WasiExecModel,
+    FunctionReturn, InliningThreshold, Input, InstrumentCoverage, InstrumentXRay,
+    LinkSelfContained, LinkerPluginLto, LocationDetail, LtoCli, MirSpanview, NextSolverConfig,
+    OomStrategy, Options, OutFileName, OutputType, OutputTypes, PAuthKey, PacRet, Passes, Polonius,
+    ProcMacroExecutionStrategy, Strip, SwitchWithOptPath, SymbolManglingVersion, WasiExecModel,
 };
 use rustc_session::lint::Level;
 use rustc_session::search_paths::SearchPath;
 use rustc_session::utils::{CanonicalizedPath, NativeLib, NativeLibKind};
-use rustc_session::{build_session, getopts, CompilerIO, EarlyErrorHandler, Session};
+use rustc_session::{build_session, getopts, CompilerIO, EarlyDiagCtxt, Session};
 use rustc_span::edition::{Edition, DEFAULT_EDITION};
 use rustc_span::symbol::sym;
 use rustc_span::{FileName, SourceFileHashAlgorithm};
@@ -24,10 +24,12 @@ use std::num::NonZeroUsize;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
-fn mk_session(handler: &mut EarlyErrorHandler, matches: getopts::Matches) -> (Session, Cfg) {
+fn mk_session(matches: getopts::Matches) -> (Session, Cfg) {
+    let mut early_dcx = EarlyDiagCtxt::new(ErrorOutputType::default());
+    early_dcx.initialize_checked_jobserver();
+
     let registry = registry::Registry::new(&[]);
-    let sessopts = build_session_options(handler, &matches);
-    let cfg = parse_cfg(handler, matches.opt_strs("cfg"));
+    let sessopts = build_session_options(&mut early_dcx, &matches);
     let temps_dir = sessopts.unstable_opts.temps_dir.as_deref().map(PathBuf::from);
     let io = CompilerIO {
         input: Input::Str { name: FileName::Custom(String::new()), input: String::new() },
@@ -36,7 +38,7 @@ fn mk_session(handler: &mut EarlyErrorHandler, matches: getopts::Matches) -> (Se
         temps_dir,
     };
     let sess = build_session(
-        handler,
+        early_dcx,
         sessopts,
         io,
         None,
@@ -50,6 +52,7 @@ fn mk_session(handler: &mut EarlyErrorHandler, matches: getopts::Matches) -> (Se
         Arc::default(),
         Default::default(),
     );
+    let cfg = parse_cfg(&sess.dcx(), matches.opt_strs("cfg"));
     (sess, cfg)
 }
 
@@ -116,8 +119,7 @@ fn assert_non_crate_hash_different(x: &Options, y: &Options) {
 fn test_switch_implies_cfg_test() {
     rustc_span::create_default_session_globals_then(|| {
         let matches = optgroups().parse(&["--test".to_string()]).unwrap();
-        let mut handler = EarlyErrorHandler::new(ErrorOutputType::default());
-        let (sess, cfg) = mk_session(&mut handler, matches);
+        let (sess, cfg) = mk_session(matches);
         let cfg = build_configuration(&sess, cfg);
         assert!(cfg.contains(&(sym::test, None)));
     });
@@ -128,8 +130,7 @@ fn test_switch_implies_cfg_test() {
 fn test_switch_implies_cfg_test_unless_cfg_test() {
     rustc_span::create_default_session_globals_then(|| {
         let matches = optgroups().parse(&["--test".to_string(), "--cfg=test".to_string()]).unwrap();
-        let mut handler = EarlyErrorHandler::new(ErrorOutputType::default());
-        let (sess, cfg) = mk_session(&mut handler, matches);
+        let (sess, cfg) = mk_session(matches);
         let cfg = build_configuration(&sess, cfg);
         let mut test_items = cfg.iter().filter(|&&(name, _)| name == sym::test);
         assert!(test_items.next().is_some());
@@ -141,24 +142,21 @@ fn test_switch_implies_cfg_test_unless_cfg_test() {
 fn test_can_print_warnings() {
     rustc_span::create_default_session_globals_then(|| {
         let matches = optgroups().parse(&["-Awarnings".to_string()]).unwrap();
-        let mut handler = EarlyErrorHandler::new(ErrorOutputType::default());
-        let (sess, _) = mk_session(&mut handler, matches);
-        assert!(!sess.diagnostic().can_emit_warnings());
+        let (sess, _) = mk_session(matches);
+        assert!(!sess.dcx().can_emit_warnings());
     });
 
     rustc_span::create_default_session_globals_then(|| {
         let matches =
             optgroups().parse(&["-Awarnings".to_string(), "-Dwarnings".to_string()]).unwrap();
-        let mut handler = EarlyErrorHandler::new(ErrorOutputType::default());
-        let (sess, _) = mk_session(&mut handler, matches);
-        assert!(sess.diagnostic().can_emit_warnings());
+        let (sess, _) = mk_session(matches);
+        assert!(sess.dcx().can_emit_warnings());
     });
 
     rustc_span::create_default_session_globals_then(|| {
         let matches = optgroups().parse(&["-Adead_code".to_string()]).unwrap();
-        let mut handler = EarlyErrorHandler::new(ErrorOutputType::default());
-        let (sess, _) = mk_session(&mut handler, matches);
-        assert!(sess.diagnostic().can_emit_warnings());
+        let (sess, _) = mk_session(matches);
+        assert!(sess.dcx().can_emit_warnings());
     });
 }
 
@@ -303,36 +301,36 @@ fn test_search_paths_tracking_hash_different_order() {
     let mut v3 = Options::default();
     let mut v4 = Options::default();
 
-    let handler = EarlyErrorHandler::new(JSON);
+    let early_dcx = EarlyDiagCtxt::new(JSON);
     const JSON: ErrorOutputType = ErrorOutputType::Json {
         pretty: false,
         json_rendered: HumanReadableErrorType::Default(ColorConfig::Never),
     };
 
     // Reference
-    v1.search_paths.push(SearchPath::from_cli_opt(&handler, "native=abc"));
-    v1.search_paths.push(SearchPath::from_cli_opt(&handler, "crate=def"));
-    v1.search_paths.push(SearchPath::from_cli_opt(&handler, "dependency=ghi"));
-    v1.search_paths.push(SearchPath::from_cli_opt(&handler, "framework=jkl"));
-    v1.search_paths.push(SearchPath::from_cli_opt(&handler, "all=mno"));
-
-    v2.search_paths.push(SearchPath::from_cli_opt(&handler, "native=abc"));
-    v2.search_paths.push(SearchPath::from_cli_opt(&handler, "dependency=ghi"));
-    v2.search_paths.push(SearchPath::from_cli_opt(&handler, "crate=def"));
-    v2.search_paths.push(SearchPath::from_cli_opt(&handler, "framework=jkl"));
-    v2.search_paths.push(SearchPath::from_cli_opt(&handler, "all=mno"));
-
-    v3.search_paths.push(SearchPath::from_cli_opt(&handler, "crate=def"));
-    v3.search_paths.push(SearchPath::from_cli_opt(&handler, "framework=jkl"));
-    v3.search_paths.push(SearchPath::from_cli_opt(&handler, "native=abc"));
-    v3.search_paths.push(SearchPath::from_cli_opt(&handler, "dependency=ghi"));
-    v3.search_paths.push(SearchPath::from_cli_opt(&handler, "all=mno"));
-
-    v4.search_paths.push(SearchPath::from_cli_opt(&handler, "all=mno"));
-    v4.search_paths.push(SearchPath::from_cli_opt(&handler, "native=abc"));
-    v4.search_paths.push(SearchPath::from_cli_opt(&handler, "crate=def"));
-    v4.search_paths.push(SearchPath::from_cli_opt(&handler, "dependency=ghi"));
-    v4.search_paths.push(SearchPath::from_cli_opt(&handler, "framework=jkl"));
+    v1.search_paths.push(SearchPath::from_cli_opt(&early_dcx, "native=abc"));
+    v1.search_paths.push(SearchPath::from_cli_opt(&early_dcx, "crate=def"));
+    v1.search_paths.push(SearchPath::from_cli_opt(&early_dcx, "dependency=ghi"));
+    v1.search_paths.push(SearchPath::from_cli_opt(&early_dcx, "framework=jkl"));
+    v1.search_paths.push(SearchPath::from_cli_opt(&early_dcx, "all=mno"));
+
+    v2.search_paths.push(SearchPath::from_cli_opt(&early_dcx, "native=abc"));
+    v2.search_paths.push(SearchPath::from_cli_opt(&early_dcx, "dependency=ghi"));
+    v2.search_paths.push(SearchPath::from_cli_opt(&early_dcx, "crate=def"));
+    v2.search_paths.push(SearchPath::from_cli_opt(&early_dcx, "framework=jkl"));
+    v2.search_paths.push(SearchPath::from_cli_opt(&early_dcx, "all=mno"));
+
+    v3.search_paths.push(SearchPath::from_cli_opt(&early_dcx, "crate=def"));
+    v3.search_paths.push(SearchPath::from_cli_opt(&early_dcx, "framework=jkl"));
+    v3.search_paths.push(SearchPath::from_cli_opt(&early_dcx, "native=abc"));
+    v3.search_paths.push(SearchPath::from_cli_opt(&early_dcx, "dependency=ghi"));
+    v3.search_paths.push(SearchPath::from_cli_opt(&early_dcx, "all=mno"));
+
+    v4.search_paths.push(SearchPath::from_cli_opt(&early_dcx, "all=mno"));
+    v4.search_paths.push(SearchPath::from_cli_opt(&early_dcx, "native=abc"));
+    v4.search_paths.push(SearchPath::from_cli_opt(&early_dcx, "crate=def"));
+    v4.search_paths.push(SearchPath::from_cli_opt(&early_dcx, "dependency=ghi"));
+    v4.search_paths.push(SearchPath::from_cli_opt(&early_dcx, "framework=jkl"));
 
     assert_same_hash(&v1, &v2);
     assert_same_hash(&v1, &v3);
@@ -679,7 +677,6 @@ fn test_unstable_options_tracking_hash() {
     untracked!(incremental_info, true);
     untracked!(incremental_verify_ich, true);
     untracked!(input_stats, true);
-    untracked!(keep_hygiene_data, true);
     untracked!(link_native_libraries, false);
     untracked!(llvm_time_trace, true);
     untracked!(ls, vec!["all".to_owned()]);
@@ -691,7 +688,6 @@ fn test_unstable_options_tracking_hash() {
     untracked!(no_leak_check, true);
     untracked!(no_parallel_llvm, true);
     untracked!(parse_only, true);
-    untracked!(perf_stats, true);
     // `pre_link_arg` is omitted because it just forwards to `pre_link_args`.
     untracked!(pre_link_args, vec![String::from("abc"), String::from("def")]);
     untracked!(print_codegen_stats, true);
@@ -751,6 +747,7 @@ fn test_unstable_options_tracking_hash() {
     tracked!(cross_crate_inline_threshold, InliningThreshold::Always);
     tracked!(debug_info_for_profiling, true);
     tracked!(debug_macros, true);
+    tracked!(default_hidden_visibility, Some(true));
     tracked!(dep_info_omit_d_target, true);
     tracked!(dual_proc_macros, true);
     tracked!(dwarf_version, Some(5));
@@ -760,6 +757,7 @@ fn test_unstable_options_tracking_hash() {
     tracked!(flatten_format_args, false);
     tracked!(force_unstable_if_unmarked, true);
     tracked!(fuel, Some(("abc".to_string(), 99)));
+    tracked!(function_return, FunctionReturn::ThunkExtern);
     tracked!(function_sections, Some(false));
     tracked!(human_readable_cgu_names, true);
     tracked!(incremental_ignore_spans, true);
@@ -771,6 +769,7 @@ fn test_unstable_options_tracking_hash() {
     tracked!(instrument_xray, Some(InstrumentXRay::default()));
     tracked!(link_directives, false);
     tracked!(link_only, true);
+    tracked!(llvm_module_flag, vec![("bar".to_string(), 123, "max".to_string())]);
     tracked!(llvm_plugins, vec![String::from("plugin_name")]);
     tracked!(location_detail, LocationDetail { file: true, line: false, column: false });
     tracked!(maximal_hir_to_mir_coverage, true);
@@ -781,6 +780,10 @@ fn test_unstable_options_tracking_hash() {
     tracked!(mir_opt_level, Some(4));
     tracked!(move_size_limit, Some(4096));
     tracked!(mutable_noalias, false);
+    tracked!(
+        next_solver,
+        Some(NextSolverConfig { coherence: true, globally: false, dump_tree: Default::default() })
+    );
     tracked!(no_generate_arange_section, true);
     tracked!(no_jump_tables, true);
     tracked!(no_link, true);
@@ -822,7 +825,6 @@ fn test_unstable_options_tracking_hash() {
     tracked!(thir_unsafeck, true);
     tracked!(tiny_const_eval_limit, true);
     tracked!(tls_model, Some(TlsModel::GeneralDynamic));
-    tracked!(trait_solver, TraitSolver::NextCoherence);
     tracked!(translate_remapped_path_to_local_path, false);
     tracked!(trap_unreachable, Some(false));
     tracked!(treat_err_as_bug, NonZeroUsize::new(1));
@@ -852,9 +854,9 @@ fn test_edition_parsing() {
     let options = Options::default();
     assert!(options.edition == DEFAULT_EDITION);
 
-    let mut handler = EarlyErrorHandler::new(ErrorOutputType::default());
+    let mut early_dcx = EarlyDiagCtxt::new(ErrorOutputType::default());
 
     let matches = optgroups().parse(&["--edition=2018".to_string()]).unwrap();
-    let sessopts = build_session_options(&mut handler, &matches);
+    let sessopts = build_session_options(&mut early_dcx, &matches);
     assert!(sessopts.edition == Edition::Edition2018)
 }
diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs
index 22d127934..bffa74321 100644
--- a/compiler/rustc_interface/src/util.rs
+++ b/compiler/rustc_interface/src/util.rs
@@ -14,7 +14,7 @@ use rustc_session::{filesearch, output, Session};
 use rustc_span::edit_distance::find_best_match_for_name;
 use rustc_span::edition::Edition;
 use rustc_span::symbol::{sym, Symbol};
-use session::EarlyErrorHandler;
+use session::EarlyDiagCtxt;
 use std::env;
 use std::env::consts::{DLL_PREFIX, DLL_SUFFIX};
 use std::mem;
@@ -107,7 +107,7 @@ pub(crate) fn run_in_thread_pool_with_globals<F: FnOnce() -> R + Send, R: Send>(
     use rustc_query_impl::QueryCtxt;
     use rustc_query_system::query::{deadlock, QueryContext};
 
-    let registry = sync::Registry::new(threads);
+    let registry = sync::Registry::new(std::num::NonZeroUsize::new(threads).unwrap());
 
     if !sync::is_dyn_thread_safe() {
         return run_in_thread_with_globals(edition, || {
@@ -126,11 +126,8 @@ pub(crate) fn run_in_thread_pool_with_globals<F: FnOnce() -> R + Send, R: Send>(
         .deadlock_handler(|| {
             // On deadlock, creates a new thread and forwards information in thread
             // locals to it. The new thread runs the deadlock handler.
-            let query_map = FromDyn::from(tls::with(|tcx| {
-                QueryCtxt::new(tcx)
-                    .try_collect_active_jobs()
-                    .expect("active jobs shouldn't be locked in deadlock handler")
-            }));
+            let query_map =
+                FromDyn::from(tls::with(|tcx| QueryCtxt::new(tcx).collect_active_jobs()));
             let registry = rayon_core::Registry::current();
             thread::spawn(move || deadlock(query_map.into_inner(), &registry));
         });
@@ -164,16 +161,16 @@ pub(crate) fn run_in_thread_pool_with_globals<F: FnOnce() -> R + Send, R: Send>(
     })
 }
 
-fn load_backend_from_dylib(handler: &EarlyErrorHandler, path: &Path) -> MakeBackendFn {
+fn load_backend_from_dylib(early_dcx: &EarlyDiagCtxt, path: &Path) -> MakeBackendFn {
     let lib = unsafe { Library::new(path) }.unwrap_or_else(|err| {
         let err = format!("couldn't load codegen backend {path:?}: {err}");
-        handler.early_error(err);
+        early_dcx.early_error(err);
     });
 
     let backend_sym = unsafe { lib.get::<MakeBackendFn>(b"__rustc_codegen_backend") }
         .unwrap_or_else(|e| {
             let err = format!("couldn't load codegen backend: {e}");
-            handler.early_error(err);
+            early_dcx.early_error(err);
         });
 
     // Intentionally leak the dynamic library. We can't ever unload it
@@ -188,7 +185,7 @@ fn load_backend_from_dylib(handler: &EarlyErrorHandler, path: &Path) -> MakeBack
 ///
 /// A name of `None` indicates that the default backend should be used.
 pub fn get_codegen_backend(
-    handler: &EarlyErrorHandler,
+    early_dcx: &EarlyDiagCtxt,
     maybe_sysroot: &Option<PathBuf>,
     backend_name: Option<&str>,
 ) -> Box<dyn CodegenBackend> {
@@ -199,11 +196,11 @@ pub fn get_codegen_backend(
 
         match backend_name.unwrap_or(default_codegen_backend) {
             filename if filename.contains('.') => {
-                load_backend_from_dylib(handler, filename.as_ref())
+                load_backend_from_dylib(early_dcx, filename.as_ref())
             }
             #[cfg(feature = "llvm")]
             "llvm" => rustc_codegen_llvm::LlvmCodegenBackend::new,
-            backend_name => get_codegen_sysroot(handler, maybe_sysroot, backend_name),
+            backend_name => get_codegen_sysroot(early_dcx, maybe_sysroot, backend_name),
         }
     });
 
@@ -236,7 +233,7 @@ fn get_rustc_path_inner(bin_path: &str) -> Option<PathBuf> {
 }
 
 fn get_codegen_sysroot(
-    handler: &EarlyErrorHandler,
+    early_dcx: &EarlyDiagCtxt,
     maybe_sysroot: &Option<PathBuf>,
     backend_name: &str,
 ) -> MakeBackendFn {
@@ -274,7 +271,7 @@ fn get_codegen_sysroot(
             "failed to find a `codegen-backends` folder \
                            in the sysroot candidates:\n* {candidates}"
         );
-        handler.early_error(err);
+        early_dcx.early_error(err);
     });
     info!("probing {} for a codegen backend", sysroot.display());
 
@@ -285,7 +282,7 @@ fn get_codegen_sysroot(
             sysroot.display(),
             e
         );
-        handler.early_error(err);
+        early_dcx.early_error(err);
     });
 
     let mut file: Option<PathBuf> = None;
@@ -313,16 +310,16 @@ fn get_codegen_sysroot(
                 prev.display(),
                 path.display()
             );
-            handler.early_error(err);
+            early_dcx.early_error(err);
         }
         file = Some(path.clone());
     }
 
     match file {
-        Some(ref s) => load_backend_from_dylib(handler, s),
+        Some(ref s) => load_backend_from_dylib(early_dcx, s),
         None => {
             let err = format!("unsupported builtin codegen backend `{backend_name}`");
-            handler.early_error(err);
+            early_dcx.early_error(err);
         }
     }
 }
@@ -415,7 +412,9 @@ pub fn collect_crate_types(session: &Session, attrs: &[ast::Attribute]) -> Vec<C
     let mut base = session.opts.crate_types.clone();
     if base.is_empty() {
         let attr_types = attrs.iter().filter_map(|a| {
-            if a.has_name(sym::crate_type) && let Some(s) = a.value_str() {
+            if a.has_name(sym::crate_type)
+                && let Some(s) = a.value_str()
+            {
                 categorize_crate_type(s)
             } else {
                 None
diff --git a/compiler/rustc_lexer/src/unescape.rs b/compiler/rustc_lexer/src/unescape.rs
index 717b042fb..abec12f52 100644
--- a/compiler/rustc_lexer/src/unescape.rs
+++ b/compiler/rustc_lexer/src/unescape.rs
@@ -4,10 +4,14 @@
 use std::ops::Range;
 use std::str::Chars;
 
+use Mode::*;
+
 #[cfg(test)]
 mod tests;
 
-/// Errors and warnings that can occur during string unescaping.
+/// Errors and warnings that can occur during string unescaping. They mostly
+/// relate to malformed escape sequences, but there are a few that are about
+/// other problems.
 #[derive(Debug, PartialEq, Eq)]
 pub enum EscapeError {
     /// Expected 1 char, but 0 were found.
@@ -73,25 +77,24 @@ impl EscapeError {
     }
 }
 
-/// Takes a contents of a literal (without quotes) and produces a
-/// sequence of escaped characters or errors.
-/// Values are returned through invoking of the provided callback.
+/// Takes a contents of a literal (without quotes) and produces a sequence of
+/// escaped characters or errors.
+///
+/// Values are returned by invoking `callback`. For `Char` and `Byte` modes,
+/// the callback will be called exactly once.
 pub fn unescape_literal<F>(src: &str, mode: Mode, callback: &mut F)
 where
     F: FnMut(Range<usize>, Result<char, EscapeError>),
 {
     match mode {
-        Mode::Char | Mode::Byte => {
+        Char | Byte => {
             let mut chars = src.chars();
-            let res = unescape_char_or_byte(&mut chars, mode == Mode::Byte);
+            let res = unescape_char_or_byte(&mut chars, mode);
             callback(0..(src.len() - chars.as_str().len()), res);
         }
-        Mode::Str | Mode::ByteStr => unescape_str_common(src, mode, callback),
-
-        Mode::RawStr | Mode::RawByteStr => {
-            unescape_raw_str_or_raw_byte_str(src, mode == Mode::RawByteStr, callback)
-        }
-        Mode::CStr | Mode::RawCStr => unreachable!(),
+        Str | ByteStr => unescape_non_raw_common(src, mode, callback),
+        RawStr | RawByteStr => check_raw_common(src, mode, callback),
+        CStr | RawCStr => unreachable!(),
     }
 }
 
@@ -117,38 +120,42 @@ pub fn unescape_c_string<F>(src: &str, mode: Mode, callback: &mut F)
 where
     F: FnMut(Range<usize>, Result<CStrUnit, EscapeError>),
 {
-    if mode == Mode::RawCStr {
-        unescape_raw_str_or_raw_byte_str(
-            src,
-            mode.characters_should_be_ascii(),
-            &mut |r, result| callback(r, result.map(CStrUnit::Char)),
-        );
-    } else {
-        unescape_str_common(src, mode, callback);
+    match mode {
+        CStr => {
+            unescape_non_raw_common(src, mode, callback);
+        }
+        RawCStr => {
+            check_raw_common(src, mode, &mut |r, result| callback(r, result.map(CStrUnit::Char)));
+        }
+        Char | Byte | Str | RawStr | ByteStr | RawByteStr => unreachable!(),
     }
 }
 
 /// Takes a contents of a char literal (without quotes), and returns an
 /// unescaped char or an error.
 pub fn unescape_char(src: &str) -> Result<char, EscapeError> {
-    unescape_char_or_byte(&mut src.chars(), false)
+    unescape_char_or_byte(&mut src.chars(), Char)
 }
 
 /// Takes a contents of a byte literal (without quotes), and returns an
 /// unescaped byte or an error.
 pub fn unescape_byte(src: &str) -> Result<u8, EscapeError> {
-    unescape_char_or_byte(&mut src.chars(), true).map(byte_from_char)
+    unescape_char_or_byte(&mut src.chars(), Byte).map(byte_from_char)
 }
 
 /// What kind of literal do we parse.
 #[derive(Debug, Clone, Copy, PartialEq)]
 pub enum Mode {
     Char,
-    Str,
+
     Byte,
-    ByteStr,
+
+    Str,
     RawStr,
+
+    ByteStr,
     RawByteStr,
+
     CStr,
     RawCStr,
 }
@@ -156,45 +163,43 @@ pub enum Mode {
 impl Mode {
     pub fn in_double_quotes(self) -> bool {
         match self {
-            Mode::Str
-            | Mode::ByteStr
-            | Mode::RawStr
-            | Mode::RawByteStr
-            | Mode::CStr
-            | Mode::RawCStr => true,
-            Mode::Char | Mode::Byte => false,
+            Str | RawStr | ByteStr | RawByteStr | CStr | RawCStr => true,
+            Char | Byte => false,
         }
     }
 
     /// Non-byte literals should have `\xXX` escapes that are within the ASCII range.
-    pub fn ascii_escapes_should_be_ascii(self) -> bool {
+    fn ascii_escapes_should_be_ascii(self) -> bool {
         match self {
-            Mode::Char | Mode::Str | Mode::RawStr => true,
-            Mode::Byte | Mode::ByteStr | Mode::RawByteStr | Mode::CStr | Mode::RawCStr => false,
+            Char | Str => true,
+            Byte | ByteStr | CStr => false,
+            RawStr | RawByteStr | RawCStr => unreachable!(),
         }
     }
 
-    /// Whether characters within the literal must be within the ASCII range
-    pub fn characters_should_be_ascii(self) -> bool {
+    /// Whether characters within the literal must be within the ASCII range.
+    #[inline]
+    fn chars_should_be_ascii(self) -> bool {
         match self {
-            Mode::Byte | Mode::ByteStr | Mode::RawByteStr => true,
-            Mode::Char | Mode::Str | Mode::RawStr | Mode::CStr | Mode::RawCStr => false,
+            Byte | ByteStr | RawByteStr => true,
+            Char | Str | RawStr | CStr | RawCStr => false,
         }
     }
 
     /// Byte literals do not allow unicode escape.
-    pub fn is_unicode_escape_disallowed(self) -> bool {
+    fn is_unicode_escape_disallowed(self) -> bool {
         match self {
-            Mode::Byte | Mode::ByteStr | Mode::RawByteStr => true,
-            Mode::Char | Mode::Str | Mode::RawStr | Mode::CStr | Mode::RawCStr => false,
+            Byte | ByteStr => true,
+            Char | Str | CStr => false,
+            RawByteStr | RawStr | RawCStr => unreachable!(),
         }
     }
 
     pub fn prefix_noraw(self) -> &'static str {
         match self {
-            Mode::Byte | Mode::ByteStr | Mode::RawByteStr => "b",
-            Mode::CStr | Mode::RawCStr => "c",
-            Mode::Char | Mode::Str | Mode::RawStr => "",
+            Char | Str | RawStr => "",
+            Byte | ByteStr | RawByteStr => "b",
+            CStr | RawCStr => "c",
         }
     }
 }
@@ -294,22 +299,21 @@ fn scan_unicode(
 }
 
 #[inline]
-fn ascii_check(c: char, characters_should_be_ascii: bool) -> Result<char, EscapeError> {
-    if characters_should_be_ascii && !c.is_ascii() {
-        // Byte literal can't be a non-ascii character.
+fn ascii_check(c: char, chars_should_be_ascii: bool) -> Result<char, EscapeError> {
+    if chars_should_be_ascii && !c.is_ascii() {
         Err(EscapeError::NonAsciiCharInByte)
     } else {
         Ok(c)
     }
 }
 
-fn unescape_char_or_byte(chars: &mut Chars<'_>, is_byte: bool) -> Result<char, EscapeError> {
+fn unescape_char_or_byte(chars: &mut Chars<'_>, mode: Mode) -> Result<char, EscapeError> {
     let c = chars.next().ok_or(EscapeError::ZeroChars)?;
     let res = match c {
-        '\\' => scan_escape(chars, if is_byte { Mode::Byte } else { Mode::Char }),
+        '\\' => scan_escape(chars, mode),
         '\n' | '\t' | '\'' => Err(EscapeError::EscapeOnlyChar),
         '\r' => Err(EscapeError::BareCarriageReturn),
-        _ => ascii_check(c, is_byte),
+        _ => ascii_check(c, mode.chars_should_be_ascii()),
     }?;
     if chars.next().is_some() {
         return Err(EscapeError::MoreThanOneChar);
@@ -319,11 +323,12 @@ fn unescape_char_or_byte(chars: &mut Chars<'_>, is_byte: bool) -> Result<char, E
 
 /// Takes a contents of a string literal (without quotes) and produces a
 /// sequence of escaped characters or errors.
-fn unescape_str_common<F, T: From<u8> + From<char>>(src: &str, mode: Mode, callback: &mut F)
+fn unescape_non_raw_common<F, T: From<u8> + From<char>>(src: &str, mode: Mode, callback: &mut F)
 where
     F: FnMut(Range<usize>, Result<T, EscapeError>),
 {
     let mut chars = src.chars();
+    let chars_should_be_ascii = mode.chars_should_be_ascii(); // get this outside the loop
 
     // The `start` and `end` computation here is complicated because
     // `skip_ascii_whitespace` makes us to skip over chars without counting
@@ -346,14 +351,12 @@ where
                     _ => scan_escape::<T>(&mut chars, mode),
                 }
             }
-            '\n' => Ok(b'\n'.into()),
-            '\t' => Ok(b'\t'.into()),
             '"' => Err(EscapeError::EscapeOnlyChar),
             '\r' => Err(EscapeError::BareCarriageReturn),
-            _ => ascii_check(c, mode.characters_should_be_ascii()).map(Into::into),
+            _ => ascii_check(c, chars_should_be_ascii).map(Into::into),
         };
         let end = src.len() - chars.as_str().len();
-        callback(start..end, res.map(Into::into));
+        callback(start..end, res);
     }
 }
 
@@ -387,20 +390,21 @@ where
 /// sequence of characters or errors.
 /// NOTE: Raw strings do not perform any explicit character escaping, here we
 /// only produce errors on bare CR.
-fn unescape_raw_str_or_raw_byte_str<F>(src: &str, is_byte: bool, callback: &mut F)
+fn check_raw_common<F>(src: &str, mode: Mode, callback: &mut F)
 where
     F: FnMut(Range<usize>, Result<char, EscapeError>),
 {
     let mut chars = src.chars();
+    let chars_should_be_ascii = mode.chars_should_be_ascii(); // get this outside the loop
 
     // The `start` and `end` computation here matches the one in
-    // `unescape_str_or_byte_str` for consistency, even though this function
+    // `unescape_non_raw_common` for consistency, even though this function
     // doesn't have to worry about skipping any chars.
     while let Some(c) = chars.next() {
         let start = src.len() - chars.as_str().len() - c.len_utf8();
         let res = match c {
             '\r' => Err(EscapeError::BareCarriageReturnInRawString),
-            _ => ascii_check(c, is_byte),
+            _ => ascii_check(c, chars_should_be_ascii),
         };
         let end = src.len() - chars.as_str().len();
         callback(start..end, res);
@@ -410,7 +414,7 @@ where
 #[inline]
 pub fn byte_from_char(c: char) -> u8 {
     let res = c as u32;
-    debug_assert!(res <= u8::MAX as u32, "guaranteed because of Mode::ByteStr");
+    debug_assert!(res <= u8::MAX as u32, "guaranteed because of ByteStr");
     res as u8
 }
 
diff --git a/compiler/rustc_lint/messages.ftl b/compiler/rustc_lint/messages.ftl
index 068f1372c..40e6b1b57 100644
--- a/compiler/rustc_lint/messages.ftl
+++ b/compiler/rustc_lint/messages.ftl
@@ -1,3 +1,7 @@
+lint_ambiguous_wide_pointer_comparisons = ambiguous wide pointer comparison, the comparison includes metadata which may not be expected
+    .addr_metadata_suggestion = use explicit `std::ptr::eq` method to compare metadata and addresses
+    .addr_suggestion = use `std::ptr::addr_eq` or untyped pointers to only compare their addresses
+
 lint_array_into_iter =
     this method call resolves to `<&{$target} as IntoIterator>::into_iter` (due to backwards compatibility), but will resolve to <{$target} as IntoIterator>::into_iter in Rust 2021
     .use_iter_suggestion = use `.iter()` instead of `.into_iter()` to avoid ambiguity
@@ -128,12 +132,6 @@ lint_builtin_type_alias_generic_bounds = bounds on generic parameters are not en
 lint_builtin_type_alias_where_clause = where clauses are not enforced in type aliases
     .suggestion = the clause will not be checked when the type alias is used, and should be removed
 
-lint_builtin_unexpected_cli_config_name = unexpected `{$name}` as condition name
-    .help = was set with `--cfg` but isn't in the `--check-cfg` expected names
-
-lint_builtin_unexpected_cli_config_value = unexpected condition value `{$value}` for condition name `{$name}`
-    .help = was set with `--cfg` but isn't in the `--check-cfg` expected values
-
 lint_builtin_unpermitted_type_init_label = this code causes undefined behavior when executed
 lint_builtin_unpermitted_type_init_label_suggestion = help: use `MaybeUninit<T>` instead, and only call `assume_init` after initialization is done
 
@@ -496,8 +494,10 @@ lint_requested_level = requested on the command line with `{$level} {$lint_name}
 
 lint_span_use_eq_ctxt = use `.eq_ctxt()` instead of `.ctxt() == .ctxt()`
 
-lint_supertrait_as_deref_target = `{$t}` implements `Deref` with supertrait `{$target_principal}` as target
-    .label = target type is set here
+lint_supertrait_as_deref_target = this `Deref` implementation is covered by an implicit supertrait coercion
+    .label = `{$self_ty}` implements `Deref<Target = dyn {$target_principal}>` which conflicts with supertrait `{$supertrait_principal}`
+    .label2 = target type is a supertrait of `{$self_ty}`
+    .help = consider removing this implementation or replacing it with a method instead
 
 lint_suspicious_double_ref_clone =
     using `.clone()` on a double reference, which returns `{$ty}` instead of cloning the inner type
@@ -523,6 +523,9 @@ lint_undropped_manually_drops = calls to `std::mem::drop` with `std::mem::Manual
 lint_ungated_async_fn_track_caller = `#[track_caller]` on async functions is a no-op
      .label = this function will not propagate the caller location
 
+lint_unit_bindings = binding has unit type `()`
+    .label = this pattern is inferred to be the unit type `()`
+
 lint_unknown_gated_lint =
     unknown lint: `{$name}`
     .note = the `{$name}` lint is unstable
diff --git a/compiler/rustc_lint/src/builtin.rs b/compiler/rustc_lint/src/builtin.rs
index 6f6150a41..045ff38c0 100644
--- a/compiler/rustc_lint/src/builtin.rs
+++ b/compiler/rustc_lint/src/builtin.rs
@@ -33,7 +33,6 @@ use crate::{
         BuiltinMutablesTransmutes, BuiltinNoMangleGeneric, BuiltinNonShorthandFieldPatterns,
         BuiltinSpecialModuleNameUsed, BuiltinTrivialBounds, BuiltinTypeAliasGenericBounds,
         BuiltinTypeAliasGenericBoundsSuggestion, BuiltinTypeAliasWhereClause,
-        BuiltinUnexpectedCliConfigName, BuiltinUnexpectedCliConfigValue,
         BuiltinUngatedAsyncFnTrackCaller, BuiltinUnpermittedTypeInit,
         BuiltinUnpermittedTypeInitSub, BuiltinUnreachablePub, BuiltinUnsafe,
         BuiltinUnstableFeatures, BuiltinUnusedDocComment, BuiltinUnusedDocCommentSub,
@@ -41,7 +40,6 @@ use crate::{
     },
     EarlyContext, EarlyLintPass, LateContext, LateLintPass, Level, LintContext,
 };
-use rustc_ast::attr;
 use rustc_ast::tokenstream::{TokenStream, TokenTree};
 use rustc_ast::visit::{FnCtxt, FnKind};
 use rustc_ast::{self as ast, *};
@@ -60,7 +58,6 @@ use rustc_middle::ty::GenericArgKind;
 use rustc_middle::ty::ToPredicate;
 use rustc_middle::ty::TypeVisitableExt;
 use rustc_middle::ty::{self, Ty, TyCtxt, VariantDef};
-use rustc_session::config::ExpectedValues;
 use rustc_session::lint::{BuiltinLintDiagnostics, FutureIncompatibilityReason};
 use rustc_span::edition::Edition;
 use rustc_span::source_map::Spanned;
@@ -265,7 +262,7 @@ impl<'tcx> LateLintPass<'tcx> for NonShorthandFieldPatterns {
                     continue;
                 }
                 if let PatKind::Binding(binding_annot, _, ident, None) = fieldpat.pat.kind {
-                    if cx.tcx.find_field_index(ident, &variant)
+                    if cx.tcx.find_field_index(ident, variant)
                         == Some(cx.typeck_results().field_index(fieldpat.hir_id))
                     {
                         cx.emit_spanned_lint(
@@ -509,7 +506,7 @@ impl MissingDoc {
             }
         }
 
-        let attrs = cx.tcx.hir().attrs(cx.tcx.hir().local_def_id_to_hir_id(def_id));
+        let attrs = cx.tcx.hir().attrs(cx.tcx.local_def_id_to_hir_id(def_id));
         let has_doc = attrs.iter().any(has_doc);
         if !has_doc {
             cx.emit_spanned_lint(
@@ -635,21 +632,21 @@ impl<'tcx> LateLintPass<'tcx> for MissingCopyImplementations {
             return;
         }
         let (def, ty) = match item.kind {
-            hir::ItemKind::Struct(_, ref ast_generics) => {
+            hir::ItemKind::Struct(_, ast_generics) => {
                 if !ast_generics.params.is_empty() {
                     return;
                 }
                 let def = cx.tcx.adt_def(item.owner_id);
                 (def, Ty::new_adt(cx.tcx, def, ty::List::empty()))
             }
-            hir::ItemKind::Union(_, ref ast_generics) => {
+            hir::ItemKind::Union(_, ast_generics) => {
                 if !ast_generics.params.is_empty() {
                     return;
                 }
                 let def = cx.tcx.adt_def(item.owner_id);
                 (def, Ty::new_adt(cx.tcx, def, ty::List::empty()))
             }
-            hir::ItemKind::Enum(_, ref ast_generics) => {
+            hir::ItemKind::Enum(_, ast_generics) => {
                 if !ast_generics.params.is_empty() {
                     return;
                 }
@@ -1002,8 +999,10 @@ impl EarlyLintPass for UnusedDocComment {
     }
 
     fn check_arm(&mut self, cx: &EarlyContext<'_>, arm: &ast::Arm) {
-        let arm_span = arm.pat.span.with_hi(arm.body.span.hi());
-        warn_if_doc(cx, arm_span, "match arms", &arm.attrs);
+        if let Some(body) = &arm.body {
+            let arm_span = arm.pat.span.with_hi(body.span.hi());
+            warn_if_doc(cx, arm_span, "match arms", &arm.attrs);
+        }
     }
 
     fn check_pat(&mut self, cx: &EarlyContext<'_>, pat: &ast::Pat) {
@@ -1029,7 +1028,7 @@ impl EarlyLintPass for UnusedDocComment {
     }
 
     fn check_block(&mut self, cx: &EarlyContext<'_>, block: &ast::Block) {
-        warn_if_doc(cx, block.span, "blocks", &block.attrs());
+        warn_if_doc(cx, block.span, "blocks", block.attrs());
     }
 
     fn check_item(&mut self, cx: &EarlyContext<'_>, item: &ast::Item) {
@@ -1119,7 +1118,7 @@ impl<'tcx> LateLintPass<'tcx> for InvalidNoMangleItems {
             }
         };
         match it.kind {
-            hir::ItemKind::Fn(.., ref generics, _) => {
+            hir::ItemKind::Fn(.., generics, _) => {
                 if let Some(no_mangle_attr) = attr::find_by_name(attrs, sym::no_mangle) {
                     check_no_mangle_on_generic_fn(no_mangle_attr, None, generics, it.span);
                 }
@@ -1446,10 +1445,10 @@ declare_lint_pass!(
 impl TypeAliasBounds {
     pub(crate) fn is_type_variable_assoc(qpath: &hir::QPath<'_>) -> bool {
         match *qpath {
-            hir::QPath::TypeRelative(ref ty, _) => {
+            hir::QPath::TypeRelative(ty, _) => {
                 // If this is a type variable, we found a `T::Assoc`.
                 match ty.kind {
-                    hir::TyKind::Path(hir::QPath::Resolved(None, ref path)) => {
+                    hir::TyKind::Path(hir::QPath::Resolved(None, path)) => {
                         matches!(path.res, Res::Def(DefKind::TyParam, _))
                     }
                     _ => false,
@@ -1716,16 +1715,16 @@ impl EarlyLintPass for EllipsisInclusiveRangePatterns {
         }
 
         let (parentheses, endpoints) = match &pat.kind {
-            PatKind::Ref(subpat, _) => (true, matches_ellipsis_pat(&subpat)),
+            PatKind::Ref(subpat, _) => (true, matches_ellipsis_pat(subpat)),
             _ => (false, matches_ellipsis_pat(pat)),
         };
 
         if let Some((start, end, join)) = endpoints {
             if parentheses {
                 self.node_id = Some(pat.id);
-                let end = expr_to_string(&end);
+                let end = expr_to_string(end);
                 let replace = match start {
-                    Some(start) => format!("&({}..={})", expr_to_string(&start), end),
+                    Some(start) => format!("&({}..={})", expr_to_string(start), end),
                     None => format!("&(..={end})"),
                 };
                 if join.edition() >= Edition::Edition2021 {
@@ -1836,7 +1835,7 @@ impl KeywordIdents {
                         self.check_ident_token(cx, UnderMacro(true), ident);
                     }
                 }
-                TokenTree::Delimited(_, _, tts) => self.check_tokens(cx, tts),
+                TokenTree::Delimited(.., tts) => self.check_tokens(cx, tts),
             }
         }
     }
@@ -1910,7 +1909,7 @@ impl ExplicitOutlivesRequirements {
             .iter()
             .filter_map(|(clause, _)| match clause.kind().skip_binder() {
                 ty::ClauseKind::RegionOutlives(ty::OutlivesPredicate(a, b)) => match *a {
-                    ty::ReEarlyBound(ebr) if ebr.def_id == def_id => Some(b),
+                    ty::ReEarlyParam(ebr) if ebr.def_id == def_id => Some(b),
                     _ => None,
                 },
                 _ => None,
@@ -1953,7 +1952,7 @@ impl ExplicitOutlivesRequirements {
                 let is_inferred = match tcx.named_bound_var(lifetime.hir_id) {
                     Some(ResolvedArg::EarlyBound(def_id)) => inferred_outlives
                         .iter()
-                        .any(|r| matches!(**r, ty::ReEarlyBound(ebr) if { ebr.def_id == def_id })),
+                        .any(|r| matches!(**r, ty::ReEarlyParam(ebr) if { ebr.def_id == def_id })),
                     _ => false,
                 };
 
@@ -2383,7 +2382,7 @@ impl<'tcx> LateLintPass<'tcx> for InvalidValue {
 
         /// Determine if this expression is a "dangerous initialization".
         fn is_dangerous_init(cx: &LateContext<'_>, expr: &hir::Expr<'_>) -> Option<InitKind> {
-            if let hir::ExprKind::Call(ref path_expr, ref args) = expr.kind {
+            if let hir::ExprKind::Call(path_expr, args) = expr.kind {
                 // Find calls to `mem::{uninitialized,zeroed}` methods.
                 if let hir::ExprKind::Path(ref qpath) = path_expr.kind {
                     let def_id = cx.qpath_res(qpath, path_expr.hir_id).opt_def_id()?;
@@ -2400,7 +2399,7 @@ impl<'tcx> LateLintPass<'tcx> for InvalidValue {
                 if cx.tcx.is_diagnostic_item(sym::assume_init, def_id) {
                     // This is a call to *some* method named `assume_init`.
                     // See if the `self` parameter is one of the dangerous constructors.
-                    if let hir::ExprKind::Call(ref path_expr, _) = receiver.kind {
+                    if let hir::ExprKind::Call(path_expr, _) = receiver.kind {
                         if let hir::ExprKind::Path(ref qpath) = path_expr.kind {
                             let def_id = cx.qpath_res(qpath, path_expr.hir_id).opt_def_id()?;
                             match cx.tcx.get_diagnostic_name(def_id) {
@@ -2542,7 +2541,7 @@ impl<'tcx> LateLintPass<'tcx> for InvalidValue {
                         return variant_find_init_error(
                             cx,
                             ty,
-                            &first_variant.0,
+                            first_variant.0,
                             args,
                             "field of the only potentially inhabited enum variant",
                             init,
@@ -2648,13 +2647,13 @@ impl<'tcx> LateLintPass<'tcx> for DerefNullPtr {
         /// test if expression is a null ptr
         fn is_null_ptr(cx: &LateContext<'_>, expr: &hir::Expr<'_>) -> bool {
             match &expr.kind {
-                rustc_hir::ExprKind::Cast(ref expr, ref ty) => {
+                rustc_hir::ExprKind::Cast(expr, ty) => {
                     if let rustc_hir::TyKind::Ptr(_) = ty.kind {
                         return is_zero(expr) || is_null_ptr(cx, expr);
                     }
                 }
                 // check for call to `core::ptr::null` or `core::ptr::null_mut`
-                rustc_hir::ExprKind::Call(ref path, _) => {
+                rustc_hir::ExprKind::Call(path, _) => {
                     if let rustc_hir::ExprKind::Path(ref qpath) = path.kind {
                         if let Some(def_id) = cx.qpath_res(qpath, path.hir_id).opt_def_id() {
                             return matches!(
@@ -2672,7 +2671,7 @@ impl<'tcx> LateLintPass<'tcx> for DerefNullPtr {
         /// test if expression is the literal `0`
         fn is_zero(expr: &hir::Expr<'_>) -> bool {
             match &expr.kind {
-                rustc_hir::ExprKind::Lit(ref lit) => {
+                rustc_hir::ExprKind::Lit(lit) => {
                     if let LitKind::Int(a, _) = lit.node {
                         return a == 0;
                     }
@@ -2801,7 +2800,7 @@ impl<'tcx> LateLintPass<'tcx> for NamedAsmLabels {
                             NAMED_ASM_LABELS,
                             Some(target_spans),
                             fluent::lint_builtin_asm_labels,
-                            |lint| lint,
+                            |_| {},
                             BuiltinLintDiagnostics::NamedAsmLabel(
                                 "only local labels of the form `<number>:` should be used in inline asm"
                                     .to_string(),
@@ -2889,26 +2888,3 @@ impl EarlyLintPass for SpecialModuleName {
         }
     }
 }
-
-pub use rustc_session::lint::builtin::UNEXPECTED_CFGS;
-
-declare_lint_pass!(UnexpectedCfgs => [UNEXPECTED_CFGS]);
-
-impl EarlyLintPass for UnexpectedCfgs {
-    fn check_crate(&mut self, cx: &EarlyContext<'_>, _: &ast::Crate) {
-        let cfg = &cx.sess().parse_sess.config;
-        let check_cfg = &cx.sess().parse_sess.check_config;
-        for &(name, value) in cfg {
-            match check_cfg.expecteds.get(&name) {
-                Some(ExpectedValues::Some(values)) if !values.contains(&value) => {
-                    let value = value.unwrap_or(kw::Empty);
-                    cx.emit_lint(UNEXPECTED_CFGS, BuiltinUnexpectedCliConfigValue { name, value });
-                }
-                None if check_cfg.exhaustive_names => {
-                    cx.emit_lint(UNEXPECTED_CFGS, BuiltinUnexpectedCliConfigName { name });
-                }
-                _ => { /* expected */ }
-            }
-        }
-    }
-}
diff --git a/compiler/rustc_lint/src/context.rs b/compiler/rustc_lint/src/context.rs
index a5f4c5ff0..c7a9d5e80 100644
--- a/compiler/rustc_lint/src/context.rs
+++ b/compiler/rustc_lint/src/context.rs
@@ -36,7 +36,7 @@ use rustc_middle::ty::{self, print::Printer, GenericArg, RegisteredTools, Ty, Ty
 use rustc_session::config::ExpectedValues;
 use rustc_session::lint::{BuiltinLintDiagnostics, LintExpectationId};
 use rustc_session::lint::{FutureIncompatibleInfo, Level, Lint, LintBuffer, LintId};
-use rustc_session::Session;
+use rustc_session::{LintStoreMarker, Session};
 use rustc_span::edit_distance::find_best_match_for_name;
 use rustc_span::symbol::{sym, Ident, Symbol};
 use rustc_span::{BytePos, Span};
@@ -77,6 +77,8 @@ pub struct LintStore {
     lint_groups: FxHashMap<&'static str, LintGroup>,
 }
 
+impl LintStoreMarker for LintStore {}
+
 /// The target of the `by_name` map, which accounts for renaming/deprecation.
 #[derive(Debug)]
 enum TargetLint {
@@ -385,7 +387,7 @@ impl LintStore {
                         };
                     }
                     Some(LintGroup { lint_ids, .. }) => {
-                        return CheckLintNameResult::Tool(Ok(&lint_ids));
+                        return CheckLintNameResult::Tool(Ok(lint_ids));
                     }
                 },
                 Some(Id(id)) => return CheckLintNameResult::Tool(Ok(slice::from_ref(id))),
@@ -406,12 +408,12 @@ impl LintStore {
                     if let Some(LintAlias { name, silent }) = depr {
                         let LintGroup { lint_ids, .. } = self.lint_groups.get(name).unwrap();
                         return if *silent {
-                            CheckLintNameResult::Ok(&lint_ids)
+                            CheckLintNameResult::Ok(lint_ids)
                         } else {
-                            CheckLintNameResult::Tool(Err((Some(&lint_ids), (*name).to_string())))
+                            CheckLintNameResult::Tool(Err((Some(lint_ids), (*name).to_string())))
                         };
                     }
-                    CheckLintNameResult::Ok(&lint_ids)
+                    CheckLintNameResult::Ok(lint_ids)
                 }
             },
             Some(Id(id)) => CheckLintNameResult::Ok(slice::from_ref(id)),
@@ -458,12 +460,12 @@ impl LintStore {
                     if let Some(LintAlias { name, silent }) = depr {
                         let LintGroup { lint_ids, .. } = self.lint_groups.get(name).unwrap();
                         return if *silent {
-                            CheckLintNameResult::Tool(Err((Some(&lint_ids), complete_name)))
+                            CheckLintNameResult::Tool(Err((Some(lint_ids), complete_name)))
                         } else {
-                            CheckLintNameResult::Tool(Err((Some(&lint_ids), (*name).to_string())))
+                            CheckLintNameResult::Tool(Err((Some(lint_ids), (*name).to_string())))
                         };
                     }
-                    CheckLintNameResult::Tool(Err((Some(&lint_ids), complete_name)))
+                    CheckLintNameResult::Tool(Err((Some(lint_ids), complete_name)))
                 }
             },
             Some(Id(id)) => {
@@ -497,9 +499,6 @@ pub struct LateContext<'tcx> {
     /// Items accessible from the crate being checked.
     pub effective_visibilities: &'tcx EffectiveVisibilities,
 
-    /// The store of registered lints and the lint levels.
-    pub lint_store: &'tcx LintStore,
-
     pub last_node_with_lint_attrs: hir::HirId,
 
     /// Generic type parameters in scope for the item we are in.
@@ -515,21 +514,11 @@ pub struct EarlyContext<'a> {
     pub buffered: LintBuffer,
 }
 
-pub trait LintPassObject: Sized {}
-
-impl LintPassObject for EarlyLintPassObject {}
-
-impl LintPassObject for LateLintPassObject<'_> {}
-
-pub trait LintContext: Sized {
-    type PassObject: LintPassObject;
-
+pub trait LintContext {
     fn sess(&self) -> &Session;
-    fn lints(&self) -> &LintStore;
 
-    /// Emit a lint at the appropriate level, with an optional associated span and an existing diagnostic.
-    ///
-    /// Return value of the `decorate` closure is ignored, see [`struct_lint_level`] for a detailed explanation.
+    /// Emit a lint at the appropriate level, with an optional associated span and an existing
+    /// diagnostic.
     ///
     /// [`struct_lint_level`]: rustc_middle::lint::struct_lint_level#decorate-signature
     #[rustc_lint_diagnostics]
@@ -538,9 +527,7 @@ pub trait LintContext: Sized {
         lint: &'static Lint,
         span: Option<impl Into<MultiSpan>>,
         msg: impl Into<DiagnosticMessage>,
-        decorate: impl for<'a, 'b> FnOnce(
-            &'b mut DiagnosticBuilder<'a, ()>,
-        ) -> &'b mut DiagnosticBuilder<'a, ()>,
+        decorate: impl for<'a, 'b> FnOnce(&'b mut DiagnosticBuilder<'a, ()>),
         diagnostic: BuiltinLintDiagnostics,
     ) {
         // We first generate a blank diagnostic.
@@ -713,10 +700,14 @@ pub trait LintContext: Sized {
                     db.note("see the asm section of Rust By Example <https://doc.rust-lang.org/nightly/rust-by-example/unsafe/asm.html#labels> for more information");
                 },
                 BuiltinLintDiagnostics::UnexpectedCfgName((name, name_span), value) => {
-                    let possibilities: Vec<Symbol> = sess.parse_sess.check_config.expecteds.keys().map(|s| *s).collect();
+                    let possibilities: Vec<Symbol> = sess.parse_sess.check_config.expecteds.keys().copied().collect();
+                    let is_from_cargo = std::env::var_os("CARGO").is_some();
+                    let mut is_feature_cfg = name == sym::feature;
 
+                    if is_feature_cfg && is_from_cargo {
+                        db.help("consider defining some features in `Cargo.toml`");
                     // Suggest the most probable if we found one
-                    if let Some(best_match) = find_best_match_for_name(&possibilities, name, None) {
+                    } else if let Some(best_match) = find_best_match_for_name(&possibilities, name, None) {
                         if let Some(ExpectedValues::Some(best_match_values)) =
                             sess.parse_sess.check_config.expecteds.get(&best_match) {
                             let mut possibilities = best_match_values.iter()
@@ -749,6 +740,8 @@ pub trait LintContext: Sized {
                         } else {
                             db.span_suggestion(name_span, "there is a config with a similar name", best_match, Applicability::MaybeIncorrect);
                         }
+
+                        is_feature_cfg |= best_match == sym::feature;
                     } else if !possibilities.is_empty() {
                         let mut possibilities = possibilities.iter()
                             .map(Symbol::as_str)
@@ -762,6 +755,23 @@ pub trait LintContext: Sized {
                         // once.
                         db.help_once(format!("expected names are: `{possibilities}`"));
                     }
+
+                    let inst = if let Some((value, _value_span)) = value {
+                        let pre = if is_from_cargo { "\\" } else { "" };
+                        format!("cfg({name}, values({pre}\"{value}{pre}\"))")
+                    } else {
+                        format!("cfg({name})")
+                    };
+
+                    if is_from_cargo {
+                        if !is_feature_cfg {
+                            db.help(format!("consider using a Cargo feature instead or adding `println!(\"cargo:rustc-check-cfg={inst}\");` to the top of a `build.rs`"));
+                        }
+                        db.note("see <https://doc.rust-lang.org/nightly/cargo/reference/unstable.html#check-cfg> for more information about checking conditional configuration");
+                    } else {
+                        db.help(format!("to expect this configuration use `--check-cfg={inst}`"));
+                        db.note("see <https://doc.rust-lang.org/nightly/unstable-book/compiler-flags/check-cfg.html> for more information about checking conditional configuration");
+                    }
                 },
                 BuiltinLintDiagnostics::UnexpectedCfgValue((name, name_span), value) => {
                     let Some(ExpectedValues::Some(values)) = &sess.parse_sess.check_config.expecteds.get(&name) else {
@@ -773,6 +783,7 @@ pub trait LintContext: Sized {
                         .copied()
                         .flatten()
                         .collect();
+                    let is_from_cargo = std::env::var_os("CARGO").is_some();
 
                     // Show the full list if all possible values for a given name, but don't do it
                     // for names as the possibilities could be very long
@@ -793,6 +804,8 @@ pub trait LintContext: Sized {
                                 db.span_suggestion(value_span, "there is a expected value with a similar name", format!("\"{best_match}\""), Applicability::MaybeIncorrect);
 
                             }
+                        } else if name == sym::feature && is_from_cargo {
+                            db.help(format!("consider defining `{name}` as feature in `Cargo.toml`"));
                         } else if let &[first_possibility] = &possibilities[..] {
                             db.span_suggestion(name_span.shrink_to_hi(), "specify a config value", format!(" = \"{first_possibility}\""), Applicability::MaybeIncorrect);
                         }
@@ -802,6 +815,27 @@ pub trait LintContext: Sized {
                             db.span_suggestion(name_span.shrink_to_hi().to(value_span), "remove the value", "", Applicability::MaybeIncorrect);
                         }
                     }
+
+                    let inst = if let Some((value, _value_span)) = value {
+                        let pre = if is_from_cargo { "\\" } else { "" };
+                        format!("cfg({name}, values({pre}\"{value}{pre}\"))")
+                    } else {
+                        format!("cfg({name})")
+                    };
+
+                    if is_from_cargo {
+                        if name == sym::feature {
+                            if let Some((value, _value_span)) = value {
+                                db.help(format!("consider adding `{value}` as a feature in `Cargo.toml`"));
+                            }
+                        } else {
+                            db.help(format!("consider using a Cargo feature instead or adding `println!(\"cargo:rustc-check-cfg={inst}\");` to the top of a `build.rs`"));
+                        }
+                        db.note("see <https://doc.rust-lang.org/nightly/cargo/reference/unstable.html#check-cfg> for more information about checking conditional configuration");
+                    } else {
+                        db.help(format!("to expect this configuration use `--check-cfg={inst}`"));
+                        db.note("see <https://doc.rust-lang.org/nightly/unstable-book/compiler-flags/check-cfg.html> for more information about checking conditional configuration");
+                    }
                 },
                 BuiltinLintDiagnostics::DeprecatedWhereclauseLocation(new_span, suggestion) => {
                     db.multipart_suggestion(
@@ -932,6 +966,10 @@ pub trait LintContext: Sized {
                         if elided { "'static " } else { "'static" },
                         Applicability::MachineApplicable
                     );
+                },
+                BuiltinLintDiagnostics::RedundantImportVisibility { max_vis, span } => {
+                    db.span_note(span, format!("the most public imported item is `{max_vis}`"));
+                    db.help("reduce the glob import's visibility or increase visibility of imported items");
                 }
             }
             // Rewrap `db`, and pass control to the user.
@@ -943,8 +981,6 @@ pub trait LintContext: Sized {
     // set the span in their `decorate` function (preferably using set_span).
     /// Emit a lint at the appropriate level, with an optional associated span.
     ///
-    /// Return value of the `decorate` closure is ignored, see [`struct_lint_level`] for a detailed explanation.
-    ///
     /// [`struct_lint_level`]: rustc_middle::lint::struct_lint_level#decorate-signature
     #[rustc_lint_diagnostics]
     fn lookup<S: Into<MultiSpan>>(
@@ -952,9 +988,7 @@ pub trait LintContext: Sized {
         lint: &'static Lint,
         span: Option<S>,
         msg: impl Into<DiagnosticMessage>,
-        decorate: impl for<'a, 'b> FnOnce(
-            &'b mut DiagnosticBuilder<'a, ()>,
-        ) -> &'b mut DiagnosticBuilder<'a, ()>,
+        decorate: impl for<'a, 'b> FnOnce(&'b mut DiagnosticBuilder<'a, ()>),
     );
 
     /// Emit a lint at `span` from a lint struct (some type that implements `DecorateLint`,
@@ -965,13 +999,13 @@ pub trait LintContext: Sized {
         span: S,
         decorator: impl for<'a> DecorateLint<'a, ()>,
     ) {
-        self.lookup(lint, Some(span), decorator.msg(), |diag| decorator.decorate_lint(diag));
+        self.lookup(lint, Some(span), decorator.msg(), |diag| {
+            decorator.decorate_lint(diag);
+        });
     }
 
     /// Emit a lint at the appropriate level, with an associated span.
     ///
-    /// Return value of the `decorate` closure is ignored, see [`struct_lint_level`] for a detailed explanation.
-    ///
     /// [`struct_lint_level`]: rustc_middle::lint::struct_lint_level#decorate-signature
     #[rustc_lint_diagnostics]
     fn struct_span_lint<S: Into<MultiSpan>>(
@@ -979,9 +1013,7 @@ pub trait LintContext: Sized {
         lint: &'static Lint,
         span: S,
         msg: impl Into<DiagnosticMessage>,
-        decorate: impl for<'a, 'b> FnOnce(
-            &'b mut DiagnosticBuilder<'a, ()>,
-        ) -> &'b mut DiagnosticBuilder<'a, ()>,
+        decorate: impl for<'a, 'b> FnOnce(&'b mut DiagnosticBuilder<'a, ()>),
     ) {
         self.lookup(lint, Some(span), msg, decorate);
     }
@@ -990,23 +1022,19 @@ pub trait LintContext: Sized {
     /// generated by `#[derive(LintDiagnostic)]`).
     fn emit_lint(&self, lint: &'static Lint, decorator: impl for<'a> DecorateLint<'a, ()>) {
         self.lookup(lint, None as Option<Span>, decorator.msg(), |diag| {
-            decorator.decorate_lint(diag)
+            decorator.decorate_lint(diag);
         });
     }
 
     /// Emit a lint at the appropriate level, with no associated span.
     ///
-    /// Return value of the `decorate` closure is ignored, see [`struct_lint_level`] for a detailed explanation.
-    ///
     /// [`struct_lint_level`]: rustc_middle::lint::struct_lint_level#decorate-signature
     #[rustc_lint_diagnostics]
     fn lint(
         &self,
         lint: &'static Lint,
         msg: impl Into<DiagnosticMessage>,
-        decorate: impl for<'a, 'b> FnOnce(
-            &'b mut DiagnosticBuilder<'a, ()>,
-        ) -> &'b mut DiagnosticBuilder<'a, ()>,
+        decorate: impl for<'a, 'b> FnOnce(&'b mut DiagnosticBuilder<'a, ()>),
     ) {
         self.lookup(lint, None as Option<Span>, msg, decorate);
     }
@@ -1059,15 +1087,9 @@ impl<'a> EarlyContext<'a> {
 }
 
 impl<'tcx> LintContext for LateContext<'tcx> {
-    type PassObject = LateLintPassObject<'tcx>;
-
     /// Gets the overall compiler `Session` object.
     fn sess(&self) -> &Session {
-        &self.tcx.sess
-    }
-
-    fn lints(&self) -> &LintStore {
-        &*self.lint_store
+        self.tcx.sess
     }
 
     #[rustc_lint_diagnostics]
@@ -1076,9 +1098,7 @@ impl<'tcx> LintContext for LateContext<'tcx> {
         lint: &'static Lint,
         span: Option<S>,
         msg: impl Into<DiagnosticMessage>,
-        decorate: impl for<'a, 'b> FnOnce(
-            &'b mut DiagnosticBuilder<'a, ()>,
-        ) -> &'b mut DiagnosticBuilder<'a, ()>,
+        decorate: impl for<'a, 'b> FnOnce(&'b mut DiagnosticBuilder<'a, ()>),
     ) {
         let hir_id = self.last_node_with_lint_attrs;
 
@@ -1094,15 +1114,9 @@ impl<'tcx> LintContext for LateContext<'tcx> {
 }
 
 impl LintContext for EarlyContext<'_> {
-    type PassObject = EarlyLintPassObject;
-
     /// Gets the overall compiler `Session` object.
     fn sess(&self) -> &Session {
-        &self.builder.sess()
-    }
-
-    fn lints(&self) -> &LintStore {
-        self.builder.lint_store()
+        self.builder.sess()
     }
 
     #[rustc_lint_diagnostics]
@@ -1111,9 +1125,7 @@ impl LintContext for EarlyContext<'_> {
         lint: &'static Lint,
         span: Option<S>,
         msg: impl Into<DiagnosticMessage>,
-        decorate: impl for<'a, 'b> FnOnce(
-            &'b mut DiagnosticBuilder<'a, ()>,
-        ) -> &'b mut DiagnosticBuilder<'a, ()>,
+        decorate: impl for<'a, 'b> FnOnce(&'b mut DiagnosticBuilder<'a, ()>),
     ) {
         self.builder.struct_lint(lint, span.map(|s| s.into()), msg, decorate)
     }
@@ -1149,7 +1161,7 @@ impl<'tcx> LateContext<'tcx> {
     /// bodies (e.g. for paths in `hir::Ty`), without any risk of ICE-ing.
     pub fn qpath_res(&self, qpath: &hir::QPath<'_>, id: hir::HirId) -> Res {
         match *qpath {
-            hir::QPath::Resolved(_, ref path) => path.res,
+            hir::QPath::Resolved(_, path) => path.res,
             hir::QPath::TypeRelative(..) | hir::QPath::LangItem(..) => self
                 .maybe_typeck_results()
                 .filter(|typeck_results| typeck_results.hir_owner == id.owner)
diff --git a/compiler/rustc_lint/src/deref_into_dyn_supertrait.rs b/compiler/rustc_lint/src/deref_into_dyn_supertrait.rs
index d2d99bc0d..4673b801d 100644
--- a/compiler/rustc_lint/src/deref_into_dyn_supertrait.rs
+++ b/compiler/rustc_lint/src/deref_into_dyn_supertrait.rs
@@ -50,7 +50,7 @@ declare_lint! {
     Warn,
     "`Deref` implementation usage with a supertrait trait object for output might be shadowed in the future",
     @future_incompatible = FutureIncompatibleInfo {
-        reason: FutureIncompatibilityReason::FutureReleaseErrorDontReportInDeps,
+        reason: FutureIncompatibilityReason::FutureReleaseSemanticsChange,
         reference: "issue #89460 <https://github.com/rust-lang/rust/issues/89460>",
     };
 }
@@ -59,32 +59,46 @@ declare_lint_pass!(DerefIntoDynSupertrait => [DEREF_INTO_DYN_SUPERTRAIT]);
 
 impl<'tcx> LateLintPass<'tcx> for DerefIntoDynSupertrait {
     fn check_item(&mut self, cx: &LateContext<'tcx>, item: &'tcx hir::Item<'tcx>) {
+        let tcx = cx.tcx;
         // `Deref` is being implemented for `t`
         if let hir::ItemKind::Impl(impl_) = item.kind
+            // the trait is a `Deref` implementation
             && let Some(trait_) = &impl_.of_trait
-            && let t = cx.tcx.type_of(item.owner_id).instantiate_identity()
-            && let opt_did @ Some(did) = trait_.trait_def_id()
-            && opt_did == cx.tcx.lang_items().deref_trait()
-            // `t` is `dyn t_principal`
-            && let ty::Dynamic(data, _, ty::Dyn) = t.kind()
-            && let Some(t_principal) = data.principal()
+            && let Some(did) = trait_.trait_def_id()
+            && Some(did) == tcx.lang_items().deref_trait()
+            // the self type is `dyn t_principal`
+            && let self_ty = tcx.type_of(item.owner_id).instantiate_identity()
+            && let ty::Dynamic(data, _, ty::Dyn) = self_ty.kind()
+            && let Some(self_principal) = data.principal()
             // `<T as Deref>::Target` is `dyn target_principal`
-            && let Some(target) = cx.get_associated_type(t, did, "Target")
+            && let Some(target) = cx.get_associated_type(self_ty, did, "Target")
             && let ty::Dynamic(data, _, ty::Dyn) = target.kind()
             && let Some(target_principal) = data.principal()
             // `target_principal` is a supertrait of `t_principal`
-            && supertraits(cx.tcx, t_principal.with_self_ty(cx.tcx, cx.tcx.types.trait_object_dummy_self))
-                .any(|sup| sup.map_bound(|x| ty::ExistentialTraitRef::erase_self_ty(cx.tcx, x)) == target_principal)
+            && let Some(supertrait_principal) = supertraits(tcx, self_principal.with_self_ty(tcx, self_ty))
+                .find(|supertrait| supertrait.def_id() == target_principal.def_id())
         {
-            let label = impl_
+            // erase regions in self type for better diagnostic presentation
+            let (self_ty, target_principal, supertrait_principal) =
+                tcx.erase_regions((self_ty, target_principal, supertrait_principal));
+            let label2 = impl_
                 .items
                 .iter()
                 .find_map(|i| (i.ident.name == sym::Target).then_some(i.span))
                 .map(|label| SupertraitAsDerefTargetLabel { label });
+            let span = tcx.def_span(item.owner_id.def_id);
             cx.emit_spanned_lint(
                 DEREF_INTO_DYN_SUPERTRAIT,
-                cx.tcx.def_span(item.owner_id.def_id),
-                SupertraitAsDerefTarget { t, target_principal, label },
+                span,
+                SupertraitAsDerefTarget {
+                    self_ty,
+                    supertrait_principal: supertrait_principal.map_bound(|trait_ref| {
+                        ty::ExistentialTraitRef::erase_self_ty(tcx, trait_ref)
+                    }),
+                    target_principal,
+                    label: span,
+                    label2,
+                },
             );
         }
     }
diff --git a/compiler/rustc_lint/src/early.rs b/compiler/rustc_lint/src/early.rs
index d102e3a6c..b9add9e9f 100644
--- a/compiler/rustc_lint/src/early.rs
+++ b/compiler/rustc_lint/src/early.rs
@@ -45,13 +45,7 @@ impl<'a, T: EarlyLintPass> EarlyContextAndPass<'a, T> {
     fn inlined_check_id(&mut self, id: ast::NodeId) {
         for early_lint in self.context.buffered.take(id) {
             let BufferedEarlyLint { span, msg, node_id: _, lint_id, diagnostic } = early_lint;
-            self.context.lookup_with_diagnostics(
-                lint_id.lint,
-                Some(span),
-                msg,
-                |lint| lint,
-                diagnostic,
-            );
+            self.context.lookup_with_diagnostics(lint_id.lint, Some(span), msg, |_| {}, diagnostic);
         }
     }
 
@@ -162,8 +156,8 @@ impl<'a, T: EarlyLintPass> ast_visit::Visitor<'a> for EarlyContextAndPass<'a, T>
         // Explicitly check for lints associated with 'closure_id', since
         // it does not have a corresponding AST node
         if let ast_visit::FnKind::Fn(_, _, sig, _, _, _) = fk {
-            if let ast::Async::Yes { closure_id, .. } = sig.header.asyncness {
-                self.check_id(closure_id);
+            if let Some(coroutine_kind) = sig.header.coroutine_kind {
+                self.check_id(coroutine_kind.closure_id());
             }
         }
     }
@@ -223,9 +217,11 @@ impl<'a, T: EarlyLintPass> ast_visit::Visitor<'a> for EarlyContextAndPass<'a, T>
         // it does not have a corresponding AST node
         match e.kind {
             ast::ExprKind::Closure(box ast::Closure {
-                asyncness: ast::Async::Yes { closure_id, .. },
+                coroutine_kind: Some(coroutine_kind),
                 ..
-            }) => self.check_id(closure_id),
+            }) => {
+                self.check_id(coroutine_kind.closure_id());
+            }
             _ => {}
         }
         lint_callback!(self, check_expr_post, e);
@@ -350,7 +346,7 @@ impl<'a> EarlyCheckNode<'a> for (&'a ast::Crate, &'a [ast::Attribute]) {
     where
         'a: 'b,
     {
-        &self.1
+        self.1
     }
     fn check<'b, T: EarlyLintPass>(self, cx: &mut EarlyContextAndPass<'b, T>)
     where
@@ -430,7 +426,7 @@ pub fn check_ast_node_inner<'a, T: EarlyLintPass>(
     // that was not lint-checked (perhaps it doesn't exist?). This is a bug.
     for (id, lints) in cx.context.buffered.map {
         for early_lint in lints {
-            sess.delay_span_bug(
+            sess.span_delayed_bug(
                 early_lint.span,
                 format!(
                     "failed to process buffered lint here (dummy = {})",
diff --git a/compiler/rustc_lint/src/expect.rs b/compiler/rustc_lint/src/expect.rs
index 740c90757..5dcc1bce5 100644
--- a/compiler/rustc_lint/src/expect.rs
+++ b/compiler/rustc_lint/src/expect.rs
@@ -16,7 +16,7 @@ fn check_expectations(tcx: TyCtxt<'_>, tool_filter: Option<Symbol>) {
     }
 
     let lint_expectations = tcx.lint_expectations(());
-    let fulfilled_expectations = tcx.sess.diagnostic().steal_fulfilled_expectation_ids();
+    let fulfilled_expectations = tcx.sess.dcx().steal_fulfilled_expectation_ids();
 
     tracing::debug!(?lint_expectations, ?fulfilled_expectations);
 
@@ -24,7 +24,7 @@ fn check_expectations(tcx: TyCtxt<'_>, tool_filter: Option<Symbol>) {
         // This check will always be true, since `lint_expectations` only
         // holds stable ids
         if let LintExpectationId::Stable { hir_id, .. } = id {
-            if !fulfilled_expectations.contains(&id)
+            if !fulfilled_expectations.contains(id)
                 && tool_filter.map_or(true, |filter| expectation.lint_tool == Some(filter))
             {
                 let rationale = expectation.reason.map(|rationale| ExpectationNote { rationale });
diff --git a/compiler/rustc_lint/src/for_loops_over_fallibles.rs b/compiler/rustc_lint/src/for_loops_over_fallibles.rs
index c8ec0458b..ea922785a 100644
--- a/compiler/rustc_lint/src/for_loops_over_fallibles.rs
+++ b/compiler/rustc_lint/src/for_loops_over_fallibles.rs
@@ -137,7 +137,7 @@ fn suggest_question_mark<'tcx>(
     // Check that the function/closure/constant we are in has a `Result` type.
     // Otherwise suggesting using `?` may not be a good idea.
     {
-        let ty = cx.typeck_results().expr_ty(&cx.tcx.hir().body(body_id).value);
+        let ty = cx.typeck_results().expr_ty(cx.tcx.hir().body(body_id).value);
         let ty::Adt(ret_adt, ..) = ty.kind() else { return false };
         if !cx.tcx.is_diagnostic_item(sym::Result, ret_adt.did()) {
             return false;
diff --git a/compiler/rustc_lint/src/foreign_modules.rs b/compiler/rustc_lint/src/foreign_modules.rs
index 86b3b4ad0..31d9c0d33 100644
--- a/compiler/rustc_lint/src/foreign_modules.rs
+++ b/compiler/rustc_lint/src/foreign_modules.rs
@@ -341,8 +341,8 @@ fn structurally_same_type_impl<'tcx>(
 
                     // We don't compare regions, but leaving bound regions around ICEs, so
                     // we erase them.
-                    let a_sig = tcx.erase_late_bound_regions(a_poly_sig);
-                    let b_sig = tcx.erase_late_bound_regions(b_poly_sig);
+                    let a_sig = tcx.instantiate_bound_regions_with_erased(a_poly_sig);
+                    let b_sig = tcx.instantiate_bound_regions_with_erased(b_poly_sig);
 
                     (a_sig.abi, a_sig.unsafety, a_sig.c_variadic)
                         == (b_sig.abi, b_sig.unsafety, b_sig.c_variadic)
diff --git a/compiler/rustc_lint/src/internal.rs b/compiler/rustc_lint/src/internal.rs
index 2d86129c4..53d99c7f7 100644
--- a/compiler/rustc_lint/src/internal.rs
+++ b/compiler/rustc_lint/src/internal.rs
@@ -34,7 +34,7 @@ declare_lint_pass!(DefaultHashTypes => [DEFAULT_HASH_TYPES]);
 impl LateLintPass<'_> for DefaultHashTypes {
     fn check_path(&mut self, cx: &LateContext<'_>, path: &Path<'_>, hir_id: HirId) {
         let Res::Def(rustc_hir::def::DefKind::Struct, def_id) = path.res else { return };
-        if matches!(cx.tcx.hir().get(hir_id), Node::Item(Item { kind: ItemKind::Use(..), .. })) {
+        if matches!(cx.tcx.hir_node(hir_id), Node::Item(Item { kind: ItemKind::Use(..), .. })) {
             // don't lint imports, only actual usages
             return;
         }
@@ -196,7 +196,7 @@ impl<'tcx> LateLintPass<'tcx> for TyTyKind {
                     }
                 } else if !ty.span.from_expansion()
                     && path.segments.len() > 1
-                    && let Some(ty) = is_ty_or_ty_ctxt(cx, &path)
+                    && let Some(ty) = is_ty_or_ty_ctxt(cx, path)
                 {
                     cx.emit_spanned_lint(
                         USAGE_OF_QUALIFIED_TY,
diff --git a/compiler/rustc_lint/src/late.rs b/compiler/rustc_lint/src/late.rs
index 6c8b60c8d..caa015565 100644
--- a/compiler/rustc_lint/src/late.rs
+++ b/compiler/rustc_lint/src/late.rs
@@ -17,22 +17,25 @@
 use crate::{passes::LateLintPassObject, LateContext, LateLintPass, LintStore};
 use rustc_ast as ast;
 use rustc_data_structures::stack::ensure_sufficient_stack;
-use rustc_data_structures::sync::join;
+use rustc_data_structures::sync::{join, Lrc};
 use rustc_hir as hir;
 use rustc_hir::def_id::{LocalDefId, LocalModDefId};
 use rustc_hir::intravisit as hir_visit;
 use rustc_middle::hir::nested_filter;
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_session::lint::LintPass;
+use rustc_session::Session;
 use rustc_span::Span;
 
 use std::any::Any;
 use std::cell::Cell;
 
-/// Extract the `LintStore` from the query context.
-/// This function exists because we've erased `LintStore` as `dyn Any` in the context.
-pub fn unerased_lint_store(tcx: TyCtxt<'_>) -> &LintStore {
-    let store: &dyn Any = &*tcx.lint_store;
+/// Extract the [`LintStore`] from [`Session`].
+///
+/// This function exists because [`Session::lint_store`] is type-erased.
+pub fn unerased_lint_store(sess: &Session) -> &LintStore {
+    let store: &Lrc<_> = sess.lint_store.as_ref().unwrap();
+    let store: &dyn Any = &**store;
     store.downcast_ref().unwrap()
 }
 
@@ -276,7 +279,7 @@ impl<'tcx, T: LateLintPass<'tcx>> hir_visit::Visitor<'tcx> for LateContextAndPas
 
     fn visit_trait_item(&mut self, trait_item: &'tcx hir::TraitItem<'tcx>) {
         let generics = self.context.generics.take();
-        self.context.generics = Some(&trait_item.generics);
+        self.context.generics = Some(trait_item.generics);
         self.with_lint_attrs(trait_item.hir_id(), |cx| {
             cx.with_param_env(trait_item.owner_id, |cx| {
                 lint_callback!(cx, check_trait_item, trait_item);
@@ -288,7 +291,7 @@ impl<'tcx, T: LateLintPass<'tcx>> hir_visit::Visitor<'tcx> for LateContextAndPas
 
     fn visit_impl_item(&mut self, impl_item: &'tcx hir::ImplItem<'tcx>) {
         let generics = self.context.generics.take();
-        self.context.generics = Some(&impl_item.generics);
+        self.context.generics = Some(impl_item.generics);
         self.with_lint_attrs(impl_item.hir_id(), |cx| {
             cx.with_param_env(impl_item.owner_id, |cx| {
                 lint_callback!(cx, check_impl_item, impl_item);
@@ -352,9 +355,8 @@ pub fn late_lint_mod<'tcx, T: LateLintPass<'tcx> + 'tcx>(
         enclosing_body: None,
         cached_typeck_results: Cell::new(None),
         param_env: ty::ParamEnv::empty(),
-        effective_visibilities: &tcx.effective_visibilities(()),
-        lint_store: unerased_lint_store(tcx),
-        last_node_with_lint_attrs: tcx.hir().local_def_id_to_hir_id(module_def_id),
+        effective_visibilities: tcx.effective_visibilities(()),
+        last_node_with_lint_attrs: tcx.local_def_id_to_hir_id(module_def_id.into()),
         generics: None,
         only_module: true,
     };
@@ -362,8 +364,11 @@ pub fn late_lint_mod<'tcx, T: LateLintPass<'tcx> + 'tcx>(
     // Note: `passes` is often empty. In that case, it's faster to run
     // `builtin_lints` directly rather than bundling it up into the
     // `RuntimeCombinedLateLintPass`.
-    let mut passes: Vec<_> =
-        unerased_lint_store(tcx).late_module_passes.iter().map(|mk_pass| (mk_pass)(tcx)).collect();
+    let mut passes: Vec<_> = unerased_lint_store(tcx.sess)
+        .late_module_passes
+        .iter()
+        .map(|mk_pass| (mk_pass)(tcx))
+        .collect();
     if passes.is_empty() {
         late_lint_mod_inner(tcx, module_def_id, context, builtin_lints);
     } else {
@@ -400,7 +405,7 @@ fn late_lint_mod_inner<'tcx, T: LateLintPass<'tcx>>(
 fn late_lint_crate<'tcx>(tcx: TyCtxt<'tcx>) {
     // Note: `passes` is often empty.
     let mut passes: Vec<_> =
-        unerased_lint_store(tcx).late_passes.iter().map(|mk_pass| (mk_pass)(tcx)).collect();
+        unerased_lint_store(tcx.sess).late_passes.iter().map(|mk_pass| (mk_pass)(tcx)).collect();
 
     if passes.is_empty() {
         return;
@@ -411,8 +416,7 @@ fn late_lint_crate<'tcx>(tcx: TyCtxt<'tcx>) {
         enclosing_body: None,
         cached_typeck_results: Cell::new(None),
         param_env: ty::ParamEnv::empty(),
-        effective_visibilities: &tcx.effective_visibilities(()),
-        lint_store: unerased_lint_store(tcx),
+        effective_visibilities: tcx.effective_visibilities(()),
         last_node_with_lint_attrs: hir::CRATE_HIR_ID,
         generics: None,
         only_module: false,
diff --git a/compiler/rustc_lint/src/levels.rs b/compiler/rustc_lint/src/levels.rs
index 0d20f6232..6eff2bfe1 100644
--- a/compiler/rustc_lint/src/levels.rs
+++ b/compiler/rustc_lint/src/levels.rs
@@ -56,7 +56,6 @@ struct LintLevelSets {
 }
 
 rustc_index::newtype_index! {
-    #[custom_encodable] // we don't need encoding
     struct LintStackIndex {
         const COMMAND_LINE = 0;
     }
@@ -123,7 +122,7 @@ impl LintLevelSets {
 }
 
 fn lint_expectations(tcx: TyCtxt<'_>, (): ()) -> Vec<(LintExpectationId, LintExpectation)> {
-    let store = unerased_lint_store(tcx);
+    let store = unerased_lint_store(tcx.sess);
 
     let mut builder = LintLevelsBuilder {
         sess: tcx.sess,
@@ -138,21 +137,21 @@ fn lint_expectations(tcx: TyCtxt<'_>, (): ()) -> Vec<(LintExpectationId, LintExp
         },
         warn_about_weird_lints: false,
         store,
-        registered_tools: &tcx.registered_tools(()),
+        registered_tools: tcx.registered_tools(()),
     };
 
     builder.add_command_line();
     builder.add_id(hir::CRATE_HIR_ID);
     tcx.hir().walk_toplevel_module(&mut builder);
 
-    tcx.sess.diagnostic().update_unstable_expectation_id(&builder.provider.unstable_to_stable_ids);
+    tcx.sess.dcx().update_unstable_expectation_id(&builder.provider.unstable_to_stable_ids);
 
     builder.provider.expectations
 }
 
 #[instrument(level = "trace", skip(tcx), ret)]
 fn shallow_lint_levels_on(tcx: TyCtxt<'_>, owner: hir::OwnerId) -> ShallowLintLevelMap {
-    let store = unerased_lint_store(tcx);
+    let store = unerased_lint_store(tcx.sess);
     let attrs = tcx.hir_attrs(owner);
 
     let mut levels = LintLevelsBuilder {
@@ -167,7 +166,7 @@ fn shallow_lint_levels_on(tcx: TyCtxt<'_>, owner: hir::OwnerId) -> ShallowLintLe
         },
         warn_about_weird_lints: false,
         store,
-        registered_tools: &tcx.registered_tools(()),
+        registered_tools: tcx.registered_tools(()),
     };
 
     if owner == hir::CRATE_OWNER_ID {
@@ -548,10 +547,6 @@ impl<'s, P: LintLevelsProvider> LintLevelsBuilder<'s, P> {
         self.features
     }
 
-    pub(crate) fn lint_store(&self) -> &LintStore {
-        self.store
-    }
-
     fn current_specs(&self) -> &FxHashMap<LintId, LevelAndSource> {
         self.provider.current_specs()
     }
@@ -609,7 +604,7 @@ impl<'s, P: LintLevelsProvider> LintLevelsBuilder<'s, P> {
             let orig_level = level;
             let lint_flag_val = Symbol::intern(lint_name);
 
-            let Ok(ids) = self.store.find_lints(&lint_name) else {
+            let Ok(ids) = self.store.find_lints(lint_name) else {
                 // errors already handled above
                 continue;
             };
@@ -633,7 +628,7 @@ impl<'s, P: LintLevelsProvider> LintLevelsBuilder<'s, P> {
     /// (e.g. if a forbid was already inserted on the same scope), then emits a
     /// diagnostic with no change to `specs`.
     fn insert_spec(&mut self, id: LintId, (mut level, src): LevelAndSource) {
-        let (old_level, old_src) = self.provider.get_lint_level(id.lint, &self.sess);
+        let (old_level, old_src) = self.provider.get_lint_level(id.lint, self.sess);
         if let Level::Expect(id) = &mut level
             && let LintExpectationId::Stable { .. } = id
         {
@@ -741,7 +736,7 @@ impl<'s, P: LintLevelsProvider> LintLevelsBuilder<'s, P> {
             if attr.has_name(sym::doc)
                 && attr
                     .meta_item_list()
-                    .map_or(false, |l| ast::attr::list_contains_name(&l, sym::hidden))
+                    .is_some_and(|l| ast::attr::list_contains_name(&l, sym::hidden))
             {
                 self.insert(LintId::of(MISSING_DOCS), (Level::Allow, LintLevelSource::Default));
                 continue;
@@ -933,12 +928,12 @@ impl<'s, P: LintLevelsProvider> LintLevelsBuilder<'s, P> {
                                     DeprecatedLintName {
                                         name,
                                         suggestion: sp,
-                                        replace: &new_lint_name,
+                                        replace: new_lint_name,
                                     },
                                 );
 
                                 let src = LintLevelSource::Node {
-                                    name: Symbol::intern(&new_lint_name),
+                                    name: Symbol::intern(new_lint_name),
                                     span: sp,
                                     reason,
                                 };
@@ -1082,7 +1077,6 @@ impl<'s, P: LintLevelsProvider> LintLevelsBuilder<'s, P> {
                             GateIssue::Language,
                             lint_from_cli,
                         );
-                        lint
                     },
                 );
                 return false;
@@ -1099,8 +1093,6 @@ impl<'s, P: LintLevelsProvider> LintLevelsBuilder<'s, P> {
     /// Used to emit a lint-related diagnostic based on the current state of
     /// this lint context.
     ///
-    /// Return value of the `decorate` closure is ignored, see [`struct_lint_level`] for a detailed explanation.
-    ///
     /// [`struct_lint_level`]: rustc_middle::lint::struct_lint_level#decorate-signature
     #[rustc_lint_diagnostics]
     #[track_caller]
@@ -1109,9 +1101,7 @@ impl<'s, P: LintLevelsProvider> LintLevelsBuilder<'s, P> {
         lint: &'static Lint,
         span: Option<MultiSpan>,
         msg: impl Into<DiagnosticMessage>,
-        decorate: impl for<'a, 'b> FnOnce(
-            &'b mut DiagnosticBuilder<'a, ()>,
-        ) -> &'b mut DiagnosticBuilder<'a, ()>,
+        decorate: impl for<'a, 'b> FnOnce(&'b mut DiagnosticBuilder<'a, ()>),
     ) {
         let (level, src) = self.lint_level(lint);
         struct_lint_level(self.sess, lint, level, src, span, msg, decorate)
@@ -1126,7 +1116,7 @@ impl<'s, P: LintLevelsProvider> LintLevelsBuilder<'s, P> {
     ) {
         let (level, src) = self.lint_level(lint);
         struct_lint_level(self.sess, lint, level, src, Some(span), decorate.msg(), |lint| {
-            decorate.decorate_lint(lint)
+            decorate.decorate_lint(lint);
         });
     }
 
@@ -1134,7 +1124,7 @@ impl<'s, P: LintLevelsProvider> LintLevelsBuilder<'s, P> {
     pub fn emit_lint(&self, lint: &'static Lint, decorate: impl for<'a> DecorateLint<'a, ()>) {
         let (level, src) = self.lint_level(lint);
         struct_lint_level(self.sess, lint, level, src, None, decorate.msg(), |lint| {
-            decorate.decorate_lint(lint)
+            decorate.decorate_lint(lint);
         });
     }
 }
diff --git a/compiler/rustc_lint/src/lib.rs b/compiler/rustc_lint/src/lib.rs
index 54adedd3c..066c88cc6 100644
--- a/compiler/rustc_lint/src/lib.rs
+++ b/compiler/rustc_lint/src/lib.rs
@@ -27,8 +27,8 @@
 
 #![allow(rustc::potential_query_instability)]
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
+#![doc(rust_logo)]
+#![feature(rustdoc_internals)]
 #![feature(array_windows)]
 #![feature(box_patterns)]
 #![feature(control_flow_enum)]
@@ -39,6 +39,7 @@
 #![feature(min_specialization)]
 #![feature(never_type)]
 #![feature(rustc_attrs)]
+#![feature(trait_upcasting)]
 #![recursion_limit = "256"]
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
@@ -85,18 +86,14 @@ mod redundant_semicolon;
 mod reference_casting;
 mod traits;
 mod types;
+mod unit_bindings;
 mod unused;
 
 pub use array_into_iter::ARRAY_INTO_ITER;
 
-use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
-use rustc_fluent_macro::fluent_messages;
 use rustc_hir::def_id::LocalModDefId;
 use rustc_middle::query::Providers;
 use rustc_middle::ty::TyCtxt;
-use rustc_session::lint::builtin::{
-    BARE_TRAIT_OBJECTS, ELIDED_LIFETIMES_IN_PATHS, EXPLICIT_OUTLIVES_REQUIREMENTS,
-};
 
 use array_into_iter::ArrayIntoIter;
 use async_fn_in_trait::AsyncFnInTrait;
@@ -123,6 +120,7 @@ use redundant_semicolon::*;
 use reference_casting::*;
 use traits::*;
 use types::*;
+use unit_bindings::*;
 use unused::*;
 
 /// Useful for other parts of the compiler / Clippy.
@@ -136,7 +134,7 @@ pub use rustc_session::lint::Level::{self, *};
 pub use rustc_session::lint::{BufferedEarlyLint, FutureIncompatibleInfo, Lint, LintId};
 pub use rustc_session::lint::{LintPass, LintVec};
 
-fluent_messages! { "../messages.ftl" }
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
 
 pub fn provide(providers: &mut Providers) {
     levels::provide(providers);
@@ -179,7 +177,6 @@ early_lint_methods!(
             IncompleteInternalFeatures: IncompleteInternalFeatures,
             RedundantSemicolons: RedundantSemicolons,
             UnusedDocComment: UnusedDocComment,
-            UnexpectedCfgs: UnexpectedCfgs,
         ]
     ]
 );
@@ -203,6 +200,7 @@ late_lint_methods!(
             InvalidReferenceCasting: InvalidReferenceCasting,
             // Depends on referenced function signatures in expressions
             UnusedResults: UnusedResults,
+            UnitBindings: UnitBindings,
             NonUpperCaseGlobals: NonUpperCaseGlobals,
             NonShorthandFieldPatterns: NonShorthandFieldPatterns,
             UnusedAllocation: UnusedAllocation,
@@ -511,6 +509,11 @@ fn register_builtins(store: &mut LintStore) {
         "converted into hard error, see PR #104616 \
          <https://github.com/rust-lang/rust/pull/104616> for more information",
     );
+    store.register_removed(
+        "implied_bounds_entailment",
+        "converted into hard error, see PR #117984 \
+        <https://github.com/rust-lang/rust/pull/117984> for more information",
+    );
 }
 
 fn register_internals(store: &mut LintStore) {
diff --git a/compiler/rustc_lint/src/lints.rs b/compiler/rustc_lint/src/lints.rs
index 756899e50..9c0d3be03 100644
--- a/compiler/rustc_lint/src/lints.rs
+++ b/compiler/rustc_lint/src/lints.rs
@@ -134,12 +134,8 @@ pub struct BuiltinMissingDebugImpl<'a> {
 
 // Needed for def_path_str
 impl<'a> DecorateLint<'a, ()> for BuiltinMissingDebugImpl<'_> {
-    fn decorate_lint<'b>(
-        self,
-        diag: &'b mut rustc_errors::DiagnosticBuilder<'a, ()>,
-    ) -> &'b mut rustc_errors::DiagnosticBuilder<'a, ()> {
+    fn decorate_lint<'b>(self, diag: &'b mut rustc_errors::DiagnosticBuilder<'a, ()>) {
         diag.set_arg("debug", self.tcx.def_path_str(self.def_id));
-        diag
     }
 
     fn msg(&self) -> DiagnosticMessage {
@@ -243,17 +239,13 @@ pub struct BuiltinUngatedAsyncFnTrackCaller<'a> {
 }
 
 impl<'a> DecorateLint<'a, ()> for BuiltinUngatedAsyncFnTrackCaller<'_> {
-    fn decorate_lint<'b>(
-        self,
-        diag: &'b mut rustc_errors::DiagnosticBuilder<'a, ()>,
-    ) -> &'b mut rustc_errors::DiagnosticBuilder<'a, ()> {
+    fn decorate_lint<'b>(self, diag: &'b mut rustc_errors::DiagnosticBuilder<'a, ()>) {
         diag.span_label(self.label, fluent::lint_label);
         rustc_session::parse::add_feature_diagnostics(
             diag,
-            &self.parse_sess,
+            self.parse_sess,
             sym::async_fn_track_caller,
         );
-        diag
     }
 
     fn msg(&self) -> DiagnosticMessage {
@@ -433,10 +425,7 @@ pub struct BuiltinUnpermittedTypeInit<'a> {
 }
 
 impl<'a> DecorateLint<'a, ()> for BuiltinUnpermittedTypeInit<'_> {
-    fn decorate_lint<'b>(
-        self,
-        diag: &'b mut rustc_errors::DiagnosticBuilder<'a, ()>,
-    ) -> &'b mut rustc_errors::DiagnosticBuilder<'a, ()> {
+    fn decorate_lint<'b>(self, diag: &'b mut rustc_errors::DiagnosticBuilder<'a, ()>) {
         diag.set_arg("ty", self.ty);
         diag.span_label(self.label, fluent::lint_builtin_unpermitted_type_init_label);
         if let InhabitedPredicate::True = self.ty.inhabited_predicate(self.tcx) {
@@ -447,7 +436,6 @@ impl<'a> DecorateLint<'a, ()> for BuiltinUnpermittedTypeInit<'_> {
             );
         }
         self.sub.add_to_diagnostic(diag);
-        diag
     }
 
     fn msg(&self) -> rustc_errors::DiagnosticMessage {
@@ -553,33 +541,21 @@ pub enum BuiltinSpecialModuleNameUsed {
     Main,
 }
 
-#[derive(LintDiagnostic)]
-#[diag(lint_builtin_unexpected_cli_config_name)]
-#[help]
-pub struct BuiltinUnexpectedCliConfigName {
-    pub name: Symbol,
-}
-
-#[derive(LintDiagnostic)]
-#[diag(lint_builtin_unexpected_cli_config_value)]
-#[help]
-pub struct BuiltinUnexpectedCliConfigValue {
-    pub name: Symbol,
-    pub value: Symbol,
-}
-
 // deref_into_dyn_supertrait.rs
 #[derive(LintDiagnostic)]
 #[diag(lint_supertrait_as_deref_target)]
 pub struct SupertraitAsDerefTarget<'a> {
-    pub t: Ty<'a>,
+    pub self_ty: Ty<'a>,
+    pub supertrait_principal: PolyExistentialTraitRef<'a>,
     pub target_principal: PolyExistentialTraitRef<'a>,
+    #[label]
+    pub label: Span,
     #[subdiagnostic]
-    pub label: Option<SupertraitAsDerefTargetLabel>,
+    pub label2: Option<SupertraitAsDerefTargetLabel>,
 }
 
 #[derive(Subdiagnostic)]
-#[label(lint_label)]
+#[label(lint_label2)]
 pub struct SupertraitAsDerefTargetLabel {
     #[primary_span]
     pub label: Span,
@@ -1170,10 +1146,7 @@ pub struct NonFmtPanicUnused {
 
 // Used because of two suggestions based on one Option<Span>
 impl<'a> DecorateLint<'a, ()> for NonFmtPanicUnused {
-    fn decorate_lint<'b>(
-        self,
-        diag: &'b mut rustc_errors::DiagnosticBuilder<'a, ()>,
-    ) -> &'b mut rustc_errors::DiagnosticBuilder<'a, ()> {
+    fn decorate_lint<'b>(self, diag: &'b mut rustc_errors::DiagnosticBuilder<'a, ()>) {
         diag.set_arg("count", self.count);
         diag.note(fluent::lint_note);
         if let Some(span) = self.suggestion {
@@ -1190,7 +1163,6 @@ impl<'a> DecorateLint<'a, ()> for NonFmtPanicUnused {
                 Applicability::MachineApplicable,
             );
         }
-        diag
     }
 
     fn msg(&self) -> rustc_errors::DiagnosticMessage {
@@ -1369,12 +1341,9 @@ pub struct DropTraitConstraintsDiag<'a> {
 
 // Needed for def_path_str
 impl<'a> DecorateLint<'a, ()> for DropTraitConstraintsDiag<'_> {
-    fn decorate_lint<'b>(
-        self,
-        diag: &'b mut rustc_errors::DiagnosticBuilder<'a, ()>,
-    ) -> &'b mut rustc_errors::DiagnosticBuilder<'a, ()> {
+    fn decorate_lint<'b>(self, diag: &'b mut rustc_errors::DiagnosticBuilder<'a, ()>) {
         diag.set_arg("predicate", self.predicate);
-        diag.set_arg("needs_drop", self.tcx.def_path_str(self.def_id))
+        diag.set_arg("needs_drop", self.tcx.def_path_str(self.def_id));
     }
 
     fn msg(&self) -> rustc_errors::DiagnosticMessage {
@@ -1389,11 +1358,8 @@ pub struct DropGlue<'a> {
 
 // Needed for def_path_str
 impl<'a> DecorateLint<'a, ()> for DropGlue<'_> {
-    fn decorate_lint<'b>(
-        self,
-        diag: &'b mut rustc_errors::DiagnosticBuilder<'a, ()>,
-    ) -> &'b mut rustc_errors::DiagnosticBuilder<'a, ()> {
-        diag.set_arg("needs_drop", self.tcx.def_path_str(self.def_id))
+    fn decorate_lint<'b>(self, diag: &'b mut rustc_errors::DiagnosticBuilder<'a, ()>) {
+        diag.set_arg("needs_drop", self.tcx.def_path_str(self.def_id));
     }
 
     fn msg(&self) -> rustc_errors::DiagnosticMessage {
@@ -1585,6 +1551,76 @@ pub enum InvalidNanComparisonsSuggestion {
     Spanless,
 }
 
+#[derive(LintDiagnostic)]
+pub enum AmbiguousWidePointerComparisons<'a> {
+    #[diag(lint_ambiguous_wide_pointer_comparisons)]
+    Spanful {
+        #[subdiagnostic]
+        addr_suggestion: AmbiguousWidePointerComparisonsAddrSuggestion<'a>,
+        #[subdiagnostic]
+        addr_metadata_suggestion: Option<AmbiguousWidePointerComparisonsAddrMetadataSuggestion<'a>>,
+    },
+    #[diag(lint_ambiguous_wide_pointer_comparisons)]
+    #[help(lint_addr_metadata_suggestion)]
+    #[help(lint_addr_suggestion)]
+    Spanless,
+}
+
+#[derive(Subdiagnostic)]
+#[multipart_suggestion(
+    lint_addr_metadata_suggestion,
+    style = "verbose",
+    applicability = "machine-applicable"
+)]
+pub struct AmbiguousWidePointerComparisonsAddrMetadataSuggestion<'a> {
+    pub ne: &'a str,
+    pub deref_left: &'a str,
+    pub deref_right: &'a str,
+    #[suggestion_part(code = "{ne}std::ptr::eq({deref_left}")]
+    pub left: Span,
+    #[suggestion_part(code = ", {deref_right}")]
+    pub middle: Span,
+    #[suggestion_part(code = ")")]
+    pub right: Span,
+}
+
+#[derive(Subdiagnostic)]
+pub enum AmbiguousWidePointerComparisonsAddrSuggestion<'a> {
+    #[multipart_suggestion(
+        lint_addr_suggestion,
+        style = "verbose",
+        applicability = "machine-applicable"
+    )]
+    AddrEq {
+        ne: &'a str,
+        deref_left: &'a str,
+        deref_right: &'a str,
+        #[suggestion_part(code = "{ne}std::ptr::addr_eq({deref_left}")]
+        left: Span,
+        #[suggestion_part(code = ", {deref_right}")]
+        middle: Span,
+        #[suggestion_part(code = ")")]
+        right: Span,
+    },
+    #[multipart_suggestion(
+        lint_addr_suggestion,
+        style = "verbose",
+        applicability = "machine-applicable"
+    )]
+    Cast {
+        deref_left: &'a str,
+        deref_right: &'a str,
+        #[suggestion_part(code = "{deref_left}")]
+        left_before: Option<Span>,
+        #[suggestion_part(code = " as *const ()")]
+        left: Span,
+        #[suggestion_part(code = "{deref_right}")]
+        right_before: Option<Span>,
+        #[suggestion_part(code = " as *const ()")]
+        right: Span,
+    },
+}
+
 pub struct ImproperCTypes<'a> {
     pub ty: Ty<'a>,
     pub desc: &'a str,
@@ -1596,10 +1632,7 @@ pub struct ImproperCTypes<'a> {
 
 // Used because of the complexity of Option<DiagnosticMessage>, DiagnosticMessage, and Option<Span>
 impl<'a> DecorateLint<'a, ()> for ImproperCTypes<'_> {
-    fn decorate_lint<'b>(
-        self,
-        diag: &'b mut rustc_errors::DiagnosticBuilder<'a, ()>,
-    ) -> &'b mut rustc_errors::DiagnosticBuilder<'a, ()> {
+    fn decorate_lint<'b>(self, diag: &'b mut rustc_errors::DiagnosticBuilder<'a, ()>) {
         diag.set_arg("ty", self.ty);
         diag.set_arg("desc", self.desc);
         diag.span_label(self.label, fluent::lint_label);
@@ -1610,7 +1643,6 @@ impl<'a> DecorateLint<'a, ()> for ImproperCTypes<'_> {
         if let Some(note) = self.span_note {
             diag.span_note(note, fluent::lint_note);
         }
-        diag
     }
 
     fn msg(&self) -> rustc_errors::DiagnosticMessage {
@@ -1743,10 +1775,7 @@ pub enum UnusedDefSuggestion {
 
 // Needed because of def_path_str
 impl<'a> DecorateLint<'a, ()> for UnusedDef<'_, '_> {
-    fn decorate_lint<'b>(
-        self,
-        diag: &'b mut rustc_errors::DiagnosticBuilder<'a, ()>,
-    ) -> &'b mut rustc_errors::DiagnosticBuilder<'a, ()> {
+    fn decorate_lint<'b>(self, diag: &'b mut rustc_errors::DiagnosticBuilder<'a, ()>) {
         diag.set_arg("pre", self.pre);
         diag.set_arg("post", self.post);
         diag.set_arg("def", self.cx.tcx.def_path_str(self.def_id));
@@ -1757,7 +1786,6 @@ impl<'a> DecorateLint<'a, ()> for UnusedDef<'_, '_> {
         if let Some(sugg) = self.suggestion {
             diag.subdiagnostic(sugg);
         }
-        diag
     }
 
     fn msg(&self) -> rustc_errors::DiagnosticMessage {
@@ -1830,18 +1858,21 @@ pub struct AsyncFnInTraitDiag {
 }
 
 impl<'a> DecorateLint<'a, ()> for AsyncFnInTraitDiag {
-    fn decorate_lint<'b>(
-        self,
-        diag: &'b mut rustc_errors::DiagnosticBuilder<'a, ()>,
-    ) -> &'b mut rustc_errors::DiagnosticBuilder<'a, ()> {
+    fn decorate_lint<'b>(self, diag: &'b mut rustc_errors::DiagnosticBuilder<'a, ()>) {
         diag.note(fluent::lint_note);
         if let Some(sugg) = self.sugg {
             diag.multipart_suggestion(fluent::lint_suggestion, sugg, Applicability::MaybeIncorrect);
         }
-        diag
     }
 
     fn msg(&self) -> rustc_errors::DiagnosticMessage {
         fluent::lint_async_fn_in_trait
     }
 }
+
+#[derive(LintDiagnostic)]
+#[diag(lint_unit_bindings)]
+pub struct UnitBindingsDiag {
+    #[label]
+    pub label: Span,
+}
diff --git a/compiler/rustc_lint/src/non_ascii_idents.rs b/compiler/rustc_lint/src/non_ascii_idents.rs
index 62bb8c2c6..4f92fcd71 100644
--- a/compiler/rustc_lint/src/non_ascii_idents.rs
+++ b/compiler/rustc_lint/src/non_ascii_idents.rs
@@ -204,7 +204,7 @@ impl EarlyLintPass for NonAsciiIdents {
 
                 // Get the skeleton as a `Symbol`.
                 skeleton_buf.clear();
-                skeleton_buf.extend(skeleton(&symbol_str));
+                skeleton_buf.extend(skeleton(symbol_str));
                 let skeleton_sym = if *symbol_str == *skeleton_buf {
                     symbol
                 } else {
diff --git a/compiler/rustc_lint/src/non_fmt_panic.rs b/compiler/rustc_lint/src/non_fmt_panic.rs
index b218cc578..9fcd70ba0 100644
--- a/compiler/rustc_lint/src/non_fmt_panic.rs
+++ b/compiler/rustc_lint/src/non_fmt_panic.rs
@@ -126,7 +126,7 @@ fn check_panic<'tcx>(cx: &LateContext<'tcx>, f: &'tcx hir::Expr<'tcx>, arg: &'tc
         lint.note(fluent::lint_more_info_note);
         if !is_arg_inside_call(arg_span, span) {
             // No clue where this argument is coming from.
-            return lint;
+            return;
         }
         if arg_macro.is_some_and(|id| cx.tcx.is_diagnostic_item(sym::format_macro, id)) {
             // A case of `panic!(format!(..))`.
@@ -154,17 +154,13 @@ fn check_panic<'tcx>(cx: &LateContext<'tcx>, f: &'tcx hir::Expr<'tcx>, arg: &'tc
 
             let infcx = cx.tcx.infer_ctxt().build();
             let suggest_display = is_str
-                || cx
-                    .tcx
-                    .get_diagnostic_item(sym::Display)
-                    .map(|t| infcx.type_implements_trait(t, [ty], cx.param_env).may_apply())
-                    == Some(true);
+                || cx.tcx.get_diagnostic_item(sym::Display).is_some_and(|t| {
+                    infcx.type_implements_trait(t, [ty], cx.param_env).may_apply()
+                });
             let suggest_debug = !suggest_display
-                && cx
-                    .tcx
-                    .get_diagnostic_item(sym::Debug)
-                    .map(|t| infcx.type_implements_trait(t, [ty], cx.param_env).may_apply())
-                    == Some(true);
+                && cx.tcx.get_diagnostic_item(sym::Debug).is_some_and(|t| {
+                    infcx.type_implements_trait(t, [ty], cx.param_env).may_apply()
+                });
 
             let suggest_panic_any = !is_str && panic == sym::std_panic_macro;
 
@@ -211,7 +207,6 @@ fn check_panic<'tcx>(cx: &LateContext<'tcx>, f: &'tcx hir::Expr<'tcx>, arg: &'tc
                 }
             }
         }
-        lint
     });
 }
 
diff --git a/compiler/rustc_lint/src/nonstandard_style.rs b/compiler/rustc_lint/src/nonstandard_style.rs
index 66dc726df..59f27a88a 100644
--- a/compiler/rustc_lint/src/nonstandard_style.rs
+++ b/compiler/rustc_lint/src/nonstandard_style.rs
@@ -334,7 +334,7 @@ impl<'tcx> LateLintPass<'tcx> for NonSnakeCase {
         let crate_ident = if let Some(name) = &cx.tcx.sess.opts.crate_name {
             Some(Ident::from_str(name))
         } else {
-            attr::find_by_name(&cx.tcx.hir().attrs(hir::CRATE_HIR_ID), sym::crate_name)
+            attr::find_by_name(cx.tcx.hir().attrs(hir::CRATE_HIR_ID), sym::crate_name)
                 .and_then(|attr| attr.meta())
                 .and_then(|meta| {
                     meta.name_value_literal().and_then(|lit| {
@@ -473,7 +473,7 @@ impl NonUpperCaseGlobals {
     fn check_upper_case(cx: &LateContext<'_>, sort: &str, ident: &Ident) {
         let name = ident.name.as_str();
         if name.chars().any(|c| c.is_lowercase()) {
-            let uc = NonSnakeCase::to_snake_case(&name).to_uppercase();
+            let uc = NonSnakeCase::to_snake_case(name).to_uppercase();
             // We cannot provide meaningful suggestions
             // if the characters are in the category of "Lowercase Letter".
             let sub = if *name != uc {
@@ -520,7 +520,7 @@ impl<'tcx> LateLintPass<'tcx> for NonUpperCaseGlobals {
 
     fn check_pat(&mut self, cx: &LateContext<'_>, p: &hir::Pat<'_>) {
         // Lint for constants that look like binding identifiers (#7526)
-        if let PatKind::Path(hir::QPath::Resolved(None, ref path)) = p.kind {
+        if let PatKind::Path(hir::QPath::Resolved(None, path)) = p.kind {
             if let Res::Def(DefKind::Const, _) = path.res {
                 if path.segments.len() == 1 {
                     NonUpperCaseGlobals::check_upper_case(
@@ -534,9 +534,9 @@ impl<'tcx> LateLintPass<'tcx> for NonUpperCaseGlobals {
     }
 
     fn check_generic_param(&mut self, cx: &LateContext<'_>, param: &hir::GenericParam<'_>) {
-        if let GenericParamKind::Const { .. } = param.kind {
-            // `rustc_host` params are explicitly allowed to be lowercase.
-            if cx.tcx.has_attr(param.def_id, sym::rustc_host) {
+        if let GenericParamKind::Const { is_host_effect, .. } = param.kind {
+            // `host` params are explicitly allowed to be lowercase.
+            if is_host_effect {
                 return;
             }
             NonUpperCaseGlobals::check_upper_case(cx, "const parameter", &param.name.ident());
diff --git a/compiler/rustc_lint/src/opaque_hidden_inferred_bound.rs b/compiler/rustc_lint/src/opaque_hidden_inferred_bound.rs
index c24846ca9..44b23b8bd 100644
--- a/compiler/rustc_lint/src/opaque_hidden_inferred_bound.rs
+++ b/compiler/rustc_lint/src/opaque_hidden_inferred_bound.rs
@@ -113,7 +113,7 @@ impl<'tcx> LateLintPass<'tcx> for OpaqueHiddenInferredBound {
             for (assoc_pred, assoc_pred_span) in cx
                 .tcx
                 .explicit_item_bounds(proj.projection_ty.def_id)
-                .iter_instantiated_copied(cx.tcx, &proj.projection_ty.args)
+                .iter_instantiated_copied(cx.tcx, proj.projection_ty.args)
             {
                 let assoc_pred = assoc_pred.fold_with(proj_replacer);
                 let Ok(assoc_pred) = traits::fully_normalize(
diff --git a/compiler/rustc_lint/src/pass_by_value.rs b/compiler/rustc_lint/src/pass_by_value.rs
index cad2cd7fa..fce750c9b 100644
--- a/compiler/rustc_lint/src/pass_by_value.rs
+++ b/compiler/rustc_lint/src/pass_by_value.rs
@@ -28,7 +28,7 @@ impl<'tcx> LateLintPass<'tcx> for PassByValue {
                         return;
                     }
                 }
-                if let Some(t) = path_for_pass_by_value(cx, &inner_ty) {
+                if let Some(t) = path_for_pass_by_value(cx, inner_ty) {
                     cx.emit_spanned_lint(
                         PASS_BY_VALUE,
                         ty.span,
diff --git a/compiler/rustc_lint/src/reference_casting.rs b/compiler/rustc_lint/src/reference_casting.rs
index d44691b5e..96290288f 100644
--- a/compiler/rustc_lint/src/reference_casting.rs
+++ b/compiler/rustc_lint/src/reference_casting.rs
@@ -37,59 +37,73 @@ declare_lint_pass!(InvalidReferenceCasting => [INVALID_REFERENCE_CASTING]);
 
 impl<'tcx> LateLintPass<'tcx> for InvalidReferenceCasting {
     fn check_expr(&mut self, cx: &LateContext<'tcx>, expr: &'tcx Expr<'tcx>) {
-        let Some((is_assignment, e)) = is_operation_we_care_about(cx, expr) else {
-            return;
-        };
-
-        let init = cx.expr_or_init(e);
-
-        let Some(ty_has_interior_mutability) = is_cast_from_const_to_mut(cx, init) else {
-            return;
-        };
-        let orig_cast = if init.span != e.span { Some(init.span) } else { None };
-        let ty_has_interior_mutability = ty_has_interior_mutability.then_some(());
-
-        cx.emit_spanned_lint(
-            INVALID_REFERENCE_CASTING,
-            expr.span,
-            if is_assignment {
-                InvalidReferenceCastingDiag::AssignToRef { orig_cast, ty_has_interior_mutability }
-            } else {
-                InvalidReferenceCastingDiag::BorrowAsMut { orig_cast, ty_has_interior_mutability }
-            },
-        );
+        if let Some((e, pat)) = borrow_or_assign(cx, expr) {
+            if matches!(pat, PatternKind::Borrow { mutbl: Mutability::Mut } | PatternKind::Assign) {
+                let init = cx.expr_or_init(e);
+
+                let Some(ty_has_interior_mutability) = is_cast_from_ref_to_mut_ptr(cx, init) else {
+                    return;
+                };
+                let orig_cast = if init.span != e.span { Some(init.span) } else { None };
+                let ty_has_interior_mutability = ty_has_interior_mutability.then_some(());
+
+                cx.emit_spanned_lint(
+                    INVALID_REFERENCE_CASTING,
+                    expr.span,
+                    if pat == PatternKind::Assign {
+                        InvalidReferenceCastingDiag::AssignToRef {
+                            orig_cast,
+                            ty_has_interior_mutability,
+                        }
+                    } else {
+                        InvalidReferenceCastingDiag::BorrowAsMut {
+                            orig_cast,
+                            ty_has_interior_mutability,
+                        }
+                    },
+                );
+            }
+        }
     }
 }
 
-fn is_operation_we_care_about<'tcx>(
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum PatternKind {
+    Borrow { mutbl: Mutability },
+    Assign,
+}
+
+fn borrow_or_assign<'tcx>(
     cx: &LateContext<'tcx>,
     e: &'tcx Expr<'tcx>,
-) -> Option<(bool, &'tcx Expr<'tcx>)> {
-    fn deref_assign_or_addr_of<'tcx>(expr: &'tcx Expr<'tcx>) -> Option<(bool, &'tcx Expr<'tcx>)> {
-        // &mut <expr>
-        let inner = if let ExprKind::AddrOf(_, Mutability::Mut, expr) = expr.kind {
-            expr
+) -> Option<(&'tcx Expr<'tcx>, PatternKind)> {
+    fn deref_assign_or_addr_of<'tcx>(
+        expr: &'tcx Expr<'tcx>,
+    ) -> Option<(&'tcx Expr<'tcx>, PatternKind)> {
+        // &(mut) <expr>
+        let (inner, pat) = if let ExprKind::AddrOf(_, mutbl, expr) = expr.kind {
+            (expr, PatternKind::Borrow { mutbl })
         // <expr> = ...
         } else if let ExprKind::Assign(expr, _, _) = expr.kind {
-            expr
+            (expr, PatternKind::Assign)
         // <expr> += ...
         } else if let ExprKind::AssignOp(_, expr, _) = expr.kind {
-            expr
+            (expr, PatternKind::Assign)
         } else {
             return None;
         };
 
-        if let ExprKind::Unary(UnOp::Deref, e) = &inner.kind {
-            Some((!matches!(expr.kind, ExprKind::AddrOf(..)), e))
-        } else {
-            None
-        }
+        // *<inner>
+        let ExprKind::Unary(UnOp::Deref, e) = &inner.kind else {
+            return None;
+        };
+        Some((e, pat))
     }
 
     fn ptr_write<'tcx>(
         cx: &LateContext<'tcx>,
         e: &'tcx Expr<'tcx>,
-    ) -> Option<(bool, &'tcx Expr<'tcx>)> {
+    ) -> Option<(&'tcx Expr<'tcx>, PatternKind)> {
         if let ExprKind::Call(path, [arg_ptr, _arg_val]) = e.kind
             && let ExprKind::Path(ref qpath) = path.kind
             && let Some(def_id) = cx.qpath_res(qpath, path.hir_id).opt_def_id()
@@ -98,7 +112,7 @@ fn is_operation_we_care_about<'tcx>(
                 Some(sym::ptr_write | sym::ptr_write_volatile | sym::ptr_write_unaligned)
             )
         {
-            Some((true, arg_ptr))
+            Some((arg_ptr, PatternKind::Assign))
         } else {
             None
         }
@@ -107,13 +121,10 @@ fn is_operation_we_care_about<'tcx>(
     deref_assign_or_addr_of(e).or_else(|| ptr_write(cx, e))
 }
 
-fn is_cast_from_const_to_mut<'tcx>(
+fn is_cast_from_ref_to_mut_ptr<'tcx>(
     cx: &LateContext<'tcx>,
     orig_expr: &'tcx Expr<'tcx>,
 ) -> Option<bool> {
-    let mut need_check_freeze = false;
-    let mut e = orig_expr;
-
     let end_ty = cx.typeck_results().node_type(orig_expr.hir_id);
 
     // Bail out early if the end type is **not** a mutable pointer.
@@ -121,6 +132,28 @@ fn is_cast_from_const_to_mut<'tcx>(
         return None;
     }
 
+    let (e, need_check_freeze) = peel_casts(cx, orig_expr);
+
+    let start_ty = cx.typeck_results().node_type(e.hir_id);
+    if let ty::Ref(_, inner_ty, Mutability::Not) = start_ty.kind() {
+        // If an UnsafeCell method is involved, we need to additionally check the
+        // inner type for the presence of the Freeze trait (ie does NOT contain
+        // an UnsafeCell), since in that case we would incorrectly lint on valid casts.
+        //
+        // Except on the presence of non concrete skeleton types (ie generics)
+        // since there is no way to make it safe for arbitrary types.
+        let inner_ty_has_interior_mutability =
+            !inner_ty.is_freeze(cx.tcx, cx.param_env) && inner_ty.has_concrete_skeleton();
+        (!need_check_freeze || !inner_ty_has_interior_mutability)
+            .then_some(inner_ty_has_interior_mutability)
+    } else {
+        None
+    }
+}
+
+fn peel_casts<'tcx>(cx: &LateContext<'tcx>, mut e: &'tcx Expr<'tcx>) -> (&'tcx Expr<'tcx>, bool) {
+    let mut gone_trough_unsafe_cell_raw_get = false;
+
     loop {
         e = e.peel_blocks();
         // <expr> as ...
@@ -145,27 +178,18 @@ fn is_cast_from_const_to_mut<'tcx>(
             )
         {
             if cx.tcx.is_diagnostic_item(sym::unsafe_cell_raw_get, def_id) {
-                need_check_freeze = true;
+                gone_trough_unsafe_cell_raw_get = true;
             }
             arg
         } else {
-            break;
+            let init = cx.expr_or_init(e);
+            if init.hir_id != e.hir_id {
+                init
+            } else {
+                break;
+            }
         };
     }
 
-    let start_ty = cx.typeck_results().node_type(e.hir_id);
-    if let ty::Ref(_, inner_ty, Mutability::Not) = start_ty.kind() {
-        // If an UnsafeCell method is involved we need to additionaly check the
-        // inner type for the presence of the Freeze trait (ie does NOT contain
-        // an UnsafeCell), since in that case we would incorrectly lint on valid casts.
-        //
-        // We also consider non concrete skeleton types (ie generics)
-        // to be an issue since there is no way to make it safe for abitrary types.
-        let inner_ty_has_interior_mutability =
-            !inner_ty.is_freeze(cx.tcx, cx.param_env) && inner_ty.has_concrete_skeleton();
-        (!need_check_freeze || !inner_ty_has_interior_mutability)
-            .then_some(inner_ty_has_interior_mutability)
-    } else {
-        None
-    }
+    (e, gone_trough_unsafe_cell_raw_get)
 }
diff --git a/compiler/rustc_lint/src/types.rs b/compiler/rustc_lint/src/types.rs
index c04053d18..6dade43a1 100644
--- a/compiler/rustc_lint/src/types.rs
+++ b/compiler/rustc_lint/src/types.rs
@@ -1,12 +1,13 @@
 use crate::{
     fluent_generated as fluent,
     lints::{
-        AtomicOrderingFence, AtomicOrderingLoad, AtomicOrderingStore, ImproperCTypes,
-        InvalidAtomicOrderingDiag, InvalidNanComparisons, InvalidNanComparisonsSuggestion,
-        OnlyCastu8ToChar, OverflowingBinHex, OverflowingBinHexSign, OverflowingBinHexSignBitSub,
-        OverflowingBinHexSub, OverflowingInt, OverflowingIntHelp, OverflowingLiteral,
-        OverflowingUInt, RangeEndpointOutOfRange, UnusedComparisons, UseInclusiveRange,
-        VariantSizeDifferencesDiag,
+        AmbiguousWidePointerComparisons, AmbiguousWidePointerComparisonsAddrMetadataSuggestion,
+        AmbiguousWidePointerComparisonsAddrSuggestion, AtomicOrderingFence, AtomicOrderingLoad,
+        AtomicOrderingStore, ImproperCTypes, InvalidAtomicOrderingDiag, InvalidNanComparisons,
+        InvalidNanComparisonsSuggestion, OnlyCastu8ToChar, OverflowingBinHex,
+        OverflowingBinHexSign, OverflowingBinHexSignBitSub, OverflowingBinHexSub, OverflowingInt,
+        OverflowingIntHelp, OverflowingLiteral, OverflowingUInt, RangeEndpointOutOfRange,
+        UnusedComparisons, UseInclusiveRange, VariantSizeDifferencesDiag,
     },
 };
 use crate::{LateContext, LateLintPass, LintContext};
@@ -17,10 +18,10 @@ use rustc_errors::DiagnosticMessage;
 use rustc_hir as hir;
 use rustc_hir::{is_range_literal, Expr, ExprKind, Node};
 use rustc_middle::ty::layout::{IntegerExt, LayoutOf, SizeSkeleton};
-use rustc_middle::ty::GenericArgsRef;
 use rustc_middle::ty::{
     self, AdtKind, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable, TypeVisitableExt,
 };
+use rustc_middle::ty::{GenericArgsRef, TypeAndMut};
 use rustc_span::def_id::LocalDefId;
 use rustc_span::source_map;
 use rustc_span::symbol::sym;
@@ -28,6 +29,7 @@ use rustc_span::{Span, Symbol};
 use rustc_target::abi::{Abi, Size, WrappingRange};
 use rustc_target::abi::{Integer, TagEncoding, Variants};
 use rustc_target::spec::abi::Abi as SpecAbi;
+use rustc_type_ir::DynKind;
 
 use std::iter;
 use std::ops::ControlFlow;
@@ -136,6 +138,37 @@ declare_lint! {
     "detects invalid floating point NaN comparisons"
 }
 
+declare_lint! {
+    /// The `ambiguous_wide_pointer_comparisons` lint checks comparison
+    /// of `*const/*mut ?Sized` as the operands.
+    ///
+    /// ### Example
+    ///
+    /// ```rust
+    /// # struct A;
+    /// # struct B;
+    ///
+    /// # trait T {}
+    /// # impl T for A {}
+    /// # impl T for B {}
+    ///
+    /// let ab = (A, B);
+    /// let a = &ab.0 as *const dyn T;
+    /// let b = &ab.1 as *const dyn T;
+    ///
+    /// let _ = a == b;
+    /// ```
+    ///
+    /// {{produces}}
+    ///
+    /// ### Explanation
+    ///
+    /// The comparison includes metadata which may not be expected.
+    AMBIGUOUS_WIDE_POINTER_COMPARISONS,
+    Warn,
+    "detects ambiguous wide pointer comparisons"
+}
+
 #[derive(Copy, Clone)]
 pub struct TypeLimits {
     /// Id of the last visited negated expression
@@ -144,7 +177,12 @@ pub struct TypeLimits {
     negated_expr_span: Option<Span>,
 }
 
-impl_lint_pass!(TypeLimits => [UNUSED_COMPARISONS, OVERFLOWING_LITERALS, INVALID_NAN_COMPARISONS]);
+impl_lint_pass!(TypeLimits => [
+    UNUSED_COMPARISONS,
+    OVERFLOWING_LITERALS,
+    INVALID_NAN_COMPARISONS,
+    AMBIGUOUS_WIDE_POINTER_COMPARISONS
+]);
 
 impl TypeLimits {
     pub fn new() -> TypeLimits {
@@ -164,7 +202,7 @@ fn lint_overflowing_range_endpoint<'tcx>(
 ) -> bool {
     // Look past casts to support cases like `0..256 as u8`
     let (expr, lit_span) = if let Node::Expr(par_expr) =
-        cx.tcx.hir().get(cx.tcx.hir().parent_id(expr.hir_id))
+        cx.tcx.hir_node(cx.tcx.hir().parent_id(expr.hir_id))
         && let ExprKind::Cast(_, _) = par_expr.kind
     {
         (par_expr, expr.span)
@@ -175,7 +213,7 @@ fn lint_overflowing_range_endpoint<'tcx>(
     // We only want to handle exclusive (`..`) ranges,
     // which are represented as `ExprKind::Struct`.
     let par_id = cx.tcx.hir().parent_id(expr.hir_id);
-    let Node::ExprField(field) = cx.tcx.hir().get(par_id) else { return false };
+    let Node::ExprField(field) = cx.tcx.hir_node(par_id) else { return false };
     let Node::Expr(struct_expr) = cx.tcx.hir().get_parent(field.hir_id) else { return false };
     if !is_range_literal(struct_expr) {
         return false;
@@ -460,7 +498,7 @@ fn lint_uint_literal<'tcx>(
     };
     if lit_val < min || lit_val > max {
         let parent_id = cx.tcx.hir().parent_id(e.hir_id);
-        if let Node::Expr(par_e) = cx.tcx.hir().get(parent_id) {
+        if let Node::Expr(par_e) = cx.tcx.hir_node(parent_id) {
             match par_e.kind {
                 hir::ExprKind::Cast(..) => {
                     if let ty::Char = cx.typeck_results().expr_ty(par_e).kind() {
@@ -620,10 +658,110 @@ fn lint_nan<'tcx>(
     cx.emit_spanned_lint(INVALID_NAN_COMPARISONS, e.span, lint);
 }
 
+fn lint_wide_pointer<'tcx>(
+    cx: &LateContext<'tcx>,
+    e: &'tcx hir::Expr<'tcx>,
+    binop: hir::BinOpKind,
+    l: &'tcx hir::Expr<'tcx>,
+    r: &'tcx hir::Expr<'tcx>,
+) {
+    let ptr_unsized = |mut ty: Ty<'tcx>| -> Option<(usize, bool)> {
+        let mut refs = 0;
+        // here we remove any "implicit" references and count the number
+        // of them to correctly suggest the right number of deref
+        while let ty::Ref(_, inner_ty, _) = ty.kind() {
+            ty = *inner_ty;
+            refs += 1;
+        }
+        match ty.kind() {
+            ty::RawPtr(TypeAndMut { mutbl: _, ty }) => (!ty.is_sized(cx.tcx, cx.param_env))
+                .then(|| (refs, matches!(ty.kind(), ty::Dynamic(_, _, DynKind::Dyn)))),
+            _ => None,
+        }
+    };
+
+    // PartialEq::{eq,ne} takes references, remove any explicit references
+    let l = l.peel_borrows();
+    let r = r.peel_borrows();
+
+    let Some(l_ty) = cx.typeck_results().expr_ty_opt(l) else {
+        return;
+    };
+    let Some(r_ty) = cx.typeck_results().expr_ty_opt(r) else {
+        return;
+    };
+
+    let Some((l_ty_refs, l_inner_ty_is_dyn)) = ptr_unsized(l_ty) else {
+        return;
+    };
+    let Some((r_ty_refs, r_inner_ty_is_dyn)) = ptr_unsized(r_ty) else {
+        return;
+    };
+
+    let (Some(l_span), Some(r_span)) =
+        (l.span.find_ancestor_inside(e.span), r.span.find_ancestor_inside(e.span))
+    else {
+        return cx.emit_spanned_lint(
+            AMBIGUOUS_WIDE_POINTER_COMPARISONS,
+            e.span,
+            AmbiguousWidePointerComparisons::Spanless,
+        );
+    };
+
+    let ne = if binop == hir::BinOpKind::Ne { "!" } else { "" };
+    let is_eq_ne = matches!(binop, hir::BinOpKind::Eq | hir::BinOpKind::Ne);
+    let is_dyn_comparison = l_inner_ty_is_dyn && r_inner_ty_is_dyn;
+
+    let left = e.span.shrink_to_lo().until(l_span.shrink_to_lo());
+    let middle = l_span.shrink_to_hi().until(r_span.shrink_to_lo());
+    let right = r_span.shrink_to_hi().until(e.span.shrink_to_hi());
+
+    let deref_left = &*"*".repeat(l_ty_refs);
+    let deref_right = &*"*".repeat(r_ty_refs);
+
+    cx.emit_spanned_lint(
+        AMBIGUOUS_WIDE_POINTER_COMPARISONS,
+        e.span,
+        AmbiguousWidePointerComparisons::Spanful {
+            addr_metadata_suggestion: (is_eq_ne && !is_dyn_comparison).then(|| {
+                AmbiguousWidePointerComparisonsAddrMetadataSuggestion {
+                    ne,
+                    deref_left,
+                    deref_right,
+                    left,
+                    middle,
+                    right,
+                }
+            }),
+            addr_suggestion: if is_eq_ne {
+                AmbiguousWidePointerComparisonsAddrSuggestion::AddrEq {
+                    ne,
+                    deref_left,
+                    deref_right,
+                    left,
+                    middle,
+                    right,
+                }
+            } else {
+                AmbiguousWidePointerComparisonsAddrSuggestion::Cast {
+                    deref_left,
+                    deref_right,
+                    // those two Options are required for correctness as having
+                    // an empty span and an empty suggestion is not permitted
+                    left_before: (l_ty_refs != 0).then_some(left),
+                    right_before: (r_ty_refs != 0).then(|| r_span.shrink_to_lo()),
+                    left: l_span.shrink_to_hi(),
+                    right,
+                }
+            },
+        },
+    );
+}
+
 impl<'tcx> LateLintPass<'tcx> for TypeLimits {
     fn check_expr(&mut self, cx: &LateContext<'tcx>, e: &'tcx hir::Expr<'tcx>) {
         match e.kind {
-            hir::ExprKind::Unary(hir::UnOp::Neg, ref expr) => {
+            hir::ExprKind::Unary(hir::UnOp::Neg, expr) => {
                 // Propagate negation, if the negation itself isn't negated
                 if self.negated_expr_id != Some(e.hir_id) {
                     self.negated_expr_id = Some(expr.hir_id);
@@ -632,14 +770,30 @@ impl<'tcx> LateLintPass<'tcx> for TypeLimits {
             }
             hir::ExprKind::Binary(binop, ref l, ref r) => {
                 if is_comparison(binop) {
-                    if !check_limits(cx, binop, &l, &r) {
+                    if !check_limits(cx, binop, l, r) {
                         cx.emit_spanned_lint(UNUSED_COMPARISONS, e.span, UnusedComparisons);
                     } else {
                         lint_nan(cx, e, binop, l, r);
+                        lint_wide_pointer(cx, e, binop.node, l, r);
                     }
                 }
             }
-            hir::ExprKind::Lit(ref lit) => lint_literal(cx, self, e, lit),
+            hir::ExprKind::Lit(lit) => lint_literal(cx, self, e, lit),
+            hir::ExprKind::Call(path, [l, r])
+                if let ExprKind::Path(ref qpath) = path.kind
+                    && let Some(def_id) = cx.qpath_res(qpath, path.hir_id).opt_def_id()
+                    && let Some(diag_item) = cx.tcx.get_diagnostic_name(def_id)
+                    && let Some(binop) = partialeq_binop(diag_item) =>
+            {
+                lint_wide_pointer(cx, e, binop, l, r);
+            }
+            hir::ExprKind::MethodCall(_, l, [r], _)
+                if let Some(def_id) = cx.typeck_results().type_dependent_def_id(e.hir_id)
+                    && let Some(diag_item) = cx.tcx.get_diagnostic_name(def_id)
+                    && let Some(binop) = partialeq_binop(diag_item) =>
+            {
+                lint_wide_pointer(cx, e, binop, l, r);
+            }
             _ => {}
         };
 
@@ -685,7 +839,7 @@ impl<'tcx> LateLintPass<'tcx> for TypeLimits {
                 ty::Int(int_ty) => {
                     let (min, max) = int_ty_range(int_ty);
                     let lit_val: i128 = match lit.kind {
-                        hir::ExprKind::Lit(ref li) => match li.node {
+                        hir::ExprKind::Lit(li) => match li.node {
                             ast::LitKind::Int(
                                 v,
                                 ast::LitIntType::Signed(_) | ast::LitIntType::Unsuffixed,
@@ -699,7 +853,7 @@ impl<'tcx> LateLintPass<'tcx> for TypeLimits {
                 ty::Uint(uint_ty) => {
                     let (min, max): (u128, u128) = uint_ty_range(uint_ty);
                     let lit_val: u128 = match lit.kind {
-                        hir::ExprKind::Lit(ref li) => match li.node {
+                        hir::ExprKind::Lit(li) => match li.node {
                             ast::LitKind::Int(v, _) => v,
                             _ => return true,
                         },
@@ -722,6 +876,16 @@ impl<'tcx> LateLintPass<'tcx> for TypeLimits {
                     | hir::BinOpKind::Gt
             )
         }
+
+        fn partialeq_binop(diag_item: Symbol) -> Option<hir::BinOpKind> {
+            if diag_item == sym::cmp_partialeq_eq {
+                Some(hir::BinOpKind::Eq)
+            } else if diag_item == sym::cmp_partialeq_ne {
+                Some(hir::BinOpKind::Ne)
+            } else {
+                None
+            }
+        }
     }
 }
 
@@ -1015,7 +1179,7 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> {
         // We can't completely trust `repr(C)` markings, so make sure the fields are actually safe.
         let mut all_phantom = !variant.fields.is_empty();
         for field in &variant.fields {
-            all_phantom &= match self.check_field_type_for_ffi(cache, &field, args) {
+            all_phantom &= match self.check_field_type_for_ffi(cache, field, args) {
                 FfiSafe => false,
                 // `()` fields are FFI-safe!
                 FfiUnsafe { ty, .. } if ty.is_unit() => false,
@@ -1234,7 +1398,7 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> {
                     };
                 }
 
-                let sig = tcx.erase_late_bound_regions(sig);
+                let sig = tcx.instantiate_bound_regions_with_erased(sig);
                 for arg in sig.inputs() {
                     match self.check_type_for_ffi(cache, *arg) {
                         FfiSafe => {}
@@ -1391,7 +1555,7 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> {
     /// types that have external ABIs, as these still need checked.
     fn check_fn(&mut self, def_id: LocalDefId, decl: &'tcx hir::FnDecl<'_>) {
         let sig = self.cx.tcx.fn_sig(def_id).instantiate_identity();
-        let sig = self.cx.tcx.erase_late_bound_regions(sig);
+        let sig = self.cx.tcx.instantiate_bound_regions_with_erased(sig);
 
         for (input_ty, input_hir) in iter::zip(sig.inputs(), decl.inputs) {
             for (fn_ptr_ty, span) in self.find_fn_ptr_ty_with_external_abi(input_hir, *input_ty) {
@@ -1399,7 +1563,7 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> {
             }
         }
 
-        if let hir::FnRetTy::Return(ref ret_hir) = decl.output {
+        if let hir::FnRetTy::Return(ret_hir) = decl.output {
             for (fn_ptr_ty, span) in self.find_fn_ptr_ty_with_external_abi(ret_hir, sig.output()) {
                 self.check_type_for_ffi_and_report_errors(span, fn_ptr_ty, false, true);
             }
@@ -1409,13 +1573,13 @@ impl<'a, 'tcx> ImproperCTypesVisitor<'a, 'tcx> {
     /// Check if a function's argument types and result type are "ffi-safe".
     fn check_foreign_fn(&mut self, def_id: LocalDefId, decl: &'tcx hir::FnDecl<'_>) {
         let sig = self.cx.tcx.fn_sig(def_id).instantiate_identity();
-        let sig = self.cx.tcx.erase_late_bound_regions(sig);
+        let sig = self.cx.tcx.instantiate_bound_regions_with_erased(sig);
 
         for (input_ty, input_hir) in iter::zip(sig.inputs(), decl.inputs) {
             self.check_type_for_ffi_and_report_errors(input_hir.span, *input_ty, false, false);
         }
 
-        if let hir::FnRetTy::Return(ref ret_hir) = decl.output {
+        if let hir::FnRetTy::Return(ret_hir) = decl.output {
             self.check_type_for_ffi_and_report_errors(ret_hir.span, sig.output(), false, true);
         }
     }
@@ -1487,13 +1651,13 @@ impl<'tcx> LateLintPass<'tcx> for ImproperCTypesDeclarations {
         let abi = cx.tcx.hir().get_foreign_abi(it.hir_id());
 
         match it.kind {
-            hir::ForeignItemKind::Fn(ref decl, _, _) if !vis.is_internal_abi(abi) => {
+            hir::ForeignItemKind::Fn(decl, _, _) if !vis.is_internal_abi(abi) => {
                 vis.check_foreign_fn(it.owner_id.def_id, decl);
             }
-            hir::ForeignItemKind::Static(ref ty, _) if !vis.is_internal_abi(abi) => {
+            hir::ForeignItemKind::Static(ty, _) if !vis.is_internal_abi(abi) => {
                 vis.check_foreign_static(it.owner_id, ty.span);
             }
-            hir::ForeignItemKind::Fn(ref decl, _, _) => vis.check_fn(it.owner_id.def_id, decl),
+            hir::ForeignItemKind::Fn(decl, _, _) => vis.check_fn(it.owner_id.def_id, decl),
             hir::ForeignItemKind::Static(..) | hir::ForeignItemKind::Type => (),
         }
     }
@@ -1705,7 +1869,7 @@ impl InvalidAtomicOrdering {
             sym::AtomicI64,
             sym::AtomicI128,
         ];
-        if let ExprKind::MethodCall(ref method_path, _, args, _) = &expr.kind
+        if let ExprKind::MethodCall(method_path, _, args, _) = &expr.kind
             && recognized_names.contains(&method_path.ident.name)
             && let Some(m_def_id) = cx.typeck_results().type_dependent_def_id(expr.hir_id)
             && let Some(impl_did) = cx.tcx.impl_of_method(m_def_id)
@@ -1766,7 +1930,7 @@ impl InvalidAtomicOrdering {
     }
 
     fn check_memory_fence(cx: &LateContext<'_>, expr: &Expr<'_>) {
-        if let ExprKind::Call(ref func, ref args) = expr.kind
+        if let ExprKind::Call(func, args) = expr.kind
             && let ExprKind::Path(ref func_qpath) = func.kind
             && let Some(def_id) = cx.qpath_res(func_qpath, func.hir_id).opt_def_id()
             && matches!(cx.tcx.get_diagnostic_name(def_id), Some(sym::fence | sym::compiler_fence))
diff --git a/compiler/rustc_lint/src/unit_bindings.rs b/compiler/rustc_lint/src/unit_bindings.rs
new file mode 100644
index 000000000..c80889f3a
--- /dev/null
+++ b/compiler/rustc_lint/src/unit_bindings.rs
@@ -0,0 +1,72 @@
+use crate::lints::UnitBindingsDiag;
+use crate::{LateLintPass, LintContext};
+use rustc_hir as hir;
+use rustc_middle::ty::Ty;
+
+declare_lint! {
+    /// The `unit_bindings` lint detects cases where bindings are useless because they have
+    /// the unit type `()` as their inferred type. The lint is suppressed if the user explicitly
+    /// annotates the let binding with the unit type `()`, or if the let binding uses an underscore
+    /// wildcard pattern, i.e. `let _ = expr`, or if the binding is produced from macro expansions.
+    ///
+    /// ### Example
+    ///
+    /// ```rust,compile_fail
+    /// #![deny(unit_bindings)]
+    ///
+    /// fn foo() {
+    ///     println!("do work");
+    /// }
+    ///
+    /// pub fn main() {
+    ///     let x = foo(); // useless binding
+    /// }
+    /// ```
+    ///
+    /// {{produces}}
+    ///
+    /// ### Explanation
+    ///
+    /// Creating a local binding with the unit type `()` does not do much and can be a sign of a
+    /// user error, such as in this example:
+    ///
+    /// ```rust,no_run
+    /// fn main() {
+    ///     let mut x = [1, 2, 3];
+    ///     x[0] = 5;
+    ///     let y = x.sort(); // useless binding as `sort` returns `()` and not the sorted array.
+    ///     println!("{:?}", y); // prints "()"
+    /// }
+    /// ```
+    pub UNIT_BINDINGS,
+    Allow,
+    "binding is useless because it has the unit `()` type"
+}
+
+declare_lint_pass!(UnitBindings => [UNIT_BINDINGS]);
+
+impl<'tcx> LateLintPass<'tcx> for UnitBindings {
+    fn check_local(&mut self, cx: &crate::LateContext<'tcx>, local: &'tcx hir::Local<'tcx>) {
+        // Suppress warning if user:
+        // - explicitly ascribes a type to the pattern
+        // - explicitly wrote `let pat = ();`
+        // - explicitly wrote `let () = init;`.
+        if !local.span.from_expansion()
+            && let Some(tyck_results) = cx.maybe_typeck_results()
+            && let Some(init) = local.init
+            && let init_ty = tyck_results.expr_ty(init)
+            && let local_ty = tyck_results.node_type(local.hir_id)
+            && init_ty == Ty::new_unit(cx.tcx)
+            && local_ty == Ty::new_unit(cx.tcx)
+            && local.ty.is_none()
+            && !matches!(init.kind, hir::ExprKind::Tup([]))
+            && !matches!(local.pat.kind, hir::PatKind::Tuple([], ..))
+        {
+            cx.emit_spanned_lint(
+                UNIT_BINDINGS,
+                local.span,
+                UnitBindingsDiag { label: local.pat.span },
+            );
+        }
+    }
+}
diff --git a/compiler/rustc_lint/src/unused.rs b/compiler/rustc_lint/src/unused.rs
index 355855b8e..34cdee4ec 100644
--- a/compiler/rustc_lint/src/unused.rs
+++ b/compiler/rustc_lint/src/unused.rs
@@ -111,7 +111,7 @@ impl<'tcx> LateLintPass<'tcx> for UnusedResults {
         }
 
         if let hir::ExprKind::Match(await_expr, _arms, hir::MatchSource::AwaitDesugar) = expr.kind
-            && let ty = cx.typeck_results().expr_ty(&await_expr)
+            && let ty = cx.typeck_results().expr_ty(await_expr)
             && let ty::Alias(ty::Opaque, ty::AliasTy { def_id: future_def_id, .. }) = ty.kind()
             && cx.tcx.ty_is_opaque_future(ty)
             && let async_fn_def_id = cx.tcx.parent(*future_def_id)
@@ -132,9 +132,9 @@ impl<'tcx> LateLintPass<'tcx> for UnusedResults {
             return;
         }
 
-        let ty = cx.typeck_results().expr_ty(&expr);
+        let ty = cx.typeck_results().expr_ty(expr);
 
-        let must_use_result = is_ty_must_use(cx, ty, &expr, expr.span);
+        let must_use_result = is_ty_must_use(cx, ty, expr, expr.span);
         let type_lint_emitted_or_suppressed = match must_use_result {
             Some(path) => {
                 emit_must_use_untranslated(cx, &path, "", "", 1, false, expr_is_from_block);
@@ -211,7 +211,7 @@ impl<'tcx> LateLintPass<'tcx> for UnusedResults {
             expr_is_from_block: bool,
         ) -> bool {
             let maybe_def_id = match expr.kind {
-                hir::ExprKind::Call(ref callee, _) => {
+                hir::ExprKind::Call(callee, _) => {
                     match callee.kind {
                         hir::ExprKind::Path(ref qpath) => {
                             match cx.qpath_res(qpath, callee.hir_id) {
@@ -251,6 +251,7 @@ impl<'tcx> LateLintPass<'tcx> for UnusedResults {
             /// The root of the normal must_use lint with an optional message.
             Def(Span, DefId, Option<Symbol>),
             Boxed(Box<Self>),
+            Pinned(Box<Self>),
             Opaque(Box<Self>),
             TraitObject(Box<Self>),
             TupleElement(Vec<(usize, Self)>),
@@ -284,8 +285,13 @@ impl<'tcx> LateLintPass<'tcx> for UnusedResults {
                     is_ty_must_use(cx, boxed_ty, expr, span)
                         .map(|inner| MustUsePath::Boxed(Box::new(inner)))
                 }
+                ty::Adt(def, args) if cx.tcx.lang_items().pin_type() == Some(def.did()) => {
+                    let pinned_ty = args.type_at(0);
+                    is_ty_must_use(cx, pinned_ty, expr, span)
+                        .map(|inner| MustUsePath::Pinned(Box::new(inner)))
+                }
                 ty::Adt(def, _) => is_def_must_use(cx, def.did(), span),
-                ty::Alias(ty::Opaque, ty::AliasTy { def_id: def, .. }) => {
+                ty::Alias(ty::Opaque | ty::Projection, ty::AliasTy { def_id: def, .. }) => {
                     elaborate(
                         cx.tcx,
                         cx.tcx.explicit_item_bounds(def).instantiate_identity_iter_copied(),
@@ -425,6 +431,18 @@ impl<'tcx> LateLintPass<'tcx> for UnusedResults {
                         expr_is_from_block,
                     );
                 }
+                MustUsePath::Pinned(path) => {
+                    let descr_pre = &format!("{descr_pre}pinned ");
+                    emit_must_use_untranslated(
+                        cx,
+                        path,
+                        descr_pre,
+                        descr_post,
+                        plural_len,
+                        true,
+                        expr_is_from_block,
+                    );
+                }
                 MustUsePath::Opaque(path) => {
                     let descr_pre = &format!("{descr_pre}implementer{plural_suffix} of ");
                     emit_must_use_untranslated(
@@ -638,7 +656,7 @@ trait UnusedDelimLint {
     ) -> bool {
         if followed_by_else {
             match inner.kind {
-                ast::ExprKind::Binary(op, ..) if op.node.lazy() => return true,
+                ast::ExprKind::Binary(op, ..) if op.node.is_lazy() => return true,
                 _ if classify::expr_trailing_brace(inner).is_some() => return true,
                 _ => {}
             }
@@ -674,7 +692,7 @@ trait UnusedDelimLint {
                 innermost = match &innermost.kind {
                     ExprKind::AddrOf(_, _, expr) => expr,
                     _ => {
-                        if parser::contains_exterior_struct_lit(&innermost) {
+                        if parser::contains_exterior_struct_lit(innermost) {
                             return true;
                         } else {
                             break;
@@ -703,7 +721,7 @@ trait UnusedDelimLint {
                     return matches!(rhs.kind, ExprKind::Block(..));
                 }
 
-                _ => return parser::contains_exterior_struct_lit(&inner),
+                _ => return parser::contains_exterior_struct_lit(inner),
             }
         }
     }
@@ -878,7 +896,7 @@ trait UnusedDelimLint {
         };
         self.check_unused_delims_expr(
             cx,
-            &value,
+            value,
             ctx,
             followed_by_block,
             left_pos,
@@ -901,7 +919,7 @@ trait UnusedDelimLint {
             StmtKind::Expr(ref expr) => {
                 self.check_unused_delims_expr(
                     cx,
-                    &expr,
+                    expr,
                     UnusedDelimsCtx::BlockRetValue,
                     false,
                     None,
@@ -998,7 +1016,7 @@ impl UnusedDelimLint for UnusedParens {
                                 rustc_span::source_map::Spanned { node, .. },
                                 _,
                                 _,
-                            ) if node.lazy()))
+                            ) if node.is_lazy()))
                 {
                     self.emit_unused_delims_expr(cx, value, ctx, left_pos, right_pos, is_kw)
                 }
@@ -1095,15 +1113,17 @@ impl EarlyLintPass for UnusedParens {
             }
             ExprKind::Match(ref _expr, ref arm) => {
                 for a in arm {
-                    self.check_unused_delims_expr(
-                        cx,
-                        &a.body,
-                        UnusedDelimsCtx::MatchArmExpr,
-                        false,
-                        None,
-                        None,
-                        true,
-                    );
+                    if let Some(body) = &a.body {
+                        self.check_unused_delims_expr(
+                            cx,
+                            body,
+                            UnusedDelimsCtx::MatchArmExpr,
+                            false,
+                            None,
+                            None,
+                            true,
+                        );
+                    }
                 }
             }
             _ => {}
@@ -1136,7 +1156,7 @@ impl EarlyLintPass for UnusedParens {
             // Do not lint on `(..)` as that will result in the other arms being useless.
             Paren(_)
             // The other cases do not contain sub-patterns.
-            | Wild | Rest | Lit(..) | MacCall(..) | Range(..) | Ident(.., None) | Path(..) => {},
+            | Wild | Never | Rest | Lit(..) | MacCall(..) | Range(..) | Ident(.., None) | Path(..) => {},
             // These are list-like patterns; parens can always be removed.
             TupleStruct(_, _, ps) | Tuple(ps) | Slice(ps) | Or(ps) => for p in ps {
                 self.check_unused_parens_pat(cx, p, false, false, keep_space);
diff --git a/compiler/rustc_lint_defs/src/builtin.rs b/compiler/rustc_lint_defs/src/builtin.rs
index bef9f469c..f9b66239b 100644
--- a/compiler/rustc_lint_defs/src/builtin.rs
+++ b/compiler/rustc_lint_defs/src/builtin.rs
@@ -8,6 +8,134 @@ use crate::{declare_lint, declare_lint_pass, FutureIncompatibilityReason};
 use rustc_span::edition::Edition;
 use rustc_span::symbol::sym;
 
+declare_lint_pass! {
+    /// Does nothing as a lint pass, but registers some `Lint`s
+    /// that are used by other parts of the compiler.
+    HardwiredLints => [
+        // tidy-alphabetical-start
+        ABSOLUTE_PATHS_NOT_STARTING_WITH_CRATE,
+        AMBIGUOUS_ASSOCIATED_ITEMS,
+        AMBIGUOUS_GLOB_IMPORTS,
+        AMBIGUOUS_GLOB_REEXPORTS,
+        ARITHMETIC_OVERFLOW,
+        ASM_SUB_REGISTER,
+        BAD_ASM_STYLE,
+        BARE_TRAIT_OBJECTS,
+        BINDINGS_WITH_VARIANT_NAME,
+        BREAK_WITH_LABEL_AND_LOOP,
+        BYTE_SLICE_IN_PACKED_STRUCT_WITH_DERIVE,
+        CENUM_IMPL_DROP_CAST,
+        COHERENCE_LEAK_CHECK,
+        COINDUCTIVE_OVERLAP_IN_COHERENCE,
+        CONFLICTING_REPR_HINTS,
+        CONST_EVALUATABLE_UNCHECKED,
+        CONST_ITEM_MUTATION,
+        CONST_PATTERNS_WITHOUT_PARTIAL_EQ,
+        DEAD_CODE,
+        DEPRECATED,
+        DEPRECATED_CFG_ATTR_CRATE_TYPE_NAME,
+        DEPRECATED_IN_FUTURE,
+        DEPRECATED_WHERE_CLAUSE_LOCATION,
+        DUPLICATE_MACRO_ATTRIBUTES,
+        ELIDED_LIFETIMES_IN_ASSOCIATED_CONSTANT,
+        ELIDED_LIFETIMES_IN_PATHS,
+        EXPORTED_PRIVATE_DEPENDENCIES,
+        FFI_UNWIND_CALLS,
+        FORBIDDEN_LINT_GROUPS,
+        FUNCTION_ITEM_REFERENCES,
+        FUZZY_PROVENANCE_CASTS,
+        HIDDEN_GLOB_REEXPORTS,
+        ILL_FORMED_ATTRIBUTE_INPUT,
+        ILLEGAL_FLOATING_POINT_LITERAL_PATTERN,
+        INCOMPLETE_INCLUDE,
+        INDIRECT_STRUCTURAL_MATCH,
+        INEFFECTIVE_UNSTABLE_TRAIT_IMPL,
+        INLINE_NO_SANITIZE,
+        INVALID_DOC_ATTRIBUTES,
+        INVALID_MACRO_EXPORT_ARGUMENTS,
+        INVALID_TYPE_PARAM_DEFAULT,
+        IRREFUTABLE_LET_PATTERNS,
+        LARGE_ASSIGNMENTS,
+        LATE_BOUND_LIFETIME_ARGUMENTS,
+        LEGACY_DERIVE_HELPERS,
+        LONG_RUNNING_CONST_EVAL,
+        LOSSY_PROVENANCE_CASTS,
+        MACRO_EXPANDED_MACRO_EXPORTS_ACCESSED_BY_ABSOLUTE_PATHS,
+        MACRO_USE_EXTERN_CRATE,
+        META_VARIABLE_MISUSE,
+        MISSING_ABI,
+        MISSING_FRAGMENT_SPECIFIER,
+        MUST_NOT_SUSPEND,
+        NAMED_ARGUMENTS_USED_POSITIONALLY,
+        NON_EXHAUSTIVE_OMITTED_PATTERNS,
+        NONTRIVIAL_STRUCTURAL_MATCH,
+        ORDER_DEPENDENT_TRAIT_OBJECTS,
+        OVERLAPPING_RANGE_ENDPOINTS,
+        PATTERNS_IN_FNS_WITHOUT_BODY,
+        POINTER_STRUCTURAL_MATCH,
+        PRIVATE_BOUNDS,
+        PRIVATE_INTERFACES,
+        PROC_MACRO_BACK_COMPAT,
+        PROC_MACRO_DERIVE_RESOLUTION_FALLBACK,
+        PUB_USE_OF_PRIVATE_EXTERN_CRATE,
+        REFINING_IMPL_TRAIT,
+        RENAMED_AND_REMOVED_LINTS,
+        REPR_TRANSPARENT_EXTERNAL_PRIVATE_FIELDS,
+        RUST_2021_INCOMPATIBLE_CLOSURE_CAPTURES,
+        RUST_2021_INCOMPATIBLE_OR_PATTERNS,
+        RUST_2021_PREFIXES_INCOMPATIBLE_SYNTAX,
+        RUST_2021_PRELUDE_COLLISIONS,
+        SEMICOLON_IN_EXPRESSIONS_FROM_MACROS,
+        SINGLE_USE_LIFETIMES,
+        SOFT_UNSTABLE,
+        STABLE_FEATURES,
+        SUSPICIOUS_AUTO_TRAIT_IMPLS,
+        TEST_UNSTABLE_LINT,
+        TEXT_DIRECTION_CODEPOINT_IN_COMMENT,
+        TRIVIAL_CASTS,
+        TRIVIAL_NUMERIC_CASTS,
+        TYVAR_BEHIND_RAW_POINTER,
+        UNCONDITIONAL_PANIC,
+        UNCONDITIONAL_RECURSION,
+        UNDEFINED_NAKED_FUNCTION_ABI,
+        UNEXPECTED_CFGS,
+        UNFULFILLED_LINT_EXPECTATIONS,
+        UNINHABITED_STATIC,
+        UNKNOWN_CRATE_TYPES,
+        UNKNOWN_LINTS,
+        UNKNOWN_OR_MALFORMED_DIAGNOSTIC_ATTRIBUTES,
+        UNNAMEABLE_TEST_ITEMS,
+        UNNAMEABLE_TYPES,
+        UNREACHABLE_CODE,
+        UNREACHABLE_PATTERNS,
+        UNSAFE_OP_IN_UNSAFE_FN,
+        UNSTABLE_NAME_COLLISIONS,
+        UNSTABLE_SYNTAX_PRE_EXPANSION,
+        UNSUPPORTED_CALLING_CONVENTIONS,
+        UNUSED_ASSIGNMENTS,
+        UNUSED_ASSOCIATED_TYPE_BOUNDS,
+        UNUSED_ATTRIBUTES,
+        UNUSED_CRATE_DEPENDENCIES,
+        UNUSED_EXTERN_CRATES,
+        UNUSED_FEATURES,
+        UNUSED_IMPORTS,
+        UNUSED_LABELS,
+        UNUSED_LIFETIMES,
+        UNUSED_MACRO_RULES,
+        UNUSED_MACROS,
+        UNUSED_MUT,
+        UNUSED_QUALIFICATIONS,
+        UNUSED_TUPLE_STRUCT_FIELDS,
+        UNUSED_UNSAFE,
+        UNUSED_VARIABLES,
+        USELESS_DEPRECATED,
+        WARNINGS,
+        WHERE_CLAUSES_OBJECT_SAFETY,
+        WRITES_THROUGH_IMMUTABLE_POINTER,
+        // tidy-alphabetical-end
+    ]
+}
+
 declare_lint! {
     /// The `forbidden_lint_groups` lint detects violations of
     /// `forbid` applied to a lint group. Due to a bug in the compiler,
@@ -3135,7 +3263,7 @@ declare_lint! {
     /// ### Example
     ///
     /// ```text
-    /// rustc --check-cfg 'names()'
+    /// rustc --check-cfg 'cfg()'
     /// ```
     ///
     /// ```rust,ignore (needs command line option)
@@ -3146,7 +3274,7 @@ declare_lint! {
     /// This will produce:
     ///
     /// ```text
-    /// warning: unknown condition name used
+    /// warning: unexpected `cfg` condition name: `widnows`
     ///  --> lint_example.rs:1:7
     ///   |
     /// 1 | #[cfg(widnows)]
@@ -3157,9 +3285,10 @@ declare_lint! {
     ///
     /// ### Explanation
     ///
-    /// This lint is only active when a `--check-cfg='names(...)'` option has been passed
-    /// to the compiler and triggers whenever an unknown condition name or value is used.
-    /// The known condition include names or values passed in `--check-cfg`, `--cfg`, and some
+    /// This lint is only active when `--check-cfg` arguments are being passed
+    /// to the compiler and triggers whenever an unexpected condition name or value is used.
+    ///
+    /// The known condition include names or values passed in `--check-cfg`, and some
     /// well-knows names and values built into the compiler.
     pub UNEXPECTED_CFGS,
     Warn,
@@ -3348,133 +3477,6 @@ declare_lint! {
     "name introduced by a private item shadows a name introduced by a public glob re-export",
 }
 
-declare_lint_pass! {
-    /// Does nothing as a lint pass, but registers some `Lint`s
-    /// that are used by other parts of the compiler.
-    HardwiredLints => [
-        // tidy-alphabetical-start
-        ABSOLUTE_PATHS_NOT_STARTING_WITH_CRATE,
-        AMBIGUOUS_ASSOCIATED_ITEMS,
-        AMBIGUOUS_GLOB_IMPORTS,
-        AMBIGUOUS_GLOB_REEXPORTS,
-        ARITHMETIC_OVERFLOW,
-        ASM_SUB_REGISTER,
-        BAD_ASM_STYLE,
-        BARE_TRAIT_OBJECTS,
-        BINDINGS_WITH_VARIANT_NAME,
-        BREAK_WITH_LABEL_AND_LOOP,
-        BYTE_SLICE_IN_PACKED_STRUCT_WITH_DERIVE,
-        CENUM_IMPL_DROP_CAST,
-        COHERENCE_LEAK_CHECK,
-        COINDUCTIVE_OVERLAP_IN_COHERENCE,
-        CONFLICTING_REPR_HINTS,
-        CONST_EVALUATABLE_UNCHECKED,
-        CONST_ITEM_MUTATION,
-        CONST_PATTERNS_WITHOUT_PARTIAL_EQ,
-        DEAD_CODE,
-        DEPRECATED,
-        DEPRECATED_CFG_ATTR_CRATE_TYPE_NAME,
-        DEPRECATED_IN_FUTURE,
-        DEPRECATED_WHERE_CLAUSE_LOCATION,
-        DUPLICATE_MACRO_ATTRIBUTES,
-        ELIDED_LIFETIMES_IN_ASSOCIATED_CONSTANT,
-        ELIDED_LIFETIMES_IN_PATHS,
-        EXPORTED_PRIVATE_DEPENDENCIES,
-        FFI_UNWIND_CALLS,
-        FORBIDDEN_LINT_GROUPS,
-        FUNCTION_ITEM_REFERENCES,
-        FUZZY_PROVENANCE_CASTS,
-        HIDDEN_GLOB_REEXPORTS,
-        ILL_FORMED_ATTRIBUTE_INPUT,
-        ILLEGAL_FLOATING_POINT_LITERAL_PATTERN,
-        IMPLIED_BOUNDS_ENTAILMENT,
-        INCOMPLETE_INCLUDE,
-        INDIRECT_STRUCTURAL_MATCH,
-        INEFFECTIVE_UNSTABLE_TRAIT_IMPL,
-        INLINE_NO_SANITIZE,
-        INVALID_DOC_ATTRIBUTES,
-        INVALID_MACRO_EXPORT_ARGUMENTS,
-        INVALID_TYPE_PARAM_DEFAULT,
-        IRREFUTABLE_LET_PATTERNS,
-        LARGE_ASSIGNMENTS,
-        LATE_BOUND_LIFETIME_ARGUMENTS,
-        LEGACY_DERIVE_HELPERS,
-        LONG_RUNNING_CONST_EVAL,
-        LOSSY_PROVENANCE_CASTS,
-        MACRO_EXPANDED_MACRO_EXPORTS_ACCESSED_BY_ABSOLUTE_PATHS,
-        MACRO_USE_EXTERN_CRATE,
-        META_VARIABLE_MISUSE,
-        MISSING_ABI,
-        MISSING_FRAGMENT_SPECIFIER,
-        MUST_NOT_SUSPEND,
-        NAMED_ARGUMENTS_USED_POSITIONALLY,
-        NON_EXHAUSTIVE_OMITTED_PATTERNS,
-        NONTRIVIAL_STRUCTURAL_MATCH,
-        ORDER_DEPENDENT_TRAIT_OBJECTS,
-        OVERLAPPING_RANGE_ENDPOINTS,
-        PATTERNS_IN_FNS_WITHOUT_BODY,
-        POINTER_STRUCTURAL_MATCH,
-        PRIVATE_BOUNDS,
-        PRIVATE_INTERFACES,
-        PROC_MACRO_BACK_COMPAT,
-        PROC_MACRO_DERIVE_RESOLUTION_FALLBACK,
-        PUB_USE_OF_PRIVATE_EXTERN_CRATE,
-        REFINING_IMPL_TRAIT,
-        RENAMED_AND_REMOVED_LINTS,
-        REPR_TRANSPARENT_EXTERNAL_PRIVATE_FIELDS,
-        RUST_2021_INCOMPATIBLE_CLOSURE_CAPTURES,
-        RUST_2021_INCOMPATIBLE_OR_PATTERNS,
-        RUST_2021_PREFIXES_INCOMPATIBLE_SYNTAX,
-        RUST_2021_PRELUDE_COLLISIONS,
-        SEMICOLON_IN_EXPRESSIONS_FROM_MACROS,
-        SINGLE_USE_LIFETIMES,
-        SOFT_UNSTABLE,
-        STABLE_FEATURES,
-        SUSPICIOUS_AUTO_TRAIT_IMPLS,
-        TEST_UNSTABLE_LINT,
-        TEXT_DIRECTION_CODEPOINT_IN_COMMENT,
-        TRIVIAL_CASTS,
-        TRIVIAL_NUMERIC_CASTS,
-        TYVAR_BEHIND_RAW_POINTER,
-        UNCONDITIONAL_PANIC,
-        UNCONDITIONAL_RECURSION,
-        UNDEFINED_NAKED_FUNCTION_ABI,
-        UNFULFILLED_LINT_EXPECTATIONS,
-        UNINHABITED_STATIC,
-        UNKNOWN_CRATE_TYPES,
-        UNKNOWN_LINTS,
-        UNKNOWN_OR_MALFORMED_DIAGNOSTIC_ATTRIBUTES,
-        UNNAMEABLE_TEST_ITEMS,
-        UNNAMEABLE_TYPES,
-        UNREACHABLE_CODE,
-        UNREACHABLE_PATTERNS,
-        UNSAFE_OP_IN_UNSAFE_FN,
-        UNSTABLE_NAME_COLLISIONS,
-        UNSTABLE_SYNTAX_PRE_EXPANSION,
-        UNSUPPORTED_CALLING_CONVENTIONS,
-        UNUSED_ASSIGNMENTS,
-        UNUSED_ASSOCIATED_TYPE_BOUNDS,
-        UNUSED_ATTRIBUTES,
-        UNUSED_CRATE_DEPENDENCIES,
-        UNUSED_EXTERN_CRATES,
-        UNUSED_FEATURES,
-        UNUSED_IMPORTS,
-        UNUSED_LABELS,
-        UNUSED_LIFETIMES,
-        UNUSED_MACRO_RULES,
-        UNUSED_MACROS,
-        UNUSED_MUT,
-        UNUSED_QUALIFICATIONS,
-        UNUSED_TUPLE_STRUCT_FIELDS,
-        UNUSED_UNSAFE,
-        UNUSED_VARIABLES,
-        USELESS_DEPRECATED,
-        WARNINGS,
-        WHERE_CLAUSES_OBJECT_SAFETY,
-        // tidy-alphabetical-end
-    ]
-}
-
 declare_lint! {
     /// The `long_running_const_eval` lint is emitted when const
     /// eval is running for a long time to ensure rustc terminates
@@ -4233,47 +4235,6 @@ declare_lint! {
 }
 
 declare_lint! {
-    /// The `implied_bounds_entailment` lint detects cases where the arguments of an impl method
-    /// have stronger implied bounds than those from the trait method it's implementing.
-    ///
-    /// ### Example
-    ///
-    /// ```rust,compile_fail
-    /// #![deny(implied_bounds_entailment)]
-    ///
-    /// trait Trait {
-    ///     fn get<'s>(s: &'s str, _: &'static &'static ()) -> &'static str;
-    /// }
-    ///
-    /// impl Trait for () {
-    ///     fn get<'s>(s: &'s str, _: &'static &'s ()) -> &'static str {
-    ///         s
-    ///     }
-    /// }
-    ///
-    /// let val = <() as Trait>::get(&String::from("blah blah blah"), &&());
-    /// println!("{}", val);
-    /// ```
-    ///
-    /// {{produces}}
-    ///
-    /// ### Explanation
-    ///
-    /// Neither the trait method, which provides no implied bounds about `'s`, nor the impl,
-    /// requires the main function to prove that 's: 'static, but the impl method is allowed
-    /// to assume that `'s: 'static` within its own body.
-    ///
-    /// This can be used to implement an unsound API if used incorrectly.
-    pub IMPLIED_BOUNDS_ENTAILMENT,
-    Deny,
-    "impl method assumes more implied bounds than its corresponding trait method",
-    @future_incompatible = FutureIncompatibleInfo {
-        reason: FutureIncompatibilityReason::FutureReleaseErrorReportInDeps,
-        reference: "issue #105572 <https://github.com/rust-lang/rust/issues/105572>",
-    };
-}
-
-declare_lint! {
     /// The `byte_slice_in_packed_struct_with_derive` lint detects cases where a byte slice field
     /// (`[u8]`) or string slice field (`str`) is used in a `packed` struct that derives one or
     /// more built-in traits.
@@ -4619,3 +4580,37 @@ declare_lint! {
         reference: "issue #115010 <https://github.com/rust-lang/rust/issues/115010>",
     };
 }
+
+declare_lint! {
+    /// The `writes_through_immutable_pointer` lint detects writes through pointers derived from
+    /// shared references.
+    ///
+    /// ### Example
+    ///
+    /// ```rust,compile_fail
+    /// #![feature(const_mut_refs)]
+    /// const WRITE_AFTER_CAST: () = unsafe {
+    ///     let mut x = 0;
+    ///     let ptr = &x as *const i32 as *mut i32;
+    ///     *ptr = 0;
+    /// };
+    /// ```
+    ///
+    /// {{produces}}
+    ///
+    /// ### Explanation
+    ///
+    /// Shared references are immutable (when there is no `UnsafeCell` involved),
+    /// and writing through them or through pointers derived from them is Undefined Behavior.
+    /// The compiler recently learned to detect such Undefined Behavior during compile-time
+    /// evaluation, and in the future this will raise a hard error.
+    ///
+    /// [future-incompatible]: ../index.md#future-incompatible-lints
+    pub WRITES_THROUGH_IMMUTABLE_POINTER,
+    Warn,
+    "shared references are immutable, and pointers derived from them must not be written to",
+    @future_incompatible = FutureIncompatibleInfo {
+        reason: FutureIncompatibilityReason::FutureReleaseErrorReportInDeps,
+        reference: "issue #X <https://github.com/rust-lang/rust/issues/X>",
+    };
+}
diff --git a/compiler/rustc_lint_defs/src/lib.rs b/compiler/rustc_lint_defs/src/lib.rs
index 878c1a65d..a25cfe68e 100644
--- a/compiler/rustc_lint_defs/src/lib.rs
+++ b/compiler/rustc_lint_defs/src/lib.rs
@@ -345,12 +345,34 @@ pub struct FutureIncompatibleInfo {
 }
 
 /// The reason for future incompatibility
+///
+/// Future-incompatible lints come in roughly two categories:
+///
+/// 1. There was a mistake in the compiler (such as a soundness issue), and
+///    we're trying to fix it, but it may be a breaking change.
+/// 2. A change across an Edition boundary, typically used for the
+///    introduction of new language features that can't otherwise be
+///    introduced in a backwards-compatible way.
+///
+/// See <https://rustc-dev-guide.rust-lang.org/bug-fix-procedure.html> and
+/// <https://rustc-dev-guide.rust-lang.org/diagnostics.html#future-incompatible-lints>
+/// for more information.
 #[derive(Copy, Clone, Debug)]
 pub enum FutureIncompatibilityReason {
     /// This will be an error in a future release for all editions
     ///
     /// This will *not* show up in cargo's future breakage report.
     /// The warning will hence only be seen in local crates, not in dependencies.
+    ///
+    /// Choose this variant when you are first introducing a "future
+    /// incompatible" warning that is intended to eventually be fixed in the
+    /// future. This allows crate developers an opportunity to fix the warning
+    /// before blasting all dependents with a warning they can't fix
+    /// (dependents have to wait for a new release of the affected crate to be
+    /// published).
+    ///
+    /// After a lint has been in this state for a while, consider graduating
+    /// it to [`FutureIncompatibilityReason::FutureReleaseErrorReportInDeps`].
     FutureReleaseErrorDontReportInDeps,
     /// This will be an error in a future release, and
     /// Cargo should create a report even for dependencies
@@ -358,17 +380,62 @@ pub enum FutureIncompatibilityReason {
     /// This is the *only* reason that will make future incompatibility warnings show up in cargo's
     /// reports. All other future incompatibility warnings are not visible when they occur in a
     /// dependency.
+    ///
+    /// Choose this variant after the lint has been sitting in the
+    /// [`FutureIncompatibilityReason::FutureReleaseErrorDontReportInDeps`]
+    /// state for a while, and you feel like it is ready to graduate to
+    /// warning everyone. It is a good signal that it is ready if you can
+    /// determine that all or most affected crates on crates.io have been
+    /// updated.
+    ///
+    /// After some period of time, lints with this variant can be turned into
+    /// hard errors (and the lint removed). Preferably when there is some
+    /// confidence that the number of impacted projects is very small (few
+    /// should have a broken dependency in their dependency tree).
     FutureReleaseErrorReportInDeps,
     /// Code that changes meaning in some way in a
     /// future release.
+    ///
+    /// Choose this variant when the semantics of existing code is changing,
+    /// (as opposed to
+    /// [`FutureIncompatibilityReason::FutureReleaseErrorDontReportInDeps`],
+    /// which is for when code is going to be rejected in the future).
     FutureReleaseSemanticsChange,
     /// Previously accepted code that will become an
     /// error in the provided edition
+    ///
+    /// Choose this variant for code that you want to start rejecting across
+    /// an edition boundary. This will automatically include the lint in the
+    /// `rust-20xx-compatibility` lint group, which is used by `cargo fix
+    /// --edition` to do migrations. The lint *should* be auto-fixable with
+    /// [`Applicability::MachineApplicable`].
+    ///
+    /// The lint can either be `Allow` or `Warn` by default. If it is `Allow`,
+    /// users usually won't see this warning unless they are doing an edition
+    /// migration manually or there is a problem during the migration (cargo's
+    /// automatic migrations will force the level to `Warn`). If it is `Warn`
+    /// by default, users on all editions will see this warning (only do this
+    /// if you think it is important for everyone to be aware of the change,
+    /// and to encourage people to update their code on all editions).
+    ///
+    /// See also [`FutureIncompatibilityReason::EditionSemanticsChange`] if
+    /// you have code that is changing semantics across the edition (as
+    /// opposed to being rejected).
     EditionError(Edition),
     /// Code that changes meaning in some way in
     /// the provided edition
+    ///
+    /// This is the same as [`FutureIncompatibilityReason::EditionError`],
+    /// except for situations where the semantics change across an edition. It
+    /// slightly changes the text of the diagnostic, but is otherwise the
+    /// same.
     EditionSemanticsChange(Edition),
     /// A custom reason.
+    ///
+    /// Choose this variant if the built-in text of the diagnostic of the
+    /// other variants doesn't match your situation. This is behaviorally
+    /// equivalent to
+    /// [`FutureIncompatibilityReason::FutureReleaseErrorDontReportInDeps`].
     Custom(&'static str),
 }
 
@@ -583,6 +650,10 @@ pub enum BuiltinLintDiagnostics {
         elided: bool,
         span: Span,
     },
+    RedundantImportVisibility {
+        span: Span,
+        max_vis: String,
+    },
 }
 
 /// Lints that are buffered up early on in the `Session` before the
diff --git a/compiler/rustc_llvm/build.rs b/compiler/rustc_llvm/build.rs
index fe13162cd..4b0c1229d 100644
--- a/compiler/rustc_llvm/build.rs
+++ b/compiler/rustc_llvm/build.rs
@@ -102,7 +102,7 @@ fn output(cmd: &mut Command) -> String {
 
 fn main() {
     for component in REQUIRED_COMPONENTS.iter().chain(OPTIONAL_COMPONENTS.iter()) {
-        println!("cargo:rustc-check-cfg=values(llvm_component,\"{component}\")");
+        println!("cargo:rustc-check-cfg=cfg(llvm_component,values(\"{component}\"))");
     }
 
     if tracked_env_var_os("RUST_CHECK").is_some() {
@@ -242,6 +242,12 @@ fn main() {
         cmd.arg("--system-libs");
     }
 
+    // We need libkstat for getHostCPUName on SPARC builds.
+    // See also: https://github.com/llvm/llvm-project/issues/64186
+    if target.starts_with("sparcv9") && target.contains("solaris") {
+        println!("cargo:rustc-link-lib=kstat");
+    }
+
     if (target.starts_with("arm") && !target.contains("freebsd"))
         || target.starts_with("mips-")
         || target.starts_with("mipsel-")
diff --git a/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp
index d61ec0b64..373bc5cc5 100644
--- a/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp
+++ b/compiler/rustc_llvm/llvm-wrapper/CoverageMappingWrapper.cpp
@@ -139,6 +139,9 @@ extern "C" void LLVMRustCoverageWriteMappingToBuffer(
            RustMappingRegions, NumMappingRegions)) {
     MappingRegions.emplace_back(
         fromRust(Region.Count), fromRust(Region.FalseCount),
+#if LLVM_VERSION_GE(18, 0)
+        coverage::CounterMappingRegion::MCDCParameters{},
+#endif
         Region.FileID, Region.ExpandedFileID,
         Region.LineStart, Region.ColumnStart, Region.LineEnd, Region.ColumnEnd,
         fromRust(Region.Kind));
diff --git a/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h b/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h
index 142384e6d..5bfffc5d9 100644
--- a/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h
+++ b/compiler/rustc_llvm/llvm-wrapper/LLVMWrapper.h
@@ -1,3 +1,5 @@
+#include "SuppressLLVMWarnings.h"
+
 #include "llvm-c/BitReader.h"
 #include "llvm-c/Core.h"
 #include "llvm-c/Object.h"
@@ -42,11 +44,7 @@
 #include "llvm/IR/IRPrintingPasses.h"
 #include "llvm/Linker/Linker.h"
 
-#if LLVM_VERSION_GE(16, 0)
 #include "llvm/TargetParser/Triple.h"
-#else
-#include "llvm/ADT/Triple.h"
-#endif
 
 extern "C" void LLVMRustSetLastError(const char *);
 
@@ -93,6 +91,7 @@ enum LLVMRustAttribute {
   AllocatedPointer = 38,
   AllocAlign = 39,
   SanitizeSafeStack = 40,
+  FnRetThunkExtern = 41,
 };
 
 typedef struct OpaqueRustString *RustStringRef;
diff --git a/compiler/rustc_llvm/llvm-wrapper/Linker.cpp b/compiler/rustc_llvm/llvm-wrapper/Linker.cpp
index 8766e96f0..533df0f75 100644
--- a/compiler/rustc_llvm/llvm-wrapper/Linker.cpp
+++ b/compiler/rustc_llvm/llvm-wrapper/Linker.cpp
@@ -1,3 +1,4 @@
+#include "SuppressLLVMWarnings.h"
 #include "llvm/Linker/Linker.h"
 
 #include "LLVMWrapper.h"
diff --git a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp
index 31565db1b..6fd0c9014 100644
--- a/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp
+++ b/compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp
@@ -37,7 +37,9 @@
 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
 #include "llvm/LTO/LTO.h"
 #include "llvm/Bitcode/BitcodeWriter.h"
-
+#if LLVM_VERSION_GE(18, 0)
+#include "llvm/TargetParser/Host.h"
+#endif
 #include "llvm/Transforms/Instrumentation.h"
 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
 #include "llvm/Support/TimeProfiler.h"
@@ -60,17 +62,17 @@ typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef;
 DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef)
 DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
 
-extern "C" void LLVMTimeTraceProfilerInitialize() {
+extern "C" void LLVMRustTimeTraceProfilerInitialize() {
   timeTraceProfilerInitialize(
       /* TimeTraceGranularity */ 0,
       /* ProcName */ "rustc");
 }
 
-extern "C" void LLVMTimeTraceProfilerFinishThread() {
+extern "C" void LLVMRustTimeTraceProfilerFinishThread() {
   timeTraceProfilerFinishThread();
 }
 
-extern "C" void LLVMTimeTraceProfilerFinish(const char* FileName) {
+extern "C" void LLVMRustTimeTraceProfilerFinish(const char* FileName) {
   StringRef FN(FileName);
   std::error_code EC;
   raw_fd_ostream OS(FN, EC, sys::fs::CD_CreateAlways);
@@ -204,11 +206,7 @@ enum class LLVMRustCodeModel {
   None,
 };
 
-#if LLVM_VERSION_LT(16, 0)
-static Optional<CodeModel::Model>
-#else
 static std::optional<CodeModel::Model>
-#endif
 fromRust(LLVMRustCodeModel Model) {
   switch (Model) {
   case LLVMRustCodeModel::Tiny:
@@ -222,11 +220,7 @@ fromRust(LLVMRustCodeModel Model) {
   case LLVMRustCodeModel::Large:
     return CodeModel::Large;
   case LLVMRustCodeModel::None:
-#if LLVM_VERSION_LT(16, 0)
-    return None;
-#else
     return std::nullopt;
-#endif
   default:
     report_fatal_error("Bad CodeModel.");
   }
@@ -418,7 +412,7 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
     const char *SplitDwarfFile,
     const char *OutputObjFile,
     const char *DebugInfoCompression,
-    bool ForceEmulatedTls,
+    bool UseEmulatedTls,
     const char *ArgsCstrBuff, size_t ArgsCstrBuffLen) {
 
   auto OptLevel = fromRust(RustOptLevel);
@@ -452,7 +446,6 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
   if (OutputObjFile) {
       Options.ObjectFilenameForDebug = OutputObjFile;
   }
-#if LLVM_VERSION_GE(16, 0)
   if (!strcmp("zlib", DebugInfoCompression) && llvm::compression::zlib::isAvailable()) {
     Options.CompressDebugSections = DebugCompressionType::Zlib;
   } else if (!strcmp("zstd", DebugInfoCompression) && llvm::compression::zstd::isAvailable()) {
@@ -460,19 +453,14 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
   } else if (!strcmp("none", DebugInfoCompression)) {
     Options.CompressDebugSections = DebugCompressionType::None;
   }
-#endif
 
   Options.RelaxELFRelocations = RelaxELFRelocations;
   Options.UseInitArray = UseInitArray;
 
 #if LLVM_VERSION_LT(17, 0)
-  if (ForceEmulatedTls) {
-    Options.ExplicitEmulatedTLS = true;
-    Options.EmulatedTLS = true;
-  }
-#else
-  Options.EmulatedTLS = ForceEmulatedTls || Trip.hasDefaultEmulatedTLS();
+  Options.ExplicitEmulatedTLS = true;
 #endif
+  Options.EmulatedTLS = UseEmulatedTls;
 
   if (TrapUnreachable) {
     // Tell LLVM to codegen `unreachable` into an explicit trap instruction.
@@ -480,6 +468,14 @@ extern "C" LLVMTargetMachineRef LLVMRustCreateTargetMachine(
     // it prevents control flow from "falling through" into whatever code
     // happens to be laid out next in memory.
     Options.TrapUnreachable = true;
+    // But don't emit traps after other traps or no-returns unnecessarily.
+    // ...except for when targeting WebAssembly, because the NoTrapAfterNoreturn
+    // option causes bugs in the LLVM WebAssembly backend. You should be able to
+    // remove this check when Rust's minimum supported LLVM version is >= 18
+    // https://github.com/llvm/llvm-project/pull/65876
+    if (!Trip.isWasm()) {
+      Options.NoTrapAfterNoreturn = true;
+    }
   }
 
   if (Singlethread) {
@@ -734,22 +730,14 @@ LLVMRustOptimize(
   bool DebugPassManager = false;
 
   PassInstrumentationCallbacks PIC;
-#if LLVM_VERSION_LT(16, 0)
-  StandardInstrumentations SI(DebugPassManager);
-#else
   StandardInstrumentations SI(TheModule->getContext(), DebugPassManager);
-#endif
   SI.registerCallbacks(PIC);
 
   if (LlvmSelfProfiler){
     LLVMSelfProfileInitializeCallbacks(PIC,LlvmSelfProfiler,BeforePassCallback,AfterPassCallback);
   }
 
-#if LLVM_VERSION_LT(16, 0)
-  Optional<PGOOptions> PGOOpt;
-#else
   std::optional<PGOOptions> PGOOpt;
-#endif
 #if LLVM_VERSION_GE(17, 0)
   auto FS = vfs::getRealFileSystem();
 #endif
@@ -868,7 +856,11 @@ LLVMRustOptimize(
         // cargo run tests in multhreading mode by default
         // so use atomics for coverage counters
         Options.Atomic = true;
+#if LLVM_VERSION_GE(18, 0)
+        MPM.addPass(InstrProfilingLoweringPass(Options, false));
+#else
         MPM.addPass(InstrProfiling(Options, false));
+#endif
       }
     );
   }
@@ -882,12 +874,7 @@ LLVMRustOptimize(
           /*EagerChecks=*/true);
       OptimizerLastEPCallbacks.push_back(
         [Options](ModulePassManager &MPM, OptimizationLevel Level) {
-#if LLVM_VERSION_LT(16, 0)
-          MPM.addPass(ModuleMemorySanitizerPass(Options));
-          MPM.addPass(createModuleToFunctionPassAdaptor(MemorySanitizerPass(Options)));
-#else
           MPM.addPass(MemorySanitizerPass(Options));
-#endif
         }
       );
     }
@@ -912,11 +899,7 @@ LLVMRustOptimize(
             /*UseAfterScope=*/true,
             AsanDetectStackUseAfterReturnMode::Runtime,
           };
-#if LLVM_VERSION_LT(16, 0)
-          MPM.addPass(ModuleAddressSanitizerPass(opts));
-#else
           MPM.addPass(AddressSanitizerPass(opts));
-#endif
         }
       );
     }
@@ -1562,32 +1545,6 @@ LLVMRustParseBitcodeForLTO(LLVMContextRef Context,
   return wrap(std::move(*SrcOrError).release());
 }
 
-// Find the bitcode section in the object file data and return it as a slice.
-// Fail if the bitcode section is present but empty.
-//
-// On success, the return value is the pointer to the start of the slice and
-// `out_len` is filled with the (non-zero) length. On failure, the return value
-// is `nullptr` and `out_len` is set to zero.
-extern "C" const char*
-LLVMRustGetBitcodeSliceFromObjectData(const char *data,
-                                      size_t len,
-                                      size_t *out_len) {
-  *out_len = 0;
-
-  StringRef Data(data, len);
-  MemoryBufferRef Buffer(Data, ""); // The id is unused.
-
-  Expected<MemoryBufferRef> BitcodeOrError =
-    object::IRObjectFile::findBitcodeInMemBuffer(Buffer);
-  if (!BitcodeOrError) {
-    LLVMRustSetLastError(toString(BitcodeOrError.takeError()).c_str());
-    return nullptr;
-  }
-
-  *out_len = BitcodeOrError->getBufferSize();
-  return BitcodeOrError->getBufferStart();
-}
-
 // Find a section of an object file by name. Fail if the section is missing or
 // empty.
 extern "C" const char *LLVMRustGetSliceFromObjectDataByName(const char *data,
diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
index 4390486b0..0df7b7eed 100644
--- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
+++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp
@@ -14,18 +14,13 @@
 #include "llvm/Remarks/RemarkSerializer.h"
 #include "llvm/Remarks/RemarkFormat.h"
 #include "llvm/Support/ToolOutputFile.h"
-#if LLVM_VERSION_GE(16, 0)
 #include "llvm/Support/ModRef.h"
-#endif
 #include "llvm/Object/Archive.h"
 #include "llvm/Object/COFFImportFile.h"
 #include "llvm/Object/ObjectFile.h"
 #include "llvm/Pass.h"
 #include "llvm/Bitcode/BitcodeWriter.h"
 #include "llvm/Support/Signals.h"
-#if LLVM_VERSION_LT(16, 0)
-#include "llvm/ADT/Optional.h"
-#endif
 
 #include <iostream>
 
@@ -283,6 +278,8 @@ static Attribute::AttrKind fromRust(LLVMRustAttribute Kind) {
     return Attribute::AllocAlign;
   case SanitizeSafeStack:
     return Attribute::SafeStack;
+  case FnRetThunkExtern:
+    return Attribute::FnRetThunkExtern;
   }
   report_fatal_error("bad AttributeKind");
 }
@@ -347,13 +344,7 @@ extern "C" LLVMAttributeRef LLVMRustCreateUWTableAttr(LLVMContextRef C, bool Asy
 }
 
 extern "C" LLVMAttributeRef LLVMRustCreateAllocSizeAttr(LLVMContextRef C, uint32_t ElementSizeArg) {
-  return wrap(Attribute::getWithAllocSizeArgs(*unwrap(C), ElementSizeArg,
-#if LLVM_VERSION_LT(16, 0)
-                                              None
-#else
-                                              std::nullopt
-#endif
-                                              ));
+  return wrap(Attribute::getWithAllocSizeArgs(*unwrap(C), ElementSizeArg, std::nullopt));
 }
 
 // These values **must** match ffi::AllocKindFlags.
@@ -416,7 +407,6 @@ enum class LLVMRustMemoryEffects {
 
 extern "C" LLVMAttributeRef LLVMRustCreateMemoryEffectsAttr(LLVMContextRef C,
                                                             LLVMRustMemoryEffects Effects) {
-#if LLVM_VERSION_GE(16, 0)
   switch (Effects) {
     case LLVMRustMemoryEffects::None:
       return wrap(Attribute::getWithMemoryEffects(*unwrap(C), MemoryEffects::none()));
@@ -428,18 +418,6 @@ extern "C" LLVMAttributeRef LLVMRustCreateMemoryEffectsAttr(LLVMContextRef C,
     default:
       report_fatal_error("bad MemoryEffects.");
   }
-#else
-  switch (Effects) {
-    case LLVMRustMemoryEffects::None:
-      return wrap(Attribute::get(*unwrap(C), Attribute::ReadNone));
-    case LLVMRustMemoryEffects::ReadOnly:
-      return wrap(Attribute::get(*unwrap(C), Attribute::ReadOnly));
-    case LLVMRustMemoryEffects::InaccessibleMemOnly:
-      return wrap(Attribute::get(*unwrap(C), Attribute::InaccessibleMemOnly));
-    default:
-      report_fatal_error("bad MemoryEffects.");
-  }
-#endif
 }
 
 // Enable a fast-math flag
@@ -719,6 +697,25 @@ static DICompileUnit::DebugEmissionKind fromRust(LLVMRustDebugEmissionKind Kind)
   }
 }
 
+enum class LLVMRustDebugNameTableKind {
+    Default,
+    GNU,
+    None,
+};
+
+static DICompileUnit::DebugNameTableKind fromRust(LLVMRustDebugNameTableKind Kind) {
+  switch (Kind) {
+  case LLVMRustDebugNameTableKind::Default:
+    return DICompileUnit::DebugNameTableKind::Default;
+  case LLVMRustDebugNameTableKind::GNU:
+    return DICompileUnit::DebugNameTableKind::GNU;
+  case LLVMRustDebugNameTableKind::None:
+    return DICompileUnit::DebugNameTableKind::None;
+  default:
+    report_fatal_error("bad DebugNameTableKind.");
+  }
+}
+
 enum class LLVMRustChecksumKind {
   None,
   MD5,
@@ -726,18 +723,10 @@ enum class LLVMRustChecksumKind {
   SHA256,
 };
 
-#if LLVM_VERSION_LT(16, 0)
-static Optional<DIFile::ChecksumKind> fromRust(LLVMRustChecksumKind Kind) {
-#else
 static std::optional<DIFile::ChecksumKind> fromRust(LLVMRustChecksumKind Kind) {
-#endif
   switch (Kind) {
   case LLVMRustChecksumKind::None:
-#if LLVM_VERSION_LT(16, 0)
-    return None;
-#else
     return std::nullopt;
-#endif
   case LLVMRustChecksumKind::MD5:
     return DIFile::ChecksumKind::CSK_MD5;
   case LLVMRustChecksumKind::SHA1:
@@ -795,13 +784,15 @@ extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateCompileUnit(
     const char *Flags, unsigned RuntimeVer,
     const char *SplitName, size_t SplitNameLen,
     LLVMRustDebugEmissionKind Kind,
-    uint64_t DWOId, bool SplitDebugInlining) {
+    uint64_t DWOId, bool SplitDebugInlining,
+    LLVMRustDebugNameTableKind TableKind) {
   auto *File = unwrapDI<DIFile>(FileRef);
 
   return wrap(Builder->createCompileUnit(Lang, File, StringRef(Producer, ProducerLen),
                                          isOptimized, Flags, RuntimeVer,
                                          StringRef(SplitName, SplitNameLen),
-                                         fromRust(Kind), DWOId, SplitDebugInlining));
+                                         fromRust(Kind), DWOId, SplitDebugInlining,
+                                         false, fromRust(TableKind)));
 }
 
 extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateFile(
@@ -810,17 +801,8 @@ extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateFile(
     const char *Directory, size_t DirectoryLen, LLVMRustChecksumKind CSKind,
     const char *Checksum, size_t ChecksumLen) {
 
-#if LLVM_VERSION_LT(16, 0)
-  Optional<DIFile::ChecksumKind> llvmCSKind = fromRust(CSKind);
-#else
   std::optional<DIFile::ChecksumKind> llvmCSKind = fromRust(CSKind);
-#endif
-
-#if LLVM_VERSION_LT(16, 0)
-  Optional<DIFile::ChecksumInfo<StringRef>> CSInfo{};
-#else
   std::optional<DIFile::ChecksumInfo<StringRef>> CSInfo{};
-#endif
   if (llvmCSKind)
     CSInfo.emplace(*llvmCSKind, StringRef{Checksum, ChecksumLen});
   return wrap(Builder->createFile(StringRef(Filename, FilenameLen),
@@ -983,6 +965,9 @@ extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateStaticMemberType(
     unwrapDI<DIType>(Ty),
     fromRust(Flags),
     unwrap<llvm::ConstantInt>(val),
+#if LLVM_VERSION_GE(18, 0)
+    llvm::dwarf::DW_TAG_member,
+#endif
     AlignInBits
   ));
 }
@@ -1105,7 +1090,11 @@ extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateEnumerationType(
       unwrapDI<DIDescriptor>(Scope), StringRef(Name, NameLen),
       unwrapDI<DIFile>(File), LineNumber,
       SizeInBits, AlignInBits, DINodeArray(unwrapDI<MDTuple>(Elements)),
-      unwrapDI<DIType>(ClassTy), "", IsScoped));
+      unwrapDI<DIType>(ClassTy),
+#if LLVM_VERSION_GE(18, 0)
+      /* RunTimeLang */ 0,
+#endif
+      "", IsScoped));
 }
 
 extern "C" LLVMMetadataRef LLVMRustDIBuilderCreateUnionType(
@@ -2046,17 +2035,9 @@ extern "C" bool LLVMRustIsNonGVFunctionPointerTy(LLVMValueRef V) {
 }
 
 extern "C" bool LLVMRustLLVMHasZlibCompressionForDebugSymbols() {
-#if LLVM_VERSION_GE(16, 0)
   return llvm::compression::zlib::isAvailable();
-#else
-  return false;
-#endif
 }
 
 extern "C" bool LLVMRustLLVMHasZstdCompressionForDebugSymbols() {
-#if LLVM_VERSION_GE(16, 0)
   return llvm::compression::zstd::isAvailable();
-#else
-  return false;
-#endif
 }
diff --git a/compiler/rustc_llvm/llvm-wrapper/SuppressLLVMWarnings.h b/compiler/rustc_llvm/llvm-wrapper/SuppressLLVMWarnings.h
new file mode 100644
index 000000000..56964e4ea
--- /dev/null
+++ b/compiler/rustc_llvm/llvm-wrapper/SuppressLLVMWarnings.h
@@ -0,0 +1,13 @@
+#ifndef _rustc_llvm_SuppressLLVMWarnings_h
+#define _rustc_llvm_SuppressLLVMWarnings_h
+
+// LLVM currently generates many warnings when compiled using MSVC. These warnings make it difficult
+// to diagnose real problems when working on C++ code, so we suppress them.
+
+#ifdef _MSC_VER
+#pragma warning(disable:4530) // C++ exception handler used, but unwind semantics are not enabled.
+#pragma warning(disable:4624) // 'xxx': destructor was implicitly defined as deleted
+#pragma warning(disable:4244) // conversion from 'xxx' to 'yyy', possible loss of data
+#endif
+
+#endif // _rustc_llvm_SuppressLLVMWarnings_h
diff --git a/compiler/rustc_llvm/llvm-wrapper/SymbolWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/SymbolWrapper.cpp
index bf00d11ed..91f84692d 100644
--- a/compiler/rustc_llvm/llvm-wrapper/SymbolWrapper.cpp
+++ b/compiler/rustc_llvm/llvm-wrapper/SymbolWrapper.cpp
@@ -7,6 +7,7 @@
 // * https://github.com/llvm/llvm-project/blob/8ef3e895ad8ab1724e2b87cabad1dacdc7a397a3/llvm/include/llvm/Object/ArchiveWriter.h
 // * https://github.com/llvm/llvm-project/blob/8ef3e895ad8ab1724e2b87cabad1dacdc7a397a3/llvm/lib/Object/ArchiveWriter.cpp
 
+#include "SuppressLLVMWarnings.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/Object/ObjectFile.h"
diff --git a/compiler/rustc_llvm/src/lib.rs b/compiler/rustc_llvm/src/lib.rs
index 518c20c9f..ca0aec710 100644
--- a/compiler/rustc_llvm/src/lib.rs
+++ b/compiler/rustc_llvm/src/lib.rs
@@ -1,9 +1,9 @@
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
-#![cfg_attr(not(bootstrap), allow(internal_features))]
+#![doc(rust_logo)]
+#![feature(rustdoc_internals)]
+#![allow(internal_features)]
 
 // NOTE: This crate only exists to allow linking on mingw targets.
 
diff --git a/compiler/rustc_log/src/lib.rs b/compiler/rustc_log/src/lib.rs
index 0c9ec5565..e3464cb8a 100644
--- a/compiler/rustc_log/src/lib.rs
+++ b/compiler/rustc_log/src/lib.rs
@@ -14,7 +14,7 @@
 //!
 //! ```
 //! fn main() {
-//!     rustc_log::init_env_logger("LOG").unwrap();
+//!     rustc_log::init_logger(rustc_log::LoggerConfig::from_env("LOG")).unwrap();
 //!
 //!     let edition = rustc_span::edition::Edition::Edition2021;
 //!     rustc_span::create_session_globals_then(edition, || {
@@ -52,13 +52,36 @@ use tracing_subscriber::fmt::{
 };
 use tracing_subscriber::layer::SubscriberExt;
 
-pub fn init_env_logger(env: &str) -> Result<(), Error> {
-    let filter = match env::var(env) {
+/// The values of all the environment variables that matter for configuring a logger.
+/// Errors are explicitly preserved so that we can share error handling.
+pub struct LoggerConfig {
+    pub filter: Result<String, VarError>,
+    pub color_logs: Result<String, VarError>,
+    pub verbose_entry_exit: Result<String, VarError>,
+    pub verbose_thread_ids: Result<String, VarError>,
+    pub backtrace: Result<String, VarError>,
+}
+
+impl LoggerConfig {
+    pub fn from_env(env: &str) -> Self {
+        LoggerConfig {
+            filter: env::var(env),
+            color_logs: env::var(format!("{env}_COLOR")),
+            verbose_entry_exit: env::var(format!("{env}_ENTRY_EXIT")),
+            verbose_thread_ids: env::var(format!("{env}_THREAD_IDS")),
+            backtrace: env::var(format!("{env}_BACKTRACE")),
+        }
+    }
+}
+
+/// Initialize the logger with the given values for the filter, coloring, and other options env variables.
+pub fn init_logger(cfg: LoggerConfig) -> Result<(), Error> {
+    let filter = match cfg.filter {
         Ok(env) => EnvFilter::new(env),
         _ => EnvFilter::default().add_directive(Directive::from(LevelFilter::WARN)),
     };
 
-    let color_logs = match env::var(String::from(env) + "_COLOR") {
+    let color_logs = match cfg.color_logs {
         Ok(value) => match value.as_ref() {
             "always" => true,
             "never" => false,
@@ -69,14 +92,14 @@ pub fn init_env_logger(env: &str) -> Result<(), Error> {
         Err(VarError::NotUnicode(_value)) => return Err(Error::NonUnicodeColorValue),
     };
 
-    let verbose_entry_exit = match env::var_os(String::from(env) + "_ENTRY_EXIT") {
-        None => false,
-        Some(v) => &v != "0",
+    let verbose_entry_exit = match cfg.verbose_entry_exit {
+        Ok(v) => &v != "0",
+        Err(_) => false,
     };
 
-    let verbose_thread_ids = match env::var_os(String::from(env) + "_THREAD_IDS") {
-        None => false,
-        Some(v) => &v == "1",
+    let verbose_thread_ids = match cfg.verbose_thread_ids {
+        Ok(v) => &v == "1",
+        Err(_) => false,
     };
 
     let layer = tracing_tree::HierarchicalLayer::default()
@@ -91,7 +114,7 @@ pub fn init_env_logger(env: &str) -> Result<(), Error> {
         .with_thread_names(verbose_thread_ids);
 
     let subscriber = tracing_subscriber::Registry::default().with(filter).with(layer);
-    match env::var(format!("{env}_BACKTRACE")) {
+    match cfg.backtrace {
         Ok(str) => {
             let fmt_layer = tracing_subscriber::fmt::layer()
                 .with_writer(io::stderr)
diff --git a/compiler/rustc_macros/src/current_version.rs b/compiler/rustc_macros/src/current_version.rs
index 5e3b91c17..42ca60a6d 100644
--- a/compiler/rustc_macros/src/current_version.rs
+++ b/compiler/rustc_macros/src/current_version.rs
@@ -1,37 +1,16 @@
 use proc_macro::TokenStream;
 use proc_macro2::Span;
 use quote::quote;
-use syn::parse::{Parse, ParseStream};
-use syn::{parenthesized, parse_macro_input, LitStr, Token};
 
-pub struct Input {
-    variable: LitStr,
-}
-
-mod kw {
-    syn::custom_keyword!(env);
-}
-
-impl Parse for Input {
-    // Input syntax is `env!("CFG_RELEASE")` to facilitate grepping.
-    fn parse(input: ParseStream<'_>) -> syn::Result<Self> {
-        let paren;
-        input.parse::<kw::env>()?;
-        input.parse::<Token![!]>()?;
-        parenthesized!(paren in input);
-        let variable: LitStr = paren.parse()?;
-        Ok(Input { variable })
-    }
-}
-
-pub(crate) fn current_version(input: TokenStream) -> TokenStream {
-    let input = parse_macro_input!(input as Input);
-
-    TokenStream::from(match RustcVersion::parse_env_var(&input.variable) {
+pub(crate) fn current_version(_input: TokenStream) -> TokenStream {
+    let env_var = "CFG_RELEASE";
+    TokenStream::from(match RustcVersion::parse_cfg_release(env_var) {
         Ok(RustcVersion { major, minor, patch }) => quote!(
+            // The produced literal has type `rustc_session::RustcVersion`.
             Self { major: #major, minor: #minor, patch: #patch }
         ),
-        Err(err) => syn::Error::new(Span::call_site(), err).into_compile_error(),
+        Err(err) => syn::Error::new(Span::call_site(), format!("{env_var} env var: {err}"))
+            .into_compile_error(),
     })
 }
 
@@ -42,8 +21,8 @@ struct RustcVersion {
 }
 
 impl RustcVersion {
-    fn parse_env_var(env_var: &LitStr) -> Result<Self, Box<dyn std::error::Error>> {
-        let value = proc_macro::tracked_env::var(env_var.value())?;
+    fn parse_cfg_release(env_var: &str) -> Result<Self, Box<dyn std::error::Error>> {
+        let value = proc_macro::tracked_env::var(env_var)?;
         Self::parse_str(&value)
             .ok_or_else(|| format!("failed to parse rustc version: {:?}", value).into())
     }
diff --git a/compiler/rustc_macros/src/diagnostics/diagnostic.rs b/compiler/rustc_macros/src/diagnostics/diagnostic.rs
index 1a8174bfd..5de0203fc 100644
--- a/compiler/rustc_macros/src/diagnostics/diagnostic.rs
+++ b/compiler/rustc_macros/src/diagnostics/diagnostic.rs
@@ -17,11 +17,11 @@ pub(crate) struct DiagnosticDerive<'a> {
 }
 
 impl<'a> DiagnosticDerive<'a> {
-    pub(crate) fn new(diag: syn::Ident, handler: syn::Ident, structure: Structure<'a>) -> Self {
+    pub(crate) fn new(diag: syn::Ident, dcx: syn::Ident, structure: Structure<'a>) -> Self {
         Self {
             builder: DiagnosticDeriveBuilder {
                 diag,
-                kind: DiagnosticDeriveKind::Diagnostic { handler },
+                kind: DiagnosticDeriveKind::Diagnostic { dcx },
             },
             structure,
         }
@@ -36,7 +36,7 @@ impl<'a> DiagnosticDerive<'a> {
             let body = builder.body(variant);
 
             let diag = &builder.parent.diag;
-            let DiagnosticDeriveKind::Diagnostic { handler } = &builder.parent.kind else {
+            let DiagnosticDeriveKind::Diagnostic { dcx } = &builder.parent.kind else {
                 unreachable!()
             };
             let init = match builder.slug.value_ref() {
@@ -62,7 +62,7 @@ impl<'a> DiagnosticDerive<'a> {
                 Some(slug) => {
                     slugs.borrow_mut().push(slug.clone());
                     quote! {
-                        let mut #diag = #handler.struct_diagnostic(crate::fluent_generated::#slug);
+                        let mut #diag = #dcx.struct_diagnostic(crate::fluent_generated::#slug);
                     }
                 }
             };
@@ -77,11 +77,12 @@ impl<'a> DiagnosticDerive<'a> {
             }
         });
 
-        let DiagnosticDeriveKind::Diagnostic { handler } = &builder.kind else { unreachable!() };
+        let DiagnosticDeriveKind::Diagnostic { dcx } = &builder.kind else { unreachable!() };
 
+        // A lifetime of `'a` causes conflicts, but `_sess` is fine.
         let mut imp = structure.gen_impl(quote! {
-            gen impl<'__diagnostic_handler_sess, G>
-                    rustc_errors::IntoDiagnostic<'__diagnostic_handler_sess, G>
+            gen impl<'_sess, G>
+                    rustc_errors::IntoDiagnostic<'_sess, G>
                     for @Self
                 where G: rustc_errors::EmissionGuarantee
             {
@@ -89,8 +90,8 @@ impl<'a> DiagnosticDerive<'a> {
                 #[track_caller]
                 fn into_diagnostic(
                     self,
-                    #handler: &'__diagnostic_handler_sess rustc_errors::Handler
-                ) -> rustc_errors::DiagnosticBuilder<'__diagnostic_handler_sess, G> {
+                    #dcx: &'_sess rustc_errors::DiagCtxt
+                ) -> rustc_errors::DiagnosticBuilder<'_sess, G> {
                     use rustc_errors::IntoDiagnosticArg;
                     #implementation
                 }
@@ -175,9 +176,9 @@ impl<'a> LintDiagnosticDerive<'a> {
                 fn decorate_lint<'__b>(
                     self,
                     #diag: &'__b mut rustc_errors::DiagnosticBuilder<'__a, ()>
-                ) -> &'__b mut rustc_errors::DiagnosticBuilder<'__a, ()> {
+                ) {
                     use rustc_errors::IntoDiagnosticArg;
-                    #implementation
+                    #implementation;
                 }
 
                 fn msg(&self) -> rustc_errors::DiagnosticMessage {
@@ -229,8 +230,8 @@ fn generate_test(slug: &syn::Path, structure: &Structure<'_>) -> TokenStream {
         }
     }
     use std::sync::atomic::{AtomicUsize, Ordering};
-    // We need to make sure that the same diagnostic slug can be used multiple times without causing an
-    // error, so just have a global counter here.
+    // We need to make sure that the same diagnostic slug can be used multiple times without
+    // causing an error, so just have a global counter here.
     static COUNTER: AtomicUsize = AtomicUsize::new(0);
     let slug = slug.get_ident().unwrap();
     let ident = quote::format_ident!("verify_{slug}_{}", COUNTER.fetch_add(1, Ordering::Relaxed));
diff --git a/compiler/rustc_macros/src/diagnostics/diagnostic_builder.rs b/compiler/rustc_macros/src/diagnostics/diagnostic_builder.rs
index e9a5cd9de..511654d99 100644
--- a/compiler/rustc_macros/src/diagnostics/diagnostic_builder.rs
+++ b/compiler/rustc_macros/src/diagnostics/diagnostic_builder.rs
@@ -19,7 +19,7 @@ use super::utils::SubdiagnosticVariant;
 /// What kind of diagnostic is being derived - a fatal/error/warning or a lint?
 #[derive(Clone, PartialEq, Eq)]
 pub(crate) enum DiagnosticDeriveKind {
-    Diagnostic { handler: syn::Ident },
+    Diagnostic { dcx: syn::Ident },
     LintDiagnostic,
 }
 
@@ -53,6 +53,7 @@ pub(crate) struct DiagnosticDeriveVariantBuilder<'parent> {
     /// Slug is a mandatory part of the struct attribute as corresponds to the Fluent message that
     /// has the actual diagnostic message.
     pub slug: SpannedOption<Path>,
+
     /// Error codes are a optional part of the struct attribute - this is only set to detect
     /// multiple specifications.
     pub code: SpannedOption<()>,
@@ -68,7 +69,7 @@ impl DiagnosticDeriveBuilder {
     /// Call `f` for the struct or for each variant of the enum, returning a `TokenStream` with the
     /// tokens from `f` wrapped in an `match` expression. Emits errors for use of derive on unions
     /// or attributes on the type itself when input is an enum.
-    pub fn each_variant<'s, F>(&mut self, structure: &mut Structure<'s>, f: F) -> TokenStream
+    pub(crate) fn each_variant<'s, F>(&mut self, structure: &mut Structure<'s>, f: F) -> TokenStream
     where
         F: for<'a, 'v> Fn(DiagnosticDeriveVariantBuilder<'a>, &VariantInfo<'v>) -> TokenStream,
     {
@@ -121,7 +122,7 @@ impl DiagnosticDeriveBuilder {
 impl<'a> DiagnosticDeriveVariantBuilder<'a> {
     /// Generates calls to `code` and similar functions based on the attributes on the type or
     /// variant.
-    pub fn preamble(&mut self, variant: &VariantInfo<'_>) -> TokenStream {
+    pub(crate) fn preamble(&mut self, variant: &VariantInfo<'_>) -> TokenStream {
         let ast = variant.ast();
         let attrs = &ast.attrs;
         let preamble = attrs.iter().map(|attr| {
@@ -135,7 +136,7 @@ impl<'a> DiagnosticDeriveVariantBuilder<'a> {
 
     /// Generates calls to `span_label` and similar functions based on the attributes on fields or
     /// calls to `set_arg` when no attributes are present.
-    pub fn body(&mut self, variant: &VariantInfo<'_>) -> TokenStream {
+    pub(crate) fn body(&mut self, variant: &VariantInfo<'_>) -> TokenStream {
         let mut body = quote! {};
         // Generate `set_arg` calls first..
         for binding in variant.bindings().iter().filter(|bi| should_generate_set_arg(bi.ast())) {
@@ -347,11 +348,11 @@ impl<'a> DiagnosticDeriveVariantBuilder<'a> {
             }
             (Meta::Path(_), "subdiagnostic") => {
                 if FieldInnerTy::from_type(&info.binding.ast().ty).will_iterate() {
-                    let DiagnosticDeriveKind::Diagnostic { handler } = &self.parent.kind else {
+                    let DiagnosticDeriveKind::Diagnostic { dcx } = &self.parent.kind else {
                         // No eager translation for lints.
                         return Ok(quote! { #diag.subdiagnostic(#binding); });
                     };
-                    return Ok(quote! { #diag.eager_subdiagnostic(#handler, #binding); });
+                    return Ok(quote! { #diag.eager_subdiagnostic(#dcx, #binding); });
                 } else {
                     return Ok(quote! { #diag.subdiagnostic(#binding); });
                 }
@@ -375,15 +376,15 @@ impl<'a> DiagnosticDeriveVariantBuilder<'a> {
                     return Ok(quote! {});
                 }
 
-                let handler = match &self.parent.kind {
-                    DiagnosticDeriveKind::Diagnostic { handler } => handler,
+                let dcx = match &self.parent.kind {
+                    DiagnosticDeriveKind::Diagnostic { dcx } => dcx,
                     DiagnosticDeriveKind::LintDiagnostic => {
                         throw_invalid_attr!(attr, |diag| {
                             diag.help("eager subdiagnostics are not supported on lints")
                         })
                     }
                 };
-                return Ok(quote! { #diag.eager_subdiagnostic(#handler, #binding); });
+                return Ok(quote! { #diag.eager_subdiagnostic(#dcx, #binding); });
             }
             _ => (),
         }
diff --git a/compiler/rustc_macros/src/diagnostics/subdiagnostic.rs b/compiler/rustc_macros/src/diagnostics/subdiagnostic.rs
index 877271ff0..0f9e68cdc 100644
--- a/compiler/rustc_macros/src/diagnostics/subdiagnostic.rs
+++ b/compiler/rustc_macros/src/diagnostics/subdiagnostic.rs
@@ -478,7 +478,7 @@ impl<'parent, 'a> SubdiagnosticDeriveVariantBuilder<'parent, 'a> {
         }
     }
 
-    pub fn into_tokens(&mut self) -> Result<TokenStream, DiagnosticDeriveError> {
+    pub(crate) fn into_tokens(&mut self) -> Result<TokenStream, DiagnosticDeriveError> {
         let kind_slugs = self.identify_kind()?;
         if kind_slugs.is_empty() {
             if self.is_enum {
diff --git a/compiler/rustc_macros/src/diagnostics/utils.rs b/compiler/rustc_macros/src/diagnostics/utils.rs
index 125632921..2700f02e3 100644
--- a/compiler/rustc_macros/src/diagnostics/utils.rs
+++ b/compiler/rustc_macros/src/diagnostics/utils.rs
@@ -17,7 +17,7 @@ use synstructure::{BindingInfo, VariantInfo};
 use super::error::invalid_attr;
 
 thread_local! {
-    pub static CODE_IDENT_COUNT: RefCell<u32> = RefCell::new(0);
+    pub(crate) static CODE_IDENT_COUNT: RefCell<u32> = RefCell::new(0);
 }
 
 /// Returns an ident of the form `__code_N` where `N` is incremented once with every call.
@@ -208,7 +208,7 @@ impl<'ty> FieldInnerTy<'ty> {
         }
     }
 
-    pub fn span(&self) -> proc_macro2::Span {
+    pub(crate) fn span(&self) -> proc_macro2::Span {
         match self {
             FieldInnerTy::Option(ty) | FieldInnerTy::Vec(ty) | FieldInnerTy::Plain(ty) => ty.span(),
         }
@@ -537,7 +537,7 @@ impl fmt::Display for SuggestionKind {
 }
 
 impl SuggestionKind {
-    pub fn to_suggestion_style(&self) -> TokenStream {
+    pub(crate) fn to_suggestion_style(&self) -> TokenStream {
         match self {
             SuggestionKind::Normal => {
                 quote! { rustc_errors::SuggestionStyle::ShowCode }
diff --git a/compiler/rustc_macros/src/hash_stable.rs b/compiler/rustc_macros/src/hash_stable.rs
index 75a2f7009..a6396ba68 100644
--- a/compiler/rustc_macros/src/hash_stable.rs
+++ b/compiler/rustc_macros/src/hash_stable.rs
@@ -1,6 +1,6 @@
-use proc_macro2::{self, Ident};
+use proc_macro2::Ident;
 use quote::quote;
-use syn::{self, parse_quote};
+use syn::parse_quote;
 
 struct Attributes {
     ignore: bool,
@@ -38,41 +38,80 @@ fn parse_attributes(field: &syn::Field) -> Attributes {
     attrs
 }
 
-pub fn hash_stable_generic_derive(mut s: synstructure::Structure<'_>) -> proc_macro2::TokenStream {
-    let generic: syn::GenericParam = parse_quote!(__CTX);
-    s.add_bounds(synstructure::AddBounds::Generics);
-    s.add_impl_generic(generic);
-    s.add_where_predicate(parse_quote! { __CTX: crate::HashStableContext });
-    let body = s.each(|bi| {
-        let attrs = parse_attributes(bi.ast());
-        if attrs.ignore {
-            quote! {}
-        } else if let Some(project) = attrs.project {
-            quote! {
-                (&#bi.#project).hash_stable(__hcx, __hasher);
-            }
-        } else {
-            quote! {
-                #bi.hash_stable(__hcx, __hasher);
-            }
-        }
+pub(crate) fn hash_stable_derive(s: synstructure::Structure<'_>) -> proc_macro2::TokenStream {
+    hash_stable_derive_with_mode(s, HashStableMode::Normal)
+}
+
+pub(crate) fn hash_stable_generic_derive(
+    s: synstructure::Structure<'_>,
+) -> proc_macro2::TokenStream {
+    hash_stable_derive_with_mode(s, HashStableMode::Generic)
+}
+
+pub(crate) fn hash_stable_no_context_derive(
+    s: synstructure::Structure<'_>,
+) -> proc_macro2::TokenStream {
+    hash_stable_derive_with_mode(s, HashStableMode::NoContext)
+}
+
+enum HashStableMode {
+    // Use the query-system aware stable hashing context.
+    Normal,
+    // Emit a generic implementation that uses a crate-local `StableHashingContext`
+    // trait, when the crate is upstream of `rustc_middle`.
+    Generic,
+    // Emit a hash-stable implementation that takes no context,
+    // and emits per-field where clauses for (almost-)perfect derives.
+    NoContext,
+}
+
+fn hash_stable_derive_with_mode(
+    mut s: synstructure::Structure<'_>,
+    mode: HashStableMode,
+) -> proc_macro2::TokenStream {
+    let generic: syn::GenericParam = match mode {
+        HashStableMode::Normal => parse_quote!('__ctx),
+        HashStableMode::Generic | HashStableMode::NoContext => parse_quote!(__CTX),
+    };
+
+    // no_context impl is able to derive by-field, which is closer to a perfect derive.
+    s.add_bounds(match mode {
+        HashStableMode::Normal | HashStableMode::Generic => synstructure::AddBounds::Generics,
+        HashStableMode::NoContext => synstructure::AddBounds::Fields,
     });
 
-    let discriminant = match s.ast().data {
-        syn::Data::Enum(_) => quote! {
-            ::std::mem::discriminant(self).hash_stable(__hcx, __hasher);
-        },
-        syn::Data::Struct(_) => quote! {},
-        syn::Data::Union(_) => panic!("cannot derive on union"),
+    // For generic impl, add `where __CTX: HashStableContext`.
+    match mode {
+        HashStableMode::Normal => {}
+        HashStableMode::Generic => {
+            s.add_where_predicate(parse_quote! { __CTX: crate::HashStableContext });
+        }
+        HashStableMode::NoContext => {}
+    }
+
+    s.add_impl_generic(generic);
+
+    let discriminant = hash_stable_discriminant(&mut s);
+    let body = hash_stable_body(&mut s);
+
+    let context: syn::Type = match mode {
+        HashStableMode::Normal => {
+            parse_quote!(::rustc_query_system::ich::StableHashingContext<'__ctx>)
+        }
+        HashStableMode::Generic | HashStableMode::NoContext => parse_quote!(__CTX),
     };
 
     s.bound_impl(
-        quote!(::rustc_data_structures::stable_hasher::HashStable<__CTX>),
+        quote!(
+            ::rustc_data_structures::stable_hasher::HashStable<
+                #context
+            >
+        ),
         quote! {
             #[inline]
             fn hash_stable(
                 &self,
-                __hcx: &mut __CTX,
+                __hcx: &mut #context,
                 __hasher: &mut ::rustc_data_structures::stable_hasher::StableHasher) {
                 #discriminant
                 match *self { #body }
@@ -81,11 +120,18 @@ pub fn hash_stable_generic_derive(mut s: synstructure::Structure<'_>) -> proc_ma
     )
 }
 
-pub fn hash_stable_derive(mut s: synstructure::Structure<'_>) -> proc_macro2::TokenStream {
-    let generic: syn::GenericParam = parse_quote!('__ctx);
-    s.add_bounds(synstructure::AddBounds::Generics);
-    s.add_impl_generic(generic);
-    let body = s.each(|bi| {
+fn hash_stable_discriminant(s: &mut synstructure::Structure<'_>) -> proc_macro2::TokenStream {
+    match s.ast().data {
+        syn::Data::Enum(_) => quote! {
+            ::std::mem::discriminant(self).hash_stable(__hcx, __hasher);
+        },
+        syn::Data::Struct(_) => quote! {},
+        syn::Data::Union(_) => panic!("cannot derive on union"),
+    }
+}
+
+fn hash_stable_body(s: &mut synstructure::Structure<'_>) -> proc_macro2::TokenStream {
+    s.each(|bi| {
         let attrs = parse_attributes(bi.ast());
         if attrs.ignore {
             quote! {}
@@ -98,31 +144,5 @@ pub fn hash_stable_derive(mut s: synstructure::Structure<'_>) -> proc_macro2::To
                 #bi.hash_stable(__hcx, __hasher);
             }
         }
-    });
-
-    let discriminant = match s.ast().data {
-        syn::Data::Enum(_) => quote! {
-            ::std::mem::discriminant(self).hash_stable(__hcx, __hasher);
-        },
-        syn::Data::Struct(_) => quote! {},
-        syn::Data::Union(_) => panic!("cannot derive on union"),
-    };
-
-    s.bound_impl(
-        quote!(
-            ::rustc_data_structures::stable_hasher::HashStable<
-                ::rustc_query_system::ich::StableHashingContext<'__ctx>,
-            >
-        ),
-        quote! {
-            #[inline]
-            fn hash_stable(
-                &self,
-                __hcx: &mut ::rustc_query_system::ich::StableHashingContext<'__ctx>,
-                __hasher: &mut ::rustc_data_structures::stable_hasher::StableHasher) {
-                #discriminant
-                match *self { #body }
-            }
-        },
-    )
+    })
 }
diff --git a/compiler/rustc_macros/src/lib.rs b/compiler/rustc_macros/src/lib.rs
index 193dbd75f..f558b74be 100644
--- a/compiler/rustc_macros/src/lib.rs
+++ b/compiler/rustc_macros/src/lib.rs
@@ -19,13 +19,15 @@ mod current_version;
 mod diagnostics;
 mod hash_stable;
 mod lift;
-mod newtype;
 mod query;
 mod serialize;
 mod symbols;
 mod type_foldable;
 mod type_visitable;
 
+// Reads the rust version (e.g. "1.75.0") from the CFG_RELEASE env var and
+// produces a `RustcVersion` literal containing that version (e.g.
+// `RustcVersion { major: 1, minor: 75, patch: 0 }`).
 #[proc_macro]
 pub fn current_rustc_version(input: TokenStream) -> TokenStream {
     current_version::current_version(input)
@@ -41,32 +43,18 @@ pub fn symbols(input: TokenStream) -> TokenStream {
     symbols::symbols(input.into()).into()
 }
 
-/// Creates a struct type `S` that can be used as an index with
-/// `IndexVec` and so on.
-///
-/// There are two ways of interacting with these indices:
-///
-/// - The `From` impls are the preferred way. So you can do
-///   `S::from(v)` with a `usize` or `u32`. And you can convert back
-///   to an integer with `u32::from(s)`.
-///
-/// - Alternatively, you can use the methods `S::new(v)` and `s.index()`
-///   to create/return a value.
-///
-/// Internally, the index uses a u32, so the index must not exceed
-/// `u32::MAX`. You can also customize things like the `Debug` impl,
-/// what traits are derived, and so forth via the macro.
-#[proc_macro]
-#[allow_internal_unstable(step_trait, rustc_attrs, trusted_step, spec_option_partial_eq)]
-pub fn newtype_index(input: TokenStream) -> TokenStream {
-    newtype::newtype(input)
-}
-
 decl_derive!([HashStable, attributes(stable_hasher)] => hash_stable::hash_stable_derive);
 decl_derive!(
     [HashStable_Generic, attributes(stable_hasher)] =>
     hash_stable::hash_stable_generic_derive
 );
+decl_derive!(
+    [HashStable_NoContext] =>
+    /// `HashStable` implementation that has no `HashStableContext` bound and
+    /// which adds `where` bounds for `HashStable` based off of fields and not
+    /// generics. This is suitable for use in crates like `rustc_type_ir`.
+    hash_stable::hash_stable_no_context_derive
+);
 
 decl_derive!([Decodable] => serialize::decodable_derive);
 decl_derive!([Encodable] => serialize::encodable_derive);
diff --git a/compiler/rustc_macros/src/lift.rs b/compiler/rustc_macros/src/lift.rs
index ad7ac7404..3dedd88fb 100644
--- a/compiler/rustc_macros/src/lift.rs
+++ b/compiler/rustc_macros/src/lift.rs
@@ -1,5 +1,5 @@
 use quote::quote;
-use syn::{self, parse_quote};
+use syn::parse_quote;
 
 pub fn lift_derive(mut s: synstructure::Structure<'_>) -> proc_macro2::TokenStream {
     s.add_bounds(synstructure::AddBounds::Generics);
diff --git a/compiler/rustc_macros/src/symbols.rs b/compiler/rustc_macros/src/symbols.rs
index 4129712a6..488d4504a 100644
--- a/compiler/rustc_macros/src/symbols.rs
+++ b/compiler/rustc_macros/src/symbols.rs
@@ -19,7 +19,9 @@
 //! ```bash
 //! cargo install cargo-expand          # this is necessary only once
 //! cd compiler/rustc_span
-//! cargo expand > /tmp/rustc_span.rs   # it's a big file
+//! # The specific version number in CFG_RELEASE doesn't matter.
+//! # The output is large.
+//! CFG_RELEASE="0.0.0" cargo +nightly expand > /tmp/rustc_span.rs
 //! ```
 
 use proc_macro2::{Span, TokenStream};
@@ -83,7 +85,9 @@ impl Parse for Value {
                 }
             }
             Expr::Macro(expr) => {
-                if expr.mac.path.is_ident("env") && let Ok(lit) = expr.mac.parse_body() {
+                if expr.mac.path.is_ident("env")
+                    && let Ok(lit) = expr.mac.parse_body()
+                {
                     return Ok(Value::Env(lit, expr.mac.clone()));
                 }
             }
@@ -318,13 +322,4 @@ fn symbols_with_errors(input: TokenStream) -> (TokenStream, Vec<syn::Error>) {
     };
 
     (output, errors.list)
-
-    // To see the generated code, use the "cargo expand" command.
-    // Do this once to install:
-    //      cargo install cargo-expand
-    //
-    // Then, cd to rustc_span and run:
-    //      cargo expand > /tmp/rustc_span_expanded.rs
-    //
-    // and read that file.
 }
diff --git a/compiler/rustc_metadata/messages.ftl b/compiler/rustc_metadata/messages.ftl
index d1815717e..8da6f0007 100644
--- a/compiler/rustc_metadata/messages.ftl
+++ b/compiler/rustc_metadata/messages.ftl
@@ -63,11 +63,8 @@ metadata_extern_location_not_file =
 metadata_fail_create_file_encoder =
     failed to create file encoder: {$err}
 
-metadata_fail_seek_file =
-    failed to seek the file: {$err}
-
 metadata_fail_write_file =
-    failed to write to the file: {$err}
+    failed to write to `{$path}`: {$err}
 
 metadata_failed_copy_to_stdout =
     failed to copy {$filename} to stdout: {$err}
@@ -272,7 +269,7 @@ metadata_unknown_import_name_type =
     unknown import name type `{$import_name_type}`, expected one of: decorated, noprefix, undecorated
 
 metadata_unknown_link_kind =
-    unknown link kind `{$kind}`, expected one of: static, dylib, framework, raw-dylib
+    unknown link kind `{$kind}`, expected one of: static, dylib, framework, raw-dylib, link-arg
     .label = unknown link kind
 
 metadata_unknown_link_modifier =
diff --git a/compiler/rustc_metadata/src/creader.rs b/compiler/rustc_metadata/src/creader.rs
index 14bbe65d5..972c84b10 100644
--- a/compiler/rustc_metadata/src/creader.rs
+++ b/compiler/rustc_metadata/src/creader.rs
@@ -7,8 +7,9 @@ use crate::rmeta::{CrateDep, CrateMetadata, CrateNumMap, CrateRoot, MetadataBlob
 use rustc_ast::expand::allocator::{alloc_error_handler_name, global_fn_name, AllocatorKind};
 use rustc_ast::{self as ast, *};
 use rustc_data_structures::fx::FxHashSet;
+use rustc_data_structures::owned_slice::OwnedSlice;
 use rustc_data_structures::svh::Svh;
-use rustc_data_structures::sync::{FreezeReadGuard, FreezeWriteGuard};
+use rustc_data_structures::sync::{self, FreezeReadGuard, FreezeWriteGuard};
 use rustc_expand::base::SyntaxExtension;
 use rustc_fs_util::try_canonicalize;
 use rustc_hir::def_id::{CrateNum, LocalDefId, StableCrateId, StableCrateIdMap, LOCAL_CRATE};
@@ -16,16 +17,14 @@ use rustc_hir::definitions::Definitions;
 use rustc_index::IndexVec;
 use rustc_middle::ty::TyCtxt;
 use rustc_session::config::{self, CrateType, ExternLocation};
-use rustc_session::cstore::{
-    CrateDepKind, CrateSource, ExternCrate, ExternCrateSource, MetadataLoaderDyn,
-};
+use rustc_session::cstore::{CrateDepKind, CrateSource, ExternCrate, ExternCrateSource};
 use rustc_session::lint;
 use rustc_session::output::validate_crate_name;
 use rustc_session::search_paths::PathKind;
 use rustc_span::edition::Edition;
 use rustc_span::symbol::{sym, Symbol};
 use rustc_span::{Span, DUMMY_SP};
-use rustc_target::spec::{PanicStrategy, TargetTriple};
+use rustc_target::spec::{PanicStrategy, Target, TargetTriple};
 
 use proc_macro::bridge::client::ProcMacro;
 use std::error::Error;
@@ -34,6 +33,17 @@ use std::path::Path;
 use std::time::Duration;
 use std::{cmp, iter};
 
+/// The backend's way to give the crate store access to the metadata in a library.
+/// Note that it returns the raw metadata bytes stored in the library file, whether
+/// it is compressed, uncompressed, some weird mix, etc.
+/// rmeta files are backend independent and not handled here.
+pub trait MetadataLoader {
+    fn get_rlib_metadata(&self, target: &Target, filename: &Path) -> Result<OwnedSlice, String>;
+    fn get_dylib_metadata(&self, target: &Target, filename: &Path) -> Result<OwnedSlice, String>;
+}
+
+pub type MetadataLoaderDyn = dyn MetadataLoader + Send + Sync + sync::DynSend + sync::DynSync;
+
 pub struct CStore {
     metadata_loader: Box<MetadataLoaderDyn>,
 
@@ -257,7 +267,7 @@ impl CStore {
             let unused_externs =
                 self.unused_externs.iter().map(|ident| ident.to_ident_string()).collect::<Vec<_>>();
             let unused_externs = unused_externs.iter().map(String::as_str).collect::<Vec<&str>>();
-            tcx.sess.parse_sess.span_diagnostic.emit_unused_externs(
+            tcx.sess.dcx().emit_unused_externs(
                 level,
                 json_unused_externs.is_loud(),
                 &unused_externs,
@@ -427,7 +437,7 @@ impl<'a, 'tcx> CrateLoader<'a, 'tcx> {
 
         let crate_metadata = CrateMetadata::new(
             self.sess,
-            &self.cstore,
+            self.cstore,
             metadata,
             crate_root,
             raw_proc_macros,
@@ -515,7 +525,7 @@ impl<'a, 'tcx> CrateLoader<'a, 'tcx> {
             Err(err) => {
                 let missing_core =
                     self.maybe_resolve_crate(sym::core, CrateDepKind::Explicit, None).is_err();
-                err.report(&self.sess, span, missing_core);
+                err.report(self.sess, span, missing_core);
                 None
             }
         }
@@ -987,7 +997,7 @@ impl<'a, 'tcx> CrateLoader<'a, 'tcx> {
 
         self.report_unused_deps(krate);
 
-        info!("{:?}", CrateDump(&self.cstore));
+        info!("{:?}", CrateDump(self.cstore));
     }
 
     pub fn process_extern_crate(
diff --git a/compiler/rustc_metadata/src/errors.rs b/compiler/rustc_metadata/src/errors.rs
index 70daee291..206c15edd 100644
--- a/compiler/rustc_metadata/src/errors.rs
+++ b/compiler/rustc_metadata/src/errors.rs
@@ -308,14 +308,9 @@ pub struct FailCreateFileEncoder {
 }
 
 #[derive(Diagnostic)]
-#[diag(metadata_fail_seek_file)]
-pub struct FailSeekFile {
-    pub err: Error,
-}
-
-#[derive(Diagnostic)]
 #[diag(metadata_fail_write_file)]
-pub struct FailWriteFile {
+pub struct FailWriteFile<'a> {
+    pub path: &'a Path,
     pub err: Error,
 }
 
@@ -503,9 +498,9 @@ pub(crate) struct MultipleCandidates {
 impl IntoDiagnostic<'_> for MultipleCandidates {
     fn into_diagnostic(
         self,
-        handler: &'_ rustc_errors::Handler,
+        dcx: &'_ rustc_errors::DiagCtxt,
     ) -> rustc_errors::DiagnosticBuilder<'_, ErrorGuaranteed> {
-        let mut diag = handler.struct_err(fluent::metadata_multiple_candidates);
+        let mut diag = dcx.struct_err(fluent::metadata_multiple_candidates);
         diag.set_arg("crate_name", self.crate_name);
         diag.set_arg("flavor", self.flavor);
         diag.code(error_code!(E0464));
@@ -602,9 +597,9 @@ impl IntoDiagnostic<'_> for InvalidMetadataFiles {
     #[track_caller]
     fn into_diagnostic(
         self,
-        handler: &'_ rustc_errors::Handler,
+        dcx: &'_ rustc_errors::DiagCtxt,
     ) -> rustc_errors::DiagnosticBuilder<'_, ErrorGuaranteed> {
-        let mut diag = handler.struct_err(fluent::metadata_invalid_meta_files);
+        let mut diag = dcx.struct_err(fluent::metadata_invalid_meta_files);
         diag.set_arg("crate_name", self.crate_name);
         diag.set_arg("add_info", self.add_info);
         diag.code(error_code!(E0786));
@@ -632,9 +627,9 @@ impl IntoDiagnostic<'_> for CannotFindCrate {
     #[track_caller]
     fn into_diagnostic(
         self,
-        handler: &'_ rustc_errors::Handler,
+        dcx: &'_ rustc_errors::DiagCtxt,
     ) -> rustc_errors::DiagnosticBuilder<'_, ErrorGuaranteed> {
-        let mut diag = handler.struct_err(fluent::metadata_cannot_find_crate);
+        let mut diag = dcx.struct_err(fluent::metadata_cannot_find_crate);
         diag.set_arg("crate_name", self.crate_name);
         diag.set_arg("current_crate", self.current_crate);
         diag.set_arg("add_info", self.add_info);
diff --git a/compiler/rustc_metadata/src/fs.rs b/compiler/rustc_metadata/src/fs.rs
index 7eb2a347d..e80afcc48 100644
--- a/compiler/rustc_metadata/src/fs.rs
+++ b/compiler/rustc_metadata/src/fs.rs
@@ -91,10 +91,7 @@ pub fn encode_and_write_metadata(tcx: TyCtxt<'_>) -> (EncodedMetadata, bool) {
             }
         };
         if tcx.sess.opts.json_artifact_notifications {
-            tcx.sess
-                .parse_sess
-                .span_diagnostic
-                .emit_artifact_notification(&out_filename.as_path(), "metadata");
+            tcx.sess.dcx().emit_artifact_notification(out_filename.as_path(), "metadata");
         }
         (filename, None)
     } else {
diff --git a/compiler/rustc_metadata/src/lib.rs b/compiler/rustc_metadata/src/lib.rs
index b06b4fb87..6c6c60af0 100644
--- a/compiler/rustc_metadata/src/lib.rs
+++ b/compiler/rustc_metadata/src/lib.rs
@@ -1,11 +1,10 @@
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
-#![cfg_attr(not(bootstrap), allow(internal_features))]
+#![doc(rust_logo)]
+#![feature(rustdoc_internals)]
+#![allow(internal_features)]
 #![feature(decl_macro)]
 #![feature(extract_if)]
-#![cfg_attr(bootstrap, feature(generators))]
-#![cfg_attr(not(bootstrap), feature(coroutines))]
+#![feature(coroutines)]
 #![feature(iter_from_coroutine)]
 #![feature(let_chains)]
 #![feature(if_let_guard)]
@@ -32,8 +31,6 @@ extern crate rustc_middle;
 extern crate tracing;
 
 pub use rmeta::provide;
-use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
-use rustc_fluent_macro::fluent_messages;
 
 mod dependency_format;
 mod foreign_modules;
@@ -49,4 +46,4 @@ pub use fs::{emit_wrapper_file, METADATA_FILENAME};
 pub use native_libs::find_native_static_library;
 pub use rmeta::{encode_metadata, rendered_const, EncodedMetadata, METADATA_HEADER};
 
-fluent_messages! { "../messages.ftl" }
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
diff --git a/compiler/rustc_metadata/src/locator.rs b/compiler/rustc_metadata/src/locator.rs
index 3a99ddc1b..f9219d12a 100644
--- a/compiler/rustc_metadata/src/locator.rs
+++ b/compiler/rustc_metadata/src/locator.rs
@@ -212,7 +212,7 @@
 //! no means all of the necessary details. Take a look at the rest of
 //! metadata::locator or metadata::creader for all the juicy details!
 
-use crate::creader::Library;
+use crate::creader::{Library, MetadataLoader};
 use crate::errors;
 use crate::rmeta::{rustc_version, MetadataBlob, METADATA_HEADER};
 
@@ -223,7 +223,7 @@ use rustc_data_structures::svh::Svh;
 use rustc_errors::{DiagnosticArgValue, IntoDiagnosticArg};
 use rustc_fs_util::try_canonicalize;
 use rustc_session::config;
-use rustc_session::cstore::{CrateSource, MetadataLoader};
+use rustc_session::cstore::CrateSource;
 use rustc_session::filesearch::FileSearch;
 use rustc_session::search_paths::PathKind;
 use rustc_session::utils::CanonicalizedPath;
@@ -783,8 +783,8 @@ fn get_metadata_section<'p>(
                 loader.get_dylib_metadata(target, filename).map_err(MetadataError::LoadFailure)?;
             // The header is uncompressed
             let header_len = METADATA_HEADER.len();
-            // header + u32 length of data
-            let data_start = header_len + 4;
+            // header + u64 length of data
+            let data_start = header_len + 8;
 
             debug!("checking {} bytes of metadata-version stamp", header_len);
             let header = &buf[..cmp::min(header_len, buf.len())];
@@ -797,13 +797,13 @@ fn get_metadata_section<'p>(
 
             // Length of the compressed stream - this allows linkers to pad the section if they want
             let Ok(len_bytes) =
-                <[u8; 4]>::try_from(&buf[header_len..cmp::min(data_start, buf.len())])
+                <[u8; 8]>::try_from(&buf[header_len..cmp::min(data_start, buf.len())])
             else {
                 return Err(MetadataError::LoadFailure(
                     "invalid metadata length found".to_string(),
                 ));
             };
-            let compressed_len = u32::from_be_bytes(len_bytes) as usize;
+            let compressed_len = u64::from_le_bytes(len_bytes) as usize;
 
             // Header is okay -> inflate the actual metadata
             let compressed_bytes = buf.slice(|buf| &buf[data_start..(data_start + compressed_len)]);
@@ -980,7 +980,7 @@ impl CrateError {
                     for CrateMismatch { path, .. } in mismatches {
                         sess.emit_err(errors::CrateLocationUnknownType {
                             span,
-                            path: &path,
+                            path: path,
                             crate_name,
                         });
                         sess.emit_err(errors::LibFilenameForm {
diff --git a/compiler/rustc_metadata/src/native_libs.rs b/compiler/rustc_metadata/src/native_libs.rs
index ab135851b..b3760b109 100644
--- a/compiler/rustc_metadata/src/native_libs.rs
+++ b/compiler/rustc_metadata/src/native_libs.rs
@@ -160,6 +160,18 @@ impl<'tcx> Collector<'tcx> {
                                 }
                                 NativeLibKind::RawDylib
                             }
+                            "link-arg" => {
+                                if !features.link_arg_attribute {
+                                    feature_err(
+                                        &sess.parse_sess,
+                                        sym::link_arg_attribute,
+                                        span,
+                                        "link kind `link-arg` is unstable",
+                                    )
+                                    .emit();
+                                }
+                                NativeLibKind::LinkArg
+                            }
                             kind => {
                                 sess.emit_err(errors::UnknownLinkKind { span, kind });
                                 continue;
@@ -470,7 +482,7 @@ impl<'tcx> Collector<'tcx> {
     }
 
     fn i686_arg_list_size(&self, item: DefId) -> usize {
-        let argument_types: &List<Ty<'_>> = self.tcx.erase_late_bound_regions(
+        let argument_types: &List<Ty<'_>> = self.tcx.instantiate_bound_regions_with_erased(
             self.tcx
                 .type_of(item)
                 .instantiate_identity()
diff --git a/compiler/rustc_metadata/src/rmeta/decoder.rs b/compiler/rustc_metadata/src/rmeta/decoder.rs
index 354023cea..281a0eafe 100644
--- a/compiler/rustc_metadata/src/rmeta/decoder.rs
+++ b/compiler/rustc_metadata/src/rmeta/decoder.rs
@@ -1,43 +1,34 @@
 // Decoding metadata from a single crate's metadata
 
-use crate::creader::{CStore, CrateMetadataRef};
+use crate::creader::CStore;
 use crate::rmeta::table::IsDefault;
 use crate::rmeta::*;
 
 use rustc_ast as ast;
 use rustc_data_structures::captures::Captures;
-use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::owned_slice::OwnedSlice;
-use rustc_data_structures::svh::Svh;
 use rustc_data_structures::sync::{AppendOnlyVec, AtomicBool, Lock, Lrc, OnceLock};
 use rustc_data_structures::unhash::UnhashMap;
 use rustc_expand::base::{SyntaxExtension, SyntaxExtensionKind};
 use rustc_expand::proc_macro::{AttrProcMacro, BangProcMacro, DeriveProcMacro};
-use rustc_hir::def::{CtorKind, DefKind, DocLinkResMap, Res};
-use rustc_hir::def_id::{CrateNum, DefId, DefIndex, CRATE_DEF_INDEX, LOCAL_CRATE};
-use rustc_hir::definitions::{DefKey, DefPath, DefPathData, DefPathHash};
+use rustc_hir::def::Res;
+use rustc_hir::def_id::{DefIdMap, CRATE_DEF_INDEX, LOCAL_CRATE};
+use rustc_hir::definitions::{DefPath, DefPathData};
 use rustc_hir::diagnostic_items::DiagnosticItems;
-use rustc_index::{Idx, IndexVec};
-use rustc_middle::metadata::ModChild;
-use rustc_middle::middle::debugger_visualizer::DebuggerVisualizerFile;
-use rustc_middle::middle::exported_symbols::{ExportedSymbol, SymbolExportInfo};
+use rustc_index::Idx;
+use rustc_middle::middle::lib_features::LibFeatures;
 use rustc_middle::mir::interpret::{AllocDecodingSession, AllocDecodingState};
 use rustc_middle::ty::codec::TyDecoder;
-use rustc_middle::ty::fast_reject::SimplifiedType;
-use rustc_middle::ty::{self, ParameterizedOverTcx, Ty, TyCtxt, Visibility};
+use rustc_middle::ty::Visibility;
 use rustc_serialize::opaque::MemDecoder;
 use rustc_serialize::{Decodable, Decoder};
-use rustc_session::cstore::{
-    CrateSource, ExternCrate, ForeignModule, LinkagePreference, NativeLib,
-};
+use rustc_session::cstore::{CrateSource, ExternCrate};
 use rustc_session::Session;
-use rustc_span::hygiene::ExpnIndex;
-use rustc_span::symbol::{kw, Ident, Symbol};
-use rustc_span::{self, BytePos, ExpnId, Pos, Span, SpanData, SyntaxContext, DUMMY_SP};
+use rustc_span::symbol::kw;
+use rustc_span::{BytePos, Pos, SpanData, SyntaxContext, DUMMY_SP};
 
 use proc_macro::bridge::client::ProcMacro;
 use std::iter::TrustedLen;
-use std::num::NonZeroUsize;
 use std::path::Path;
 use std::sync::atomic::Ordering;
 use std::{io, iter, mem};
@@ -699,28 +690,25 @@ impl MetadataBlob {
     }
 
     pub(crate) fn get_rustc_version(&self) -> String {
-        LazyValue::<String>::from_position(NonZeroUsize::new(METADATA_HEADER.len() + 4).unwrap())
+        LazyValue::<String>::from_position(NonZeroUsize::new(METADATA_HEADER.len() + 8).unwrap())
             .decode(self)
     }
 
-    pub(crate) fn get_header(&self) -> CrateHeader {
-        let slice = &self.blob()[..];
+    fn root_pos(&self) -> NonZeroUsize {
         let offset = METADATA_HEADER.len();
+        let pos_bytes = self.blob()[offset..][..8].try_into().unwrap();
+        let pos = u64::from_le_bytes(pos_bytes);
+        NonZeroUsize::new(pos as usize).unwrap()
+    }
 
-        let pos_bytes = slice[offset..][..4].try_into().unwrap();
-        let pos = u32::from_be_bytes(pos_bytes) as usize;
-
-        LazyValue::<CrateHeader>::from_position(NonZeroUsize::new(pos).unwrap()).decode(self)
+    pub(crate) fn get_header(&self) -> CrateHeader {
+        let pos = self.root_pos();
+        LazyValue::<CrateHeader>::from_position(pos).decode(self)
     }
 
     pub(crate) fn get_root(&self) -> CrateRoot {
-        let slice = &self.blob()[..];
-        let offset = METADATA_HEADER.len();
-
-        let pos_bytes = slice[offset..][..4].try_into().unwrap();
-        let pos = u32::from_be_bytes(pos_bytes) as usize;
-
-        LazyValue::<CrateRoot>::from_position(NonZeroUsize::new(pos).unwrap()).decode(self)
+        let pos = self.root_pos();
+        LazyValue::<CrateRoot>::from_position(pos).decode(self)
     }
 
     pub(crate) fn list_crate_metadata(
@@ -828,7 +816,7 @@ impl MetadataBlob {
                             out,
                             "{}{}",
                             feature,
-                            if let Some(since) = since {
+                            if let FeatureStability::AcceptedSince(since) = since {
                                 format!(" since {since}")
                             } else {
                                 String::new()
@@ -849,7 +837,7 @@ impl MetadataBlob {
                     ) -> io::Result<()> {
                         let root = blob.get_root();
 
-                        let def_kind = root.tables.opt_def_kind.get(blob, item).unwrap();
+                        let def_kind = root.tables.def_kind.get(blob, item).unwrap();
                         let def_key = root.tables.def_keys.get(blob, item).unwrap().decode(blob);
                         let def_name = if item == CRATE_DEF_INDEX {
                             rustc_span::symbol::kw::Crate
@@ -1000,14 +988,11 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
     }
 
     fn def_kind(self, item_id: DefIndex) -> DefKind {
-        self.root.tables.opt_def_kind.get(self, item_id).unwrap_or_else(|| {
-            bug!(
-                "CrateMetadata::def_kind({:?}): id not found, in crate {:?} with number {}",
-                item_id,
-                self.root.name(),
-                self.cnum,
-            )
-        })
+        self.root
+            .tables
+            .def_kind
+            .get(self, item_id)
+            .unwrap_or_else(|| self.missing("def_kind", item_id))
     }
 
     fn get_span(self, index: DefIndex, sess: &Session) -> Span {
@@ -1176,8 +1161,15 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
     }
 
     /// Iterates over all the stability attributes in the given crate.
-    fn get_lib_features(self, tcx: TyCtxt<'tcx>) -> &'tcx [(Symbol, Option<Symbol>)] {
-        tcx.arena.alloc_from_iter(self.root.lib_features.decode(self))
+    fn get_lib_features(self) -> LibFeatures {
+        LibFeatures {
+            stability: self
+                .root
+                .lib_features
+                .decode(self)
+                .map(|(sym, stab)| (sym, (stab, DUMMY_SP)))
+                .collect(),
+        }
     }
 
     /// Iterates over the stability implications in the given crate (when a `#[unstable]` attribute
@@ -1208,7 +1200,7 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
 
     /// Iterates over the diagnostic items in the given crate.
     fn get_diagnostic_items(self) -> DiagnosticItems {
-        let mut id_to_name = FxHashMap::default();
+        let mut id_to_name = DefIdMap::default();
         let name_to_id = self
             .root
             .diagnostic_items
diff --git a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs
index 595d816e9..f6cd013d2 100644
--- a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs
+++ b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs
@@ -231,7 +231,7 @@ provide! { tcx, def_id, other, cdata,
     lookup_deprecation_entry => { table }
     params_in_repr => { table }
     unused_generic_params => { cdata.root.tables.unused_generic_params.get(cdata, def_id.index) }
-    opt_def_kind => { table_direct }
+    def_kind => { cdata.def_kind(def_id.index) }
     impl_parent => { table }
     impl_polarity => { table_direct }
     defaultness => { table_direct }
@@ -346,7 +346,7 @@ provide! { tcx, def_id, other, cdata,
     module_children => {
         tcx.arena.alloc_from_iter(cdata.get_module_children(def_id.index, tcx.sess))
     }
-    defined_lib_features => { cdata.get_lib_features(tcx) }
+    lib_features => { cdata.get_lib_features() }
     stability_implications => {
         cdata.get_stability_implications(tcx).iter().copied().collect()
     }
@@ -394,7 +394,7 @@ pub(in crate::rmeta) fn provide(providers: &mut Providers) {
         native_library: |tcx, id| {
             tcx.native_libraries(id.krate)
                 .iter()
-                .filter(|lib| native_libs::relevant_lib(&tcx.sess, lib))
+                .filter(|lib| native_libs::relevant_lib(tcx.sess, lib))
                 .find(|lib| {
                     let Some(fm_id) = lib.foreign_module else {
                         return false;
diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs
index 2042863d1..ad3fea65e 100644
--- a/compiler/rustc_metadata/src/rmeta/encoder.rs
+++ b/compiler/rustc_metadata/src/rmeta/encoder.rs
@@ -1,49 +1,37 @@
-use crate::errors::{FailCreateFileEncoder, FailSeekFile, FailWriteFile};
-use crate::rmeta::def_path_hash_map::DefPathHashMapRef;
-use crate::rmeta::table::TableBuilder;
+use crate::errors::{FailCreateFileEncoder, FailWriteFile};
 use crate::rmeta::*;
 
-use rustc_ast::expand::StrippedCfgItem;
 use rustc_ast::Attribute;
 use rustc_data_structures::fingerprint::Fingerprint;
-use rustc_data_structures::fx::{FxHashMap, FxIndexSet};
+use rustc_data_structures::fx::FxIndexSet;
 use rustc_data_structures::memmap::{Mmap, MmapMut};
 use rustc_data_structures::stable_hasher::{Hash128, HashStable, StableHasher};
 use rustc_data_structures::sync::{join, par_for_each_in, Lrc};
 use rustc_data_structures::temp_dir::MaybeTempDir;
 use rustc_hir as hir;
-use rustc_hir::def::DefKind;
-use rustc_hir::def_id::{
-    CrateNum, DefId, DefIndex, LocalDefId, LocalDefIdSet, CRATE_DEF_ID, CRATE_DEF_INDEX,
-    LOCAL_CRATE,
-};
+use rustc_hir::def_id::{LocalDefId, LocalDefIdSet, CRATE_DEF_ID, CRATE_DEF_INDEX, LOCAL_CRATE};
 use rustc_hir::definitions::DefPathData;
-use rustc_hir::lang_items::LangItem;
 use rustc_hir_pretty::id_to_string;
-use rustc_middle::middle::debugger_visualizer::DebuggerVisualizerFile;
 use rustc_middle::middle::dependency_format::Linkage;
-use rustc_middle::middle::exported_symbols::{
-    metadata_symbol_name, ExportedSymbol, SymbolExportInfo,
-};
+use rustc_middle::middle::exported_symbols::metadata_symbol_name;
 use rustc_middle::mir::interpret;
 use rustc_middle::query::LocalCrate;
 use rustc_middle::query::Providers;
 use rustc_middle::traits::specialization_graph;
 use rustc_middle::ty::codec::TyEncoder;
-use rustc_middle::ty::fast_reject::{self, SimplifiedType, TreatParams};
-use rustc_middle::ty::{self, AssocItemContainer, SymbolName, Ty, TyCtxt};
+use rustc_middle::ty::fast_reject::{self, TreatParams};
+use rustc_middle::ty::{AssocItemContainer, SymbolName};
 use rustc_middle::util::common::to_readable_str;
 use rustc_serialize::{opaque, Decodable, Decoder, Encodable, Encoder};
 use rustc_session::config::{CrateType, OptLevel};
-use rustc_session::cstore::{ForeignModule, LinkagePreference, NativeLib};
-use rustc_span::hygiene::{ExpnIndex, HygieneEncodeContext, MacroKind};
-use rustc_span::symbol::{sym, Symbol};
-use rustc_span::{self, ExternalSource, FileName, SourceFile, Span, SpanData, SyntaxContext};
+use rustc_span::hygiene::HygieneEncodeContext;
+use rustc_span::symbol::sym;
+use rustc_span::{ExternalSource, FileName, SourceFile, SpanData, SyntaxContext};
 use std::borrow::Borrow;
 use std::collections::hash_map::Entry;
+use std::fs::File;
 use std::hash::Hash;
 use std::io::{Read, Seek, Write};
-use std::num::NonZeroUsize;
 use std::path::{Path, PathBuf};
 
 pub(super) struct EncodeContext<'a, 'tcx> {
@@ -159,7 +147,7 @@ impl<'a, 'tcx> Encodable<EncodeContext<'a, 'tcx>> for ExpnIndex {
 
 impl<'a, 'tcx> Encodable<EncodeContext<'a, 'tcx>> for SyntaxContext {
     fn encode(&self, s: &mut EncodeContext<'a, 'tcx>) {
-        rustc_span::hygiene::raw_encode_syntax_context(*self, &s.hygiene_ctxt, s);
+        rustc_span::hygiene::raw_encode_syntax_context(*self, s.hygiene_ctxt, s);
     }
 }
 
@@ -656,10 +644,9 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         let debugger_visualizers =
             stat!("debugger-visualizers", || self.encode_debugger_visualizers());
 
-        // Encode exported symbols info. This is prefetched in `encode_metadata` so we encode
-        // this as late as possible to give the prefetching as much time as possible to complete.
+        // Encode exported symbols info. This is prefetched in `encode_metadata`.
         let exported_symbols = stat!("exported-symbols", || {
-            self.encode_exported_symbols(&tcx.exported_symbols(LOCAL_CRATE))
+            self.encode_exported_symbols(tcx.exported_symbols(LOCAL_CRATE))
         });
 
         // Encode the hygiene data.
@@ -693,15 +680,15 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 has_global_allocator: tcx.has_global_allocator(LOCAL_CRATE),
                 has_alloc_error_handler: tcx.has_alloc_error_handler(LOCAL_CRATE),
                 has_panic_handler: tcx.has_panic_handler(LOCAL_CRATE),
-                has_default_lib_allocator: attr::contains_name(&attrs, sym::default_lib_allocator),
+                has_default_lib_allocator: attr::contains_name(attrs, sym::default_lib_allocator),
                 proc_macro_data,
                 debugger_visualizers,
-                compiler_builtins: attr::contains_name(&attrs, sym::compiler_builtins),
-                needs_allocator: attr::contains_name(&attrs, sym::needs_allocator),
-                needs_panic_runtime: attr::contains_name(&attrs, sym::needs_panic_runtime),
-                no_builtins: attr::contains_name(&attrs, sym::no_builtins),
-                panic_runtime: attr::contains_name(&attrs, sym::panic_runtime),
-                profiler_runtime: attr::contains_name(&attrs, sym::profiler_runtime),
+                compiler_builtins: attr::contains_name(attrs, sym::compiler_builtins),
+                needs_allocator: attr::contains_name(attrs, sym::needs_allocator),
+                needs_panic_runtime: attr::contains_name(attrs, sym::needs_panic_runtime),
+                no_builtins: attr::contains_name(attrs, sym::no_builtins),
+                panic_runtime: attr::contains_name(attrs, sym::panic_runtime),
+                profiler_runtime: attr::contains_name(attrs, sym::profiler_runtime),
                 symbol_mangling_version: tcx.sess.opts.get_symbol_mangling_version(),
 
                 crate_deps,
@@ -855,8 +842,7 @@ fn should_encode_span(def_kind: DefKind) -> bool {
         | DefKind::OpaqueTy
         | DefKind::Field
         | DefKind::Impl { .. }
-        | DefKind::Closure
-        | DefKind::Coroutine => true,
+        | DefKind::Closure => true,
         DefKind::ForeignMod | DefKind::GlobalAsm => false,
     }
 }
@@ -896,8 +882,7 @@ fn should_encode_attrs(def_kind: DefKind) -> bool {
         | DefKind::InlineConst
         | DefKind::OpaqueTy
         | DefKind::LifetimeParam
-        | DefKind::GlobalAsm
-        | DefKind::Coroutine => false,
+        | DefKind::GlobalAsm => false,
     }
 }
 
@@ -932,8 +917,7 @@ fn should_encode_expn_that_defined(def_kind: DefKind) -> bool {
         | DefKind::Field
         | DefKind::LifetimeParam
         | DefKind::GlobalAsm
-        | DefKind::Closure
-        | DefKind::Coroutine => false,
+        | DefKind::Closure => false,
     }
 }
 
@@ -968,7 +952,6 @@ fn should_encode_visibility(def_kind: DefKind) -> bool {
         | DefKind::GlobalAsm
         | DefKind::Impl { .. }
         | DefKind::Closure
-        | DefKind::Coroutine
         | DefKind::ExternCrate => false,
     }
 }
@@ -1004,7 +987,6 @@ fn should_encode_stability(def_kind: DefKind) -> bool {
         | DefKind::InlineConst
         | DefKind::GlobalAsm
         | DefKind::Closure
-        | DefKind::Coroutine
         | DefKind::ExternCrate => false,
     }
 }
@@ -1047,6 +1029,8 @@ fn should_encode_mir(
         | DefKind::AssocConst
         | DefKind::Static(..)
         | DefKind::Const => (true, false),
+        // Coroutines require optimized MIR to compute layout.
+        DefKind::Closure if tcx.is_coroutine(def_id.to_def_id()) => (false, true),
         // Full-fledged functions + closures
         DefKind::AssocFn | DefKind::Fn | DefKind::Closure => {
             let generics = tcx.generics_of(def_id);
@@ -1060,8 +1044,6 @@ fn should_encode_mir(
                 || tcx.is_const_default_method(def_id.to_def_id());
             (is_const_fn, opt)
         }
-        // Coroutines require optimized MIR to compute layout.
-        DefKind::Coroutine => (false, true),
         // The others don't have MIR.
         _ => (false, false),
     }
@@ -1097,7 +1079,6 @@ fn should_encode_variances<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, def_kind: Def
         | DefKind::InlineConst
         | DefKind::GlobalAsm
         | DefKind::Closure
-        | DefKind::Coroutine
         | DefKind::ExternCrate => false,
         DefKind::TyAlias => tcx.type_alias_is_lazy(def_id),
     }
@@ -1126,8 +1107,7 @@ fn should_encode_generics(def_kind: DefKind) -> bool {
         | DefKind::Impl { .. }
         | DefKind::Field
         | DefKind::TyParam
-        | DefKind::Closure
-        | DefKind::Coroutine => true,
+        | DefKind::Closure => true,
         DefKind::Mod
         | DefKind::ForeignMod
         | DefKind::ConstParam
@@ -1156,7 +1136,6 @@ fn should_encode_type(tcx: TyCtxt<'_>, def_id: LocalDefId, def_kind: DefKind) ->
         | DefKind::AssocFn
         | DefKind::AssocConst
         | DefKind::Closure
-        | DefKind::Coroutine
         | DefKind::ConstParam
         | DefKind::AnonConst
         | DefKind::InlineConst => true,
@@ -1165,7 +1144,7 @@ fn should_encode_type(tcx: TyCtxt<'_>, def_id: LocalDefId, def_kind: DefKind) ->
             let origin = tcx.opaque_type_origin(def_id);
             if let hir::OpaqueTyOrigin::FnReturn(fn_def_id)
             | hir::OpaqueTyOrigin::AsyncFn(fn_def_id) = origin
-                && let hir::Node::TraitItem(trait_item) = tcx.hir().get_by_def_id(fn_def_id)
+                && let hir::Node::TraitItem(trait_item) = tcx.hir_node_by_def_id(fn_def_id)
                 && let (_, hir::TraitFn::Required(..)) = trait_item.expect_fn()
             {
                 false
@@ -1182,7 +1161,7 @@ fn should_encode_type(tcx: TyCtxt<'_>, def_id: LocalDefId, def_kind: DefKind) ->
             }
         }
         DefKind::TyParam => {
-            let hir::Node::GenericParam(param) = tcx.hir().get_by_def_id(def_id) else { bug!() };
+            let hir::Node::GenericParam(param) = tcx.hir_node_by_def_id(def_id) else { bug!() };
             let hir::GenericParamKind::Type { default, .. } = param.kind else { bug!() };
             default.is_some()
         }
@@ -1217,7 +1196,6 @@ fn should_encode_fn_sig(def_kind: DefKind) -> bool {
         | DefKind::Impl { .. }
         | DefKind::AssocConst
         | DefKind::Closure
-        | DefKind::Coroutine
         | DefKind::ConstParam
         | DefKind::AnonConst
         | DefKind::InlineConst
@@ -1256,7 +1234,6 @@ fn should_encode_constness(def_kind: DefKind) -> bool {
         | DefKind::OpaqueTy
         | DefKind::Impl { of_trait: false }
         | DefKind::ForeignTy
-        | DefKind::Coroutine
         | DefKind::ConstParam
         | DefKind::InlineConst
         | DefKind::AssocTy
@@ -1291,7 +1268,6 @@ fn should_encode_const(def_kind: DefKind) -> bool {
         | DefKind::Impl { .. }
         | DefKind::AssocFn
         | DefKind::Closure
-        | DefKind::Coroutine
         | DefKind::ConstParam
         | DefKind::AssocTy
         | DefKind::TyParam
@@ -1353,9 +1329,8 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
 
         for local_id in tcx.iter_local_def_id() {
             let def_id = local_id.to_def_id();
-            let def_kind = tcx.opt_def_kind(local_id);
-            let Some(def_kind) = def_kind else { continue };
-            self.tables.opt_def_kind.set_some(def_id.index, def_kind);
+            let def_kind = tcx.def_kind(local_id);
+            self.tables.def_kind.set_some(def_id.index, def_kind);
             if should_encode_span(def_kind) {
                 let def_span = tcx.def_span(local_id);
                 record!(self.tables.def_span[def_id] <- def_span);
@@ -1392,7 +1367,13 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             if should_encode_fn_sig(def_kind) {
                 record!(self.tables.fn_sig[def_id] <- tcx.fn_sig(def_id));
             }
-            if should_encode_generics(def_kind) {
+            // FIXME: Some anonymous constants produced by `#[rustc_legacy_const_generics]`
+            // do not have corresponding HIR nodes, so some queries usually making sense for
+            // anonymous constants will not work on them and panic. It's not clear whether it
+            // can cause any observable issues or not.
+            let anon_const_without_hir = def_kind == DefKind::AnonConst
+                && tcx.opt_hir_node(tcx.local_def_id_to_hir_id(local_id)).is_none();
+            if should_encode_generics(def_kind) && !anon_const_without_hir {
                 let g = tcx.generics_of(def_id);
                 record!(self.tables.generics_of[def_id] <- g);
                 record!(self.tables.explicit_predicates_of[def_id] <- self.tcx.explicit_predicates_of(def_id));
@@ -1406,7 +1387,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                     }
                 }
             }
-            if should_encode_type(tcx, local_id, def_kind) {
+            if should_encode_type(tcx, local_id, def_kind) && !anon_const_without_hir {
                 record!(self.tables.type_of[def_id] <- self.tcx.type_of(def_id));
             }
             if should_encode_constness(def_kind) {
@@ -1446,8 +1427,9 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                     self.encode_info_for_assoc_item(def_id);
                 }
             }
-            if let DefKind::Coroutine = def_kind {
-                let data = self.tcx.coroutine_kind(def_id).unwrap();
+            if def_kind == DefKind::Closure
+                && let Some(data) = self.tcx.coroutine_kind(def_id)
+            {
                 record!(self.tables.coroutine_kind[def_id] <- data);
             }
             if let DefKind::Enum | DefKind::Struct | DefKind::Union = def_kind {
@@ -1629,7 +1611,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 record!(self.tables.closure_saved_names_of_captured_variables[def_id.to_def_id()]
                     <- tcx.closure_saved_names_of_captured_variables(def_id));
 
-                if let DefKind::Coroutine = self.tcx.def_kind(def_id)
+                if self.tcx.is_coroutine(def_id.to_def_id())
                     && let Some(witnesses) = tcx.mir_coroutine_witnesses(def_id)
                 {
                     record!(self.tables.mir_coroutine_witnesses[def_id.to_def_id()] <- witnesses);
@@ -1656,7 +1638,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             }
             record!(self.tables.promoted_mir[def_id.to_def_id()] <- tcx.promoted_mir(def_id));
 
-            if let DefKind::Coroutine = self.tcx.def_kind(def_id)
+            if self.tcx.is_coroutine(def_id.to_def_id())
                 && let Some(witnesses) = tcx.mir_coroutine_witnesses(def_id)
             {
                 record!(self.tables.mir_coroutine_witnesses[def_id.to_def_id()] <- witnesses);
@@ -1727,9 +1709,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
     fn encode_info_for_macro(&mut self, def_id: LocalDefId) {
         let tcx = self.tcx;
 
-        let hir::ItemKind::Macro(ref macro_def, _) = tcx.hir().expect_item(def_id).kind else {
-            bug!()
-        };
+        let hir::ItemKind::Macro(macro_def, _) = tcx.hir().expect_item(def_id).kind else { bug!() };
         self.tables.is_macro_rules.set(def_id.local_def_index, macro_def.macro_rules);
         record!(self.tables.macro_definition[def_id.to_def_id()] <- &*macro_def.body);
     }
@@ -1786,7 +1766,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 self.tables.proc_macro_quoted_spans.set_some(i, span);
             }
 
-            self.tables.opt_def_kind.set_some(LOCAL_CRATE.as_def_id().index, DefKind::Mod);
+            self.tables.def_kind.set_some(LOCAL_CRATE.as_def_id().index, DefKind::Mod);
             record!(self.tables.def_span[LOCAL_CRATE.as_def_id()] <- tcx.def_span(LOCAL_CRATE.as_def_id()));
             self.encode_attrs(LOCAL_CRATE.as_def_id().expect_local());
             let vis = tcx.local_visibility(CRATE_DEF_ID).map_id(|def_id| def_id.local_def_index);
@@ -1807,7 +1787,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             // so we manually encode just the information that we need
             for &proc_macro in &tcx.resolutions(()).proc_macros {
                 let id = proc_macro;
-                let proc_macro = hir.local_def_id_to_hir_id(proc_macro);
+                let proc_macro = tcx.local_def_id_to_hir_id(proc_macro);
                 let mut name = hir.name(proc_macro);
                 let span = hir.span(proc_macro);
                 // Proc-macros may have attributes like `#[allow_internal_unstable]`,
@@ -1834,7 +1814,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
                 def_key.disambiguated_data.data = DefPathData::MacroNs(name);
 
                 let def_id = id.to_def_id();
-                self.tables.opt_def_kind.set_some(def_id.index, DefKind::Macro(macro_kind));
+                self.tables.def_kind.set_some(def_id.index, DefKind::Macro(macro_kind));
                 self.tables.proc_macro.set_some(def_id.index, macro_kind);
                 self.encode_attrs(id);
                 record!(self.tables.def_keys[def_id] <- def_key);
@@ -1902,10 +1882,10 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         self.lazy_array(deps.iter().map(|(_, dep)| dep))
     }
 
-    fn encode_lib_features(&mut self) -> LazyArray<(Symbol, Option<Symbol>)> {
+    fn encode_lib_features(&mut self) -> LazyArray<(Symbol, FeatureStability)> {
         empty_proc_macro!(self);
         let tcx = self.tcx;
-        let lib_features = tcx.lib_features(());
+        let lib_features = tcx.lib_features(LOCAL_CRATE);
         self.lazy_array(lib_features.to_vec())
     }
 
@@ -2198,27 +2178,19 @@ pub fn encode_metadata(tcx: TyCtxt<'_>, path: &Path) {
     // there's no need to do dep-graph tracking for any of it.
     tcx.dep_graph.assert_ignored();
 
-    join(
-        || encode_metadata_impl(tcx, path),
-        || {
-            if tcx.sess.threads() == 1 {
-                return;
-            }
-            // Prefetch some queries used by metadata encoding.
-            // This is not necessary for correctness, but is only done for performance reasons.
-            // It can be removed if it turns out to cause trouble or be detrimental to performance.
-            join(|| prefetch_mir(tcx), || tcx.exported_symbols(LOCAL_CRATE));
-        },
-    );
-}
+    if tcx.sess.threads() != 1 {
+        // Prefetch some queries used by metadata encoding.
+        // This is not necessary for correctness, but is only done for performance reasons.
+        // It can be removed if it turns out to cause trouble or be detrimental to performance.
+        join(|| prefetch_mir(tcx), || tcx.exported_symbols(LOCAL_CRATE));
+    }
 
-fn encode_metadata_impl(tcx: TyCtxt<'_>, path: &Path) {
     let mut encoder = opaque::FileEncoder::new(path)
         .unwrap_or_else(|err| tcx.sess.emit_fatal(FailCreateFileEncoder { err }));
     encoder.emit_raw_bytes(METADATA_HEADER);
 
     // Will be filled with the root position after encoding everything.
-    encoder.emit_raw_bytes(&[0, 0, 0, 0]);
+    encoder.emit_raw_bytes(&0u64.to_le_bytes());
 
     let source_map_files = tcx.sess.source_map().files();
     let source_file_cache = (source_map_files[0].clone(), 0);
@@ -2251,29 +2223,34 @@ fn encode_metadata_impl(tcx: TyCtxt<'_>, path: &Path) {
     // culminating in the `CrateRoot` which points to all of it.
     let root = ecx.encode_crate_root();
 
-    ecx.opaque.flush();
+    // Make sure we report any errors from writing to the file.
+    // If we forget this, compilation can succeed with an incomplete rmeta file,
+    // causing an ICE when the rmeta file is read by another compilation.
+    if let Err((path, err)) = ecx.opaque.finish() {
+        tcx.sess.emit_err(FailWriteFile { path: &path, err });
+    }
+
+    let file = ecx.opaque.file();
+    if let Err(err) = encode_root_position(file, root.position.get()) {
+        tcx.sess.emit_err(FailWriteFile { path: ecx.opaque.path(), err });
+    }
+
+    // Record metadata size for self-profiling
+    tcx.prof.artifact_size("crate_metadata", "crate_metadata", file.metadata().unwrap().len());
+}
 
-    let mut file = ecx.opaque.file();
+fn encode_root_position(mut file: &File, pos: usize) -> Result<(), std::io::Error> {
     // We will return to this position after writing the root position.
     let pos_before_seek = file.stream_position().unwrap();
 
     // Encode the root position.
     let header = METADATA_HEADER.len();
-    file.seek(std::io::SeekFrom::Start(header as u64))
-        .unwrap_or_else(|err| tcx.sess.emit_fatal(FailSeekFile { err }));
-    let pos = root.position.get();
-    file.write_all(&[(pos >> 24) as u8, (pos >> 16) as u8, (pos >> 8) as u8, (pos >> 0) as u8])
-        .unwrap_or_else(|err| tcx.sess.emit_fatal(FailWriteFile { err }));
+    file.seek(std::io::SeekFrom::Start(header as u64))?;
+    file.write_all(&pos.to_le_bytes())?;
 
     // Return to the position where we are before writing the root position.
-    file.seek(std::io::SeekFrom::Start(pos_before_seek)).unwrap();
-
-    // Record metadata size for self-profiling
-    tcx.prof.artifact_size(
-        "crate_metadata",
-        "crate_metadata",
-        file.metadata().unwrap().len() as u64,
-    );
+    file.seek(std::io::SeekFrom::Start(pos_before_seek))?;
+    Ok(())
 }
 
 pub fn provide(providers: &mut Providers) {
@@ -2396,8 +2373,10 @@ pub fn rendered_const<'tcx>(tcx: TyCtxt<'tcx>, body: hir::BodyId) -> String {
         // * character escapes
         //
         // FIXME: This passes through `-/*spacer*/0` verbatim.
-        Literal if !value.span.from_expansion()
-            && let Ok(snippet) = tcx.sess.source_map().span_to_snippet(value.span) => {
+        Literal
+            if !value.span.from_expansion()
+                && let Ok(snippet) = tcx.sess.source_map().span_to_snippet(value.span) =>
+        {
             snippet
         }
 
@@ -2408,10 +2387,12 @@ pub fn rendered_const<'tcx>(tcx: TyCtxt<'tcx>, body: hir::BodyId) -> String {
         // FIXME: Omit the curly braces if the enclosing expression is an array literal
         //        with a repeated element (an `ExprKind::Repeat`) as in such case it
         //        would not actually need any disambiguation.
-        Complex => if tcx.def_kind(hir.body_owner_def_id(body).to_def_id()) == DefKind::AnonConst {
-            "{ _ }".to_owned()
-        } else {
-            "_".to_owned()
+        Complex => {
+            if tcx.def_kind(hir.body_owner_def_id(body).to_def_id()) == DefKind::AnonConst {
+                "{ _ }".to_owned()
+            } else {
+                "_".to_owned()
+            }
         }
     }
 }
diff --git a/compiler/rustc_metadata/src/rmeta/mod.rs b/compiler/rustc_metadata/src/rmeta/mod.rs
index 9ae5c0af0..a85822848 100644
--- a/compiler/rustc_metadata/src/rmeta/mod.rs
+++ b/compiler/rustc_metadata/src/rmeta/mod.rs
@@ -3,6 +3,7 @@ use decoder::Metadata;
 use def_path_hash_map::DefPathHashMapRef;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_middle::middle::debugger_visualizer::DebuggerVisualizerFile;
+use rustc_middle::middle::lib_features::FeatureStability;
 use table::TableBuilder;
 
 use rustc_ast as ast;
@@ -56,12 +57,12 @@ pub(crate) fn rustc_version(cfg_version: &'static str) -> String {
 /// Metadata encoding version.
 /// N.B., increment this if you change the format of metadata such that
 /// the rustc version can't be found to compare with `rustc_version()`.
-const METADATA_VERSION: u8 = 8;
+const METADATA_VERSION: u8 = 9;
 
 /// Metadata header which includes `METADATA_VERSION`.
 ///
 /// This header is followed by the length of the compressed data, then
-/// the position of the `CrateRoot`, which is encoded as a 32-bit big-endian
+/// the position of the `CrateRoot`, which is encoded as a 64-bit little-endian
 /// unsigned integer, and further followed by the rustc version string.
 pub const METADATA_HEADER: &[u8] = &[b'r', b'u', b's', b't', 0, 0, 0, METADATA_VERSION];
 
@@ -263,7 +264,7 @@ pub(crate) struct CrateRoot {
 
     crate_deps: LazyArray<CrateDep>,
     dylib_dependency_formats: LazyArray<Option<LinkagePreference>>,
-    lib_features: LazyArray<(Symbol, Option<Symbol>)>,
+    lib_features: LazyArray<(Symbol, FeatureStability)>,
     stability_implications: LazyArray<(Symbol, Symbol)>,
     lang_items: LazyArray<(DefIndex, LangItem)>,
     lang_items_missing: LazyArray<LangItem>,
@@ -404,7 +405,7 @@ define_tables! {
     // so we can take their names, visibilities etc from other encoded tables.
     module_children_non_reexports: Table<DefIndex, LazyArray<DefIndex>>,
     associated_item_or_field_def_ids: Table<DefIndex, LazyArray<DefIndex>>,
-    opt_def_kind: Table<DefIndex, DefKind>,
+    def_kind: Table<DefIndex, DefKind>,
     visibility: Table<DefIndex, LazyValue<ty::Visibility<DefIndex>>>,
     def_span: Table<DefIndex, LazyValue<Span>>,
     def_ident_span: Table<DefIndex, LazyValue<Span>>,
diff --git a/compiler/rustc_metadata/src/rmeta/table.rs b/compiler/rustc_metadata/src/rmeta/table.rs
index 027994c40..3fc6d9db3 100644
--- a/compiler/rustc_metadata/src/rmeta/table.rs
+++ b/compiler/rustc_metadata/src/rmeta/table.rs
@@ -1,13 +1,8 @@
 use crate::rmeta::*;
 
 use rustc_data_structures::fingerprint::Fingerprint;
-use rustc_hir::def::{CtorKind, CtorOf};
+use rustc_hir::def::CtorOf;
 use rustc_index::Idx;
-use rustc_middle::ty::{ParameterizedOverTcx, UnusedGenericParams};
-use rustc_serialize::opaque::FileEncoder;
-use rustc_span::hygiene::MacroKind;
-use std::marker::PhantomData;
-use std::num::NonZeroUsize;
 
 pub(super) trait IsDefault: Default {
     fn is_default(&self) -> bool;
@@ -167,7 +162,6 @@ fixed_size_enum! {
         ( Impl { of_trait: false }                 )
         ( Impl { of_trait: true }                  )
         ( Closure                                  )
-        ( Coroutine                                )
         ( Static(ast::Mutability::Not)             )
         ( Static(ast::Mutability::Mut)             )
         ( Ctor(CtorOf::Struct, CtorKind::Fn)       )
@@ -382,8 +376,8 @@ impl<T> LazyArray<T> {
     }
 
     fn from_bytes_impl(position: &[u8; 8], meta: &[u8; 8]) -> Option<LazyArray<T>> {
-        let position = NonZeroUsize::new(u64::from_bytes(&position) as usize)?;
-        let len = u64::from_bytes(&meta) as usize;
+        let position = NonZeroUsize::new(u64::from_bytes(position) as usize)?;
+        let len = u64::from_bytes(meta) as usize;
         Some(LazyArray::from_position_and_num_elems(position, len))
     }
 }
@@ -497,7 +491,7 @@ impl<I: Idx, const N: usize, T: FixedSizeEncoding<ByteArray = [u8; N]>> TableBui
         }
 
         LazyTable::from_position_and_encoded_size(
-            NonZeroUsize::new(pos as usize).unwrap(),
+            NonZeroUsize::new(pos).unwrap(),
             width,
             self.blocks.len(),
         )
diff --git a/compiler/rustc_middle/src/arena.rs b/compiler/rustc_middle/src/arena.rs
index dd761b4e3..52fd494a1 100644
--- a/compiler/rustc_middle/src/arena.rs
+++ b/compiler/rustc_middle/src/arena.rs
@@ -27,19 +27,12 @@ macro_rules! arena_types {
                     rustc_middle::mir::Promoted,
                     rustc_middle::mir::Body<'tcx>
                 >,
-            [decode] closure_debuginfo:
-                rustc_index::IndexVec<
-                    rustc_target::abi::FieldIdx,
-                    rustc_span::symbol::Symbol,
-                >,
             [decode] typeck_results: rustc_middle::ty::TypeckResults<'tcx>,
-            [decode] borrowck_result:
-                rustc_middle::mir::BorrowCheckResult<'tcx>,
+            [decode] borrowck_result: rustc_middle::mir::BorrowCheckResult<'tcx>,
             [] resolver: rustc_data_structures::steal::Steal<(
                 rustc_middle::ty::ResolverAstLowering,
                 rustc_data_structures::sync::Lrc<rustc_ast::Crate>,
             )>,
-            [] output_filenames: std::sync::Arc<rustc_session::config::OutputFilenames>,
             [] crate_for_resolver: rustc_data_structures::steal::Steal<(rustc_ast::Crate, rustc_ast::AttrVec)>,
             [] resolutions: rustc_middle::ty::ResolverGlobalCtxt,
             [decode] unsafety_check_result: rustc_middle::mir::UnsafetyCheckResult,
@@ -91,29 +84,19 @@ macro_rules! arena_types {
                 rustc_middle::infer::canonical::Canonical<'tcx,
                     rustc_middle::infer::canonical::QueryResponse<'tcx, rustc_middle::ty::Ty<'tcx>>
                 >,
-            [] all_traits: Vec<rustc_hir::def_id::DefId>,
             [] effective_visibilities: rustc_middle::middle::privacy::EffectiveVisibilities,
-            [] foreign_module: rustc_session::cstore::ForeignModule,
-            [] foreign_modules: Vec<rustc_session::cstore::ForeignModule>,
             [] upvars_mentioned: rustc_data_structures::fx::FxIndexMap<rustc_hir::HirId, rustc_hir::Upvar>,
             [] object_safety_violations: rustc_middle::traits::ObjectSafetyViolation,
             [] codegen_unit: rustc_middle::mir::mono::CodegenUnit<'tcx>,
             [decode] attribute: rustc_ast::Attribute,
             [] name_set: rustc_data_structures::unord::UnordSet<rustc_span::symbol::Symbol>,
             [] ordered_name_set: rustc_data_structures::fx::FxIndexSet<rustc_span::symbol::Symbol>,
-            [] hir_id_set: rustc_hir::HirIdSet,
-
-            // Interned types
-            [] tys: rustc_type_ir::WithCachedTypeInfo<rustc_middle::ty::TyKind<'tcx>>,
-            [] predicates: rustc_type_ir::WithCachedTypeInfo<rustc_middle::ty::PredicateKind<'tcx>>,
-            [] consts: rustc_middle::ty::ConstData<'tcx>,
 
             // Note that this deliberately duplicates items in the `rustc_hir::arena`,
             // since we need to allocate this type on both the `rustc_hir` arena
             // (during lowering) and the `librustc_middle` arena (for decoding MIR)
             [decode] asm_template: rustc_ast::InlineAsmTemplatePiece,
             [decode] used_trait_imports: rustc_data_structures::unord::UnordSet<rustc_hir::def_id::LocalDefId>,
-            [decode] registered_tools: rustc_middle::ty::RegisteredTools,
             [decode] is_late_bound_map: rustc_data_structures::fx::FxIndexSet<rustc_hir::ItemLocalId>,
             [decode] impl_source: rustc_middle::traits::ImplSource<'tcx, ()>,
 
@@ -124,11 +107,9 @@ macro_rules! arena_types {
                     rustc_hir::def_id::DefId,
                     rustc_middle::ty::EarlyBinder<rustc_middle::ty::Ty<'tcx>>
                 >,
-            [] bit_set_u32: rustc_index::bit_set::BitSet<u32>,
             [] external_constraints: rustc_middle::traits::solve::ExternalConstraintsData<'tcx>,
             [] predefined_opaques_in_body: rustc_middle::traits::solve::PredefinedOpaquesData<'tcx>,
             [decode] doc_link_resolutions: rustc_hir::def::DocLinkResMap,
-            [] closure_kind_origin: (rustc_span::Span, rustc_middle::hir::place::Place<'tcx>),
             [] stripped_cfg_items: rustc_ast::expand::StrippedCfgItem,
             [] mod_child: rustc_middle::metadata::ModChild,
             [] features: rustc_feature::Features,
diff --git a/compiler/rustc_middle/src/dep_graph/mod.rs b/compiler/rustc_middle/src/dep_graph/mod.rs
index 76ef62f9f..dc0da165a 100644
--- a/compiler/rustc_middle/src/dep_graph/mod.rs
+++ b/compiler/rustc_middle/src/dep_graph/mod.rs
@@ -8,8 +8,8 @@ mod dep_node;
 
 pub use rustc_query_system::dep_graph::debug::EdgeFilter;
 pub use rustc_query_system::dep_graph::{
-    debug::DepNodeFilter, hash_result, DepContext, DepGraphQuery, DepNodeColor, DepNodeIndex, Deps,
-    SerializedDepGraph, SerializedDepNodeIndex, TaskDeps, TaskDepsRef, WorkProduct, WorkProductId,
+    debug::DepNodeFilter, hash_result, DepContext, DepGraphQuery, DepNodeIndex, Deps,
+    SerializedDepGraph, SerializedDepNodeIndex, TaskDepsRef, WorkProduct, WorkProductId,
     WorkProductMap,
 };
 
diff --git a/compiler/rustc_middle/src/hir/map/mod.rs b/compiler/rustc_middle/src/hir/map/mod.rs
index 58c0c6bab..81f34c7b8 100644
--- a/compiler/rustc_middle/src/hir/map/mod.rs
+++ b/compiler/rustc_middle/src/hir/map/mod.rs
@@ -9,8 +9,8 @@ use rustc_data_structures::svh::Svh;
 use rustc_data_structures::sync::{par_for_each_in, try_par_for_each_in, DynSend, DynSync};
 use rustc_hir::def::{DefKind, Res};
 use rustc_hir::def_id::{DefId, LocalDefId, LocalModDefId, LOCAL_CRATE};
-use rustc_hir::definitions::{DefKey, DefPath, DefPathData, DefPathHash};
-use rustc_hir::intravisit::{self, Visitor};
+use rustc_hir::definitions::{DefKey, DefPath, DefPathHash};
+use rustc_hir::intravisit::Visitor;
 use rustc_hir::*;
 use rustc_index::Idx;
 use rustc_middle::hir::nested_filter;
@@ -57,6 +57,10 @@ fn is_body_owner(node: Node<'_>, hir_id: HirId) -> bool {
     }
 }
 
+// FIXME: the structure was necessary in the past but now it
+// only serves as "namespace" for HIR-related methods, and can be
+// removed if all the methods are reasonably renamed and moved to tcx
+// (https://github.com/rust-lang/rust/pull/118256#issuecomment-1826442834).
 #[derive(Copy, Clone)]
 pub struct Map<'hir> {
     pub(super) tcx: TyCtxt<'hir>,
@@ -116,7 +120,7 @@ impl<'hir> Iterator for ParentOwnerIterator<'hir> {
 
             let parent_id = parent_id.map_or(CRATE_OWNER_ID, |local_def_index| {
                 let def_id = LocalDefId { local_def_index };
-                self.map.local_def_id_to_hir_id(def_id).owner
+                self.map.tcx.local_def_id_to_hir_id(def_id).owner
             });
             self.current_id = HirId::make_owner(parent_id.def_id);
 
@@ -128,6 +132,40 @@ impl<'hir> Iterator for ParentOwnerIterator<'hir> {
     }
 }
 
+impl<'tcx> TyCtxt<'tcx> {
+    /// Retrieves the `hir::Node` corresponding to `id`, returning `None` if cannot be found.
+    pub fn opt_hir_node(self, id: HirId) -> Option<Node<'tcx>> {
+        if id.local_id == ItemLocalId::from_u32(0) {
+            let owner = self.hir_owner(id.owner)?;
+            Some(owner.node.into())
+        } else {
+            let owner = self.hir_owner_nodes(id.owner).as_owner()?;
+            let node = owner.nodes[id.local_id].as_ref()?;
+            Some(node.node)
+        }
+    }
+
+    /// Retrieves the `hir::Node` corresponding to `id`, returning `None` if cannot be found.
+    #[inline]
+    pub fn opt_hir_node_by_def_id(self, id: LocalDefId) -> Option<Node<'tcx>> {
+        self.opt_hir_node(self.opt_local_def_id_to_hir_id(id)?)
+    }
+
+    /// Retrieves the `hir::Node` corresponding to `id`, panicking if it cannot be found.
+    #[track_caller]
+    pub fn hir_node(self, id: HirId) -> Node<'tcx> {
+        self.opt_hir_node(id).unwrap_or_else(|| bug!("couldn't find HIR node for hir id {id:?}"))
+    }
+
+    /// Retrieves the `hir::Node` corresponding to `id`, panicking if it cannot be found.
+    #[inline]
+    #[track_caller]
+    pub fn hir_node_by_def_id(self, id: LocalDefId) -> Node<'tcx> {
+        self.opt_hir_node_by_def_id(id)
+            .unwrap_or_else(|| bug!("couldn't find HIR node for def id {id:?}"))
+    }
+}
+
 impl<'hir> Map<'hir> {
     #[inline]
     pub fn krate(self) -> &'hir Crate<'hir> {
@@ -168,108 +206,6 @@ impl<'hir> Map<'hir> {
         self.tcx.definitions_untracked().def_path_hash(def_id)
     }
 
-    #[inline]
-    pub fn local_def_id_to_hir_id(self, def_id: impl Into<LocalDefId>) -> HirId {
-        self.tcx.local_def_id_to_hir_id(def_id.into())
-    }
-
-    /// Do not call this function directly. The query should be called.
-    pub(super) fn opt_def_kind(self, local_def_id: LocalDefId) -> Option<DefKind> {
-        let hir_id = self.local_def_id_to_hir_id(local_def_id);
-        let node = match self.find(hir_id) {
-            Some(node) => node,
-            None => match self.def_key(local_def_id).disambiguated_data.data {
-                // FIXME: Some anonymous constants do not have corresponding HIR nodes,
-                // so many local queries will panic on their def ids. `None` is currently
-                // returned here instead of `DefKind::{Anon,Inline}Const` to avoid such panics.
-                // Ideally all def ids should have `DefKind`s, we need to create the missing
-                // HIR nodes or feed relevant query results to achieve that.
-                DefPathData::AnonConst => return None,
-                _ => bug!("no HIR node for def id {local_def_id:?}"),
-            },
-        };
-        let def_kind = match node {
-            Node::Item(item) => match item.kind {
-                ItemKind::Static(_, mt, _) => DefKind::Static(mt),
-                ItemKind::Const(..) => DefKind::Const,
-                ItemKind::Fn(..) => DefKind::Fn,
-                ItemKind::Macro(_, macro_kind) => DefKind::Macro(macro_kind),
-                ItemKind::Mod(..) => DefKind::Mod,
-                ItemKind::OpaqueTy(..) => DefKind::OpaqueTy,
-                ItemKind::TyAlias(..) => DefKind::TyAlias,
-                ItemKind::Enum(..) => DefKind::Enum,
-                ItemKind::Struct(..) => DefKind::Struct,
-                ItemKind::Union(..) => DefKind::Union,
-                ItemKind::Trait(..) => DefKind::Trait,
-                ItemKind::TraitAlias(..) => DefKind::TraitAlias,
-                ItemKind::ExternCrate(_) => DefKind::ExternCrate,
-                ItemKind::Use(..) => DefKind::Use,
-                ItemKind::ForeignMod { .. } => DefKind::ForeignMod,
-                ItemKind::GlobalAsm(..) => DefKind::GlobalAsm,
-                ItemKind::Impl(impl_) => DefKind::Impl { of_trait: impl_.of_trait.is_some() },
-            },
-            Node::ForeignItem(item) => match item.kind {
-                ForeignItemKind::Fn(..) => DefKind::Fn,
-                ForeignItemKind::Static(_, mt) => DefKind::Static(mt),
-                ForeignItemKind::Type => DefKind::ForeignTy,
-            },
-            Node::TraitItem(item) => match item.kind {
-                TraitItemKind::Const(..) => DefKind::AssocConst,
-                TraitItemKind::Fn(..) => DefKind::AssocFn,
-                TraitItemKind::Type(..) => DefKind::AssocTy,
-            },
-            Node::ImplItem(item) => match item.kind {
-                ImplItemKind::Const(..) => DefKind::AssocConst,
-                ImplItemKind::Fn(..) => DefKind::AssocFn,
-                ImplItemKind::Type(..) => DefKind::AssocTy,
-            },
-            Node::Variant(_) => DefKind::Variant,
-            Node::Ctor(variant_data) => {
-                let ctor_of = match self.find_parent(hir_id) {
-                    Some(Node::Item(..)) => def::CtorOf::Struct,
-                    Some(Node::Variant(..)) => def::CtorOf::Variant,
-                    _ => unreachable!(),
-                };
-                match variant_data.ctor_kind() {
-                    Some(kind) => DefKind::Ctor(ctor_of, kind),
-                    None => bug!("constructor node without a constructor"),
-                }
-            }
-            Node::AnonConst(_) => DefKind::AnonConst,
-            Node::ConstBlock(_) => DefKind::InlineConst,
-            Node::Field(_) => DefKind::Field,
-            Node::Expr(expr) => match expr.kind {
-                ExprKind::Closure(Closure { movability: None, .. }) => DefKind::Closure,
-                ExprKind::Closure(Closure { movability: Some(_), .. }) => DefKind::Coroutine,
-                _ => bug!("def_kind: unsupported node: {}", self.node_to_string(hir_id)),
-            },
-            Node::GenericParam(param) => match param.kind {
-                GenericParamKind::Lifetime { .. } => DefKind::LifetimeParam,
-                GenericParamKind::Type { .. } => DefKind::TyParam,
-                GenericParamKind::Const { .. } => DefKind::ConstParam,
-            },
-            Node::Crate(_) => DefKind::Mod,
-            Node::Stmt(_)
-            | Node::PathSegment(_)
-            | Node::Ty(_)
-            | Node::TypeBinding(_)
-            | Node::Infer(_)
-            | Node::TraitRef(_)
-            | Node::Pat(_)
-            | Node::PatField(_)
-            | Node::ExprField(_)
-            | Node::Local(_)
-            | Node::Param(_)
-            | Node::Arm(_)
-            | Node::Lifetime(_)
-            | Node::Block(_) => span_bug!(
-                self.span(hir_id),
-                "unexpected node with def id {local_def_id:?}: {node:?}"
-            ),
-        };
-        Some(def_kind)
-    }
-
     /// Finds the id of the parent node to this one.
     ///
     /// If calling repeatedly and iterating over parents, prefer [`Map::parent_iter`].
@@ -293,46 +229,15 @@ impl<'hir> Map<'hir> {
     }
 
     pub fn get_parent(self, hir_id: HirId) -> Node<'hir> {
-        self.get(self.parent_id(hir_id))
+        self.tcx.hir_node(self.parent_id(hir_id))
     }
 
     pub fn find_parent(self, hir_id: HirId) -> Option<Node<'hir>> {
-        self.find(self.opt_parent_id(hir_id)?)
-    }
-
-    /// Retrieves the `Node` corresponding to `id`, returning `None` if cannot be found.
-    pub fn find(self, id: HirId) -> Option<Node<'hir>> {
-        if id.local_id == ItemLocalId::from_u32(0) {
-            let owner = self.tcx.hir_owner(id.owner)?;
-            Some(owner.node.into())
-        } else {
-            let owner = self.tcx.hir_owner_nodes(id.owner).as_owner()?;
-            let node = owner.nodes[id.local_id].as_ref()?;
-            Some(node.node)
-        }
-    }
-
-    /// Retrieves the `Node` corresponding to `id`, returning `None` if cannot be found.
-    #[inline]
-    pub fn find_by_def_id(self, id: LocalDefId) -> Option<Node<'hir>> {
-        self.find(self.tcx.opt_local_def_id_to_hir_id(id)?)
-    }
-
-    /// Retrieves the `Node` corresponding to `id`, panicking if it cannot be found.
-    #[track_caller]
-    pub fn get(self, id: HirId) -> Node<'hir> {
-        self.find(id).unwrap_or_else(|| bug!("couldn't find hir id {} in the HIR map", id))
-    }
-
-    /// Retrieves the `Node` corresponding to `id`, panicking if it cannot be found.
-    #[inline]
-    #[track_caller]
-    pub fn get_by_def_id(self, id: LocalDefId) -> Node<'hir> {
-        self.find_by_def_id(id).unwrap_or_else(|| bug!("couldn't find {:?} in the HIR map", id))
+        self.tcx.opt_hir_node(self.opt_parent_id(hir_id)?)
     }
 
     pub fn get_if_local(self, id: DefId) -> Option<Node<'hir>> {
-        id.as_local().and_then(|id| self.find(self.tcx.opt_local_def_id_to_hir_id(id)?))
+        id.as_local().and_then(|id| self.tcx.opt_hir_node(self.tcx.opt_local_def_id_to_hir_id(id)?))
     }
 
     pub fn get_generics(self, id: LocalDefId) -> Option<&'hir Generics<'hir>> {
@@ -366,7 +271,7 @@ impl<'hir> Map<'hir> {
 
     #[track_caller]
     pub fn fn_decl_by_hir_id(self, hir_id: HirId) -> Option<&'hir FnDecl<'hir>> {
-        if let Some(node) = self.find(hir_id) {
+        if let Some(node) = self.tcx.opt_hir_node(hir_id) {
             node.fn_decl()
         } else {
             bug!("no node for hir_id `{}`", hir_id)
@@ -375,7 +280,7 @@ impl<'hir> Map<'hir> {
 
     #[track_caller]
     pub fn fn_sig_by_hir_id(self, hir_id: HirId) -> Option<&'hir FnSig<'hir>> {
-        if let Some(node) = self.find(hir_id) {
+        if let Some(node) = self.tcx.opt_hir_node(hir_id) {
             node.fn_sig()
         } else {
             bug!("no node for hir_id `{}`", hir_id)
@@ -398,19 +303,22 @@ impl<'hir> Map<'hir> {
     /// item (possibly associated), a closure, or a `hir::AnonConst`.
     pub fn body_owner(self, BodyId { hir_id }: BodyId) -> HirId {
         let parent = self.parent_id(hir_id);
-        assert!(self.find(parent).is_some_and(|n| is_body_owner(n, hir_id)), "{hir_id:?}");
+        assert!(
+            self.tcx.opt_hir_node(parent).is_some_and(|n| is_body_owner(n, hir_id)),
+            "{hir_id:?}"
+        );
         parent
     }
 
     pub fn body_owner_def_id(self, BodyId { hir_id }: BodyId) -> LocalDefId {
         let parent = self.parent_id(hir_id);
-        associated_body(self.get(parent)).unwrap().0
+        associated_body(self.tcx.hir_node(parent)).unwrap().0
     }
 
     /// Given a `LocalDefId`, returns the `BodyId` associated with it,
     /// if the node is a body owner, otherwise returns `None`.
     pub fn maybe_body_owned_by(self, id: LocalDefId) -> Option<BodyId> {
-        let node = self.find_by_def_id(id)?;
+        let node = self.tcx.opt_hir_node_by_def_id(id)?;
         let (_, body_id) = associated_body(node)?;
         Some(body_id)
     }
@@ -419,7 +327,7 @@ impl<'hir> Map<'hir> {
     #[track_caller]
     pub fn body_owned_by(self, id: LocalDefId) -> BodyId {
         self.maybe_body_owned_by(id).unwrap_or_else(|| {
-            let hir_id = self.local_def_id_to_hir_id(id);
+            let hir_id = self.tcx.local_def_id_to_hir_id(id);
             span_bug!(
                 self.span(hir_id),
                 "body_owned_by: {} has no associated body",
@@ -438,14 +346,15 @@ impl<'hir> Map<'hir> {
     /// Returns the `BodyOwnerKind` of this `LocalDefId`.
     ///
     /// Panics if `LocalDefId` does not have an associated body.
-    pub fn body_owner_kind(self, def_id: LocalDefId) -> BodyOwnerKind {
+    pub fn body_owner_kind(self, def_id: impl Into<DefId>) -> BodyOwnerKind {
+        let def_id = def_id.into();
         match self.tcx.def_kind(def_id) {
             DefKind::Const | DefKind::AssocConst | DefKind::AnonConst => {
                 BodyOwnerKind::Const { inline: false }
             }
             DefKind::InlineConst => BodyOwnerKind::Const { inline: true },
             DefKind::Ctor(..) | DefKind::Fn | DefKind::AssocFn => BodyOwnerKind::Fn,
-            DefKind::Closure | DefKind::Coroutine => BodyOwnerKind::Closure,
+            DefKind::Closure => BodyOwnerKind::Closure,
             DefKind::Static(mt) => BodyOwnerKind::Static(mt),
             dk => bug!("{:?} is not a body node: {:?}", def_id, dk),
         }
@@ -458,20 +367,17 @@ impl<'hir> Map<'hir> {
     /// This should only be used for determining the context of a body, a return
     /// value of `Some` does not always suggest that the owner of the body is `const`,
     /// just that it has to be checked as if it were.
-    pub fn body_const_context(self, def_id: LocalDefId) -> Option<ConstContext> {
+    pub fn body_const_context(self, def_id: impl Into<DefId>) -> Option<ConstContext> {
+        let def_id = def_id.into();
         let ccx = match self.body_owner_kind(def_id) {
             BodyOwnerKind::Const { inline } => ConstContext::Const { inline },
             BodyOwnerKind::Static(mt) => ConstContext::Static(mt),
 
-            BodyOwnerKind::Fn if self.tcx.is_constructor(def_id.to_def_id()) => return None,
-            BodyOwnerKind::Fn | BodyOwnerKind::Closure
-                if self.tcx.is_const_fn_raw(def_id.to_def_id()) =>
-            {
-                ConstContext::ConstFn
-            }
-            BodyOwnerKind::Fn if self.tcx.is_const_default_method(def_id.to_def_id()) => {
+            BodyOwnerKind::Fn if self.tcx.is_constructor(def_id) => return None,
+            BodyOwnerKind::Fn | BodyOwnerKind::Closure if self.tcx.is_const_fn_raw(def_id) => {
                 ConstContext::ConstFn
             }
+            BodyOwnerKind::Fn if self.tcx.is_const_default_method(def_id) => ConstContext::ConstFn,
             BodyOwnerKind::Fn | BodyOwnerKind::Closure => return None,
         };
 
@@ -531,9 +437,7 @@ impl<'hir> Map<'hir> {
     pub fn get_module(self, module: LocalModDefId) -> (&'hir Mod<'hir>, Span, HirId) {
         let hir_id = HirId::make_owner(module.to_local_def_id());
         match self.tcx.hir_owner(hir_id.owner).map(|o| o.node) {
-            Some(OwnerNode::Item(&Item { span, kind: ItemKind::Mod(ref m), .. })) => {
-                (m, span, hir_id)
-            }
+            Some(OwnerNode::Item(&Item { span, kind: ItemKind::Mod(m), .. })) => (m, span, hir_id),
             Some(OwnerNode::Crate(item)) => (item, item.spans.inner_span, hir_id),
             node => panic!("not a module: {node:?}"),
         }
@@ -654,7 +558,7 @@ impl<'hir> Map<'hir> {
     /// until the crate root is reached. Prefer this over your own loop using `parent_id`.
     #[inline]
     pub fn parent_iter(self, current_id: HirId) -> impl Iterator<Item = (HirId, Node<'hir>)> {
-        self.parent_id_iter(current_id).filter_map(move |id| Some((id, self.find(id)?)))
+        self.parent_id_iter(current_id).filter_map(move |id| Some((id, self.tcx.opt_hir_node(id)?)))
     }
 
     /// Returns an iterator for the nodes in the ancestor tree of the `current_id`
@@ -706,7 +610,7 @@ impl<'hir> Map<'hir> {
     pub fn get_return_block(self, id: HirId) -> Option<HirId> {
         let mut iter = self.parent_iter(id).peekable();
         let mut ignore_tail = false;
-        if let Some(Node::Expr(Expr { kind: ExprKind::Ret(_), .. })) = self.find(id) {
+        if let Some(Node::Expr(Expr { kind: ExprKind::Ret(_), .. })) = self.tcx.opt_hir_node(id) {
             // When dealing with `return` statements, we don't care about climbing only tail
             // expressions.
             ignore_tail = true;
@@ -814,7 +718,7 @@ impl<'hir> Map<'hir> {
         let mut scope = id;
         loop {
             scope = self.get_enclosing_scope(scope).unwrap_or(CRATE_HIR_ID);
-            if scope == CRATE_HIR_ID || !matches!(self.get(scope), Node::Block(_)) {
+            if scope == CRATE_HIR_ID || !matches!(self.tcx.hir_node(scope), Node::Block(_)) {
                 return scope;
             }
         }
@@ -870,7 +774,7 @@ impl<'hir> Map<'hir> {
     }
 
     pub fn expect_variant(self, id: HirId) -> &'hir Variant<'hir> {
-        match self.find(id) {
+        match self.tcx.opt_hir_node(id) {
             Some(Node::Variant(variant)) => variant,
             _ => bug!("expected variant, found {}", self.node_to_string(id)),
         }
@@ -889,7 +793,7 @@ impl<'hir> Map<'hir> {
     }
 
     pub fn expect_expr(self, id: HirId) -> &'hir Expr<'hir> {
-        match self.find(id) {
+        match self.tcx.opt_hir_node(id) {
             Some(Node::Expr(expr)) => expr,
             _ => bug!("expected expr, found {}", self.node_to_string(id)),
         }
@@ -897,7 +801,7 @@ impl<'hir> Map<'hir> {
 
     #[inline]
     fn opt_ident(self, id: HirId) -> Option<Ident> {
-        match self.get(id) {
+        match self.tcx.opt_hir_node(id)? {
             Node::Pat(&Pat { kind: PatKind::Binding(_, _, ident, _), .. }) => Some(ident),
             // A `Ctor` doesn't have an identifier itself, but its parent
             // struct/variant does. Compare with `hir::Map::opt_span`.
@@ -966,7 +870,7 @@ impl<'hir> Map<'hir> {
             }
         }
 
-        let span = match self.find(hir_id)? {
+        let span = match self.tcx.opt_hir_node(hir_id)? {
             // Function-like.
             Node::Item(Item { kind: ItemKind::Fn(sig, ..), span: outer_span, .. })
             | Node::TraitItem(TraitItem {
@@ -1056,7 +960,7 @@ impl<'hir> Map<'hir> {
     /// Like `hir.span()`, but includes the body of items
     /// (instead of just the item header)
     pub fn span_with_body(self, hir_id: HirId) -> Span {
-        match self.get(hir_id) {
+        match self.tcx.hir_node(hir_id) {
             Node::Param(param) => param.span,
             Node::Item(item) => item.span,
             Node::ForeignItem(foreign_item) => foreign_item.span,
@@ -1151,7 +1055,7 @@ impl<'hir> Map<'hir> {
 
 impl<'hir> intravisit::Map<'hir> for Map<'hir> {
     fn find(&self, hir_id: HirId) -> Option<Node<'hir>> {
-        (*self).find(hir_id)
+        self.tcx.opt_hir_node(hir_id)
     }
 
     fn body(&self, id: BodyId) -> &'hir Body<'hir> {
@@ -1238,7 +1142,10 @@ pub(super) fn crate_hash(tcx: TyCtxt<'_>, _: LocalCrate) -> Svh {
         tcx.sess.opts.dep_tracking_hash(true).hash_stable(&mut hcx, &mut stable_hasher);
         tcx.stable_crate_id(LOCAL_CRATE).hash_stable(&mut hcx, &mut stable_hasher);
         // Hash visibility information since it does not appear in HIR.
-        resolutions.visibilities.hash_stable(&mut hcx, &mut stable_hasher);
+        // FIXME: Figure out how to remove `visibilities_for_hashing` by hashing visibilities on
+        // the fly in the resolver, storing only their accumulated hash in `ResolverGlobalCtxt`,
+        // and combining it with other hashes here.
+        resolutions.visibilities_for_hashing.hash_stable(&mut hcx, &mut stable_hasher);
         stable_hasher.finish()
     });
 
@@ -1265,7 +1172,7 @@ fn hir_id_to_string(map: Map<'_>, id: HirId) -> String {
     let span_str = || map.tcx.sess.source_map().span_to_snippet(map.span(id)).unwrap_or_default();
     let node_str = |prefix| format!("{id} ({prefix} `{}`)", span_str());
 
-    match map.find(id) {
+    match map.tcx.opt_hir_node(id) {
         Some(Node::Item(item)) => {
             let item_str = match item.kind {
                 ItemKind::ExternCrate(..) => "extern crate",
@@ -1278,7 +1185,7 @@ fn hir_id_to_string(map: Map<'_>, id: HirId) -> String {
                 ItemKind::ForeignMod { .. } => "foreign mod",
                 ItemKind::GlobalAsm(..) => "global asm",
                 ItemKind::TyAlias(..) => "ty",
-                ItemKind::OpaqueTy(ref opaque) => {
+                ItemKind::OpaqueTy(opaque) => {
                     if opaque.in_trait {
                         "opaque type in trait"
                     } else {
@@ -1314,10 +1221,10 @@ fn hir_id_to_string(map: Map<'_>, id: HirId) -> String {
 
             format!("{id} ({kind} `{}` in {})", ti.ident, path_str(ti.owner_id.def_id))
         }
-        Some(Node::Variant(ref variant)) => {
+        Some(Node::Variant(variant)) => {
             format!("{id} (variant `{}` in {})", variant.ident, path_str(variant.def_id))
         }
-        Some(Node::Field(ref field)) => {
+        Some(Node::Field(field)) => {
             format!("{id} (field `{}` in {})", field.ident, path_str(field.def_id))
         }
         Some(Node::AnonConst(_)) => node_str("const"),
@@ -1341,7 +1248,7 @@ fn hir_id_to_string(map: Map<'_>, id: HirId) -> String {
             ctor.ctor_def_id().map_or("<missing path>".into(), |def_id| path_str(def_id)),
         ),
         Some(Node::Lifetime(_)) => node_str("lifetime"),
-        Some(Node::GenericParam(ref param)) => {
+        Some(Node::GenericParam(param)) => {
             format!("{id} (generic_param {})", path_str(param.def_id))
         }
         Some(Node::Crate(..)) => String::from("(root_crate)"),
diff --git a/compiler/rustc_middle/src/hir/mod.rs b/compiler/rustc_middle/src/hir/mod.rs
index f28ec7711..af99c7d55 100644
--- a/compiler/rustc_middle/src/hir/mod.rs
+++ b/compiler/rustc_middle/src/hir/mod.rs
@@ -166,7 +166,7 @@ pub fn provide(providers: &mut Providers) {
     providers.hir_owner_parent = |tcx, id| {
         // Accessing the local_parent is ok since its value is hashed as part of `id`'s DefPathHash.
         tcx.opt_local_parent(id.def_id).map_or(CRATE_HIR_ID, |parent| {
-            let mut parent_hir_id = tcx.hir().local_def_id_to_hir_id(parent);
+            let mut parent_hir_id = tcx.local_def_id_to_hir_id(parent);
             parent_hir_id.local_id =
                 tcx.hir_crate(()).owners[parent_hir_id.owner.def_id].unwrap().parenting[&id.def_id];
             parent_hir_id
@@ -176,16 +176,16 @@ pub fn provide(providers: &mut Providers) {
         tcx.hir_crate(()).owners[id.def_id].as_owner().map_or(AttributeMap::EMPTY, |o| &o.attrs)
     };
     providers.def_span = |tcx, def_id| {
-        let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
+        let hir_id = tcx.local_def_id_to_hir_id(def_id);
         tcx.hir().opt_span(hir_id).unwrap_or(DUMMY_SP)
     };
     providers.def_ident_span = |tcx, def_id| {
-        let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
+        let hir_id = tcx.local_def_id_to_hir_id(def_id);
         tcx.hir().opt_ident_span(hir_id)
     };
     providers.fn_arg_names = |tcx, def_id| {
         let hir = tcx.hir();
-        let hir_id = hir.local_def_id_to_hir_id(def_id);
+        let hir_id = tcx.local_def_id_to_hir_id(def_id);
         if let Some(body_id) = hir.maybe_body_owned_by(def_id) {
             tcx.arena.alloc_from_iter(hir.body_param_names(body_id))
         } else if let Node::TraitItem(&TraitItem {
@@ -195,14 +195,13 @@ pub fn provide(providers: &mut Providers) {
         | Node::ForeignItem(&ForeignItem {
             kind: ForeignItemKind::Fn(_, idents, _),
             ..
-        }) = hir.get(hir_id)
+        }) = tcx.hir_node(hir_id)
         {
             idents
         } else {
             span_bug!(hir.span(hir_id), "fn_arg_names: unexpected item {:?}", def_id);
         }
     };
-    providers.opt_def_kind = |tcx, def_id| tcx.hir().opt_def_kind(def_id);
     providers.all_local_trait_impls = |tcx, ()| &tcx.resolutions(()).trait_impls;
     providers.expn_that_defined =
         |tcx, id| tcx.resolutions(()).expn_that_defined.get(&id).copied().unwrap_or(ExpnId::root());
diff --git a/compiler/rustc_middle/src/infer/canonical.rs b/compiler/rustc_middle/src/infer/canonical.rs
index 64b63f4c5..e544c2a26 100644
--- a/compiler/rustc_middle/src/infer/canonical.rs
+++ b/compiler/rustc_middle/src/infer/canonical.rs
@@ -21,18 +21,25 @@
 //!
 //! [c]: https://rust-lang.github.io/chalk/book/canonical_queries/canonicalization.html
 
+use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::sync::Lock;
 use rustc_macros::HashStable;
 use rustc_type_ir::Canonical as IrCanonical;
+use rustc_type_ir::CanonicalVarInfo as IrCanonicalVarInfo;
+pub use rustc_type_ir::{CanonicalTyVarKind, CanonicalVarKind};
 use smallvec::SmallVec;
+use std::collections::hash_map::Entry;
 use std::ops::Index;
 
 use crate::infer::MemberConstraint;
 use crate::mir::ConstraintCategory;
 use crate::ty::GenericArg;
-use crate::ty::{self, BoundVar, List, Region, Ty, TyCtxt};
+use crate::ty::{self, BoundVar, List, Region, Ty, TyCtxt, TypeFlags, TypeVisitableExt};
 
 pub type Canonical<'tcx, V> = IrCanonical<TyCtxt<'tcx>, V>;
 
+pub type CanonicalVarInfo<'tcx> = IrCanonicalVarInfo<TyCtxt<'tcx>>;
+
 pub type CanonicalVarInfos<'tcx> = &'tcx List<CanonicalVarInfo<'tcx>>;
 
 impl<'tcx> ty::TypeFoldable<TyCtxt<'tcx>> for CanonicalVarInfos<'tcx> {
@@ -63,7 +70,7 @@ impl CanonicalVarValues<'_> {
     pub fn is_identity(&self) -> bool {
         self.var_values.iter().enumerate().all(|(bv, arg)| match arg.unpack() {
             ty::GenericArgKind::Lifetime(r) => {
-                matches!(*r, ty::ReLateBound(ty::INNERMOST, br) if br.var.as_usize() == bv)
+                matches!(*r, ty::ReBound(ty::INNERMOST, br) if br.var.as_usize() == bv)
             }
             ty::GenericArgKind::Type(ty) => {
                 matches!(*ty.kind(), ty::Bound(ty::INNERMOST, bt) if bt.var.as_usize() == bv)
@@ -79,7 +86,7 @@ impl CanonicalVarValues<'_> {
         for arg in self.var_values {
             match arg.unpack() {
                 ty::GenericArgKind::Lifetime(r) => {
-                    if let ty::ReLateBound(ty::INNERMOST, br) = *r
+                    if let ty::ReBound(ty::INNERMOST, br) = *r
                         && var == br.var
                     {
                         var = var + 1;
@@ -138,158 +145,6 @@ impl<'tcx> Default for OriginalQueryValues<'tcx> {
     }
 }
 
-/// Information about a canonical variable that is included with the
-/// canonical value. This is sufficient information for code to create
-/// a copy of the canonical value in some other inference context,
-/// with fresh inference variables replacing the canonical values.
-#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, TyDecodable, TyEncodable, HashStable)]
-#[derive(TypeFoldable, TypeVisitable)]
-pub struct CanonicalVarInfo<'tcx> {
-    pub kind: CanonicalVarKind<'tcx>,
-}
-
-impl<'tcx> CanonicalVarInfo<'tcx> {
-    pub fn universe(&self) -> ty::UniverseIndex {
-        self.kind.universe()
-    }
-
-    #[must_use]
-    pub fn with_updated_universe(self, ui: ty::UniverseIndex) -> CanonicalVarInfo<'tcx> {
-        CanonicalVarInfo { kind: self.kind.with_updated_universe(ui) }
-    }
-
-    pub fn is_existential(&self) -> bool {
-        match self.kind {
-            CanonicalVarKind::Ty(_) => true,
-            CanonicalVarKind::PlaceholderTy(_) => false,
-            CanonicalVarKind::Region(_) => true,
-            CanonicalVarKind::PlaceholderRegion(..) => false,
-            CanonicalVarKind::Const(..) => true,
-            CanonicalVarKind::PlaceholderConst(_, _) => false,
-            CanonicalVarKind::Effect => true,
-        }
-    }
-
-    pub fn is_region(&self) -> bool {
-        match self.kind {
-            CanonicalVarKind::Region(_) | CanonicalVarKind::PlaceholderRegion(_) => true,
-            CanonicalVarKind::Ty(_)
-            | CanonicalVarKind::PlaceholderTy(_)
-            | CanonicalVarKind::Const(_, _)
-            | CanonicalVarKind::PlaceholderConst(_, _)
-            | CanonicalVarKind::Effect => false,
-        }
-    }
-
-    pub fn expect_placeholder_index(self) -> usize {
-        match self.kind {
-            CanonicalVarKind::Ty(_)
-            | CanonicalVarKind::Region(_)
-            | CanonicalVarKind::Const(_, _)
-            | CanonicalVarKind::Effect => bug!("expected placeholder: {self:?}"),
-
-            CanonicalVarKind::PlaceholderRegion(placeholder) => placeholder.bound.var.as_usize(),
-            CanonicalVarKind::PlaceholderTy(placeholder) => placeholder.bound.var.as_usize(),
-            CanonicalVarKind::PlaceholderConst(placeholder, _) => placeholder.bound.as_usize(),
-        }
-    }
-}
-
-/// Describes the "kind" of the canonical variable. This is a "kind"
-/// in the type-theory sense of the term -- i.e., a "meta" type system
-/// that analyzes type-like values.
-#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, TyDecodable, TyEncodable, HashStable)]
-#[derive(TypeFoldable, TypeVisitable)]
-pub enum CanonicalVarKind<'tcx> {
-    /// Some kind of type inference variable.
-    Ty(CanonicalTyVarKind),
-
-    /// A "placeholder" that represents "any type".
-    PlaceholderTy(ty::PlaceholderType),
-
-    /// Region variable `'?R`.
-    Region(ty::UniverseIndex),
-
-    /// A "placeholder" that represents "any region". Created when you
-    /// are solving a goal like `for<'a> T: Foo<'a>` to represent the
-    /// bound region `'a`.
-    PlaceholderRegion(ty::PlaceholderRegion),
-
-    /// Some kind of const inference variable.
-    Const(ty::UniverseIndex, Ty<'tcx>),
-
-    /// Effect variable `'?E`.
-    Effect,
-
-    /// A "placeholder" that represents "any const".
-    PlaceholderConst(ty::PlaceholderConst, Ty<'tcx>),
-}
-
-impl<'tcx> CanonicalVarKind<'tcx> {
-    pub fn universe(self) -> ty::UniverseIndex {
-        match self {
-            CanonicalVarKind::Ty(CanonicalTyVarKind::General(ui)) => ui,
-            CanonicalVarKind::Ty(CanonicalTyVarKind::Float | CanonicalTyVarKind::Int) => {
-                ty::UniverseIndex::ROOT
-            }
-            CanonicalVarKind::Effect => ty::UniverseIndex::ROOT,
-            CanonicalVarKind::PlaceholderTy(placeholder) => placeholder.universe,
-            CanonicalVarKind::Region(ui) => ui,
-            CanonicalVarKind::PlaceholderRegion(placeholder) => placeholder.universe,
-            CanonicalVarKind::Const(ui, _) => ui,
-            CanonicalVarKind::PlaceholderConst(placeholder, _) => placeholder.universe,
-        }
-    }
-
-    /// Replaces the universe of this canonical variable with `ui`.
-    ///
-    /// In case this is a float or int variable, this causes an ICE if
-    /// the updated universe is not the root.
-    pub fn with_updated_universe(self, ui: ty::UniverseIndex) -> CanonicalVarKind<'tcx> {
-        match self {
-            CanonicalVarKind::Ty(CanonicalTyVarKind::General(_)) => {
-                CanonicalVarKind::Ty(CanonicalTyVarKind::General(ui))
-            }
-            CanonicalVarKind::Ty(CanonicalTyVarKind::Int | CanonicalTyVarKind::Float)
-            | CanonicalVarKind::Effect => {
-                assert_eq!(ui, ty::UniverseIndex::ROOT);
-                self
-            }
-            CanonicalVarKind::PlaceholderTy(placeholder) => {
-                CanonicalVarKind::PlaceholderTy(ty::Placeholder { universe: ui, ..placeholder })
-            }
-            CanonicalVarKind::Region(_) => CanonicalVarKind::Region(ui),
-            CanonicalVarKind::PlaceholderRegion(placeholder) => {
-                CanonicalVarKind::PlaceholderRegion(ty::Placeholder { universe: ui, ..placeholder })
-            }
-            CanonicalVarKind::Const(_, ty) => CanonicalVarKind::Const(ui, ty),
-            CanonicalVarKind::PlaceholderConst(placeholder, ty) => {
-                CanonicalVarKind::PlaceholderConst(
-                    ty::Placeholder { universe: ui, ..placeholder },
-                    ty,
-                )
-            }
-        }
-    }
-}
-
-/// Rust actually has more than one category of type variables;
-/// notably, the type variables we create for literals (e.g., 22 or
-/// 22.) can only be instantiated with integral/float types (e.g.,
-/// usize or f32). In order to faithfully reproduce a type, we need to
-/// know what set of types a given type variable can be unified with.
-#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, TyDecodable, TyEncodable, HashStable)]
-pub enum CanonicalTyVarKind {
-    /// General type variable `?T` that can be unified with arbitrary types.
-    General(ty::UniverseIndex),
-
-    /// Integral type variable `?I` (that can only be unified with integral types).
-    Int,
-
-    /// Floating-point type variable `?F` (that can only be unified with float types).
-    Float,
-}
-
 /// After we execute a query with a canonicalized key, we get back a
 /// `Canonical<QueryResponse<..>>`. You can use
 /// `instantiate_query_result` to access the data in this result.
@@ -366,7 +221,6 @@ pub type QueryOutlivesConstraint<'tcx> =
 
 TrivialTypeTraversalImpls! {
     crate::infer::canonical::Certainty,
-    crate::infer::canonical::CanonicalTyVarKind,
 }
 
 impl<'tcx> CanonicalVarValues<'tcx> {
@@ -389,7 +243,7 @@ impl<'tcx> CanonicalVarValues<'tcx> {
                                 var: ty::BoundVar::from_usize(i),
                                 kind: ty::BrAnon,
                             };
-                            ty::Region::new_late_bound(tcx, ty::INNERMOST, br).into()
+                            ty::Region::new_bound(tcx, ty::INNERMOST, br).into()
                         }
                         CanonicalVarKind::Effect => ty::Const::new_bound(
                             tcx,
@@ -440,3 +294,62 @@ impl<'tcx> Index<BoundVar> for CanonicalVarValues<'tcx> {
         &self.var_values[value.as_usize()]
     }
 }
+
+#[derive(Default)]
+pub struct CanonicalParamEnvCache<'tcx> {
+    map: Lock<
+        FxHashMap<
+            ty::ParamEnv<'tcx>,
+            (Canonical<'tcx, ty::ParamEnv<'tcx>>, &'tcx [GenericArg<'tcx>]),
+        >,
+    >,
+}
+
+impl<'tcx> CanonicalParamEnvCache<'tcx> {
+    /// Gets the cached canonical form of `key` or executes
+    /// `canonicalize_op` and caches the result if not present.
+    ///
+    /// `canonicalize_op` is intentionally not allowed to be a closure to
+    /// statically prevent it from capturing `InferCtxt` and resolving
+    /// inference variables, which invalidates the cache.
+    pub fn get_or_insert(
+        &self,
+        tcx: TyCtxt<'tcx>,
+        key: ty::ParamEnv<'tcx>,
+        state: &mut OriginalQueryValues<'tcx>,
+        canonicalize_op: fn(
+            TyCtxt<'tcx>,
+            ty::ParamEnv<'tcx>,
+            &mut OriginalQueryValues<'tcx>,
+        ) -> Canonical<'tcx, ty::ParamEnv<'tcx>>,
+    ) -> Canonical<'tcx, ty::ParamEnv<'tcx>> {
+        if !key.has_type_flags(
+            TypeFlags::HAS_INFER | TypeFlags::HAS_PLACEHOLDER | TypeFlags::HAS_FREE_REGIONS,
+        ) {
+            return Canonical {
+                max_universe: ty::UniverseIndex::ROOT,
+                variables: List::empty(),
+                value: key,
+            };
+        }
+
+        assert_eq!(state.var_values.len(), 0);
+        assert_eq!(state.universe_map.len(), 1);
+        debug_assert_eq!(&*state.universe_map, &[ty::UniverseIndex::ROOT]);
+
+        match self.map.borrow().entry(key) {
+            Entry::Occupied(e) => {
+                let (canonical, var_values) = e.get();
+                state.var_values.extend_from_slice(var_values);
+                *canonical
+            }
+            Entry::Vacant(e) => {
+                let canonical = canonicalize_op(tcx, key, state);
+                let OriginalQueryValues { var_values, universe_map } = state;
+                assert_eq!(universe_map.len(), 1);
+                e.insert((canonical, tcx.arena.alloc_slice(var_values)));
+                canonical
+            }
+        }
+    }
+}
diff --git a/compiler/rustc_middle/src/infer/unify_key.rs b/compiler/rustc_middle/src/infer/unify_key.rs
index 041a63776..6e50e8940 100644
--- a/compiler/rustc_middle/src/infer/unify_key.rs
+++ b/compiler/rustc_middle/src/infer/unify_key.rs
@@ -58,18 +58,17 @@ impl<'tcx> UnifyValue for UnifiedRegion<'tcx> {
 
     fn unify_values(value1: &Self, value2: &Self) -> Result<Self, NoError> {
         // We pick the value of the least universe because it is compatible with more variables.
-        // This is *not* necessary for soundness, but it allows more region variables to be
-        // resolved to the said value.
+        // This is *not* necessary for completeness.
         #[cold]
         fn min_universe<'tcx>(r1: Region<'tcx>, r2: Region<'tcx>) -> Region<'tcx> {
             cmp::min_by_key(r1, r2, |r| match r.kind() {
                 ty::ReStatic
                 | ty::ReErased
-                | ty::ReFree(..)
-                | ty::ReEarlyBound(..)
+                | ty::ReLateParam(..)
+                | ty::ReEarlyParam(..)
                 | ty::ReError(_) => ty::UniverseIndex::ROOT,
                 ty::RePlaceholder(placeholder) => placeholder.universe,
-                ty::ReVar(..) | ty::ReLateBound(..) => bug!("not a universal region"),
+                ty::ReVar(..) | ty::ReBound(..) => bug!("not a universal region"),
             })
         }
 
diff --git a/compiler/rustc_middle/src/lib.rs b/compiler/rustc_middle/src/lib.rs
index 448a3029a..3206c6cf6 100644
--- a/compiler/rustc_middle/src/lib.rs
+++ b/compiler/rustc_middle/src/lib.rs
@@ -23,8 +23,8 @@
 //! This API is completely unstable and subject to change.
 
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
+#![doc(rust_logo)]
+#![feature(rustdoc_internals)]
 #![feature(allocator_api)]
 #![feature(array_windows)]
 #![feature(assert_matches)]
@@ -32,8 +32,7 @@
 #![feature(core_intrinsics)]
 #![feature(discriminant_kind)]
 #![feature(exhaustive_patterns)]
-#![cfg_attr(bootstrap, feature(generators))]
-#![cfg_attr(not(bootstrap), feature(coroutines))]
+#![feature(coroutines)]
 #![feature(get_mut_unchecked)]
 #![feature(if_let_guard)]
 #![feature(inline_const)]
@@ -59,7 +58,6 @@
 #![feature(extract_if)]
 #![feature(intra_doc_pointers)]
 #![feature(yeet_expr)]
-#![feature(result_option_inspect)]
 #![feature(const_option)]
 #![feature(trait_alias)]
 #![feature(ptr_alignment_type)]
@@ -79,9 +77,6 @@ extern crate tracing;
 #[macro_use]
 extern crate smallvec;
 
-use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
-use rustc_fluent_macro::fluent_messages;
-
 #[cfg(test)]
 mod tests;
 
@@ -112,4 +107,4 @@ pub mod dep_graph;
 // Allows macros to refer to this crate as `::rustc_middle`
 extern crate self as rustc_middle;
 
-fluent_messages! { "../messages.ftl" }
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
diff --git a/compiler/rustc_middle/src/lint.rs b/compiler/rustc_middle/src/lint.rs
index eada116f8..c49c4ee81 100644
--- a/compiler/rustc_middle/src/lint.rs
+++ b/compiler/rustc_middle/src/lint.rs
@@ -253,29 +253,10 @@ pub fn explain_lint_level_source(
 /// - [`TyCtxt::struct_lint_node`]
 /// - `LintContext::lookup`
 ///
-/// ## `decorate` signature
+/// ## `decorate`
 ///
-/// The return value of `decorate` is ignored by this function. So what is the
-/// point of returning `&'b mut DiagnosticBuilder<'a, ()>`?
-///
-/// There are 2 reasons for this signature.
-///
-/// First of all, it prevents accidental use of `.emit()` -- it's clear that the
-/// builder will be later used and shouldn't be emitted right away (this is
-/// especially important because the old API expected you to call `.emit()` in
-/// the closure).
-///
-/// Second of all, it makes the most common case of adding just a single label
-/// /suggestion much nicer, since [`DiagnosticBuilder`] methods return
-/// `&mut DiagnosticBuilder`, you can just chain methods, without needed
-/// awkward `{ ...; }`:
-/// ```ignore pseudo-code
-/// struct_lint_level(
-///     ...,
-///     |lint| lint.span_label(sp, "lbl")
-///     //          ^^^^^^^^^^^^^^^^^^^^^ returns `&mut DiagnosticBuilder` by default
-/// )
-/// ```
+/// It is not intended to call `emit`/`cancel` on the `DiagnosticBuilder` passed
+/// in the `decorate` callback.
 #[track_caller]
 pub fn struct_lint_level(
     sess: &Session,
@@ -284,9 +265,7 @@ pub fn struct_lint_level(
     src: LintLevelSource,
     span: Option<MultiSpan>,
     msg: impl Into<DiagnosticMessage>,
-    decorate: impl for<'a, 'b> FnOnce(
-        &'b mut DiagnosticBuilder<'a, ()>,
-    ) -> &'b mut DiagnosticBuilder<'a, ()>,
+    decorate: impl for<'a, 'b> FnOnce(&'b mut DiagnosticBuilder<'a, ()>),
 ) {
     // Avoid codegen bloat from monomorphization by immediately doing dyn dispatch of `decorate` to
     // the "real" work.
@@ -298,12 +277,7 @@ pub fn struct_lint_level(
         src: LintLevelSource,
         span: Option<MultiSpan>,
         msg: impl Into<DiagnosticMessage>,
-        decorate: Box<
-            dyn '_
-                + for<'a, 'b> FnOnce(
-                    &'b mut DiagnosticBuilder<'a, ()>,
-                ) -> &'b mut DiagnosticBuilder<'a, ()>,
-        >,
+        decorate: Box<dyn '_ + for<'a, 'b> FnOnce(&'b mut DiagnosticBuilder<'a, ()>)>,
     ) {
         // Check for future incompatibility lints and issue a stronger warning.
         let future_incompatible = lint.future_incompatible;
@@ -334,7 +308,7 @@ pub fn struct_lint_level(
             (Level::Expect(expect_id), _) => {
                 // This case is special as we actually allow the lint itself in this context, but
                 // we can't return early like in the case for `Level::Allow` because we still
-                // need the lint diagnostic to be emitted to `rustc_error::HandlerInner`.
+                // need the lint diagnostic to be emitted to `rustc_error::DiagCtxtInner`.
                 //
                 // We can also not mark the lint expectation as fulfilled here right away, as it
                 // can still be cancelled in the decorate function. All of this means that we simply
@@ -350,11 +324,11 @@ pub fn struct_lint_level(
             (Level::Warn | Level::ForceWarn(None), Some(span)) => sess.struct_span_warn(span, ""),
             (Level::Warn | Level::ForceWarn(None), None) => sess.struct_warn(""),
             (Level::Deny | Level::Forbid, Some(span)) => {
-                let mut builder = sess.diagnostic().struct_err_lint("");
+                let mut builder = sess.dcx().struct_err_lint("");
                 builder.set_span(span);
                 builder
             }
-            (Level::Deny | Level::Forbid, None) => sess.diagnostic().struct_err_lint(""),
+            (Level::Deny | Level::Forbid, None) => sess.dcx().struct_err_lint(""),
         };
 
         err.set_is_lint();
diff --git a/compiler/rustc_middle/src/macros.rs b/compiler/rustc_middle/src/macros.rs
index c1884bb80..8c1e58fef 100644
--- a/compiler/rustc_middle/src/macros.rs
+++ b/compiler/rustc_middle/src/macros.rs
@@ -4,9 +4,10 @@
 ///
 /// If you have a span available, you should use [`span_bug`] instead.
 ///
-/// If the bug should only be emitted when compilation didn't fail, [`Session::delay_span_bug`] may be useful.
+/// If the bug should only be emitted when compilation didn't fail, [`Session::span_delayed_bug`]
+/// may be useful.
 ///
-/// [`Session::delay_span_bug`]: rustc_session::Session::delay_span_bug
+/// [`Session::span_delayed_bug`]: rustc_session::Session::span_delayed_bug
 /// [`span_bug`]: crate::span_bug
 #[macro_export]
 macro_rules! bug {
@@ -23,9 +24,10 @@ macro_rules! bug {
 /// at the code the compiler was compiling when it ICEd. This is the preferred way to trigger
 /// ICEs.
 ///
-/// If the bug should only be emitted when compilation didn't fail, [`Session::delay_span_bug`] may be useful.
+/// If the bug should only be emitted when compilation didn't fail, [`Session::span_delayed_bug`]
+/// may be useful.
 ///
-/// [`Session::delay_span_bug`]: rustc_session::Session::delay_span_bug
+/// [`Session::span_delayed_bug`]: rustc_session::Session::span_delayed_bug
 #[macro_export]
 macro_rules! span_bug {
     ($span:expr, $msg:expr) => ({ $crate::util::bug::span_bug_fmt($span, ::std::format_args!($msg)) });
diff --git a/compiler/rustc_middle/src/middle/lang_items.rs b/compiler/rustc_middle/src/middle/lang_items.rs
index 9a633e04c..2899e629d 100644
--- a/compiler/rustc_middle/src/middle/lang_items.rs
+++ b/compiler/rustc_middle/src/middle/lang_items.rs
@@ -36,6 +36,17 @@ impl<'tcx> TyCtxt<'tcx> {
         }
     }
 
+    /// Given a [`ty::ClosureKind`], get the [`DefId`] of its corresponding `Fn`-family
+    /// trait, if it is defined.
+    pub fn fn_trait_kind_to_def_id(self, kind: ty::ClosureKind) -> Option<DefId> {
+        let items = self.lang_items();
+        match kind {
+            ty::ClosureKind::Fn => items.fn_trait(),
+            ty::ClosureKind::FnMut => items.fn_mut_trait(),
+            ty::ClosureKind::FnOnce => items.fn_once_trait(),
+        }
+    }
+
     /// Returns `true` if `id` is a `DefId` of [`Fn`], [`FnMut`] or [`FnOnce`] traits.
     pub fn is_fn_trait(self, id: DefId) -> bool {
         self.fn_trait_kind_from_def_id(id).is_some()
diff --git a/compiler/rustc_middle/src/middle/limits.rs b/compiler/rustc_middle/src/middle/limits.rs
index d4f023958..b29be92ae 100644
--- a/compiler/rustc_middle/src/middle/limits.rs
+++ b/compiler/rustc_middle/src/middle/limits.rs
@@ -8,7 +8,6 @@
 //! this via an attribute on the crate like `#![recursion_limit="22"]`. This pass
 //! just peeks and looks for that attribute.
 
-use crate::bug;
 use crate::error::LimitInvalid;
 use crate::query::Providers;
 use rustc_ast::Attribute;
diff --git a/compiler/rustc_middle/src/middle/mod.rs b/compiler/rustc_middle/src/middle/mod.rs
index 85c5af9ca..8c1b1ff12 100644
--- a/compiler/rustc_middle/src/middle/mod.rs
+++ b/compiler/rustc_middle/src/middle/mod.rs
@@ -7,22 +7,23 @@ pub mod lib_features {
     use rustc_data_structures::fx::FxHashMap;
     use rustc_span::{symbol::Symbol, Span};
 
-    #[derive(HashStable, Debug)]
+    #[derive(Copy, Clone, Debug, PartialEq, Eq)]
+    #[derive(HashStable, TyEncodable, TyDecodable)]
+    pub enum FeatureStability {
+        AcceptedSince(Symbol),
+        Unstable,
+    }
+
+    #[derive(HashStable, Debug, Default)]
     pub struct LibFeatures {
-        /// A map from feature to stabilisation version.
-        pub stable: FxHashMap<Symbol, (Symbol, Span)>,
-        pub unstable: FxHashMap<Symbol, Span>,
+        pub stability: FxHashMap<Symbol, (FeatureStability, Span)>,
     }
 
     impl LibFeatures {
-        pub fn to_vec(&self) -> Vec<(Symbol, Option<Symbol>)> {
-            let mut all_features: Vec<_> = self
-                .stable
-                .iter()
-                .map(|(f, (s, _))| (*f, Some(*s)))
-                .chain(self.unstable.keys().map(|f| (*f, None)))
-                .collect();
-            all_features.sort_unstable_by(|a, b| a.0.as_str().partial_cmp(b.0.as_str()).unwrap());
+        pub fn to_vec(&self) -> Vec<(Symbol, FeatureStability)> {
+            let mut all_features: Vec<_> =
+                self.stability.iter().map(|(&sym, &(stab, _))| (sym, stab)).collect();
+            all_features.sort_unstable_by(|(a, _), (b, _)| a.as_str().cmp(b.as_str()));
             all_features
         }
     }
diff --git a/compiler/rustc_middle/src/middle/region.rs b/compiler/rustc_middle/src/middle/region.rs
index 56fed05c6..3f6dc2b9f 100644
--- a/compiler/rustc_middle/src/middle/region.rs
+++ b/compiler/rustc_middle/src/middle/region.rs
@@ -77,7 +77,7 @@ use std::ops::Deref;
 /// picture, but rather the ending point.
 //
 // FIXME(pnkfelix): this currently derives `PartialOrd` and `Ord` to
-// placate the same deriving in `ty::FreeRegion`, but we may want to
+// placate the same deriving in `ty::LateParamRegion`, but we may want to
 // actually attach a more meaningful ordering to scopes than the one
 // generated via deriving here.
 #[derive(Clone, PartialEq, PartialOrd, Eq, Ord, Hash, Copy, TyEncodable, TyDecodable)]
@@ -148,6 +148,8 @@ rustc_index::newtype_index! {
     /// * The subscope with `first_statement_index == 1` is scope of `c`,
     ///   and thus does not include EXPR_2, but covers the `...`.
     #[derive(HashStable)]
+    #[encodable]
+    #[orderable]
     pub struct FirstStatementIndex {}
 }
 
@@ -178,7 +180,7 @@ impl Scope {
         };
         let span = tcx.hir().span(hir_id);
         if let ScopeData::Remainder(first_statement_index) = self.data {
-            if let Node::Block(ref blk) = tcx.hir().get(hir_id) {
+            if let Node::Block(blk) = tcx.hir_node(hir_id) {
                 // Want span for scope starting after the
                 // indexed statement and ending at end of
                 // `blk`; reuse span of `blk` and shift `lo`
@@ -347,10 +349,6 @@ impl ScopeTree {
         }
     }
 
-    pub fn opt_destruction_scope(&self, n: hir::ItemLocalId) -> Option<Scope> {
-        self.destruction_scopes.get(&n).cloned()
-    }
-
     pub fn record_var_scope(&mut self, var: hir::ItemLocalId, lifetime: Scope) {
         debug!("record_var_scope(sub={:?}, sup={:?})", var, lifetime);
         assert!(var != lifetime.item_local_id());
diff --git a/compiler/rustc_middle/src/middle/stability.rs b/compiler/rustc_middle/src/middle/stability.rs
index f7a55fa95..0cba6d5b5 100644
--- a/compiler/rustc_middle/src/middle/stability.rs
+++ b/compiler/rustc_middle/src/middle/stability.rs
@@ -219,11 +219,10 @@ fn late_report_deprecation(
     }
     let method_span = method_span.unwrap_or(span);
     tcx.struct_span_lint_hir(lint, hir_id, method_span, message, |diag| {
-        if let hir::Node::Expr(_) = tcx.hir().get(hir_id) {
+        if let hir::Node::Expr(_) = tcx.hir_node(hir_id) {
             let kind = tcx.def_descr(def_id);
             deprecation_suggestion(diag, kind, suggestion, method_span);
         }
-        diag
     });
 }
 
@@ -566,7 +565,7 @@ impl<'tcx> TyCtxt<'tcx> {
             |span, def_id| {
                 // The API could be uncallable for other reasons, for example when a private module
                 // was referenced.
-                self.sess.delay_span_bug(span, format!("encountered unmarked API: {def_id:?}"));
+                self.sess.span_delayed_bug(span, format!("encountered unmarked API: {def_id:?}"));
             },
         )
     }
@@ -587,7 +586,7 @@ impl<'tcx> TyCtxt<'tcx> {
         unmarked: impl FnOnce(Span, DefId),
     ) -> bool {
         let soft_handler = |lint, span, msg: String| {
-            self.struct_span_lint_hir(lint, id.unwrap_or(hir::CRATE_HIR_ID), span, msg, |lint| lint)
+            self.struct_span_lint_hir(lint, id.unwrap_or(hir::CRATE_HIR_ID), span, msg, |_| {})
         };
         let eval_result =
             self.eval_stability_allow_unstable(def_id, id, span, method_span, allow_unstable);
diff --git a/compiler/rustc_middle/src/mir/consts.rs b/compiler/rustc_middle/src/mir/consts.rs
index a9d09709e..2c38f998c 100644
--- a/compiler/rustc_middle/src/mir/consts.rs
+++ b/compiler/rustc_middle/src/mir/consts.rs
@@ -1,17 +1,15 @@
 use std::fmt::{self, Debug, Display, Formatter};
 
-use rustc_hir;
-use rustc_hir::def_id::{DefId, LocalDefId};
-use rustc_hir::{self as hir};
+use rustc_hir::def_id::DefId;
 use rustc_session::RemapFileNameExt;
 use rustc_span::Span;
 use rustc_target::abi::{HasDataLayout, Size};
 
 use crate::mir::interpret::{alloc_range, AllocId, ConstAllocation, ErrorHandled, Scalar};
 use crate::mir::{pretty_print_const_value, Promoted};
+use crate::ty::GenericArgsRef;
 use crate::ty::ScalarInt;
-use crate::ty::{self, print::pretty_print_const, List, Ty, TyCtxt};
-use crate::ty::{GenericArgs, GenericArgsRef};
+use crate::ty::{self, print::pretty_print_const, Ty, TyCtxt};
 
 ///////////////////////////////////////////////////////////////////////////
 /// Evaluated Constants
@@ -76,7 +74,7 @@ static_assert_size!(ConstValue<'_>, 24);
 
 impl<'tcx> ConstValue<'tcx> {
     #[inline]
-    pub fn try_to_scalar(&self) -> Option<Scalar<AllocId>> {
+    pub fn try_to_scalar(&self) -> Option<Scalar> {
         match *self {
             ConstValue::Indirect { .. } | ConstValue::Slice { .. } | ConstValue::ZeroSized => None,
             ConstValue::Scalar(val) => Some(val),
@@ -162,8 +160,8 @@ impl<'tcx> ConstValue<'tcx> {
                     return Some(&[]);
                 }
                 // Non-empty slice, must have memory. We know this is a relative pointer.
-                let (inner_alloc_id, offset) = ptr.into_parts();
-                let data = tcx.global_alloc(inner_alloc_id?).unwrap_memory();
+                let (inner_prov, offset) = ptr.into_parts();
+                let data = tcx.global_alloc(inner_prov?.alloc_id()).unwrap_memory();
                 (data, offset.bytes(), offset.bytes() + len)
             }
         };
@@ -220,6 +218,17 @@ pub enum Const<'tcx> {
 }
 
 impl<'tcx> Const<'tcx> {
+    pub fn identity_unevaluated(tcx: TyCtxt<'tcx>, def_id: DefId) -> ty::EarlyBinder<Const<'tcx>> {
+        ty::EarlyBinder::bind(Const::Unevaluated(
+            UnevaluatedConst {
+                def: def_id,
+                args: ty::GenericArgs::identity_for_item(tcx, def_id),
+                promoted: None,
+            },
+            tcx.type_of(def_id).skip_binder(),
+        ))
+    }
+
     #[inline(always)]
     pub fn ty(&self) -> Ty<'tcx> {
         match self {
@@ -399,101 +408,6 @@ impl<'tcx> Const<'tcx> {
         Self::Val(val, ty)
     }
 
-    /// Literals are converted to `Const::Val`, const generic parameters are eagerly
-    /// converted to a constant, everything else becomes `Unevaluated`.
-    #[instrument(skip(tcx), level = "debug", ret)]
-    pub fn from_anon_const(
-        tcx: TyCtxt<'tcx>,
-        def: LocalDefId,
-        param_env: ty::ParamEnv<'tcx>,
-    ) -> Self {
-        let body_id = match tcx.hir().get_by_def_id(def) {
-            hir::Node::AnonConst(ac) => ac.body,
-            _ => {
-                span_bug!(tcx.def_span(def), "from_anon_const can only process anonymous constants")
-            }
-        };
-
-        let expr = &tcx.hir().body(body_id).value;
-        debug!(?expr);
-
-        // Unwrap a block, so that e.g. `{ P }` is recognised as a parameter. Const arguments
-        // currently have to be wrapped in curly brackets, so it's necessary to special-case.
-        let expr = match &expr.kind {
-            hir::ExprKind::Block(block, _) if block.stmts.is_empty() && block.expr.is_some() => {
-                block.expr.as_ref().unwrap()
-            }
-            _ => expr,
-        };
-        debug!("expr.kind: {:?}", expr.kind);
-
-        let ty = tcx.type_of(def).instantiate_identity();
-        debug!(?ty);
-
-        // FIXME(const_generics): We currently have to special case parameters because `min_const_generics`
-        // does not provide the parents generics to anonymous constants. We still allow generic const
-        // parameters by themselves however, e.g. `N`. These constants would cause an ICE if we were to
-        // ever try to substitute the generic parameters in their bodies.
-        //
-        // While this doesn't happen as these constants are always used as `ty::ConstKind::Param`, it does
-        // cause issues if we were to remove that special-case and try to evaluate the constant instead.
-        use hir::{def::DefKind::ConstParam, def::Res, ExprKind, Path, QPath};
-        match expr.kind {
-            ExprKind::Path(QPath::Resolved(_, &Path { res: Res::Def(ConstParam, def_id), .. })) => {
-                // Find the name and index of the const parameter by indexing the generics of
-                // the parent item and construct a `ParamConst`.
-                let item_def_id = tcx.parent(def_id);
-                let generics = tcx.generics_of(item_def_id);
-                let index = generics.param_def_id_to_index[&def_id];
-                let name = tcx.item_name(def_id);
-                let ty_const = ty::Const::new_param(tcx, ty::ParamConst::new(index, name), ty);
-                debug!(?ty_const);
-
-                return Self::Ty(ty_const);
-            }
-            _ => {}
-        }
-
-        let hir_id = tcx.hir().local_def_id_to_hir_id(def);
-        let parent_args = if let Some(parent_hir_id) = tcx.hir().opt_parent_id(hir_id)
-            && let Some(parent_did) = parent_hir_id.as_owner()
-        {
-            GenericArgs::identity_for_item(tcx, parent_did)
-        } else {
-            List::empty()
-        };
-        debug!(?parent_args);
-
-        let did = def.to_def_id();
-        let child_args = GenericArgs::identity_for_item(tcx, did);
-        let args = tcx.mk_args_from_iter(parent_args.into_iter().chain(child_args.into_iter()));
-        debug!(?args);
-
-        let span = tcx.def_span(def);
-        let uneval = UnevaluatedConst::new(did, args);
-        debug!(?span, ?param_env);
-
-        match tcx.const_eval_resolve(param_env, uneval, Some(span)) {
-            Ok(val) => {
-                debug!("evaluated const value");
-                Self::Val(val, ty)
-            }
-            Err(_) => {
-                debug!("error encountered during evaluation");
-                // Error was handled in `const_eval_resolve`. Here we just create a
-                // new unevaluated const and error hard later in codegen
-                Self::Unevaluated(
-                    UnevaluatedConst {
-                        def: did,
-                        args: GenericArgs::identity_for_item(tcx, did),
-                        promoted: None,
-                    },
-                    ty,
-                )
-            }
-        }
-    }
-
     pub fn from_ty_const(c: ty::Const<'tcx>, tcx: TyCtxt<'tcx>) -> Self {
         match c.kind() {
             ty::ConstKind::Value(valtree) => {
@@ -592,7 +506,7 @@ impl<'tcx> TyCtxt<'tcx> {
         let caller = self.sess.source_map().lookup_char_pos(topmost.lo());
         self.const_caller_location(
             rustc_span::symbol::Symbol::intern(
-                &caller.file.name.for_codegen(&self.sess).to_string_lossy(),
+                &caller.file.name.for_codegen(self.sess).to_string_lossy(),
             ),
             caller.line as u32,
             caller.col_display as u32 + 1,
diff --git a/compiler/rustc_middle/src/mir/coverage.rs b/compiler/rustc_middle/src/mir/coverage.rs
index 08d377a86..ec5edceb2 100644
--- a/compiler/rustc_middle/src/mir/coverage.rs
+++ b/compiler/rustc_middle/src/mir/coverage.rs
@@ -17,6 +17,8 @@ rustc_index::newtype_index! {
     /// Note that LLVM handles counter IDs as `uint32_t`, so there is no need
     /// to use a larger representation on the Rust side.
     #[derive(HashStable)]
+    #[encodable]
+    #[orderable]
     #[max = 0xFFFF_FFFF]
     #[debug_format = "CounterId({})"]
     pub struct CounterId {}
@@ -37,6 +39,8 @@ rustc_index::newtype_index! {
     /// Note that LLVM handles expression IDs as `uint32_t`, so there is no need
     /// to use a larger representation on the Rust side.
     #[derive(HashStable)]
+    #[encodable]
+    #[orderable]
     #[max = 0xFFFF_FFFF]
     #[debug_format = "ExpressionId({})"]
     pub struct ExpressionId {}
@@ -72,6 +76,13 @@ impl Debug for CovTerm {
 
 #[derive(Clone, PartialEq, TyEncodable, TyDecodable, Hash, HashStable, TypeFoldable, TypeVisitable)]
 pub enum CoverageKind {
+    /// Marks a span that might otherwise not be represented in MIR, so that
+    /// coverage instrumentation can associate it with its enclosing block/BCB.
+    ///
+    /// Only used by the `InstrumentCoverage` pass, and has no effect during
+    /// codegen.
+    SpanMarker,
+
     /// Marks the point in MIR control flow represented by a coverage counter.
     ///
     /// This is eventually lowered to `llvm.instrprof.increment` in LLVM IR.
@@ -95,6 +106,7 @@ impl Debug for CoverageKind {
     fn fmt(&self, fmt: &mut Formatter<'_>) -> fmt::Result {
         use CoverageKind::*;
         match self {
+            SpanMarker => write!(fmt, "SpanMarker"),
             CounterIncrement { id } => write!(fmt, "CounterIncrement({:?})", id.index()),
             ExpressionUsed { id } => write!(fmt, "ExpressionUsed({:?})", id.index()),
         }
diff --git a/compiler/rustc_middle/src/mir/generic_graph.rs b/compiler/rustc_middle/src/mir/generic_graph.rs
index d1753427e..51e26ab79 100644
--- a/compiler/rustc_middle/src/mir/generic_graph.rs
+++ b/compiler/rustc_middle/src/mir/generic_graph.rs
@@ -1,7 +1,5 @@
 use gsgdt::{Edge, Graph, Node, NodeStyle};
-use rustc_hir::def_id::DefId;
 use rustc_middle::mir::*;
-use rustc_middle::ty::TyCtxt;
 
 /// Convert an MIR function into a gsgdt Graph
 pub fn mir_fn_to_generic_graph<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'_>) -> Graph {
diff --git a/compiler/rustc_middle/src/mir/graphviz.rs b/compiler/rustc_middle/src/mir/graphviz.rs
index 5c7de8644..96bef40da 100644
--- a/compiler/rustc_middle/src/mir/graphviz.rs
+++ b/compiler/rustc_middle/src/mir/graphviz.rs
@@ -1,9 +1,6 @@
 use gsgdt::GraphvizSettings;
 use rustc_graphviz as dot;
-use rustc_hir::def_id::DefId;
 use rustc_middle::mir::*;
-use rustc_middle::ty::{self, TyCtxt};
-use std::fmt::Debug;
 use std::io::{self, Write};
 
 use super::generic_graph::mir_fn_to_generic_graph;
diff --git a/compiler/rustc_middle/src/mir/interpret/allocation.rs b/compiler/rustc_middle/src/mir/interpret/allocation.rs
index aded3e495..2a9e0847f 100644
--- a/compiler/rustc_middle/src/mir/interpret/allocation.rs
+++ b/compiler/rustc_middle/src/mir/interpret/allocation.rs
@@ -18,9 +18,9 @@ use rustc_span::DUMMY_SP;
 use rustc_target::abi::{Align, HasDataLayout, Size};
 
 use super::{
-    read_target_uint, write_target_uint, AllocId, BadBytesAccess, InterpError, InterpResult,
-    Pointer, PointerArithmetic, Provenance, ResourceExhaustionInfo, Scalar, ScalarSizeMismatch,
-    UndefinedBehaviorInfo, UnsupportedOpInfo,
+    read_target_uint, write_target_uint, AllocId, BadBytesAccess, CtfeProvenance, InterpError,
+    InterpResult, Pointer, PointerArithmetic, Provenance, ResourceExhaustionInfo, Scalar,
+    ScalarSizeMismatch, UndefinedBehaviorInfo, UnsupportedOpInfo,
 };
 use crate::ty;
 use init_mask::*;
@@ -63,7 +63,7 @@ impl AllocBytes for Box<[u8]> {
 // hashed. (see the `Hash` impl below for more details), so the impl is not derived.
 #[derive(Clone, Eq, PartialEq, TyEncodable, TyDecodable)]
 #[derive(HashStable)]
-pub struct Allocation<Prov: Provenance = AllocId, Extra = (), Bytes = Box<[u8]>> {
+pub struct Allocation<Prov: Provenance = CtfeProvenance, Extra = (), Bytes = Box<[u8]>> {
     /// The actual bytes of the allocation.
     /// Note that the bytes of a pointer represent the offset of the pointer.
     bytes: Bytes,
@@ -315,7 +315,7 @@ impl<Prov: Provenance, Bytes: AllocBytes> Allocation<Prov, (), Bytes> {
     pub fn try_uninit<'tcx>(size: Size, align: Align) -> InterpResult<'tcx, Self> {
         Self::uninit_inner(size, align, || {
             ty::tls::with(|tcx| {
-                tcx.sess.delay_span_bug(DUMMY_SP, "exhausted memory during interpretation")
+                tcx.sess.span_delayed_bug(DUMMY_SP, "exhausted memory during interpretation")
             });
             InterpError::ResourceExhaustion(ResourceExhaustionInfo::MemoryExhausted).into()
         })
@@ -336,14 +336,14 @@ impl<Prov: Provenance, Bytes: AllocBytes> Allocation<Prov, (), Bytes> {
     }
 }
 
-impl<Bytes: AllocBytes> Allocation<AllocId, (), Bytes> {
-    /// Adjust allocation from the ones in tcx to a custom Machine instance
-    /// with a different Provenance and Extra type.
+impl<Bytes: AllocBytes> Allocation<CtfeProvenance, (), Bytes> {
+    /// Adjust allocation from the ones in `tcx` to a custom Machine instance
+    /// with a different `Provenance` and `Extra` type.
     pub fn adjust_from_tcx<Prov: Provenance, Extra, Err>(
         self,
         cx: &impl HasDataLayout,
         extra: Extra,
-        mut adjust_ptr: impl FnMut(Pointer<AllocId>) -> Result<Pointer<Prov>, Err>,
+        mut adjust_ptr: impl FnMut(Pointer<CtfeProvenance>) -> Result<Pointer<Prov>, Err>,
     ) -> Result<Allocation<Prov, Extra, Bytes>, Err> {
         let mut bytes = self.bytes;
         // Adjust provenance of pointers stored in this allocation.
diff --git a/compiler/rustc_middle/src/mir/interpret/allocation/provenance_map.rs b/compiler/rustc_middle/src/mir/interpret/allocation/provenance_map.rs
index d504af6b7..9459af490 100644
--- a/compiler/rustc_middle/src/mir/interpret/allocation/provenance_map.rs
+++ b/compiler/rustc_middle/src/mir/interpret/allocation/provenance_map.rs
@@ -6,14 +6,14 @@ use std::cmp;
 use rustc_data_structures::sorted_map::SortedMap;
 use rustc_target::abi::{HasDataLayout, Size};
 
-use super::{alloc_range, AllocError, AllocId, AllocRange, AllocResult, Provenance};
+use super::{alloc_range, AllocError, AllocRange, AllocResult, CtfeProvenance, Provenance};
 use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
 
 /// Stores the provenance information of pointers stored in memory.
 #[derive(Clone, PartialEq, Eq, Hash, Debug)]
 #[derive(HashStable)]
-pub struct ProvenanceMap<Prov = AllocId> {
-    /// Provenance in this map applies from the given offset for an entire pointer-size worth of
+pub struct ProvenanceMap<Prov = CtfeProvenance> {
+    /// `Provenance` in this map applies from the given offset for an entire pointer-size worth of
     /// bytes. Two entries in this map are always at least a pointer size apart.
     ptrs: SortedMap<Size, Prov>,
     /// Provenance in this map only applies to the given single byte.
@@ -22,18 +22,19 @@ pub struct ProvenanceMap<Prov = AllocId> {
     bytes: Option<Box<SortedMap<Size, Prov>>>,
 }
 
+// These impls are generic over `Prov` since `CtfeProvenance` is only decodable/encodable
+// for some particular `D`/`S`.
 impl<D: Decoder, Prov: Provenance + Decodable<D>> Decodable<D> for ProvenanceMap<Prov> {
     fn decode(d: &mut D) -> Self {
-        assert!(!Prov::OFFSET_IS_ADDR); // only `AllocId` is ever serialized
+        assert!(!Prov::OFFSET_IS_ADDR); // only `CtfeProvenance` is ever serialized
         Self { ptrs: Decodable::decode(d), bytes: None }
     }
 }
-
 impl<S: Encoder, Prov: Provenance + Encodable<S>> Encodable<S> for ProvenanceMap<Prov> {
     fn encode(&self, s: &mut S) {
         let Self { ptrs, bytes } = self;
-        assert!(!Prov::OFFSET_IS_ADDR); // only `AllocId` is ever serialized
-        debug_assert!(bytes.is_none());
+        assert!(!Prov::OFFSET_IS_ADDR); // only `CtfeProvenance` is ever serialized
+        debug_assert!(bytes.is_none()); // without `OFFSET_IS_ADDR`, this is always empty
         ptrs.encode(s)
     }
 }
@@ -54,10 +55,10 @@ impl ProvenanceMap {
     /// Give access to the ptr-sized provenances (which can also be thought of as relocations, and
     /// indeed that is how codegen treats them).
     ///
-    /// Only exposed with `AllocId` provenance, since it panics if there is bytewise provenance.
+    /// Only exposed with `CtfeProvenance` provenance, since it panics if there is bytewise provenance.
     #[inline]
-    pub fn ptrs(&self) -> &SortedMap<Size, AllocId> {
-        debug_assert!(self.bytes.is_none()); // `AllocId::OFFSET_IS_ADDR` is false so this cannot fail
+    pub fn ptrs(&self) -> &SortedMap<Size, CtfeProvenance> {
+        debug_assert!(self.bytes.is_none()); // `CtfeProvenance::OFFSET_IS_ADDR` is false so this cannot fail
         &self.ptrs
     }
 }
diff --git a/compiler/rustc_middle/src/mir/interpret/error.rs b/compiler/rustc_middle/src/mir/interpret/error.rs
index 44b22e2d3..413867939 100644
--- a/compiler/rustc_middle/src/mir/interpret/error.rs
+++ b/compiler/rustc_middle/src/mir/interpret/error.rs
@@ -290,7 +290,7 @@ macro_rules! impl_into_diagnostic_arg_through_debug {
 // These types have nice `Debug` output so we can just use them in diagnostics.
 impl_into_diagnostic_arg_through_debug! {
     AllocId,
-    Pointer,
+    Pointer<AllocId>,
     AllocRange,
 }
 
@@ -323,7 +323,7 @@ pub enum UndefinedBehaviorInfo<'tcx> {
     /// Invalid metadata in a wide pointer
     InvalidMeta(InvalidMetaKind),
     /// Reading a C string that does not end within its allocation.
-    UnterminatedCString(Pointer),
+    UnterminatedCString(Pointer<AllocId>),
     /// Using a pointer after it got freed.
     PointerUseAfterFree(AllocId, CheckInAllocMsg),
     /// Used a pointer outside the bounds it is valid for.
@@ -350,11 +350,11 @@ pub enum UndefinedBehaviorInfo<'tcx> {
     /// Using a non-character `u32` as character.
     InvalidChar(u32),
     /// The tag of an enum does not encode an actual discriminant.
-    InvalidTag(Scalar),
+    InvalidTag(Scalar<AllocId>),
     /// Using a pointer-not-to-a-function as function pointer.
-    InvalidFunctionPointer(Pointer),
+    InvalidFunctionPointer(Pointer<AllocId>),
     /// Using a pointer-not-to-a-vtable as vtable pointer.
-    InvalidVTablePointer(Pointer),
+    InvalidVTablePointer(Pointer<AllocId>),
     /// Using a string that is not valid UTF-8,
     InvalidStr(std::str::Utf8Error),
     /// Using uninitialized data where it is not allowed.
diff --git a/compiler/rustc_middle/src/mir/interpret/mod.rs b/compiler/rustc_middle/src/mir/interpret/mod.rs
index e360fb3ea..2db560085 100644
--- a/compiler/rustc_middle/src/mir/interpret/mod.rs
+++ b/compiler/rustc_middle/src/mir/interpret/mod.rs
@@ -157,7 +157,7 @@ pub use self::allocation::{
     InitChunk, InitChunkIter,
 };
 
-pub use self::pointer::{Pointer, PointerArithmetic, Provenance};
+pub use self::pointer::{CtfeProvenance, Pointer, PointerArithmetic, Provenance};
 
 /// Uniquely identifies one of the following:
 /// - A constant
@@ -199,7 +199,7 @@ pub struct LitToConstInput<'tcx> {
 #[derive(Copy, Clone, Debug, Eq, PartialEq, HashStable)]
 pub enum LitToConstError {
     /// The literal's inferred type did not match the expected `ty` in the input.
-    /// This is used for graceful error handling (`delay_span_bug`) in
+    /// This is used for graceful error handling (`span_delayed_bug`) in
     /// type checking (`Const::from_anon_const`).
     TypeError,
     Reported(ErrorGuaranteed),
diff --git a/compiler/rustc_middle/src/mir/interpret/pointer.rs b/compiler/rustc_middle/src/mir/interpret/pointer.rs
index 1c9ce1cb1..689338773 100644
--- a/compiler/rustc_middle/src/mir/interpret/pointer.rs
+++ b/compiler/rustc_middle/src/mir/interpret/pointer.rs
@@ -3,7 +3,7 @@ use super::{AllocId, InterpResult};
 use rustc_macros::HashStable;
 use rustc_target::abi::{HasDataLayout, Size};
 
-use std::fmt;
+use std::{fmt, num::NonZeroU64};
 
 ////////////////////////////////////////////////////////////////////////////////
 // Pointer arithmetic
@@ -114,22 +114,7 @@ pub trait Provenance: Copy + fmt::Debug + 'static {
     const OFFSET_IS_ADDR: bool;
 
     /// Determines how a pointer should be printed.
-    ///
-    /// Default impl is only good for when `OFFSET_IS_ADDR == true`.
-    fn fmt(ptr: &Pointer<Self>, f: &mut fmt::Formatter<'_>) -> fmt::Result
-    where
-        Self: Sized,
-    {
-        assert!(Self::OFFSET_IS_ADDR);
-        let (prov, addr) = ptr.into_parts(); // address is absolute
-        write!(f, "{:#x}", addr.bytes())?;
-        if f.alternate() {
-            write!(f, "{prov:#?}")?;
-        } else {
-            write!(f, "{prov:?}")?;
-        }
-        Ok(())
-    }
+    fn fmt(ptr: &Pointer<Self>, f: &mut fmt::Formatter<'_>) -> fmt::Result;
 
     /// If `OFFSET_IS_ADDR == false`, provenance must always be able to
     /// identify the allocation this ptr points to (i.e., this must return `Some`).
@@ -141,6 +126,80 @@ pub trait Provenance: Copy + fmt::Debug + 'static {
     fn join(left: Option<Self>, right: Option<Self>) -> Option<Self>;
 }
 
+/// The type of provenance in the compile-time interpreter.
+/// This is a packed representation of an `AllocId` and an `immutable: bool`.
+#[derive(Copy, Clone, Eq, Hash, Ord, PartialEq, PartialOrd)]
+pub struct CtfeProvenance(NonZeroU64);
+
+impl From<AllocId> for CtfeProvenance {
+    fn from(value: AllocId) -> Self {
+        let prov = CtfeProvenance(value.0);
+        assert!(!prov.immutable(), "`AllocId` with the highest bit set cannot be used in CTFE");
+        prov
+    }
+}
+
+impl fmt::Debug for CtfeProvenance {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt::Debug::fmt(&self.alloc_id(), f)?; // propagates `alternate` flag
+        if self.immutable() {
+            write!(f, "<imm>")?;
+        }
+        Ok(())
+    }
+}
+
+const IMMUTABLE_MASK: u64 = 1 << 63; // the highest bit
+
+impl CtfeProvenance {
+    /// Returns the `AllocId` of this provenance.
+    #[inline(always)]
+    pub fn alloc_id(self) -> AllocId {
+        AllocId(NonZeroU64::new(self.0.get() & !IMMUTABLE_MASK).unwrap())
+    }
+
+    /// Returns whether this provenance is immutable.
+    #[inline]
+    pub fn immutable(self) -> bool {
+        self.0.get() & IMMUTABLE_MASK != 0
+    }
+
+    /// Returns an immutable version of this provenance.
+    #[inline]
+    pub fn as_immutable(self) -> Self {
+        CtfeProvenance(self.0 | IMMUTABLE_MASK)
+    }
+}
+
+impl Provenance for CtfeProvenance {
+    // With the `AllocId` as provenance, the `offset` is interpreted *relative to the allocation*,
+    // so ptr-to-int casts are not possible (since we do not know the global physical offset).
+    const OFFSET_IS_ADDR: bool = false;
+
+    fn fmt(ptr: &Pointer<Self>, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        // Print AllocId.
+        fmt::Debug::fmt(&ptr.provenance.alloc_id(), f)?; // propagates `alternate` flag
+        // Print offset only if it is non-zero.
+        if ptr.offset.bytes() > 0 {
+            write!(f, "+{:#x}", ptr.offset.bytes())?;
+        }
+        // Print immutable status.
+        if ptr.provenance.immutable() {
+            write!(f, "<imm>")?;
+        }
+        Ok(())
+    }
+
+    fn get_alloc_id(self) -> Option<AllocId> {
+        Some(self.alloc_id())
+    }
+
+    fn join(_left: Option<Self>, _right: Option<Self>) -> Option<Self> {
+        panic!("merging provenance is not supported when `OFFSET_IS_ADDR` is false")
+    }
+}
+
+// We also need this impl so that one can debug-print `Pointer<AllocId>`
 impl Provenance for AllocId {
     // With the `AllocId` as provenance, the `offset` is interpreted *relative to the allocation*,
     // so ptr-to-int casts are not possible (since we do not know the global physical offset).
@@ -174,7 +233,7 @@ impl Provenance for AllocId {
 /// Pointers are "tagged" with provenance information; typically the `AllocId` they belong to.
 #[derive(Copy, Clone, Eq, PartialEq, TyEncodable, TyDecodable, Hash)]
 #[derive(HashStable)]
-pub struct Pointer<Prov = AllocId> {
+pub struct Pointer<Prov = CtfeProvenance> {
     pub(super) offset: Size, // kept private to avoid accidental misinterpretation (meaning depends on `Prov` type)
     pub provenance: Prov,
 }
@@ -182,7 +241,7 @@ pub struct Pointer<Prov = AllocId> {
 static_assert_size!(Pointer, 16);
 // `Option<Prov>` pointers are also passed around quite a bit
 // (but not stored in permanent machine state).
-static_assert_size!(Pointer<Option<AllocId>>, 16);
+static_assert_size!(Pointer<Option<CtfeProvenance>>, 16);
 
 // We want the `Debug` output to be readable as it is used by `derive(Debug)` for
 // all the Miri types.
@@ -215,7 +274,7 @@ impl<Prov: Provenance> fmt::Display for Pointer<Option<Prov>> {
 impl From<AllocId> for Pointer {
     #[inline(always)]
     fn from(alloc_id: AllocId) -> Self {
-        Pointer::new(alloc_id, Size::ZERO)
+        Pointer::new(alloc_id.into(), Size::ZERO)
     }
 }
 
diff --git a/compiler/rustc_middle/src/mir/interpret/queries.rs b/compiler/rustc_middle/src/mir/interpret/queries.rs
index fbf6403ea..092b59dee 100644
--- a/compiler/rustc_middle/src/mir/interpret/queries.rs
+++ b/compiler/rustc_middle/src/mir/interpret/queries.rs
@@ -110,10 +110,10 @@ impl<'tcx> TyCtxt<'tcx> {
                             let Some(local_def_id) = ct.def.as_local() else { return };
                             self.struct_span_lint_hir(
                                 lint::builtin::CONST_EVALUATABLE_UNCHECKED,
-                                self.hir().local_def_id_to_hir_id(local_def_id),
+                                self.local_def_id_to_hir_id(local_def_id),
                                 self.def_span(ct.def),
                                 "cannot use constants which depend on generic parameters in types",
-                                |err| err,
+                                |_| {},
                             )
                         }
                     }
diff --git a/compiler/rustc_middle/src/mir/interpret/value.rs b/compiler/rustc_middle/src/mir/interpret/value.rs
index 0d548f886..5ecff04f3 100644
--- a/compiler/rustc_middle/src/mir/interpret/value.rs
+++ b/compiler/rustc_middle/src/mir/interpret/value.rs
@@ -11,7 +11,10 @@ use rustc_target::abi::{HasDataLayout, Size};
 
 use crate::ty::ScalarInt;
 
-use super::{AllocId, InterpResult, Pointer, PointerArithmetic, Provenance, ScalarSizeMismatch};
+use super::{
+    AllocId, CtfeProvenance, InterpResult, Pointer, PointerArithmetic, Provenance,
+    ScalarSizeMismatch,
+};
 
 /// A `Scalar` represents an immediate, primitive value existing outside of a
 /// `memory::Allocation`. It is in many ways like a small chunk of an `Allocation`, up to 16 bytes in
@@ -22,7 +25,7 @@ use super::{AllocId, InterpResult, Pointer, PointerArithmetic, Provenance, Scala
 /// Do *not* match on a `Scalar`! Use the various `to_*` methods instead.
 #[derive(Clone, Copy, Eq, PartialEq, TyEncodable, TyDecodable, Hash)]
 #[derive(HashStable)]
-pub enum Scalar<Prov = AllocId> {
+pub enum Scalar<Prov = CtfeProvenance> {
     /// The raw bytes of a simple value.
     Int(ScalarInt),
 
@@ -267,6 +270,9 @@ impl<'tcx, Prov: Provenance> Scalar<Prov> {
     /// Will perform ptr-to-int casts if needed and possible.
     /// If that fails, we know the offset is relative, so we return an "erased" Scalar
     /// (which is useful for error messages but not much else).
+    ///
+    /// The error type is `AllocId`, not `CtfeProvenance`, since `AllocId` is the "minimal"
+    /// component all provenance types must have.
     #[inline]
     pub fn try_to_int(self) -> Result<ScalarInt, Scalar<AllocId>> {
         match self {
diff --git a/compiler/rustc_middle/src/mir/mod.rs b/compiler/rustc_middle/src/mir/mod.rs
index 7054cede2..1e5a7401c 100644
--- a/compiler/rustc_middle/src/mir/mod.rs
+++ b/compiler/rustc_middle/src/mir/mod.rs
@@ -736,6 +736,8 @@ impl SourceInfo {
 
 rustc_index::newtype_index! {
     #[derive(HashStable)]
+    #[encodable]
+    #[orderable]
     #[debug_format = "_{}"]
     pub struct Local {
         const RETURN_PLACE = 0;
@@ -973,7 +975,7 @@ pub enum LocalInfo<'tcx> {
 
 impl<'tcx> LocalDecl<'tcx> {
     pub fn local_info(&self) -> &LocalInfo<'tcx> {
-        &self.local_info.as_ref().assert_crate_local()
+        self.local_info.as_ref().assert_crate_local()
     }
 
     /// Returns `true` only if local is a binding that can itself be
@@ -1171,6 +1173,8 @@ rustc_index::newtype_index! {
     /// [`CriticalCallEdges`]: ../../rustc_const_eval/transform/add_call_guards/enum.AddCallGuards.html#variant.CriticalCallEdges
     /// [guide-mir]: https://rustc-dev-guide.rust-lang.org/mir/
     #[derive(HashStable)]
+    #[encodable]
+    #[orderable]
     #[debug_format = "bb{}"]
     pub struct BasicBlock {
         const START_BLOCK = 0;
@@ -1305,6 +1309,7 @@ impl<'tcx> BasicBlockData<'tcx> {
 
 rustc_index::newtype_index! {
     #[derive(HashStable)]
+    #[encodable]
     #[debug_format = "scope[{}]"]
     pub struct SourceScope {
         const OUTERMOST_SOURCE_SCOPE = 0;
@@ -1533,6 +1538,8 @@ impl UserTypeProjection {
 
 rustc_index::newtype_index! {
     #[derive(HashStable)]
+    #[encodable]
+    #[orderable]
     #[debug_format = "promoted[{}]"]
     pub struct Promoted {}
 }
@@ -1611,14 +1618,29 @@ impl Location {
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
 pub enum DefLocation {
     Argument,
-    Body(Location),
+    Assignment(Location),
+    CallReturn { call: BasicBlock, target: Option<BasicBlock> },
 }
 
 impl DefLocation {
     pub fn dominates(self, location: Location, dominators: &Dominators<BasicBlock>) -> bool {
         match self {
             DefLocation::Argument => true,
-            DefLocation::Body(def) => def.successor_within_block().dominates(location, dominators),
+            DefLocation::Assignment(def) => {
+                def.successor_within_block().dominates(location, dominators)
+            }
+            DefLocation::CallReturn { target: None, .. } => false,
+            DefLocation::CallReturn { call, target: Some(target) } => {
+                // The definition occurs on the call -> target edge. The definition dominates a use
+                // if and only if the edge is on all paths from the entry to the use.
+                //
+                // Note that a call terminator has only one edge that can reach the target, so when
+                // the call strongly dominates the target, all paths from the entry to the target
+                // go through the call -> target edge.
+                call != target
+                    && dominators.dominates(call, target)
+                    && dominators.dominates(target, location.block)
+            }
         }
     }
 }
diff --git a/compiler/rustc_middle/src/mir/patch.rs b/compiler/rustc_middle/src/mir/patch.rs
index eb4aa9eb9..b81e9fa1a 100644
--- a/compiler/rustc_middle/src/mir/patch.rs
+++ b/compiler/rustc_middle/src/mir/patch.rs
@@ -1,7 +1,4 @@
-use rustc_index::{Idx, IndexVec};
 use rustc_middle::mir::*;
-use rustc_middle::ty::Ty;
-use rustc_span::Span;
 
 /// This struct represents a patch to MIR, which can add
 /// new statements and basic blocks and patch over block
diff --git a/compiler/rustc_middle/src/mir/pretty.rs b/compiler/rustc_middle/src/mir/pretty.rs
index a13248584..071c6a755 100644
--- a/compiler/rustc_middle/src/mir/pretty.rs
+++ b/compiler/rustc_middle/src/mir/pretty.rs
@@ -1,23 +1,18 @@
 use std::collections::BTreeSet;
-use std::fmt::{self, Debug, Display, Write as _};
+use std::fmt::{Display, Write as _};
 use std::fs;
 use std::io::{self, Write as _};
 use std::path::{Path, PathBuf};
 
 use super::graphviz::write_mir_fn_graphviz;
 use super::spanview::write_mir_fn_spanview;
-use either::Either;
 use rustc_ast::InlineAsmTemplatePiece;
-use rustc_data_structures::fx::FxHashMap;
-use rustc_hir::def_id::DefId;
-use rustc_index::Idx;
 use rustc_middle::mir::interpret::{
-    alloc_range, read_target_uint, AllocBytes, AllocId, Allocation, ConstAllocation, GlobalAlloc,
-    Pointer, Provenance,
+    alloc_range, read_target_uint, AllocBytes, AllocId, Allocation, GlobalAlloc, Pointer,
+    Provenance,
 };
 use rustc_middle::mir::visit::Visitor;
 use rustc_middle::mir::{self, *};
-use rustc_middle::ty::{self, TyCtxt};
 use rustc_target::abi::Size;
 
 const INDENT: &str = "    ";
@@ -1337,13 +1332,13 @@ pub fn write_allocations<'tcx>(
     fn alloc_ids_from_alloc(
         alloc: ConstAllocation<'_>,
     ) -> impl DoubleEndedIterator<Item = AllocId> + '_ {
-        alloc.inner().provenance().ptrs().values().map(|id| *id)
+        alloc.inner().provenance().ptrs().values().map(|p| p.alloc_id())
     }
 
     fn alloc_ids_from_const_val(val: ConstValue<'_>) -> impl Iterator<Item = AllocId> + '_ {
         match val {
             ConstValue::Scalar(interpret::Scalar::Ptr(ptr, _)) => {
-                Either::Left(std::iter::once(ptr.provenance))
+                Either::Left(std::iter::once(ptr.provenance.alloc_id()))
             }
             ConstValue::Scalar(interpret::Scalar::Int { .. }) => Either::Right(std::iter::empty()),
             ConstValue::ZeroSized => Either::Right(std::iter::empty()),
diff --git a/compiler/rustc_middle/src/mir/query.rs b/compiler/rustc_middle/src/mir/query.rs
index 0540eb0ef..98642adc1 100644
--- a/compiler/rustc_middle/src/mir/query.rs
+++ b/compiler/rustc_middle/src/mir/query.rs
@@ -27,7 +27,7 @@ pub enum UnsafetyViolationKind {
     UnsafeFn,
 }
 
-#[derive(Copy, Clone, PartialEq, TyEncodable, TyDecodable, HashStable, Debug)]
+#[derive(Clone, PartialEq, TyEncodable, TyDecodable, HashStable, Debug)]
 pub enum UnsafetyViolationDetails {
     CallToUnsafeFunction,
     UseOfInlineAssembly,
@@ -39,68 +39,17 @@ pub enum UnsafetyViolationDetails {
     AccessToUnionField,
     MutationOfLayoutConstrainedField,
     BorrowOfLayoutConstrainedField,
-    CallToFunctionWith,
+    CallToFunctionWith {
+        /// Target features enabled in callee's `#[target_feature]` but missing in
+        /// caller's `#[target_feature]`.
+        missing: Vec<Symbol>,
+        /// Target features in `missing` that are enabled at compile time
+        /// (e.g., with `-C target-feature`).
+        build_enabled: Vec<Symbol>,
+    },
 }
 
-impl UnsafetyViolationDetails {
-    pub fn description_and_note(&self) -> (&'static str, &'static str) {
-        use UnsafetyViolationDetails::*;
-        match self {
-            CallToUnsafeFunction => (
-                "call to unsafe function",
-                "consult the function's documentation for information on how to avoid undefined \
-                 behavior",
-            ),
-            UseOfInlineAssembly => (
-                "use of inline assembly",
-                "inline assembly is entirely unchecked and can cause undefined behavior",
-            ),
-            InitializingTypeWith => (
-                "initializing type with `rustc_layout_scalar_valid_range` attr",
-                "initializing a layout restricted type's field with a value outside the valid \
-                 range is undefined behavior",
-            ),
-            CastOfPointerToInt => {
-                ("cast of pointer to int", "casting pointers to integers in constants")
-            }
-            UseOfMutableStatic => (
-                "use of mutable static",
-                "mutable statics can be mutated by multiple threads: aliasing violations or data \
-                 races will cause undefined behavior",
-            ),
-            UseOfExternStatic => (
-                "use of extern static",
-                "extern statics are not controlled by the Rust type system: invalid data, \
-                 aliasing violations or data races will cause undefined behavior",
-            ),
-            DerefOfRawPointer => (
-                "dereference of raw pointer",
-                "raw pointers may be null, dangling or unaligned; they can violate aliasing rules \
-                 and cause data races: all of these are undefined behavior",
-            ),
-            AccessToUnionField => (
-                "access to union field",
-                "the field may not be properly initialized: using uninitialized data will cause \
-                 undefined behavior",
-            ),
-            MutationOfLayoutConstrainedField => (
-                "mutation of layout constrained field",
-                "mutating layout constrained fields cannot statically be checked for valid values",
-            ),
-            BorrowOfLayoutConstrainedField => (
-                "borrow of layout constrained field with interior mutability",
-                "references to fields of layout constrained fields lose the constraints. Coupled \
-                 with interior mutability, the field can be changed to invalid values",
-            ),
-            CallToFunctionWith => (
-                "call to function with `#[target_feature]`",
-                "can only be called if the required target features are available",
-            ),
-        }
-    }
-}
-
-#[derive(Copy, Clone, PartialEq, TyEncodable, TyDecodable, HashStable, Debug)]
+#[derive(Clone, PartialEq, TyEncodable, TyDecodable, HashStable, Debug)]
 pub struct UnsafetyViolation {
     pub source_info: SourceInfo,
     pub lint_root: hir::HirId,
@@ -132,6 +81,7 @@ pub struct UnsafetyCheckResult {
 
 rustc_index::newtype_index! {
     #[derive(HashStable)]
+    #[encodable]
     #[debug_format = "_{}"]
     pub struct CoroutineSavedLocal {}
 }
@@ -341,7 +291,11 @@ pub enum ConstraintCategory<'tcx> {
     UseAsConst,
     UseAsStatic,
     TypeAnnotation,
-    Cast,
+    Cast {
+        /// Whether this is an unsizing cast and if yes, this contains the target type.
+        /// Region variables are erased to ReErased.
+        unsize_to: Option<Ty<'tcx>>,
+    },
 
     /// A constraint that came from checking the body of a closure.
     ///
@@ -416,7 +370,7 @@ impl<'tcx> ClosureOutlivesSubjectTy<'tcx> {
         let inner = tcx.fold_regions(ty, |r, depth| match r.kind() {
             ty::ReVar(vid) => {
                 let br = ty::BoundRegion { var: ty::BoundVar::new(vid.index()), kind: ty::BrAnon };
-                ty::Region::new_late_bound(tcx, depth, br)
+                ty::Region::new_bound(tcx, depth, br)
             }
             _ => bug!("unexpected region in ClosureOutlivesSubjectTy: {r:?}"),
         });
@@ -430,7 +384,7 @@ impl<'tcx> ClosureOutlivesSubjectTy<'tcx> {
         mut map: impl FnMut(ty::RegionVid) -> ty::Region<'tcx>,
     ) -> Ty<'tcx> {
         tcx.fold_regions(self.inner, |r, depth| match r.kind() {
-            ty::ReLateBound(debruijn, br) => {
+            ty::ReBound(debruijn, br) => {
                 debug_assert_eq!(debruijn, depth);
                 map(ty::RegionVid::new(br.var.index()))
             }
diff --git a/compiler/rustc_middle/src/mir/spanview.rs b/compiler/rustc_middle/src/mir/spanview.rs
index a5358687c..cb9fc0d37 100644
--- a/compiler/rustc_middle/src/mir/spanview.rs
+++ b/compiler/rustc_middle/src/mir/spanview.rs
@@ -1,9 +1,7 @@
-use rustc_hir::def_id::DefId;
 use rustc_middle::hir;
 use rustc_middle::mir::*;
-use rustc_middle::ty::TyCtxt;
 use rustc_session::config::MirSpanview;
-use rustc_span::{BytePos, Pos, Span};
+use rustc_span::{BytePos, Pos};
 
 use std::cmp;
 use std::io::{self, Write};
@@ -369,7 +367,7 @@ where
                 debug_indent, span, viewable.span
             );
             from_pos = span.hi();
-            make_html_snippet(tcx, span, Some(&viewable))
+            make_html_snippet(tcx, span, Some(viewable))
         } else {
             None
         };
@@ -439,7 +437,7 @@ where
             tcx,
             from_pos,
             to_pos,
-            &remaining_viewables,
+            remaining_viewables,
             subalt,
             layer + 1,
             w,
@@ -472,7 +470,7 @@ where
             "{}After overlaps, writing (end span?) {:?} of viewable.span {:?}",
             debug_indent, span, viewable.span
         );
-        if let Some(ref html_snippet) = make_html_snippet(tcx, span, Some(&viewable)) {
+        if let Some(ref html_snippet) = make_html_snippet(tcx, span, Some(viewable)) {
             from_pos = span.hi();
             write_span(html_snippet, &viewable.tooltip, alt, layer, w)?;
         }
@@ -507,11 +505,7 @@ fn write_span<W>(
 where
     W: Write,
 {
-    let maybe_alt_class = if layer > 0 {
-        if alt { " odd" } else { " even" }
-    } else {
-        ""
-    };
+    let maybe_alt_class = if layer > 0 { if alt { " odd" } else { " even" } } else { "" };
     let maybe_title_attr = if !tooltip.is_empty() {
         format!(" title=\"{}\"", escape_attr(tooltip))
     } else {
diff --git a/compiler/rustc_middle/src/mir/statement.rs b/compiler/rustc_middle/src/mir/statement.rs
index 3471d620e..f929a5cec 100644
--- a/compiler/rustc_middle/src/mir/statement.rs
+++ b/compiler/rustc_middle/src/mir/statement.rs
@@ -159,7 +159,7 @@ impl<'tcx> Place<'tcx> {
 
     #[inline]
     pub fn as_ref(&self) -> PlaceRef<'tcx> {
-        PlaceRef { local: self.local, projection: &self.projection }
+        PlaceRef { local: self.local, projection: self.projection }
     }
 
     /// Iterate over the projections in evaluation order, i.e., the first element is the base with
@@ -381,7 +381,7 @@ impl<'tcx> Operand<'tcx> {
 impl<'tcx> ConstOperand<'tcx> {
     pub fn check_static_ptr(&self, tcx: TyCtxt<'_>) -> Option<DefId> {
         match self.const_.try_to_scalar() {
-            Some(Scalar::Ptr(ptr, _size)) => match tcx.global_alloc(ptr.provenance) {
+            Some(Scalar::Ptr(ptr, _size)) => match tcx.global_alloc(ptr.provenance.alloc_id()) {
                 GlobalAlloc::Static(def_id) => {
                     assert!(!tcx.is_thread_local_static(def_id));
                     Some(def_id)
diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs
index 7b0f27f9b..8cf9e55f0 100644
--- a/compiler/rustc_middle/src/mir/syntax.rs
+++ b/compiler/rustc_middle/src/mir/syntax.rs
@@ -1404,18 +1404,18 @@ pub enum BinOp {
     BitOr,
     /// The `<<` operator (shift left)
     ///
-    /// The offset is truncated to the size of the first operand before shifting.
+    /// The offset is truncated to the size of the first operand and made unsigned before shifting.
     Shl,
-    /// Like `Shl`, but is UB if the RHS >= LHS::BITS
+    /// Like `Shl`, but is UB if the RHS >= LHS::BITS or RHS < 0
     ShlUnchecked,
     /// The `>>` operator (shift right)
     ///
-    /// The offset is truncated to the size of the first operand before shifting.
+    /// The offset is truncated to the size of the first operand and made unsigned before shifting.
     ///
     /// This is an arithmetic shift if the LHS is signed
     /// and a logical shift if the LHS is unsigned.
     Shr,
-    /// Like `Shl`, but is UB if the RHS >= LHS::BITS
+    /// Like `Shl`, but is UB if the RHS >= LHS::BITS or RHS < 0
     ShrUnchecked,
     /// The `==` operator (equality)
     Eq,
diff --git a/compiler/rustc_middle/src/mir/tcx.rs b/compiler/rustc_middle/src/mir/tcx.rs
index 6ab2da23a..f9b2a6ee8 100644
--- a/compiler/rustc_middle/src/mir/tcx.rs
+++ b/compiler/rustc_middle/src/mir/tcx.rs
@@ -4,9 +4,7 @@
  */
 
 use crate::mir::*;
-use crate::ty::{self, Ty, TyCtxt};
 use rustc_hir as hir;
-use rustc_target::abi::{FieldIdx, VariantIdx};
 
 #[derive(Copy, Clone, Debug, TypeFoldable, TypeVisitable)]
 pub struct PlaceTy<'tcx> {
@@ -40,7 +38,7 @@ impl<'tcx> PlaceTy<'tcx> {
                     None => adt_def.non_enum_variant(),
                     Some(variant_index) => {
                         assert!(adt_def.is_enum());
-                        &adt_def.variant(variant_index)
+                        adt_def.variant(variant_index)
                     }
                 };
                 let field_def = &variant_def.fields[f];
@@ -95,9 +93,7 @@ impl<'tcx> PlaceTy<'tcx> {
             ProjectionElem::Subslice { from, to, from_end } => {
                 PlaceTy::from_ty(match self.ty.kind() {
                     ty::Slice(..) => self.ty,
-                    ty::Array(inner, _) if !from_end => {
-                        Ty::new_array(tcx, *inner, (to - from) as u64)
-                    }
+                    ty::Array(inner, _) if !from_end => Ty::new_array(tcx, *inner, to - from),
                     ty::Array(inner, size) if from_end => {
                         let size = size.eval_target_usize(tcx, param_env);
                         let len = size - from - to;
@@ -143,7 +139,7 @@ impl<'tcx> Place<'tcx> {
     where
         D: HasLocalDecls<'tcx>,
     {
-        Place::ty_from(self.local, &self.projection, local_decls, tcx)
+        Place::ty_from(self.local, self.projection, local_decls, tcx)
     }
 }
 
@@ -152,7 +148,7 @@ impl<'tcx> PlaceRef<'tcx> {
     where
         D: HasLocalDecls<'tcx>,
     {
-        Place::ty_from(self.local, &self.projection, local_decls, tcx)
+        Place::ty_from(self.local, self.projection, local_decls, tcx)
     }
 }
 
diff --git a/compiler/rustc_middle/src/mir/terminator.rs b/compiler/rustc_middle/src/mir/terminator.rs
index 9dfbe1733..98e3a1f60 100644
--- a/compiler/rustc_middle/src/mir/terminator.rs
+++ b/compiler/rustc_middle/src/mir/terminator.rs
@@ -2,9 +2,8 @@
 use rustc_hir::LangItem;
 use smallvec::SmallVec;
 
-use super::{BasicBlock, InlineAsmOperand, Operand, SourceInfo, TerminatorKind, UnwindAction};
+use super::TerminatorKind;
 use rustc_macros::HashStable;
-use std::iter;
 use std::slice;
 
 use super::*;
@@ -28,7 +27,9 @@ impl SwitchTargets {
 
     /// Inverse of `SwitchTargets::static_if`.
     pub fn as_static_if(&self) -> Option<(u128, BasicBlock, BasicBlock)> {
-        if let &[value] = &self.values[..] && let &[then, else_] = &self.targets[..] {
+        if let &[value] = &self.values[..]
+            && let &[then, else_] = &self.targets[..]
+        {
             Some((value, then, else_))
         } else {
             None
@@ -148,11 +149,17 @@ impl<O> AssertKind<O> {
             RemainderByZero(_) => "attempt to calculate the remainder with a divisor of zero",
             ResumedAfterReturn(CoroutineKind::Coroutine) => "coroutine resumed after completion",
             ResumedAfterReturn(CoroutineKind::Async(_)) => "`async fn` resumed after completion",
+            ResumedAfterReturn(CoroutineKind::AsyncGen(_)) => {
+                "`async gen fn` resumed after completion"
+            }
             ResumedAfterReturn(CoroutineKind::Gen(_)) => {
                 "`gen fn` should just keep returning `None` after completion"
             }
             ResumedAfterPanic(CoroutineKind::Coroutine) => "coroutine resumed after panicking",
             ResumedAfterPanic(CoroutineKind::Async(_)) => "`async fn` resumed after panicking",
+            ResumedAfterPanic(CoroutineKind::AsyncGen(_)) => {
+                "`async gen fn` resumed after panicking"
+            }
             ResumedAfterPanic(CoroutineKind::Gen(_)) => {
                 "`gen fn` should just keep returning `None` after panicking"
             }
@@ -243,6 +250,7 @@ impl<O> AssertKind<O> {
             DivisionByZero(_) => middle_assert_divide_by_zero,
             RemainderByZero(_) => middle_assert_remainder_by_zero,
             ResumedAfterReturn(CoroutineKind::Async(_)) => middle_assert_async_resume_after_return,
+            ResumedAfterReturn(CoroutineKind::AsyncGen(_)) => todo!(),
             ResumedAfterReturn(CoroutineKind::Gen(_)) => {
                 bug!("gen blocks can be resumed after they return and will keep returning `None`")
             }
@@ -250,6 +258,7 @@ impl<O> AssertKind<O> {
                 middle_assert_coroutine_resume_after_return
             }
             ResumedAfterPanic(CoroutineKind::Async(_)) => middle_assert_async_resume_after_panic,
+            ResumedAfterPanic(CoroutineKind::AsyncGen(_)) => todo!(),
             ResumedAfterPanic(CoroutineKind::Gen(_)) => middle_assert_gen_resume_after_panic,
             ResumedAfterPanic(CoroutineKind::Coroutine) => {
                 middle_assert_coroutine_resume_after_panic
diff --git a/compiler/rustc_middle/src/mir/type_foldable.rs b/compiler/rustc_middle/src/mir/type_foldable.rs
index d5c81b6cd..93a9bbf64 100644
--- a/compiler/rustc_middle/src/mir/type_foldable.rs
+++ b/compiler/rustc_middle/src/mir/type_foldable.rs
@@ -3,7 +3,6 @@
 use rustc_ast::InlineAsmTemplatePiece;
 
 use super::*;
-use crate::ty;
 
 TrivialTypeTraversalImpls! {
     BlockTailInfo,
@@ -16,7 +15,6 @@ TrivialTypeTraversalImpls! {
     UserTypeAnnotationIndex,
     BorrowKind,
     CastKind,
-    hir::Movability,
     BasicBlock,
     SwitchTargets,
     CoroutineKind,
diff --git a/compiler/rustc_middle/src/mir/visit.rs b/compiler/rustc_middle/src/mir/visit.rs
index d47cfd571..9059936f4 100644
--- a/compiler/rustc_middle/src/mir/visit.rs
+++ b/compiler/rustc_middle/src/mir/visit.rs
@@ -63,9 +63,7 @@
 //! `is_cleanup` above.
 
 use crate::mir::*;
-use crate::ty::GenericArgsRef;
-use crate::ty::{self, CanonicalUserTypeAnnotation, Ty};
-use rustc_span::Span;
+use crate::ty::CanonicalUserTypeAnnotation;
 
 macro_rules! make_mir_visitor {
     ($visitor_trait_name:ident, $($mutability:ident)?) => {
diff --git a/compiler/rustc_middle/src/query/erase.rs b/compiler/rustc_middle/src/query/erase.rs
index cdde6a596..b9200f1ab 100644
--- a/compiler/rustc_middle/src/query/erase.rs
+++ b/compiler/rustc_middle/src/query/erase.rs
@@ -264,6 +264,7 @@ trivial! {
     rustc_middle::middle::stability::DeprecationEntry,
     rustc_middle::mir::ConstQualifs,
     rustc_middle::mir::interpret::AllocId,
+    rustc_middle::mir::interpret::CtfeProvenance,
     rustc_middle::mir::interpret::ErrorHandled,
     rustc_middle::mir::interpret::LitToConstError,
     rustc_middle::thir::ExprId,
diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs
index f9ec36836..3a54f5f6b 100644
--- a/compiler/rustc_middle/src/query/mod.rs
+++ b/compiler/rustc_middle/src/query/mod.rs
@@ -61,7 +61,6 @@ use rustc_data_structures::fx::{FxHashMap, FxIndexMap, FxIndexSet};
 use rustc_data_structures::steal::Steal;
 use rustc_data_structures::svh::Svh;
 use rustc_data_structures::sync::Lrc;
-use rustc_data_structures::sync::WorkerLocal;
 use rustc_data_structures::unord::UnordSet;
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir as hir;
@@ -70,7 +69,7 @@ use rustc_hir::def_id::{
     CrateNum, DefId, DefIdMap, DefIdSet, LocalDefId, LocalDefIdMap, LocalDefIdSet, LocalModDefId,
 };
 use rustc_hir::lang_items::{LangItem, LanguageItems};
-use rustc_hir::{Crate, ItemLocalId, TraitCandidate};
+use rustc_hir::{Crate, ItemLocalId, ItemLocalMap, TraitCandidate};
 use rustc_index::IndexVec;
 use rustc_query_system::ich::StableHashingContext;
 use rustc_query_system::query::{try_get_cached, CacheSelector, QueryCache, QueryMode, QueryState};
@@ -109,10 +108,12 @@ pub use plumbing::{IntoQueryParam, TyCtxtAt, TyCtxtEnsure, TyCtxtEnsureWithValue
 // Queries marked with `fatal_cycle` do not need the latter implementation,
 // as they will raise an fatal error on query cycles instead.
 rustc_queries! {
-    query trigger_delay_span_bug(key: DefId) -> () {
-        desc { "triggering a delay span bug" }
+    /// This exists purely for testing the interactions between span_delayed_bug and incremental.
+    query trigger_span_delayed_bug(key: DefId) -> () {
+        desc { "triggering a span delayed bug for testing incremental" }
     }
 
+    /// Collects the list of all tools registered using `#![register_tool]`.
     query registered_tools(_: ()) -> &'tcx ty::RegisteredTools {
         arena_cache
         desc { "compute registered tools for crate" }
@@ -286,6 +287,7 @@ rustc_queries! {
         }
     }
 
+    /// The root query triggering all analysis passes like typeck or borrowck.
     query analysis(key: ()) -> Result<(), ErrorGuaranteed> {
         eval_always
         desc { "running analysis passes on this crate" }
@@ -564,7 +566,7 @@ rustc_queries! {
         separate_provide_extern
     }
 
-    query check_coroutine_obligations(key: LocalDefId) {
+    query check_coroutine_obligations(key: LocalDefId) -> Result<(), ErrorGuaranteed> {
         desc { |tcx| "verify auto trait bounds for coroutine interior type `{}`", tcx.def_path_str(key) }
     }
 
@@ -1149,7 +1151,7 @@ rustc_queries! {
         cache_on_disk_if { true }
     }
 
-    query opt_def_kind(def_id: DefId) -> Option<DefKind> {
+    query def_kind(def_id: DefId) -> DefKind {
         desc { |tcx| "looking up definition kind of `{}`", tcx.def_path_str(def_id) }
         cache_on_disk_if { def_id.is_local() }
         separate_provide_extern
@@ -1488,7 +1490,7 @@ rustc_queries! {
         desc { "computing whether impls specialize one another" }
     }
     query in_scope_traits_map(_: hir::OwnerId)
-        -> Option<&'tcx FxHashMap<ItemLocalId, Box<[TraitCandidate]>>> {
+        -> Option<&'tcx ItemLocalMap<Box<[TraitCandidate]>>> {
         desc { "getting traits in scope at a block" }
     }
 
@@ -1732,13 +1734,10 @@ rustc_queries! {
         desc { |tcx| "computing crate imported by `{}`", tcx.def_path_str(def_id) }
     }
 
-    query lib_features(_: ()) -> &'tcx LibFeatures {
-        arena_cache
-        desc { "calculating the lib features map" }
-    }
-    query defined_lib_features(_: CrateNum) -> &'tcx [(Symbol, Option<Symbol>)] {
+    query lib_features(_: CrateNum) -> &'tcx LibFeatures {
         desc { "calculating the lib features defined in a crate" }
         separate_provide_extern
+        arena_cache
     }
     query stability_implications(_: CrateNum) -> &'tcx FxHashMap<Symbol, Symbol> {
         arena_cache
@@ -1781,10 +1780,17 @@ rustc_queries! {
         desc { "calculating the missing lang items in a crate" }
         separate_provide_extern
     }
+
+    /// The visible parent map is a map from every item to a visible parent.
+    /// It prefers the shortest visible path to an item.
+    /// Used for diagnostics, for example path trimming.
+    /// The parents are modules, enums or traits.
     query visible_parent_map(_: ()) -> &'tcx DefIdMap<DefId> {
         arena_cache
         desc { "calculating the visible parent map" }
     }
+    /// Collects the "trimmed", shortest accessible paths to all items for diagnostics.
+    /// See the [provider docs](`rustc_middle::ty::print::trimmed_def_paths`) for more info.
     query trimmed_def_paths(_: ()) -> &'tcx FxHashMap<DefId, Symbol> {
         arena_cache
         desc { "calculating trimmed def paths" }
@@ -2171,7 +2177,9 @@ rustc_queries! {
     /// Used in `super_combine_consts` to ICE if the type of the two consts are definitely not going to end up being
     /// equal to eachother. This might return `Ok` even if the types are not equal, but will never return `Err` if
     /// the types might be equal.
-    query check_tys_might_be_eq(arg: Canonical<'tcx, (ty::ParamEnv<'tcx>, Ty<'tcx>, Ty<'tcx>)>) -> Result<(), NoSolution> {
+    query check_tys_might_be_eq(
+        arg: Canonical<'tcx, ty::ParamEnvAnd<'tcx, (Ty<'tcx>, Ty<'tcx>)>>
+    ) -> Result<(), NoSolution> {
         desc { "check whether two const param are definitely not equal to eachother"}
     }
 
diff --git a/compiler/rustc_middle/src/query/on_disk_cache.rs b/compiler/rustc_middle/src/query/on_disk_cache.rs
index 280f5d0a8..f37cfe8b0 100644
--- a/compiler/rustc_middle/src/query/on_disk_cache.rs
+++ b/compiler/rustc_middle/src/query/on_disk_cache.rs
@@ -25,7 +25,6 @@ use rustc_span::source_map::{SourceMap, StableSourceFileId};
 use rustc_span::{BytePos, ExpnData, ExpnHash, Pos, RelativeBytePos, SourceFile, Span};
 use rustc_span::{CachingSourceMapView, Symbol};
 use std::collections::hash_map::Entry;
-use std::io;
 use std::mem;
 
 const TAG_FILE_FOOTER: u128 = 0xC0FFEE_C0FFEE_C0FFEE_C0FFEE_C0FFEE;
@@ -246,7 +245,7 @@ impl<'sess> OnDiskCache<'sess> {
                     let index = SourceFileIndex(index as u32);
                     let file_ptr: *const SourceFile = &**file as *const _;
                     file_to_file_index.insert(file_ptr, index);
-                    let source_file_id = EncodedSourceFileId::new(tcx, &file);
+                    let source_file_id = EncodedSourceFileId::new(tcx, file);
                     file_index_to_stable_id.insert(index, source_file_id);
                 }
 
@@ -482,13 +481,8 @@ pub struct CacheDecoder<'a, 'tcx> {
 impl<'a, 'tcx> CacheDecoder<'a, 'tcx> {
     #[inline]
     fn file_index_to_file(&self, index: SourceFileIndex) -> Lrc<SourceFile> {
-        let CacheDecoder {
-            tcx,
-            ref file_index_to_file,
-            ref file_index_to_stable_id,
-            ref source_map,
-            ..
-        } = *self;
+        let CacheDecoder { tcx, file_index_to_file, file_index_to_stable_id, source_map, .. } =
+            *self;
 
         file_index_to_file
             .borrow_mut()
@@ -867,7 +861,7 @@ impl<'a, 'tcx> CacheEncoder<'a, 'tcx> {
     }
 
     #[inline]
-    fn finish(self) -> Result<usize, io::Error> {
+    fn finish(mut self) -> FileEncodeResult {
         self.encoder.finish()
     }
 }
diff --git a/compiler/rustc_middle/src/query/plumbing.rs b/compiler/rustc_middle/src/query/plumbing.rs
index f4a8ada8f..0d1f3c1f8 100644
--- a/compiler/rustc_middle/src/query/plumbing.rs
+++ b/compiler/rustc_middle/src/query/plumbing.rs
@@ -11,7 +11,7 @@ use field_offset::FieldOffset;
 use measureme::StringId;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sync::AtomicU64;
-use rustc_hir::def::DefKind;
+use rustc_data_structures::sync::WorkerLocal;
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::hir_id::OwnerId;
 use rustc_query_system::dep_graph::DepNodeIndex;
@@ -71,7 +71,7 @@ pub struct QuerySystemFns<'tcx> {
 
 pub struct QuerySystem<'tcx> {
     pub states: QueryStates<'tcx>,
-    pub arenas: QueryArenas<'tcx>,
+    pub arenas: WorkerLocal<QueryArenas<'tcx>>,
     pub caches: QueryCaches<'tcx>,
     pub dynamic_queries: DynamicQueries<'tcx>,
 
@@ -370,7 +370,7 @@ macro_rules! define_callbacks {
 
         pub struct QueryArenas<'tcx> {
             $($(#[$attr])* pub $name: query_if_arena!([$($modifiers)*]
-                (WorkerLocal<TypedArena<<$V as Deref>::Target>>)
+                (TypedArena<<$V as Deref>::Target>)
                 ()
             ),)*
         }
@@ -379,7 +379,7 @@ macro_rules! define_callbacks {
             fn default() -> Self {
                 Self {
                     $($name: query_if_arena!([$($modifiers)*]
-                        (WorkerLocal::new(|_| Default::default()))
+                        (Default::default())
                         ()
                     ),)*
                 }
@@ -551,7 +551,7 @@ macro_rules! define_feedable {
                                 // We have an inconsistency. This can happen if one of the two
                                 // results is tainted by errors. In this case, delay a bug to
                                 // ensure compilation is doomed, and keep the `old` value.
-                                tcx.sess.delay_span_bug(DUMMY_SP, format!(
+                                tcx.sess.span_delayed_bug(DUMMY_SP, format!(
                                     "Trying to feed an already recorded value for query {} key={key:?}:\n\
                                     old value: {old:?}\nnew value: {value:?}",
                                     stringify!($name),
@@ -667,21 +667,5 @@ mod sealed {
 
 pub use sealed::IntoQueryParam;
 
-impl<'tcx> TyCtxt<'tcx> {
-    pub fn def_kind(self, def_id: impl IntoQueryParam<DefId>) -> DefKind {
-        let def_id = def_id.into_query_param();
-        self.opt_def_kind(def_id)
-            .unwrap_or_else(|| bug!("def_kind: unsupported node: {:?}", def_id))
-    }
-}
-
-impl<'tcx> TyCtxtAt<'tcx> {
-    pub fn def_kind(self, def_id: impl IntoQueryParam<DefId>) -> DefKind {
-        let def_id = def_id.into_query_param();
-        self.opt_def_kind(def_id)
-            .unwrap_or_else(|| bug!("def_kind: unsupported node: {:?}", def_id))
-    }
-}
-
 #[derive(Copy, Clone, Debug, HashStable)]
 pub struct CyclePlaceholder(pub ErrorGuaranteed);
diff --git a/compiler/rustc_middle/src/thir.rs b/compiler/rustc_middle/src/thir.rs
index 3086082fe..b6759d352 100644
--- a/compiler/rustc_middle/src/thir.rs
+++ b/compiler/rustc_middle/src/thir.rs
@@ -134,7 +134,6 @@ pub struct Block {
     /// This does *not* include labels on loops, e.g. `'label: loop {}`.
     pub targeted_by_break: bool,
     pub region_scope: region::Scope,
-    pub opt_destruction_scope: Option<region::Scope>,
     /// The span of the block, including the opening braces,
     /// the label, and the `unsafe` keyword, if present.
     pub span: Span,
@@ -193,7 +192,6 @@ pub enum BlockSafety {
 #[derive(Clone, Debug, HashStable)]
 pub struct Stmt<'tcx> {
     pub kind: StmtKind<'tcx>,
-    pub opt_destruction_scope: Option<region::Scope>,
 }
 
 #[derive(Clone, Debug, HashStable)]
@@ -635,7 +633,12 @@ impl<'tcx> Pat<'tcx> {
 
         use PatKind::*;
         match &self.kind {
-            Wild | Range(..) | Binding { subpattern: None, .. } | Constant { .. } | Error(_) => {}
+            Wild
+            | Never
+            | Range(..)
+            | Binding { subpattern: None, .. }
+            | Constant { .. }
+            | Error(_) => {}
             AscribeUserType { subpattern, .. }
             | Binding { subpattern: Some(subpattern), .. }
             | Deref { subpattern }
@@ -655,7 +658,9 @@ impl<'tcx> Pat<'tcx> {
     pub fn pat_error_reported(&self) -> Result<(), ErrorGuaranteed> {
         let mut error = None;
         self.walk(|pat| {
-            if let PatKind::Error(e) = pat.kind && error.is_none() {
+            if let PatKind::Error(e) = pat.kind
+                && error.is_none()
+            {
                 error = Some(e);
             }
             error.is_none()
@@ -807,6 +812,9 @@ pub enum PatKind<'tcx> {
         pats: Box<[Box<Pat<'tcx>>]>,
     },
 
+    /// A never pattern `!`.
+    Never,
+
     /// An error has been encountered during lowering. We probably shouldn't report more lints
     /// related to this pattern.
     Error(ErrorGuaranteed),
@@ -1067,6 +1075,7 @@ impl<'tcx> fmt::Display for Pat<'tcx> {
 
         match self.kind {
             PatKind::Wild => write!(f, "_"),
+            PatKind::Never => write!(f, "!"),
             PatKind::AscribeUserType { ref subpattern, .. } => write!(f, "{subpattern}: _"),
             PatKind::Binding { mutability, name, mode, ref subpattern, .. } => {
                 let is_mut = match mode {
@@ -1213,12 +1222,12 @@ impl<'tcx> fmt::Display for Pat<'tcx> {
 mod size_asserts {
     use super::*;
     // tidy-alphabetical-start
-    static_assert_size!(Block, 56);
+    static_assert_size!(Block, 48);
     static_assert_size!(Expr<'_>, 64);
     static_assert_size!(ExprKind<'_>, 40);
     static_assert_size!(Pat<'_>, 64);
     static_assert_size!(PatKind<'_>, 48);
-    static_assert_size!(Stmt<'_>, 56);
+    static_assert_size!(Stmt<'_>, 48);
     static_assert_size!(StmtKind<'_>, 48);
     // tidy-alphabetical-end
 }
diff --git a/compiler/rustc_middle/src/thir/visit.rs b/compiler/rustc_middle/src/thir/visit.rs
index 8feefb4c0..4943c1184 100644
--- a/compiler/rustc_middle/src/thir/visit.rs
+++ b/compiler/rustc_middle/src/thir/visit.rs
@@ -226,8 +226,8 @@ pub fn walk_pat<'a, 'tcx: 'a, V: Visitor<'a, 'tcx>>(visitor: &mut V, pat: &Pat<'
             ty: _,
             is_primary: _,
             name: _,
-        } => visitor.visit_pat(&subpattern),
-        Binding { .. } | Wild | Error(_) => {}
+        } => visitor.visit_pat(subpattern),
+        Binding { .. } | Wild | Never | Error(_) => {}
         Variant { subpatterns, adt_def: _, args: _, variant_index: _ } | Leaf { subpatterns } => {
             for subpattern in subpatterns {
                 visitor.visit_pat(&subpattern.pattern);
@@ -249,7 +249,7 @@ pub fn walk_pat<'a, 'tcx: 'a, V: Visitor<'a, 'tcx>>(visitor: &mut V, pat: &Pat<'
         }
         Or { pats } => {
             for pat in pats.iter() {
-                visitor.visit_pat(&pat);
+                visitor.visit_pat(pat);
             }
         }
     };
diff --git a/compiler/rustc_middle/src/traits/mod.rs b/compiler/rustc_middle/src/traits/mod.rs
index 6cd75e087..5d0187a85 100644
--- a/compiler/rustc_middle/src/traits/mod.rs
+++ b/compiler/rustc_middle/src/traits/mod.rs
@@ -250,9 +250,6 @@ pub enum ObligationCauseCode<'tcx> {
     /// A tuple is WF only if its middle elements are `Sized`.
     TupleElem,
 
-    /// This is the trait reference from the given projection.
-    ProjectionWf(ty::AliasTy<'tcx>),
-
     /// Must satisfy all of the where-clause predicates of the
     /// given item.
     ItemObligation(DefId),
@@ -343,7 +340,8 @@ pub enum ObligationCauseCode<'tcx> {
         parent_code: InternedObligationCauseCode<'tcx>,
     },
 
-    /// Error derived when matching traits/impls; see ObligationCause for more details
+    /// Error derived when checking an impl item is compatible with
+    /// its corresponding trait item's definition
     CompareImplItemObligation {
         impl_item_def_id: LocalDefId,
         trait_item_def_id: DefId,
@@ -372,9 +370,6 @@ pub enum ObligationCauseCode<'tcx> {
         origin_expr: bool,
     },
 
-    /// Constants in patterns must have `Structural` type.
-    ConstPatternStructural,
-
     /// Computing common supertype in an if expression
     IfExpression(Box<IfExpressionCause<'tcx>>),
 
@@ -407,9 +402,6 @@ pub enum ObligationCauseCode<'tcx> {
     /// `return` with an expression
     ReturnValue(hir::HirId),
 
-    /// Return type of this function
-    ReturnType,
-
     /// Opaque return type of this function
     OpaqueReturnType(Option<(Ty<'tcx>, Span)>),
 
@@ -419,10 +411,7 @@ pub enum ObligationCauseCode<'tcx> {
     /// #[feature(trivial_bounds)] is not enabled
     TrivialBound,
 
-    /// If `X` is the concrete type of an opaque type `impl Y`, then `X` must implement `Y`
-    OpaqueType,
-
-    AwaitableExpr(Option<hir::HirId>),
+    AwaitableExpr(hir::HirId),
 
     ForLoopIterator,
 
@@ -686,7 +675,7 @@ impl<'tcx, N> ImplSource<'tcx, N> {
     pub fn borrow_nested_obligations(&self) -> &[N] {
         match self {
             ImplSource::UserDefined(i) => &i.nested,
-            ImplSource::Param(n) | ImplSource::Builtin(_, n) => &n,
+            ImplSource::Param(n) | ImplSource::Builtin(_, n) => n,
         }
     }
 
@@ -843,50 +832,31 @@ impl ObjectSafetyViolation {
         }
     }
 
-    pub fn solution(&self, err: &mut Diagnostic) {
+    pub fn solution(&self) -> ObjectSafetyViolationSolution {
         match self {
             ObjectSafetyViolation::SizedSelf(_)
             | ObjectSafetyViolation::SupertraitSelf(_)
-            | ObjectSafetyViolation::SupertraitNonLifetimeBinder(..) => {}
+            | ObjectSafetyViolation::SupertraitNonLifetimeBinder(..) => {
+                ObjectSafetyViolationSolution::None
+            }
             ObjectSafetyViolation::Method(
                 name,
                 MethodViolationCode::StaticMethod(Some((add_self_sugg, make_sized_sugg))),
                 _,
-            ) => {
-                err.span_suggestion(
-                    add_self_sugg.1,
-                    format!(
-                        "consider turning `{name}` into a method by giving it a `&self` argument"
-                    ),
-                    add_self_sugg.0.to_string(),
-                    Applicability::MaybeIncorrect,
-                );
-                err.span_suggestion(
-                    make_sized_sugg.1,
-                    format!(
-                        "alternatively, consider constraining `{name}` so it does not apply to \
-                             trait objects"
-                    ),
-                    make_sized_sugg.0.to_string(),
-                    Applicability::MaybeIncorrect,
-                );
-            }
+            ) => ObjectSafetyViolationSolution::AddSelfOrMakeSized {
+                name: *name,
+                add_self_sugg: add_self_sugg.clone(),
+                make_sized_sugg: make_sized_sugg.clone(),
+            },
             ObjectSafetyViolation::Method(
                 name,
                 MethodViolationCode::UndispatchableReceiver(Some(span)),
                 _,
-            ) => {
-                err.span_suggestion(
-                    *span,
-                    format!("consider changing method `{name}`'s `self` parameter to be `&self`"),
-                    "&Self",
-                    Applicability::MachineApplicable,
-                );
-            }
+            ) => ObjectSafetyViolationSolution::ChangeToRefSelf(*name, *span),
             ObjectSafetyViolation::AssocConst(name, _)
             | ObjectSafetyViolation::GAT(name, _)
             | ObjectSafetyViolation::Method(name, ..) => {
-                err.help(format!("consider moving `{name}` to another trait"));
+                ObjectSafetyViolationSolution::MoveToAnotherTrait(*name)
             }
         }
     }
@@ -910,6 +880,60 @@ impl ObjectSafetyViolation {
     }
 }
 
+#[derive(Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
+pub enum ObjectSafetyViolationSolution {
+    None,
+    AddSelfOrMakeSized {
+        name: Symbol,
+        add_self_sugg: (String, Span),
+        make_sized_sugg: (String, Span),
+    },
+    ChangeToRefSelf(Symbol, Span),
+    MoveToAnotherTrait(Symbol),
+}
+
+impl ObjectSafetyViolationSolution {
+    pub fn add_to(self, err: &mut Diagnostic) {
+        match self {
+            ObjectSafetyViolationSolution::None => {}
+            ObjectSafetyViolationSolution::AddSelfOrMakeSized {
+                name,
+                add_self_sugg,
+                make_sized_sugg,
+            } => {
+                err.span_suggestion(
+                    add_self_sugg.1,
+                    format!(
+                        "consider turning `{name}` into a method by giving it a `&self` argument"
+                    ),
+                    add_self_sugg.0,
+                    Applicability::MaybeIncorrect,
+                );
+                err.span_suggestion(
+                    make_sized_sugg.1,
+                    format!(
+                        "alternatively, consider constraining `{name}` so it does not apply to \
+                             trait objects"
+                    ),
+                    make_sized_sugg.0,
+                    Applicability::MaybeIncorrect,
+                );
+            }
+            ObjectSafetyViolationSolution::ChangeToRefSelf(name, span) => {
+                err.span_suggestion(
+                    span,
+                    format!("consider changing method `{name}`'s `self` parameter to be `&self`"),
+                    "&Self",
+                    Applicability::MachineApplicable,
+                );
+            }
+            ObjectSafetyViolationSolution::MoveToAnotherTrait(name) => {
+                err.help(format!("consider moving `{name}` to another trait"));
+            }
+        }
+    }
+}
+
 /// Reasons a method might not be object-safe.
 #[derive(Clone, Debug, PartialEq, Eq, Hash, HashStable, PartialOrd, Ord)]
 pub enum MethodViolationCode {
@@ -956,13 +980,26 @@ pub enum CodegenObligationError {
     FulfillmentError,
 }
 
+/// Defines the treatment of opaque types in a given inference context.
+///
+/// This affects both what opaques are allowed to be defined, but also whether
+/// opaques are replaced with inference vars eagerly in the old solver (e.g.
+/// in projection, and in the signature during function type-checking).
 #[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, HashStable, TypeFoldable, TypeVisitable)]
 pub enum DefiningAnchor {
-    /// `DefId` of the item.
+    /// Define opaques which are in-scope of the `LocalDefId`. Also, eagerly
+    /// replace opaque types in `replace_opaque_types_with_inference_vars`.
     Bind(LocalDefId),
-    /// When opaque types are not resolved, we `Bubble` up, meaning
-    /// return the opaque/hidden type pair from query, for caller of query to handle it.
+    /// In contexts where we don't currently know what opaques are allowed to be
+    /// defined, such as (old solver) canonical queries, we will simply allow
+    /// opaques to be defined, but "bubble" them up in the canonical response or
+    /// otherwise treat them to be handled later.
+    ///
+    /// We do not eagerly replace opaque types in `replace_opaque_types_with_inference_vars`,
+    /// which may affect what predicates pass and fail in the old trait solver.
     Bubble,
-    /// Used to catch type mismatch errors when handling opaque types.
+    /// Do not allow any opaques to be defined. This is used to catch type mismatch
+    /// errors when handling opaque types, and also should be used when we would
+    /// otherwise reveal opaques (such as [`Reveal::All`] reveal mode).
     Error,
 }
diff --git a/compiler/rustc_middle/src/traits/query.rs b/compiler/rustc_middle/src/traits/query.rs
index 3cceb8b2c..34c56eb0d 100644
--- a/compiler/rustc_middle/src/traits/query.rs
+++ b/compiler/rustc_middle/src/traits/query.rs
@@ -108,17 +108,6 @@ impl<'tcx> DropckOutlivesResult<'tcx> {
             tcx.sess.emit_err(DropCheckOverflow { span, ty, overflow_ty: *overflow_ty });
         }
     }
-
-    pub fn into_kinds_reporting_overflows(
-        self,
-        tcx: TyCtxt<'tcx>,
-        span: Span,
-        ty: Ty<'tcx>,
-    ) -> Vec<GenericArg<'tcx>> {
-        self.report_overflows(tcx, span, ty);
-        let DropckOutlivesResult { kinds, overflows: _ } = self;
-        kinds
-    }
 }
 
 /// A set of constraints that need to be satisfied in order for
diff --git a/compiler/rustc_middle/src/traits/select.rs b/compiler/rustc_middle/src/traits/select.rs
index f33421bba..734c2b61c 100644
--- a/compiler/rustc_middle/src/traits/select.rs
+++ b/compiler/rustc_middle/src/traits/select.rs
@@ -125,10 +125,8 @@ pub enum SelectionCandidate<'tcx> {
 
     /// This is a trait matching with a projected type as `Self`, and we found
     /// an applicable bound in the trait definition. The `usize` is an index
-    /// into the list returned by `tcx.item_bounds`. The constness is the
-    /// constness of the bound in the trait.
-    // FIXME(effects) do we need this constness
-    ProjectionCandidate(usize, ty::BoundConstness),
+    /// into the list returned by `tcx.item_bounds`.
+    ProjectionCandidate(usize),
 
     /// Implementation of a `Fn`-family trait by one of the anonymous types
     /// generated for an `||` expression.
@@ -144,14 +142,18 @@ pub enum SelectionCandidate<'tcx> {
     /// generated for an async construct.
     FutureCandidate,
 
-    /// Implementation of an `Iterator` trait by one of the generator types
-    /// generated for a gen construct.
+    /// Implementation of an `Iterator` trait by one of the coroutine types
+    /// generated for a `gen` construct.
     IteratorCandidate,
 
+    /// Implementation of an `AsyncIterator` trait by one of the coroutine types
+    /// generated for a `async gen` construct.
+    AsyncIteratorCandidate,
+
     /// Implementation of a `Fn`-family trait by one of the anonymous
     /// types generated for a fn pointer type (e.g., `fn(int) -> int`)
     FnPointerCandidate {
-        is_const: bool,
+        fn_host_effect: ty::Const<'tcx>,
     },
 
     TraitAliasCandidate,
@@ -180,8 +182,8 @@ pub enum SelectionCandidate<'tcx> {
 ///
 /// The evaluation results are ordered:
 ///     - `EvaluatedToOk` implies `EvaluatedToOkModuloRegions`
-///       implies `EvaluatedToAmbig` implies `EvaluatedToUnknown`
-///     - `EvaluatedToErr` implies `EvaluatedToRecur`
+///       implies `EvaluatedToAmbig` implies `EvaluatedToAmbigStackDependent`
+///     - `EvaluatedToErr` implies `EvaluatedToErrStackDependent`
 ///     - the "union" of evaluation results is equal to their maximum -
 ///     all the "potential success" candidates can potentially succeed,
 ///     so they are noops when unioned with a definite error, and within
@@ -199,7 +201,7 @@ pub enum EvaluationResult {
     /// Evaluation is known to be ambiguous -- it *might* hold for some
     /// assignment of inference variables, but it might not.
     ///
-    /// While this has the same meaning as `EvaluatedToUnknown` -- we can't
+    /// While this has the same meaning as `EvaluatedToAmbigStackDependent` -- we can't
     /// know whether this obligation holds or not -- it is the result we
     /// would get with an empty stack, and therefore is cacheable.
     EvaluatedToAmbig,
@@ -207,8 +209,8 @@ pub enum EvaluationResult {
     /// variables. We are somewhat imprecise there, so we don't actually
     /// know the real result.
     ///
-    /// This can't be trivially cached for the same reason as `EvaluatedToRecur`.
-    EvaluatedToUnknown,
+    /// This can't be trivially cached for the same reason as `EvaluatedToErrStackDependent`.
+    EvaluatedToAmbigStackDependent,
     /// Evaluation failed because we encountered an obligation we are already
     /// trying to prove on this branch.
     ///
@@ -247,12 +249,12 @@ pub enum EvaluationResult {
     /// does not hold, because of the bound (which can indeed be satisfied
     /// by `SomeUnsizedType` from another crate).
     //
-    // FIXME: when an `EvaluatedToRecur` goes past its parent root, we
+    // FIXME: when an `EvaluatedToErrStackDependent` goes past its parent root, we
     // ought to convert it to an `EvaluatedToErr`, because we know
     // there definitely isn't a proof tree for that obligation. Not
     // doing so is still sound -- there isn't any proof tree, so the
     // branch still can't be a part of a minimal one -- but does not re-enable caching.
-    EvaluatedToRecur,
+    EvaluatedToErrStackDependent,
     /// Evaluation failed.
     EvaluatedToErr,
 }
@@ -276,15 +278,15 @@ impl EvaluationResult {
             | EvaluatedToOk
             | EvaluatedToOkModuloRegions
             | EvaluatedToAmbig
-            | EvaluatedToUnknown => true,
+            | EvaluatedToAmbigStackDependent => true,
 
-            EvaluatedToErr | EvaluatedToRecur => false,
+            EvaluatedToErr | EvaluatedToErrStackDependent => false,
         }
     }
 
     pub fn is_stack_dependent(self) -> bool {
         match self {
-            EvaluatedToUnknown | EvaluatedToRecur => true,
+            EvaluatedToAmbigStackDependent | EvaluatedToErrStackDependent => true,
 
             EvaluatedToOkModuloOpaqueTypes
             | EvaluatedToOk
diff --git a/compiler/rustc_middle/src/traits/solve.rs b/compiler/rustc_middle/src/traits/solve.rs
index 27a1e64a7..048df367b 100644
--- a/compiler/rustc_middle/src/traits/solve.rs
+++ b/compiler/rustc_middle/src/traits/solve.rs
@@ -233,6 +233,27 @@ impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for PredefinedOpaques<'tcx> {
     }
 }
 
+/// Why a specific goal has to be proven.
+///
+/// This is necessary as we treat nested goals different depending on
+/// their source.
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub enum GoalSource {
+    Misc,
+    /// We're proving a where-bound of an impl.
+    ///
+    /// FIXME(-Znext-solver=coinductive): Explain how and why this
+    /// changes whether cycles are coinductive.
+    ///
+    /// This also impacts whether we erase constraints on overflow.
+    /// Erasing constraints is generally very useful for perf and also
+    /// results in better error messages by avoiding spurious errors.
+    /// We do not erase overflow constraints in `normalizes-to` goals unless
+    /// they are from an impl where-clause. This is necessary due to
+    /// backwards compatability, cc trait-system-refactor-initiatitive#70.
+    ImplWhereBound,
+}
+
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, HashStable)]
 pub enum IsNormalizesToHack {
     Yes,
diff --git a/compiler/rustc_middle/src/traits/solve/cache.rs b/compiler/rustc_middle/src/traits/solve/cache.rs
index e9e9cc418..4f90af0a2 100644
--- a/compiler/rustc_middle/src/traits/solve/cache.rs
+++ b/compiler/rustc_middle/src/traits/solve/cache.rs
@@ -5,7 +5,7 @@ use rustc_data_structures::sync::Lock;
 use rustc_query_system::cache::WithDepNode;
 use rustc_query_system::dep_graph::DepNodeIndex;
 use rustc_session::Limit;
-/// The trait solver cache used by `-Ztrait-solver=next`.
+/// The trait solver cache used by `-Znext-solver`.
 ///
 /// FIXME(@lcnr): link to some official documentation of how
 /// this works.
diff --git a/compiler/rustc_middle/src/traits/solve/inspect.rs b/compiler/rustc_middle/src/traits/solve/inspect.rs
index a5916c4ab..77d112d0a 100644
--- a/compiler/rustc_middle/src/traits/solve/inspect.rs
+++ b/compiler/rustc_middle/src/traits/solve/inspect.rs
@@ -19,8 +19,8 @@
 //! [canonicalized]: https://rustc-dev-guide.rust-lang.org/solve/canonicalization.html
 
 use super::{
-    CandidateSource, Canonical, CanonicalInput, Certainty, Goal, IsNormalizesToHack, NoSolution,
-    QueryInput, QueryResult,
+    CandidateSource, Canonical, CanonicalInput, Certainty, Goal, GoalSource, IsNormalizesToHack,
+    NoSolution, QueryInput, QueryResult,
 };
 use crate::{infer::canonical::CanonicalVarValues, ty};
 use format::ProofTreeFormatter;
@@ -115,13 +115,15 @@ impl Debug for Probe<'_> {
 pub enum ProbeStep<'tcx> {
     /// We added a goal to the `EvalCtxt` which will get proven
     /// the next time `EvalCtxt::try_evaluate_added_goals` is called.
-    AddGoal(CanonicalState<'tcx, Goal<'tcx, ty::Predicate<'tcx>>>),
+    AddGoal(GoalSource, CanonicalState<'tcx, Goal<'tcx, ty::Predicate<'tcx>>>),
     /// The inside of a `EvalCtxt::try_evaluate_added_goals` call.
     EvaluateGoals(AddedGoalsEvaluation<'tcx>),
     /// A call to `probe` while proving the current goal. This is
     /// used whenever there are multiple candidates to prove the
     /// current goalby .
     NestedProbe(Probe<'tcx>),
+    CommitIfOkStart,
+    CommitIfOkSuccess,
 }
 
 /// What kind of probe we're in. In case the probe represents a candidate, or
@@ -142,6 +144,9 @@ pub enum ProbeKind<'tcx> {
     /// Used in the probe that wraps normalizing the non-self type for the unsize
     /// trait, which is also structurally matched on.
     UnsizeAssembly,
+    /// A call to `EvalCtxt::commit_if_ok` which failed, causing the work
+    /// to be discarded.
+    CommitIfOk,
     /// During upcasting from some source object to target object type, used to
     /// do a probe to find out what projection type(s) may be used to prove that
     /// the source type upholds all of the target type's object bounds.
diff --git a/compiler/rustc_middle/src/traits/solve/inspect/format.rs b/compiler/rustc_middle/src/traits/solve/inspect/format.rs
index 4b73d8e41..4e2207ed5 100644
--- a/compiler/rustc_middle/src/traits/solve/inspect/format.rs
+++ b/compiler/rustc_middle/src/traits/solve/inspect/format.rs
@@ -109,6 +109,9 @@ impl<'a, 'b> ProofTreeFormatter<'a, 'b> {
             ProbeKind::UpcastProjectionCompatibility => {
                 writeln!(self.f, "PROBING FOR PROJECTION COMPATIBILITY FOR UPCASTING:")
             }
+            ProbeKind::CommitIfOk => {
+                writeln!(self.f, "COMMIT_IF_OK:")
+            }
             ProbeKind::MiscCandidate { name, result } => {
                 writeln!(self.f, "CANDIDATE {name}: {result:?}")
             }
@@ -120,9 +123,17 @@ impl<'a, 'b> ProofTreeFormatter<'a, 'b> {
         self.nested(|this| {
             for step in &probe.steps {
                 match step {
-                    ProbeStep::AddGoal(goal) => writeln!(this.f, "ADDED GOAL: {goal:?}")?,
+                    ProbeStep::AddGoal(source, goal) => {
+                        let source = match source {
+                            GoalSource::Misc => "misc",
+                            GoalSource::ImplWhereBound => "impl where-bound",
+                        };
+                        writeln!(this.f, "ADDED GOAL ({source}): {goal:?}")?
+                    }
                     ProbeStep::EvaluateGoals(eval) => this.format_added_goals_evaluation(eval)?,
                     ProbeStep::NestedProbe(probe) => this.format_probe(probe)?,
+                    ProbeStep::CommitIfOkStart => writeln!(this.f, "COMMIT_IF_OK START")?,
+                    ProbeStep::CommitIfOkSuccess => writeln!(this.f, "COMMIT_IF_OK SUCCESS")?,
                 }
             }
             Ok(())
diff --git a/compiler/rustc_middle/src/traits/specialization_graph.rs b/compiler/rustc_middle/src/traits/specialization_graph.rs
index e48b46d12..6db427064 100644
--- a/compiler/rustc_middle/src/traits/specialization_graph.rs
+++ b/compiler/rustc_middle/src/traits/specialization_graph.rs
@@ -72,7 +72,7 @@ impl OverlapMode {
                     .as_local()
                     .into_iter()
                     .flat_map(|local_def_id| {
-                        tcx.hir().attrs(tcx.hir().local_def_id_to_hir_id(local_def_id))
+                        tcx.hir().attrs(tcx.local_def_id_to_hir_id(local_def_id))
                     })
                     .find(|attr| attr.has_name(sym::rustc_strict_coherence))
                     .map(|attr| attr.span);
diff --git a/compiler/rustc_middle/src/ty/_match.rs b/compiler/rustc_middle/src/ty/_match.rs
index 85181720d..a2794a100 100644
--- a/compiler/rustc_middle/src/ty/_match.rs
+++ b/compiler/rustc_middle/src/ty/_match.rs
@@ -20,12 +20,11 @@ use crate::ty::{self, InferConst, Ty, TyCtxt};
 /// affects any type variables or unification state.
 pub struct MatchAgainstFreshVars<'tcx> {
     tcx: TyCtxt<'tcx>,
-    param_env: ty::ParamEnv<'tcx>,
 }
 
 impl<'tcx> MatchAgainstFreshVars<'tcx> {
-    pub fn new(tcx: TyCtxt<'tcx>, param_env: ty::ParamEnv<'tcx>) -> MatchAgainstFreshVars<'tcx> {
-        MatchAgainstFreshVars { tcx, param_env }
+    pub fn new(tcx: TyCtxt<'tcx>) -> MatchAgainstFreshVars<'tcx> {
+        MatchAgainstFreshVars { tcx }
     }
 }
 
@@ -33,13 +32,11 @@ impl<'tcx> TypeRelation<'tcx> for MatchAgainstFreshVars<'tcx> {
     fn tag(&self) -> &'static str {
         "MatchAgainstFreshVars"
     }
+
     fn tcx(&self) -> TyCtxt<'tcx> {
         self.tcx
     }
 
-    fn param_env(&self) -> ty::ParamEnv<'tcx> {
-        self.param_env
-    }
     fn a_is_expected(&self) -> bool {
         true
     } // irrelevant
diff --git a/compiler/rustc_middle/src/ty/adt.rs b/compiler/rustc_middle/src/ty/adt.rs
index f50969dd9..1d9a25628 100644
--- a/compiler/rustc_middle/src/ty/adt.rs
+++ b/compiler/rustc_middle/src/ty/adt.rs
@@ -99,7 +99,7 @@ pub struct AdtDefData {
 
 impl PartialOrd for AdtDefData {
     fn partial_cmp(&self, other: &AdtDefData) -> Option<Ordering> {
-        Some(self.cmp(&other))
+        Some(self.cmp(other))
     }
 }
 
@@ -375,7 +375,7 @@ impl<'tcx> AdtDef<'tcx> {
     /// Asserts this is a struct or union and returns its unique variant.
     pub fn non_enum_variant(self) -> &'tcx VariantDef {
         assert!(self.is_struct() || self.is_union());
-        &self.variant(FIRST_VARIANT)
+        self.variant(FIRST_VARIANT)
     }
 
     #[inline]
@@ -481,7 +481,7 @@ impl<'tcx> AdtDef<'tcx> {
                     ErrorHandled::Reported(..) => "enum discriminant evaluation failed",
                     ErrorHandled::TooGeneric(..) => "enum discriminant depends on generics",
                 };
-                tcx.sess.delay_span_bug(tcx.def_span(expr_did), msg);
+                tcx.sess.span_delayed_bug(tcx.def_span(expr_did), msg);
                 None
             }
         }
diff --git a/compiler/rustc_middle/src/ty/closure.rs b/compiler/rustc_middle/src/ty/closure.rs
index 74bdd07a1..8c29bc5a4 100644
--- a/compiler/rustc_middle/src/ty/closure.rs
+++ b/compiler/rustc_middle/src/ty/closure.rs
@@ -7,14 +7,13 @@ use std::fmt::Write;
 
 use crate::query::Providers;
 use rustc_data_structures::fx::FxIndexMap;
-use rustc_errors::{DiagnosticArgValue, IntoDiagnosticArg};
-use rustc_hir::def_id::{DefId, LocalDefId};
-use rustc_hir::{self as hir, LangItem};
+use rustc_hir as hir;
+use rustc_hir::def_id::LocalDefId;
 use rustc_span::def_id::LocalDefIdMap;
 use rustc_span::symbol::Ident;
 use rustc_span::{Span, Symbol};
 
-use super::{Ty, TyCtxt};
+use super::TyCtxt;
 
 use self::BorrowKind::*;
 
@@ -73,72 +72,6 @@ pub type RootVariableMinCaptureList<'tcx> = FxIndexMap<hir::HirId, MinCaptureLis
 /// Part of `MinCaptureInformationMap`; List of `CapturePlace`s.
 pub type MinCaptureList<'tcx> = Vec<CapturedPlace<'tcx>>;
 
-/// Represents the various closure traits in the language. This
-/// will determine the type of the environment (`self`, in the
-/// desugaring) argument that the closure expects.
-///
-/// You can get the environment type of a closure using
-/// `tcx.closure_env_ty()`.
-#[derive(Clone, Copy, PartialOrd, Ord, PartialEq, Eq, Hash, Debug, TyEncodable, TyDecodable)]
-#[derive(HashStable)]
-pub enum ClosureKind {
-    // Warning: Ordering is significant here! The ordering is chosen
-    // because the trait Fn is a subtrait of FnMut and so in turn, and
-    // hence we order it so that Fn < FnMut < FnOnce.
-    Fn,
-    FnMut,
-    FnOnce,
-}
-
-impl ClosureKind {
-    /// This is the initial value used when doing upvar inference.
-    pub const LATTICE_BOTTOM: ClosureKind = ClosureKind::Fn;
-
-    pub const fn as_str(self) -> &'static str {
-        match self {
-            ClosureKind::Fn => "Fn",
-            ClosureKind::FnMut => "FnMut",
-            ClosureKind::FnOnce => "FnOnce",
-        }
-    }
-
-    /// Returns `true` if a type that impls this closure kind
-    /// must also implement `other`.
-    pub fn extends(self, other: ty::ClosureKind) -> bool {
-        self <= other
-    }
-
-    /// Converts `self` to a [`DefId`] of the corresponding trait.
-    ///
-    /// Note: the inverse of this function is [`TyCtxt::fn_trait_kind_from_def_id`].
-    pub fn to_def_id(&self, tcx: TyCtxt<'_>) -> DefId {
-        tcx.require_lang_item(
-            match self {
-                ClosureKind::Fn => LangItem::Fn,
-                ClosureKind::FnMut => LangItem::FnMut,
-                ClosureKind::FnOnce => LangItem::FnOnce,
-            },
-            None,
-        )
-    }
-
-    /// Returns the representative scalar type for this closure kind.
-    /// See `Ty::to_opt_closure_kind` for more details.
-    pub fn to_ty<'tcx>(self, tcx: TyCtxt<'tcx>) -> Ty<'tcx> {
-        match self {
-            ClosureKind::Fn => tcx.types.i8,
-            ClosureKind::FnMut => tcx.types.i16,
-            ClosureKind::FnOnce => tcx.types.i32,
-        }
-    }
-}
-
-impl IntoDiagnosticArg for ClosureKind {
-    fn into_diagnostic_arg(self) -> DiagnosticArgValue<'static> {
-        DiagnosticArgValue::Str(self.as_str().into())
-    }
-}
-
 /// A composite describing a `Place` that is captured by a closure.
 #[derive(PartialEq, Clone, Debug, TyEncodable, TyDecodable, HashStable)]
 #[derive(TypeFoldable, TypeVisitable)]
@@ -247,6 +180,13 @@ impl<'tcx> CapturedPlace<'tcx> {
                 .span
         }
     }
+
+    pub fn is_by_ref(&self) -> bool {
+        match self.info.capture_kind {
+            ty::UpvarCapture::ByValue => false,
+            ty::UpvarCapture::ByRef(..) => true,
+        }
+    }
 }
 
 #[derive(Copy, Clone, Debug, HashStable)]
@@ -262,7 +202,7 @@ fn closure_typeinfo<'tcx>(tcx: TyCtxt<'tcx>, def: LocalDefId) -> ClosureTypeInfo
     let user_provided_sig = typeck_results.user_provided_sigs[&def];
     let captures = typeck_results.closure_min_captures_flattened(def);
     let captures = tcx.arena.alloc_from_iter(captures);
-    let hir_id = tcx.hir().local_def_id_to_hir_id(def);
+    let hir_id = tcx.local_def_id_to_hir_id(def);
     let kind_origin = typeck_results.closure_kind_origins().get(hir_id);
     ClosureTypeInfo { user_provided_sig, captures, kind_origin }
 }
diff --git a/compiler/rustc_middle/src/ty/codec.rs b/compiler/rustc_middle/src/ty/codec.rs
index 8b67e3966..69ae05ca8 100644
--- a/compiler/rustc_middle/src/ty/codec.rs
+++ b/compiler/rustc_middle/src/ty/codec.rs
@@ -8,6 +8,7 @@
 
 use crate::arena::ArenaAllocatable;
 use crate::infer::canonical::{CanonicalVarInfo, CanonicalVarInfos};
+use crate::mir::interpret::CtfeProvenance;
 use crate::mir::{
     self,
     interpret::{AllocId, ConstAllocation},
@@ -164,6 +165,13 @@ impl<'tcx, E: TyEncoder<I = TyCtxt<'tcx>>> Encodable<E> for AllocId {
     }
 }
 
+impl<'tcx, E: TyEncoder<I = TyCtxt<'tcx>>> Encodable<E> for CtfeProvenance {
+    fn encode(&self, e: &mut E) {
+        self.alloc_id().encode(e);
+        self.immutable().encode(e);
+    }
+}
+
 impl<'tcx, E: TyEncoder<I = TyCtxt<'tcx>>> Encodable<E> for ty::ParamEnv<'tcx> {
     fn encode(&self, e: &mut E) {
         self.caller_bounds().encode(e);
@@ -295,9 +303,18 @@ impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> Decodable<D> for AllocId {
     }
 }
 
+impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> Decodable<D> for CtfeProvenance {
+    fn decode(decoder: &mut D) -> Self {
+        let alloc_id: AllocId = Decodable::decode(decoder);
+        let prov = CtfeProvenance::from(alloc_id);
+        let immutable: bool = Decodable::decode(decoder);
+        if immutable { prov.as_immutable() } else { prov }
+    }
+}
+
 impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> Decodable<D> for ty::SymbolName<'tcx> {
     fn decode(decoder: &mut D) -> Self {
-        ty::SymbolName::new(decoder.interner(), &decoder.read_str())
+        ty::SymbolName::new(decoder.interner(), decoder.read_str())
     }
 }
 
diff --git a/compiler/rustc_middle/src/ty/consts.rs b/compiler/rustc_middle/src/ty/consts.rs
index af5ffc20d..2d3ee3348 100644
--- a/compiler/rustc_middle/src/ty/consts.rs
+++ b/compiler/rustc_middle/src/ty/consts.rs
@@ -1,5 +1,5 @@
 use crate::middle::resolve_bound_vars as rbv;
-use crate::mir::interpret::{AllocId, ErrorHandled, LitToConstInput, Scalar};
+use crate::mir::interpret::{ErrorHandled, LitToConstInput, Scalar};
 use crate::ty::{self, GenericArgs, ParamEnv, ParamEnvAnd, Ty, TyCtxt, TypeVisitableExt};
 use rustc_data_structures::intern::Interned;
 use rustc_error_messages::MultiSpan;
@@ -7,6 +7,7 @@ use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
 use rustc_hir::def_id::LocalDefId;
 use rustc_macros::HashStable;
+use rustc_type_ir::{ConstTy, IntoKind, TypeFlags, WithCachedTypeInfo};
 
 mod int;
 mod kind;
@@ -23,10 +24,25 @@ use super::sty::ConstKind;
 /// Use this rather than `ConstData`, whenever possible.
 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, HashStable)]
 #[rustc_pass_by_value]
-pub struct Const<'tcx>(pub(super) Interned<'tcx, ConstData<'tcx>>);
+pub struct Const<'tcx>(pub(super) Interned<'tcx, WithCachedTypeInfo<ConstData<'tcx>>>);
+
+impl<'tcx> IntoKind for Const<'tcx> {
+    type Kind = ConstKind<'tcx>;
+
+    fn kind(self) -> ConstKind<'tcx> {
+        self.kind()
+    }
+}
+
+impl<'tcx> ConstTy<TyCtxt<'tcx>> for Const<'tcx> {
+    fn ty(self) -> Ty<'tcx> {
+        self.ty()
+    }
+}
 
 /// Typed constant value.
-#[derive(PartialEq, Eq, PartialOrd, Ord, Hash, HashStable, TyEncodable, TyDecodable)]
+#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[derive(HashStable, TyEncodable, TyDecodable)]
 pub struct ConstData<'tcx> {
     pub ty: Ty<'tcx>,
     pub kind: ConstKind<'tcx>,
@@ -43,7 +59,17 @@ impl<'tcx> Const<'tcx> {
 
     #[inline]
     pub fn kind(self) -> ConstKind<'tcx> {
-        self.0.kind.clone()
+        self.0.kind
+    }
+
+    #[inline]
+    pub fn flags(self) -> TypeFlags {
+        self.0.flags
+    }
+
+    #[inline]
+    pub fn outer_exclusive_binder(self) -> ty::DebruijnIndex {
+        self.0.outer_exclusive_binder
     }
 
     #[inline]
@@ -133,7 +159,7 @@ impl<'tcx> Const<'tcx> {
         span: S,
         msg: &'static str,
     ) -> Const<'tcx> {
-        let reported = tcx.sess.delay_span_bug(span, msg);
+        let reported = tcx.sess.span_delayed_bug(span, msg);
         Const::new_error(tcx, reported, ty)
     }
 
@@ -141,7 +167,7 @@ impl<'tcx> Const<'tcx> {
     /// becomes `Unevaluated`.
     #[instrument(skip(tcx), level = "debug")]
     pub fn from_anon_const(tcx: TyCtxt<'tcx>, def: LocalDefId) -> Self {
-        let body_id = match tcx.hir().get_by_def_id(def) {
+        let body_id = match tcx.hir_node_by_def_id(def) {
             hir::Node::AnonConst(ac) => ac.body,
             _ => span_bug!(
                 tcx.def_span(def.to_def_id()),
@@ -183,11 +209,9 @@ impl<'tcx> Const<'tcx> {
         };
 
         let lit_input = match expr.kind {
-            hir::ExprKind::Lit(ref lit) => Some(LitToConstInput { lit: &lit.node, ty, neg: false }),
-            hir::ExprKind::Unary(hir::UnOp::Neg, ref expr) => match expr.kind {
-                hir::ExprKind::Lit(ref lit) => {
-                    Some(LitToConstInput { lit: &lit.node, ty, neg: true })
-                }
+            hir::ExprKind::Lit(lit) => Some(LitToConstInput { lit: &lit.node, ty, neg: false }),
+            hir::ExprKind::Unary(hir::UnOp::Neg, expr) => match expr.kind {
+                hir::ExprKind::Lit(lit) => Some(LitToConstInput { lit: &lit.node, ty, neg: true }),
                 _ => None,
             },
             _ => None,
@@ -199,7 +223,7 @@ impl<'tcx> Const<'tcx> {
             match tcx.at(expr.span).lit_to_const(lit_input) {
                 Ok(c) => return Some(c),
                 Err(e) => {
-                    tcx.sess.delay_span_bug(
+                    tcx.sess.span_delayed_bug(
                         expr.span,
                         format!("Const::from_anon_const: couldn't lit_to_const {e:?}"),
                     );
@@ -207,6 +231,10 @@ impl<'tcx> Const<'tcx> {
             }
         }
 
+        // FIXME(const_generics): We currently have to special case parameters because `min_const_generics`
+        // does not provide the parents generics to anonymous constants. We still allow generic const
+        // parameters by themselves however, e.g. `N`. These constants would cause an ICE if we were to
+        // ever try to substitute the generic parameters in their bodies.
         match expr.kind {
             hir::ExprKind::Path(hir::QPath::Resolved(
                 _,
@@ -291,8 +319,16 @@ impl<'tcx> Const<'tcx> {
                 let (param_env, unevaluated) = unevaluated.prepare_for_eval(tcx, param_env);
                 // try to resolve e.g. associated constants to their definition on an impl, and then
                 // evaluate the const.
-                let c = tcx.const_eval_resolve_for_typeck(param_env, unevaluated, span)?;
-                Ok(c.expect("`ty::Const::eval` called on a non-valtree-compatible type"))
+                let Some(c) = tcx.const_eval_resolve_for_typeck(param_env, unevaluated, span)?
+                else {
+                    // This can happen when we run on ill-typed code.
+                    let e = tcx.sess.span_delayed_bug(
+                        span.unwrap_or(DUMMY_SP),
+                        "`ty::Const::eval` called on a non-valtree-compatible type",
+                    );
+                    return Err(e.into());
+                };
+                Ok(c)
             }
             ConstKind::Value(val) => Ok(val),
             ConstKind::Error(g) => Err(g.into()),
@@ -392,7 +428,7 @@ impl<'tcx> Const<'tcx> {
     }
 
     #[inline]
-    pub fn try_to_scalar(self) -> Option<Scalar<AllocId>> {
+    pub fn try_to_scalar(self) -> Option<Scalar> {
         self.try_to_valtree()?.try_to_scalar()
     }
 
@@ -407,7 +443,7 @@ impl<'tcx> Const<'tcx> {
 }
 
 pub fn const_param_default(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::EarlyBinder<Const<'_>> {
-    let default_def_id = match tcx.hir().get_by_def_id(def_id) {
+    let default_def_id = match tcx.hir_node_by_def_id(def_id) {
         hir::Node::GenericParam(hir::GenericParam {
             kind: hir::GenericParamKind::Const { default: Some(ac), .. },
             ..
diff --git a/compiler/rustc_middle/src/ty/consts/int.rs b/compiler/rustc_middle/src/ty/consts/int.rs
index 9d99344d5..310cf113b 100644
--- a/compiler/rustc_middle/src/ty/consts/int.rs
+++ b/compiler/rustc_middle/src/ty/consts/int.rs
@@ -71,7 +71,7 @@ impl std::fmt::Debug for ConstInt {
                         (4, _) => write!(fmt, "_i32")?,
                         (8, _) => write!(fmt, "_i64")?,
                         (16, _) => write!(fmt, "_i128")?,
-                        _ => bug!(),
+                        (sz, _) => bug!("unexpected int size i{sz}"),
                     }
                 }
                 Ok(())
@@ -105,7 +105,7 @@ impl std::fmt::Debug for ConstInt {
                         (4, _) => write!(fmt, "_u32")?,
                         (8, _) => write!(fmt, "_u64")?,
                         (16, _) => write!(fmt, "_u128")?,
-                        _ => bug!(),
+                        (sz, _) => bug!("unexpected unsigned int size u{sz}"),
                     }
                 }
                 Ok(())
diff --git a/compiler/rustc_middle/src/ty/consts/kind.rs b/compiler/rustc_middle/src/ty/consts/kind.rs
index 4af841fcf..16e7b16a0 100644
--- a/compiler/rustc_middle/src/ty/consts/kind.rs
+++ b/compiler/rustc_middle/src/ty/consts/kind.rs
@@ -3,7 +3,6 @@ use crate::mir;
 use crate::ty::abstract_const::CastKind;
 use crate::ty::GenericArgsRef;
 use crate::ty::{self, visit::TypeVisitableExt as _, List, Ty, TyCtxt};
-use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_hir::def_id::DefId;
 use rustc_macros::HashStable;
 
@@ -77,28 +76,3 @@ static_assert_size!(Expr<'_>, 24);
 
 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 static_assert_size!(super::ConstKind<'_>, 32);
-
-/// An inference variable for a const, for use in const generics.
-#[derive(Copy, Clone, Eq, PartialEq, PartialOrd, Ord, TyEncodable, TyDecodable, Hash)]
-pub enum InferConst {
-    /// Infer the value of the const.
-    Var(ty::ConstVid),
-    /// Infer the value of the effect.
-    ///
-    /// For why this is separate from the `Var` variant above, see the
-    /// documentation on `EffectVid`.
-    EffectVar(ty::EffectVid),
-    /// A fresh const variable. See `infer::freshen` for more details.
-    Fresh(u32),
-}
-
-impl<CTX> HashStable<CTX> for InferConst {
-    fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) {
-        match self {
-            InferConst::Var(_) | InferConst::EffectVar(_) => {
-                panic!("const variables should not be hashed: {self:?}")
-            }
-            InferConst::Fresh(i) => i.hash_stable(hcx, hasher),
-        }
-    }
-}
diff --git a/compiler/rustc_middle/src/ty/consts/valtree.rs b/compiler/rustc_middle/src/ty/consts/valtree.rs
index fb7bf78ba..ffa0e89c4 100644
--- a/compiler/rustc_middle/src/ty/consts/valtree.rs
+++ b/compiler/rustc_middle/src/ty/consts/valtree.rs
@@ -1,5 +1,5 @@
 use super::ScalarInt;
-use crate::mir::interpret::{AllocId, Scalar};
+use crate::mir::interpret::Scalar;
 use crate::ty::{self, Ty, TyCtxt};
 use rustc_macros::{HashStable, TyDecodable, TyEncodable};
 
@@ -67,7 +67,7 @@ impl<'tcx> ValTree<'tcx> {
         Self::Leaf(i)
     }
 
-    pub fn try_to_scalar(self) -> Option<Scalar<AllocId>> {
+    pub fn try_to_scalar(self) -> Option<Scalar> {
         self.try_to_scalar_int().map(Scalar::Int)
     }
 
diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index 551c4a15d..b5ca700c2 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -6,7 +6,7 @@ pub mod tls;
 
 use crate::arena::Arena;
 use crate::dep_graph::{DepGraph, DepKindStruct};
-use crate::infer::canonical::{CanonicalVarInfo, CanonicalVarInfos};
+use crate::infer::canonical::{CanonicalParamEnvCache, CanonicalVarInfo, CanonicalVarInfos};
 use crate::lint::struct_lint_level;
 use crate::metadata::ModChild;
 use crate::middle::codegen_fn_attrs::CodegenFnAttrs;
@@ -26,9 +26,9 @@ use crate::traits::solve::{
 };
 use crate::ty::{
     self, AdtDef, AdtDefData, AdtKind, Binder, Clause, Const, ConstData, GenericParamDefKind,
-    ImplPolarity, InferTy, List, ParamConst, ParamTy, PolyExistentialPredicate, PolyFnSig,
-    Predicate, PredicateKind, Region, RegionKind, ReprOptions, TraitObjectVisitor, Ty, TyKind,
-    TyVid, TypeAndMut, Visibility,
+    ImplPolarity, List, ParamConst, ParamTy, PolyExistentialPredicate, PolyFnSig, Predicate,
+    PredicateKind, Region, RegionKind, ReprOptions, TraitObjectVisitor, Ty, TyKind, TyVid,
+    Visibility,
 };
 use crate::ty::{GenericArg, GenericArgs, GenericArgsRef};
 use rustc_ast::{self as ast, attr};
@@ -39,7 +39,9 @@ use rustc_data_structures::profiling::SelfProfilerRef;
 use rustc_data_structures::sharded::{IntoPointer, ShardedHashMap};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::steal::Steal;
-use rustc_data_structures::sync::{self, FreezeReadGuard, Lock, Lrc, WorkerLocal};
+use rustc_data_structures::sync::{self, FreezeReadGuard, Lock, WorkerLocal};
+#[cfg(parallel_compiler)]
+use rustc_data_structures::sync::{DynSend, DynSync};
 use rustc_data_structures::unord::UnordSet;
 use rustc_errors::{
     DecorateLint, DiagnosticBuilder, DiagnosticMessage, ErrorGuaranteed, MultiSpan,
@@ -69,7 +71,6 @@ use rustc_type_ir::TyKind::*;
 use rustc_type_ir::WithCachedTypeInfo;
 use rustc_type_ir::{CollectAndApply, Interner, TypeFlags};
 
-use std::any::Any;
 use std::borrow::Borrow;
 use std::cmp::Ordering;
 use std::fmt;
@@ -87,7 +88,6 @@ impl<'tcx> Interner for TyCtxt<'tcx> {
     type Term = ty::Term<'tcx>;
 
     type Binder<T> = Binder<'tcx, T>;
-    type TypeAndMut = TypeAndMut<'tcx>;
     type CanonicalVars = CanonicalVarInfos<'tcx>;
 
     type Ty = Ty<'tcx>;
@@ -96,7 +96,6 @@ impl<'tcx> Interner for TyCtxt<'tcx> {
     type ParamTy = ParamTy;
     type BoundTy = ty::BoundTy;
     type PlaceholderTy = ty::PlaceholderType;
-    type InferTy = InferTy;
 
     type ErrorGuaranteed = ErrorGuaranteed;
     type BoundExistentialPredicates = &'tcx List<PolyExistentialPredicate<'tcx>>;
@@ -104,7 +103,6 @@ impl<'tcx> Interner for TyCtxt<'tcx> {
     type AllocId = crate::mir::interpret::AllocId;
 
     type Const = ty::Const<'tcx>;
-    type InferConst = ty::InferConst;
     type AliasConst = ty::UnevaluatedConst<'tcx>;
     type PlaceholderConst = ty::PlaceholderConst;
     type ParamConst = ty::ParamConst;
@@ -113,9 +111,9 @@ impl<'tcx> Interner for TyCtxt<'tcx> {
     type ExprConst = ty::Expr<'tcx>;
 
     type Region = Region<'tcx>;
-    type EarlyBoundRegion = ty::EarlyBoundRegion;
+    type EarlyParamRegion = ty::EarlyParamRegion;
     type BoundRegion = ty::BoundRegion;
-    type FreeRegion = ty::FreeRegion;
+    type LateParamRegion = ty::LateParamRegion;
     type InferRegion = ty::RegionVid;
     type PlaceholderRegion = ty::PlaceholderRegion;
 
@@ -124,14 +122,34 @@ impl<'tcx> Interner for TyCtxt<'tcx> {
     type RegionOutlivesPredicate = ty::RegionOutlivesPredicate<'tcx>;
     type TypeOutlivesPredicate = ty::TypeOutlivesPredicate<'tcx>;
     type ProjectionPredicate = ty::ProjectionPredicate<'tcx>;
+    type NormalizesTo = ty::NormalizesTo<'tcx>;
     type SubtypePredicate = ty::SubtypePredicate<'tcx>;
     type CoercePredicate = ty::CoercePredicate<'tcx>;
     type ClosureKind = ty::ClosureKind;
 
-    fn ty_and_mut_to_parts(
-        TypeAndMut { ty, mutbl }: TypeAndMut<'tcx>,
-    ) -> (Self::Ty, ty::Mutability) {
-        (ty, mutbl)
+    fn mk_canonical_var_infos(self, infos: &[ty::CanonicalVarInfo<Self>]) -> Self::CanonicalVars {
+        self.mk_canonical_var_infos(infos)
+    }
+
+    fn mk_bound_ty(self, debruijn: ty::DebruijnIndex, var: ty::BoundVar) -> Self::Ty {
+        Ty::new_bound(self, debruijn, ty::BoundTy { var, kind: ty::BoundTyKind::Anon })
+    }
+
+    fn mk_bound_region(self, debruijn: ty::DebruijnIndex, var: ty::BoundVar) -> Self::Region {
+        Region::new_bound(
+            self,
+            debruijn,
+            ty::BoundRegion { var, kind: ty::BoundRegionKind::BrAnon },
+        )
+    }
+
+    fn mk_bound_const(
+        self,
+        debruijn: ty::DebruijnIndex,
+        var: ty::BoundVar,
+        ty: Self::Ty,
+    ) -> Self::Const {
+        Const::new_bound(self, debruijn, var, ty)
     }
 }
 
@@ -154,7 +172,7 @@ pub struct CtxtInterners<'tcx> {
     clauses: InternedSet<'tcx, List<Clause<'tcx>>>,
     projs: InternedSet<'tcx, List<ProjectionKind>>,
     place_elems: InternedSet<'tcx, List<PlaceElem<'tcx>>>,
-    const_: InternedSet<'tcx, ConstData<'tcx>>,
+    const_: InternedSet<'tcx, WithCachedTypeInfo<ConstData<'tcx>>>,
     const_allocation: InternedSet<'tcx, Allocation>,
     bound_variable_kinds: InternedSet<'tcx, List<ty::BoundVariableKind>>,
     layout: InternedSet<'tcx, LayoutS<FieldIdx, VariantIdx>>,
@@ -215,6 +233,32 @@ impl<'tcx> CtxtInterners<'tcx> {
         ))
     }
 
+    /// Interns a const. (Use `mk_*` functions instead, where possible.)
+    #[allow(rustc::usage_of_ty_tykind)]
+    #[inline(never)]
+    fn intern_const(
+        &self,
+        data: ty::ConstData<'tcx>,
+        sess: &Session,
+        untracked: &Untracked,
+    ) -> Const<'tcx> {
+        Const(Interned::new_unchecked(
+            self.const_
+                .intern(data, |data: ConstData<'_>| {
+                    let flags = super::flags::FlagComputation::for_const(&data.kind, data.ty);
+                    let stable_hash = self.stable_hash(&flags, sess, untracked, &data);
+
+                    InternedInSet(self.arena.alloc(WithCachedTypeInfo {
+                        internee: data,
+                        stable_hash,
+                        flags: flags.flags,
+                        outer_exclusive_binder: flags.outer_exclusive_binder,
+                    }))
+                })
+                .0,
+        ))
+    }
+
     fn stable_hash<'a, T: HashStable<StableHashingContext<'a>>>(
         &self,
         flags: &ty::flags::FlagComputation,
@@ -327,7 +371,7 @@ pub struct CommonLifetimes<'tcx> {
     pub re_vars: Vec<Region<'tcx>>,
 
     /// Pre-interned values of the form:
-    /// `ReLateBound(DebruijnIndex(i), BoundRegion { var: v, kind: BrAnon })`
+    /// `ReBound(DebruijnIndex(i), BoundRegion { var: v, kind: BrAnon })`
     /// for small values of `i` and `v`.
     pub re_late_bounds: Vec<Vec<Region<'tcx>>>,
 }
@@ -402,7 +446,7 @@ impl<'tcx> CommonLifetimes<'tcx> {
             .map(|i| {
                 (0..NUM_PREINTERNED_RE_LATE_BOUNDS_V)
                     .map(|v| {
-                        mk(ty::ReLateBound(
+                        mk(ty::ReBound(
                             ty::DebruijnIndex::from(i),
                             ty::BoundRegion { var: ty::BoundVar::from(v), kind: ty::BrAnon },
                         ))
@@ -421,11 +465,17 @@ impl<'tcx> CommonLifetimes<'tcx> {
 }
 
 impl<'tcx> CommonConsts<'tcx> {
-    fn new(interners: &CtxtInterners<'tcx>, types: &CommonTypes<'tcx>) -> CommonConsts<'tcx> {
+    fn new(
+        interners: &CtxtInterners<'tcx>,
+        types: &CommonTypes<'tcx>,
+        sess: &Session,
+        untracked: &Untracked,
+    ) -> CommonConsts<'tcx> {
         let mk_const = |c| {
-            Const(Interned::new_unchecked(
-                interners.const_.intern(c, |c| InternedInSet(interners.arena.alloc(c))).0,
-            ))
+            interners.intern_const(
+                c, sess, // This is only used to create a stable hashing context.
+                untracked,
+            )
         };
 
         CommonConsts {
@@ -445,14 +495,14 @@ impl<'tcx> CommonConsts<'tcx> {
     }
 }
 
-/// This struct contains information regarding the `ReFree(FreeRegion)` corresponding to a lifetime
-/// conflict.
+/// This struct contains information regarding a free parameter region,
+/// either a `ReEarlyParam` or `ReLateParam`.
 #[derive(Debug)]
 pub struct FreeRegionInfo {
-    /// `LocalDefId` corresponding to FreeRegion
+    /// `LocalDefId` of the free region.
     pub def_id: LocalDefId,
-    /// the bound region corresponding to FreeRegion
-    pub boundregion: ty::BoundRegionKind,
+    /// the bound region corresponding to free region.
+    pub bound_region: ty::BoundRegionKind,
     /// checks if bound region is in Impl Item
     pub is_impl_item: bool,
 }
@@ -472,6 +522,9 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn feed_local_crate(self) -> TyCtxtFeed<'tcx, CrateNum> {
         TyCtxtFeed { tcx: self, key: LOCAL_CRATE }
     }
+    pub fn feed_local_def_id(self, key: LocalDefId) -> TyCtxtFeed<'tcx, LocalDefId> {
+        TyCtxtFeed { tcx: self, key }
+    }
 
     /// In order to break cycles involving `AnonConst`, we need to set the expected type by side
     /// effect. However, we do not want this as a general capability, so this interface restricts
@@ -520,6 +573,16 @@ pub struct TyCtxt<'tcx> {
     gcx: &'tcx GlobalCtxt<'tcx>,
 }
 
+// Explicitly implement `DynSync` and `DynSend` for `TyCtxt` to short circuit trait resolution.
+#[cfg(parallel_compiler)]
+unsafe impl DynSend for TyCtxt<'_> {}
+#[cfg(parallel_compiler)]
+unsafe impl DynSync for TyCtxt<'_> {}
+fn _assert_tcx_fields() {
+    sync::assert_dyn_sync::<&'_ GlobalCtxt<'_>>();
+    sync::assert_dyn_send::<&'_ GlobalCtxt<'_>>();
+}
+
 impl<'tcx> Deref for TyCtxt<'tcx> {
     type Target = &'tcx GlobalCtxt<'tcx>;
     #[inline(always)]
@@ -544,12 +607,6 @@ pub struct GlobalCtxt<'tcx> {
     /// `rustc_symbol_mangling` crate for more information.
     stable_crate_id: StableCrateId,
 
-    /// This only ever stores a `LintStore` but we don't want a dependency on that type here.
-    ///
-    /// FIXME(Centril): consider `dyn LintStoreMarker` once
-    /// we can upcast to `Any` for some additional type safety.
-    pub lint_store: Lrc<dyn Any + sync::DynSync + sync::DynSend>,
-
     pub dep_graph: DepGraph,
 
     pub prof: SelfProfilerRef,
@@ -589,6 +646,8 @@ pub struct GlobalCtxt<'tcx> {
     pub new_solver_evaluation_cache: solve::EvaluationCache<'tcx>,
     pub new_solver_coherence_evaluation_cache: solve::EvaluationCache<'tcx>,
 
+    pub canonical_param_env_cache: CanonicalParamEnvCache<'tcx>,
+
     /// Data layout specification for the current target.
     pub data_layout: TargetDataLayout,
 
@@ -606,6 +665,10 @@ impl<'tcx> GlobalCtxt<'tcx> {
         let icx = tls::ImplicitCtxt::new(self);
         tls::enter_context(&icx, || f(icx.tcx))
     }
+
+    pub fn finish(&self) -> FileEncodeResult {
+        self.dep_graph.finish_encoding(&self.sess.prof)
+    }
 }
 
 impl<'tcx> TyCtxt<'tcx> {
@@ -684,8 +747,10 @@ impl<'tcx> TyCtxt<'tcx> {
             {
                 Bound::Included(a)
             } else {
-                self.sess
-                    .delay_span_bug(attr.span, "invalid rustc_layout_scalar_valid_range attribute");
+                self.sess.span_delayed_bug(
+                    attr.span,
+                    "invalid rustc_layout_scalar_valid_range attribute",
+                );
                 Bound::Unbounded
             }
         };
@@ -709,7 +774,6 @@ impl<'tcx> TyCtxt<'tcx> {
         s: &'tcx Session,
         crate_types: Vec<CrateType>,
         stable_crate_id: StableCrateId,
-        lint_store: Lrc<dyn Any + sync::DynSend + sync::DynSync>,
         arena: &'tcx WorkerLocal<Arena<'tcx>>,
         hir_arena: &'tcx WorkerLocal<hir::Arena<'tcx>>,
         untracked: Untracked,
@@ -724,13 +788,12 @@ impl<'tcx> TyCtxt<'tcx> {
         let interners = CtxtInterners::new(arena);
         let common_types = CommonTypes::new(&interners, s, &untracked);
         let common_lifetimes = CommonLifetimes::new(&interners);
-        let common_consts = CommonConsts::new(&interners, &common_types);
+        let common_consts = CommonConsts::new(&interners, &common_types, s, &untracked);
 
         GlobalCtxt {
             sess: s,
             crate_types,
             stable_crate_id,
-            lint_store,
             arena,
             hir_arena,
             interners,
@@ -749,6 +812,7 @@ impl<'tcx> TyCtxt<'tcx> {
             evaluation_cache: Default::default(),
             new_solver_evaluation_cache: Default::default(),
             new_solver_coherence_evaluation_cache: Default::default(),
+            canonical_param_env_cache: Default::default(),
             data_layout,
             alloc_map: Lock::new(interpret::AllocMap::new()),
         }
@@ -779,6 +843,10 @@ impl<'tcx> TyCtxt<'tcx> {
         self.diagnostic_items(did.krate).name_to_id.get(&name) == Some(&did)
     }
 
+    pub fn is_coroutine(self, def_id: DefId) -> bool {
+        self.coroutine_kind(def_id).is_some()
+    }
+
     /// Returns `true` if the node pointed to by `def_id` is a coroutine for an async construct.
     pub fn coroutine_is_async(self, def_id: DefId) -> bool {
         matches!(self.coroutine_kind(def_id), Some(hir::CoroutineKind::Async(_)))
@@ -790,11 +858,16 @@ impl<'tcx> TyCtxt<'tcx> {
         matches!(self.coroutine_kind(def_id), Some(hir::CoroutineKind::Coroutine))
     }
 
-    /// Returns `true` if the node pointed to by `def_id` is a coroutine for a gen construct.
+    /// Returns `true` if the node pointed to by `def_id` is a coroutine for a `gen` construct.
     pub fn coroutine_is_gen(self, def_id: DefId) -> bool {
         matches!(self.coroutine_kind(def_id), Some(hir::CoroutineKind::Gen(_)))
     }
 
+    /// Returns `true` if the node pointed to by `def_id` is a coroutine for a `async gen` construct.
+    pub fn coroutine_is_async_gen(self, def_id: DefId) -> bool {
+        matches!(self.coroutine_kind(def_id), Some(hir::CoroutineKind::AsyncGen(_)))
+    }
+
     pub fn stability(self) -> &'tcx stability::Index {
         self.stability_index(())
     }
@@ -943,7 +1016,8 @@ impl<'tcx> TyCtxtAt<'tcx> {
     pub fn create_def(
         self,
         parent: LocalDefId,
-        data: hir::definitions::DefPathData,
+        name: Symbol,
+        def_kind: DefKind,
     ) -> TyCtxtFeed<'tcx, LocalDefId> {
         // This function modifies `self.definitions` using a side-effect.
         // We need to ensure that these side effects are re-run by the incr. comp. engine.
@@ -965,15 +1039,34 @@ impl<'tcx> TyCtxtAt<'tcx> {
         // This is fine because:
         // - those queries are `eval_always` so we won't miss their result changing;
         // - this write will have happened before these queries are called.
-        let key = self.untracked.definitions.write().create_def(parent, data);
+        let def_id = self.tcx.create_def(parent, name, def_kind);
 
-        let feed = TyCtxtFeed { tcx: self.tcx, key };
+        let feed = self.tcx.feed_local_def_id(def_id);
         feed.def_span(self.span);
         feed
     }
 }
 
 impl<'tcx> TyCtxt<'tcx> {
+    /// `tcx`-dependent operations performed for every created definition.
+    pub fn create_def(self, parent: LocalDefId, name: Symbol, def_kind: DefKind) -> LocalDefId {
+        let data = def_kind.def_path_data(name);
+        let def_id = self.untracked.definitions.write().create_def(parent, data);
+
+        let feed = self.feed_local_def_id(def_id);
+        feed.def_kind(def_kind);
+        // Unique types created for closures participate in type privacy checking.
+        // They have visibilities inherited from the module they are defined in.
+        // Visibilities for opaque types are meaningless, but still provided
+        // so that all items have visibilities.
+        if matches!(def_kind, DefKind::Closure | DefKind::OpaqueTy) {
+            let parent_mod = self.parent_module_from_def_id(def_id).to_def_id();
+            feed.visibility(ty::Visibility::Restricted(parent_mod));
+        }
+
+        def_id
+    }
+
     pub fn iter_local_def_id(self) -> impl Iterator<Item = LocalDefId> + 'tcx {
         // Create a dependency to the red node to be sure we re-execute this when the amount of
         // definitions change.
@@ -1080,8 +1173,8 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn is_suitable_region(self, mut region: Region<'tcx>) -> Option<FreeRegionInfo> {
         let (suitable_region_binding_scope, bound_region) = loop {
             let def_id = match region.kind() {
-                ty::ReFree(fr) => fr.bound_region.get_id()?.as_local()?,
-                ty::ReEarlyBound(ebr) => ebr.def_id.expect_local(),
+                ty::ReLateParam(fr) => fr.bound_region.get_id()?.as_local()?,
+                ty::ReEarlyParam(ebr) => ebr.def_id.expect_local(),
                 _ => return None, // not a free region
             };
             let scope = self.local_parent(def_id);
@@ -1094,7 +1187,7 @@ impl<'tcx> TyCtxt<'tcx> {
             break (scope, ty::BrNamed(def_id.into(), self.item_name(def_id.into())));
         };
 
-        let is_impl_item = match self.hir().find_by_def_id(suitable_region_binding_scope) {
+        let is_impl_item = match self.opt_hir_node_by_def_id(suitable_region_binding_scope) {
             Some(Node::Item(..) | Node::TraitItem(..)) => false,
             Some(Node::ImplItem(..)) => {
                 self.is_bound_region_in_impl_item(suitable_region_binding_scope)
@@ -1102,11 +1195,7 @@ impl<'tcx> TyCtxt<'tcx> {
             _ => false,
         };
 
-        Some(FreeRegionInfo {
-            def_id: suitable_region_binding_scope,
-            boundregion: bound_region,
-            is_impl_item,
-        })
+        Some(FreeRegionInfo { def_id: suitable_region_binding_scope, bound_region, is_impl_item })
     }
 
     /// Given a `DefId` for an `fn`, return all the `dyn` and `impl` traits in its return type.
@@ -1114,7 +1203,7 @@ impl<'tcx> TyCtxt<'tcx> {
         self,
         scope_def_id: LocalDefId,
     ) -> Vec<&'tcx hir::Ty<'tcx>> {
-        let hir_id = self.hir().local_def_id_to_hir_id(scope_def_id);
+        let hir_id = self.local_def_id_to_hir_id(scope_def_id);
         let Some(hir::FnDecl { output: hir::FnRetTy::Return(hir_output), .. }) =
             self.hir().fn_decl_by_hir_id(hir_id)
         else {
@@ -1133,7 +1222,7 @@ impl<'tcx> TyCtxt<'tcx> {
         self,
         scope_def_id: LocalDefId,
     ) -> Option<(Vec<&'tcx hir::Ty<'tcx>>, Span, Option<Span>)> {
-        let hir_id = self.hir().local_def_id_to_hir_id(scope_def_id);
+        let hir_id = self.local_def_id_to_hir_id(scope_def_id);
         let mut v = TraitObjectVisitor(vec![], self.hir());
         // when the return type is a type alias
         if let Some(hir::FnDecl { output: hir::FnRetTy::Return(hir_output), .. }) = self.hir().fn_decl_by_hir_id(hir_id)
@@ -1141,8 +1230,8 @@ impl<'tcx> TyCtxt<'tcx> {
                 None,
                 hir::Path { res: hir::def::Res::Def(DefKind::TyAlias, def_id), .. }, )) = hir_output.kind
             && let Some(local_id) = def_id.as_local()
-            && let Some(alias_ty) = self.hir().get_by_def_id(local_id).alias_ty() // it is type alias
-            && let Some(alias_generics) = self.hir().get_by_def_id(local_id).generics()
+            && let Some(alias_ty) = self.hir_node_by_def_id(local_id).alias_ty() // it is type alias
+            && let Some(alias_generics) = self.hir_node_by_def_id(local_id).generics()
         {
             v.visit_ty(alias_ty);
             if !v.0.is_empty() {
@@ -1548,7 +1637,6 @@ macro_rules! direct_interners {
 // crate only, and have a corresponding `mk_` function.
 direct_interners! {
     region: pub(crate) intern_region(RegionKind<'tcx>): Region -> Region<'tcx>,
-    const_: intern_const(ConstData<'tcx>): Const -> Const<'tcx>,
     const_allocation: pub mk_const_alloc(Allocation): ConstAllocation -> ConstAllocation<'tcx>,
     layout: pub mk_layout(LayoutS<FieldIdx, VariantIdx>): Layout -> Layout<'tcx>,
     adt_def: pub mk_adt_def_from_data(AdtDefData): AdtDef -> AdtDef<'tcx>,
@@ -1725,7 +1813,12 @@ impl<'tcx> TyCtxt<'tcx> {
 
     #[inline]
     pub fn mk_ct_from_kind(self, kind: ty::ConstKind<'tcx>, ty: Ty<'tcx>) -> Const<'tcx> {
-        self.intern_const(ty::ConstData { kind, ty })
+        self.interners.intern_const(
+            ty::ConstData { kind, ty },
+            self.sess,
+            // This is only used to create a stable hashing context.
+            &self.untracked,
+        )
     }
 
     // Avoid this in favour of more specific `Ty::new_*` methods, where possible.
@@ -1743,7 +1836,7 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn mk_param_from_def(self, param: &ty::GenericParamDef) -> GenericArg<'tcx> {
         match param.kind {
             GenericParamDefKind::Lifetime => {
-                ty::Region::new_early_bound(self, param.to_early_bound_region_data()).into()
+                ty::Region::new_early_param(self, param.to_early_bound_region_data()).into()
             }
             GenericParamDefKind::Type { .. } => Ty::new_param(self, param.index, param.name).into(),
             GenericParamDefKind::Const { .. } => ty::Const::new_param(
@@ -1954,14 +2047,12 @@ impl<'tcx> TyCtxt<'tcx> {
         let msg = decorator.msg();
         let (level, src) = self.lint_level_at_node(lint, hir_id);
         struct_lint_level(self.sess, lint, level, src, Some(span.into()), msg, |diag| {
-            decorator.decorate_lint(diag)
+            decorator.decorate_lint(diag);
         })
     }
 
     /// Emit a lint at the appropriate level for a hir node, with an associated span.
     ///
-    /// Return value of the `decorate` closure is ignored, see [`struct_lint_level`] for a detailed explanation.
-    ///
     /// [`struct_lint_level`]: rustc_middle::lint::struct_lint_level#decorate-signature
     #[rustc_lint_diagnostics]
     #[track_caller]
@@ -1971,9 +2062,7 @@ impl<'tcx> TyCtxt<'tcx> {
         hir_id: HirId,
         span: impl Into<MultiSpan>,
         msg: impl Into<DiagnosticMessage>,
-        decorate: impl for<'a, 'b> FnOnce(
-            &'b mut DiagnosticBuilder<'a, ()>,
-        ) -> &'b mut DiagnosticBuilder<'a, ()>,
+        decorate: impl for<'a, 'b> FnOnce(&'b mut DiagnosticBuilder<'a, ()>),
     ) {
         let (level, src) = self.lint_level_at_node(lint, hir_id);
         struct_lint_level(self.sess, lint, level, src, Some(span.into()), msg, decorate);
@@ -1988,13 +2077,13 @@ impl<'tcx> TyCtxt<'tcx> {
         id: HirId,
         decorator: impl for<'a> DecorateLint<'a, ()>,
     ) {
-        self.struct_lint_node(lint, id, decorator.msg(), |diag| decorator.decorate_lint(diag))
+        self.struct_lint_node(lint, id, decorator.msg(), |diag| {
+            decorator.decorate_lint(diag);
+        })
     }
 
     /// Emit a lint at the appropriate level for a hir node.
     ///
-    /// Return value of the `decorate` closure is ignored, see [`struct_lint_level`] for a detailed explanation.
-    ///
     /// [`struct_lint_level`]: rustc_middle::lint::struct_lint_level#decorate-signature
     #[rustc_lint_diagnostics]
     #[track_caller]
@@ -2003,9 +2092,7 @@ impl<'tcx> TyCtxt<'tcx> {
         lint: &'static Lint,
         id: HirId,
         msg: impl Into<DiagnosticMessage>,
-        decorate: impl for<'a, 'b> FnOnce(
-            &'b mut DiagnosticBuilder<'a, ()>,
-        ) -> &'b mut DiagnosticBuilder<'a, ()>,
+        decorate: impl for<'a, 'b> FnOnce(&'b mut DiagnosticBuilder<'a, ()>),
     ) {
         let (level, src) = self.lint_level_at_node(lint, id);
         struct_lint_level(self.sess, lint, level, src, None, msg, decorate);
@@ -2040,7 +2127,7 @@ impl<'tcx> TyCtxt<'tcx> {
     /// Given the def-id of an early-bound lifetime on an RPIT corresponding to
     /// a duplicated captured lifetime, map it back to the early- or late-bound
     /// lifetime of the function from which it originally as captured. If it is
-    /// a late-bound lifetime, this will represent the liberated (`ReFree`) lifetime
+    /// a late-bound lifetime, this will represent the liberated (`ReLateParam`) lifetime
     /// of the signature.
     // FIXME(RPITIT): if we ever synthesize new lifetimes for RPITITs and not just
     // re-use the generics of the opaque, this function will need to be tweaked slightly.
@@ -2057,7 +2144,7 @@ impl<'tcx> TyCtxt<'tcx> {
         loop {
             let parent = self.local_parent(rpit_lifetime_param_def_id);
             let hir::OpaqueTy { lifetime_mapping, .. } =
-                self.hir().get_by_def_id(parent).expect_item().expect_opaque_ty();
+                self.hir_node_by_def_id(parent).expect_item().expect_opaque_ty();
 
             let Some((lifetime, _)) = lifetime_mapping
                 .iter()
@@ -2079,9 +2166,9 @@ impl<'tcx> TyCtxt<'tcx> {
                     }
 
                     let generics = self.generics_of(new_parent);
-                    return ty::Region::new_early_bound(
+                    return ty::Region::new_early_param(
                         self,
-                        ty::EarlyBoundRegion {
+                        ty::EarlyParamRegion {
                             def_id: ebv,
                             index: generics
                                 .param_def_id_to_index(self, ebv)
@@ -2092,7 +2179,7 @@ impl<'tcx> TyCtxt<'tcx> {
                 }
                 Some(resolve_bound_vars::ResolvedArg::LateBound(_, _, lbv)) => {
                     let new_parent = self.parent(lbv);
-                    return ty::Region::new_free(
+                    return ty::Region::new_late_param(
                         self,
                         new_parent,
                         ty::BoundRegionKind::BrNamed(
@@ -2141,15 +2228,14 @@ impl<'tcx> TyCtxt<'tcx> {
     /// Whether the trait impl is marked const. This does not consider stability or feature gates.
     pub fn is_const_trait_impl_raw(self, def_id: DefId) -> bool {
         let Some(local_def_id) = def_id.as_local() else { return false };
-        let hir_id = self.local_def_id_to_hir_id(local_def_id);
-        let node = self.hir().get(hir_id);
+        let node = self.hir_node_by_def_id(local_def_id);
 
         matches!(
             node,
             hir::Node::Item(hir::Item {
                 kind: hir::ItemKind::Impl(hir::Impl { generics, .. }),
                 ..
-            }) if generics.params.iter().any(|p| self.has_attr(p.def_id, sym::rustc_host))
+            }) if generics.params.iter().any(|p| matches!(p.kind, hir::GenericParamKind::Const { is_host_effect: true, .. }))
         )
     }
 
@@ -2158,15 +2244,11 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     pub fn next_trait_solver_globally(self) -> bool {
-        self.sess.opts.unstable_opts.trait_solver == rustc_session::config::TraitSolver::Next
+        self.sess.opts.unstable_opts.next_solver.map_or(false, |c| c.globally)
     }
 
     pub fn next_trait_solver_in_coherence(self) -> bool {
-        matches!(
-            self.sess.opts.unstable_opts.trait_solver,
-            rustc_session::config::TraitSolver::Next
-                | rustc_session::config::TraitSolver::NextCoherence
-        )
+        self.sess.opts.unstable_opts.next_solver.map_or(false, |c| c.coherence)
     }
 
     pub fn is_impl_trait_in_trait(self, def_id: DefId) -> bool {
diff --git a/compiler/rustc_middle/src/ty/erase_regions.rs b/compiler/rustc_middle/src/ty/erase_regions.rs
index 3371ea3be..cfd36fd8c 100644
--- a/compiler/rustc_middle/src/ty/erase_regions.rs
+++ b/compiler/rustc_middle/src/ty/erase_regions.rs
@@ -53,16 +53,11 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for RegionEraserVisitor<'tcx> {
     }
 
     fn fold_region(&mut self, r: ty::Region<'tcx>) -> ty::Region<'tcx> {
-        // because late-bound regions affect subtyping, we can't
-        // erase the bound/free distinction, but we can replace
-        // all free regions with 'erased.
-        //
-        // Note that we *CAN* replace early-bound regions -- the
-        // type system never "sees" those, they get substituted
-        // away. In codegen, they will always be erased to 'erased
-        // whenever a substitution occurs.
+        // We must not erase bound regions. `for<'a> fn(&'a ())` and
+        // `fn(&'free ())` are different types: they may implement different
+        // traits and have a different `TypeId`.
         match *r {
-            ty::ReLateBound(..) => r,
+            ty::ReBound(..) => r,
             _ => self.tcx.lifetimes.re_erased,
         }
     }
diff --git a/compiler/rustc_middle/src/ty/error.rs b/compiler/rustc_middle/src/ty/error.rs
index 738bb5e8b..0e4487852 100644
--- a/compiler/rustc_middle/src/ty/error.rs
+++ b/compiler/rustc_middle/src/ty/error.rs
@@ -7,8 +7,7 @@ use rustc_hir::def_id::DefId;
 use rustc_span::symbol::Symbol;
 use rustc_target::spec::abi;
 use std::borrow::Cow;
-use std::collections::hash_map::DefaultHasher;
-use std::hash::{Hash, Hasher};
+use std::hash::{DefaultHasher, Hash, Hasher};
 use std::path::PathBuf;
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq, TypeFoldable, TypeVisitable)]
@@ -236,7 +235,7 @@ impl<'tcx> Ty<'tcx> {
                 _ => "fn item".into(),
             },
             ty::FnPtr(_) => "fn pointer".into(),
-            ty::Dynamic(ref inner, ..) if let Some(principal) = inner.principal() => {
+            ty::Dynamic(inner, ..) if let Some(principal) = inner.principal() => {
                 format!("`dyn {}`", tcx.def_path_str(principal.def_id())).into()
             }
             ty::Dynamic(..) => "trait object".into(),
@@ -282,7 +281,7 @@ impl<'tcx> Ty<'tcx> {
             | ty::Float(_)
             | ty::Str
             | ty::Never => "type".into(),
-            ty::Tuple(ref tys) if tys.is_empty() => "unit type".into(),
+            ty::Tuple(tys) if tys.is_empty() => "unit type".into(),
             ty::Adt(def, _) => def.descr().into(),
             ty::Foreign(_) => "extern type".into(),
             ty::Array(..) => "array".into(),
@@ -346,33 +345,35 @@ impl<'tcx> TyCtxt<'tcx> {
         short
     }
 
-    pub fn short_ty_string(self, ty: Ty<'tcx>) -> (String, Option<PathBuf>) {
+    pub fn short_ty_string(self, ty: Ty<'tcx>, path: &mut Option<PathBuf>) -> String {
         let regular = FmtPrinter::print_string(self, hir::def::Namespace::TypeNS, |cx| {
             cx.pretty_print_type(ty)
         })
         .expect("could not write to `String`");
 
         if !self.sess.opts.unstable_opts.write_long_types_to_disk {
-            return (regular, None);
+            return regular;
         }
 
         let width = self.sess.diagnostic_width();
         let length_limit = width.saturating_sub(30);
         if regular.len() <= width {
-            return (regular, None);
+            return regular;
         }
         let short = self.ty_string_with_limit(ty, length_limit);
         if regular == short {
-            return (regular, None);
+            return regular;
         }
-        // Multiple types might be shortened in a single error, ensure we create a file for each.
+        // Ensure we create an unique file for the type passed in when we create a file.
         let mut s = DefaultHasher::new();
         ty.hash(&mut s);
         let hash = s.finish();
-        let path = self.output_filenames(()).temp_path_ext(&format!("long-type-{hash}.txt"), None);
-        match std::fs::write(&path, &regular) {
-            Ok(_) => (short, Some(path)),
-            Err(_) => (regular, None),
+        *path = Some(path.take().unwrap_or_else(|| {
+            self.output_filenames(()).temp_path_ext(&format!("long-type-{hash}.txt"), None)
+        }));
+        match std::fs::write(path.as_ref().unwrap(), &format!("{regular}\n")) {
+            Ok(_) => short,
+            Err(_) => regular,
         }
     }
 }
diff --git a/compiler/rustc_middle/src/ty/fast_reject.rs b/compiler/rustc_middle/src/ty/fast_reject.rs
index 75ea53195..38a9cabca 100644
--- a/compiler/rustc_middle/src/ty/fast_reject.rs
+++ b/compiler/rustc_middle/src/ty/fast_reject.rs
@@ -62,7 +62,7 @@ pub enum TreatParams {
     ///
     /// N.B. during deep rejection, this acts identically to `ForLookup`.
     ///
-    /// FIXME(-Ztrait-solver=next): Remove this variant and cleanup
+    /// FIXME(-Znext-solver): Remove this variant and cleanup
     /// the code.
     NextSolverLookup,
 }
@@ -189,14 +189,14 @@ pub struct DeepRejectCtxt {
 }
 
 impl DeepRejectCtxt {
-    pub fn args_refs_may_unify<'tcx>(
+    pub fn args_may_unify<'tcx>(
         self,
         obligation_args: GenericArgsRef<'tcx>,
         impl_args: GenericArgsRef<'tcx>,
     ) -> bool {
         iter::zip(obligation_args, impl_args).all(|(obl, imp)| {
             match (obl.unpack(), imp.unpack()) {
-                // We don't fast reject based on regions for now.
+                // We don't fast reject based on regions.
                 (GenericArgKind::Lifetime(_), GenericArgKind::Lifetime(_)) => true,
                 (GenericArgKind::Type(obl), GenericArgKind::Type(imp)) => {
                     self.types_may_unify(obl, imp)
@@ -231,7 +231,7 @@ impl DeepRejectCtxt {
             | ty::Never
             | ty::Tuple(..)
             | ty::FnPtr(..)
-            | ty::Foreign(..) => {}
+            | ty::Foreign(..) => debug_assert!(impl_ty.is_known_rigid()),
             ty::FnDef(..)
             | ty::Closure(..)
             | ty::Coroutine(..)
@@ -260,7 +260,7 @@ impl DeepRejectCtxt {
             },
             ty::Adt(obl_def, obl_args) => match k {
                 &ty::Adt(impl_def, impl_args) => {
-                    obl_def == impl_def && self.args_refs_may_unify(obl_args, impl_args)
+                    obl_def == impl_def && self.args_may_unify(obl_args, impl_args)
                 }
                 _ => false,
             },
diff --git a/compiler/rustc_middle/src/ty/flags.rs b/compiler/rustc_middle/src/ty/flags.rs
index ec36bdc5a..f9a2385b1 100644
--- a/compiler/rustc_middle/src/ty/flags.rs
+++ b/compiler/rustc_middle/src/ty/flags.rs
@@ -28,10 +28,11 @@ impl FlagComputation {
         result
     }
 
-    pub fn for_const(c: ty::Const<'_>) -> TypeFlags {
+    pub fn for_const(c: &ty::ConstKind<'_>, t: Ty<'_>) -> FlagComputation {
         let mut result = FlagComputation::new();
-        result.add_const(c);
-        result.flags
+        result.add_const_kind(c);
+        result.add_ty(t);
+        result
     }
 
     fn add_flags(&mut self, flags: TypeFlags) {
@@ -137,7 +138,7 @@ impl FlagComputation {
 
             &ty::Bound(debruijn, _) => {
                 self.add_bound_var(debruijn);
-                self.add_flags(TypeFlags::HAS_TY_LATE_BOUND);
+                self.add_flags(TypeFlags::HAS_TY_BOUND);
             }
 
             &ty::Placeholder(..) => {
@@ -263,9 +264,6 @@ impl FlagComputation {
                 self.add_args(slice::from_ref(&arg));
             }
             ty::PredicateKind::ObjectSafe(_def_id) => {}
-            ty::PredicateKind::ClosureKind(_def_id, args, _kind) => {
-                self.add_args(args);
-            }
             ty::PredicateKind::Clause(ty::ClauseKind::ConstEvaluatable(uv)) => {
                 self.add_const(uv);
             }
@@ -274,6 +272,10 @@ impl FlagComputation {
                 self.add_const(found);
             }
             ty::PredicateKind::Ambiguous => {}
+            ty::PredicateKind::NormalizesTo(ty::NormalizesTo { alias, term }) => {
+                self.add_alias_ty(alias);
+                self.add_term(term);
+            }
             ty::PredicateKind::AliasRelate(t1, t2, _) => {
                 self.add_term(t1);
                 self.add_term(t2);
@@ -294,14 +296,18 @@ impl FlagComputation {
 
     fn add_region(&mut self, r: ty::Region<'_>) {
         self.add_flags(r.type_flags());
-        if let ty::ReLateBound(debruijn, _) = *r {
+        if let ty::ReBound(debruijn, _) = *r {
             self.add_bound_var(debruijn);
         }
     }
 
     fn add_const(&mut self, c: ty::Const<'_>) {
-        self.add_ty(c.ty());
-        match c.kind() {
+        self.add_flags(c.flags());
+        self.add_exclusive_binder(c.outer_exclusive_binder());
+    }
+
+    fn add_const_kind(&mut self, c: &ty::ConstKind<'_>) {
+        match *c {
             ty::ConstKind::Unevaluated(uv) => {
                 self.add_args(uv.args);
                 self.add_flags(TypeFlags::HAS_CT_PROJECTION);
@@ -317,7 +323,7 @@ impl FlagComputation {
             }
             ty::ConstKind::Bound(debruijn, _) => {
                 self.add_bound_var(debruijn);
-                self.add_flags(TypeFlags::HAS_CT_LATE_BOUND);
+                self.add_flags(TypeFlags::HAS_CT_BOUND);
             }
             ty::ConstKind::Param(_) => {
                 self.add_flags(TypeFlags::HAS_CT_PARAM);
diff --git a/compiler/rustc_middle/src/ty/fold.rs b/compiler/rustc_middle/src/ty/fold.rs
index 00529a1e0..3e64f9a2a 100644
--- a/compiler/rustc_middle/src/ty/fold.rs
+++ b/compiler/rustc_middle/src/ty/fold.rs
@@ -68,12 +68,10 @@ impl<'tcx> TyCtxt<'tcx> {
 /// Folds over the substructure of a type, visiting its component
 /// types and all regions that occur *free* within it.
 ///
-/// That is, `Ty` can contain function or method types that bind
-/// regions at the call site (`ReLateBound`), and occurrences of
-/// regions (aka "lifetimes") that are bound within a type are not
-/// visited by this folder; only regions that occur free will be
+/// That is, function pointer types and trait object can introduce
+/// new bound regions which are not visited by this visitors as
+/// they are not free; only regions that occur free will be
 /// visited by `fld_r`.
-
 pub struct RegionFolder<'a, 'tcx> {
     tcx: TyCtxt<'tcx>,
 
@@ -117,7 +115,7 @@ impl<'a, 'tcx> TypeFolder<TyCtxt<'tcx>> for RegionFolder<'a, 'tcx> {
     #[instrument(skip(self), level = "debug", ret)]
     fn fold_region(&mut self, r: ty::Region<'tcx>) -> ty::Region<'tcx> {
         match *r {
-            ty::ReLateBound(debruijn, _) if debruijn < self.current_index => {
+            ty::ReBound(debruijn, _) if debruijn < self.current_index => {
                 debug!(?self.current_index, "skipped bound region");
                 r
             }
@@ -205,15 +203,15 @@ where
 
     fn fold_region(&mut self, r: ty::Region<'tcx>) -> ty::Region<'tcx> {
         match *r {
-            ty::ReLateBound(debruijn, br) if debruijn == self.current_index => {
+            ty::ReBound(debruijn, br) if debruijn == self.current_index => {
                 let region = self.delegate.replace_region(br);
-                if let ty::ReLateBound(debruijn1, br) = *region {
-                    // If the callback returns a late-bound region,
+                if let ty::ReBound(debruijn1, br) = *region {
+                    // If the callback returns a bound region,
                     // that region should always use the INNERMOST
                     // debruijn index. Then we adjust it to the
                     // correct depth.
                     assert_eq!(debruijn1, ty::INNERMOST);
-                    ty::Region::new_late_bound(self.tcx, debruijn, br)
+                    ty::Region::new_bound(self.tcx, debruijn, br)
                 } else {
                     region
                 }
@@ -252,7 +250,7 @@ impl<'tcx> TyCtxt<'tcx> {
     ///
     /// This method only replaces late bound regions. Any types or
     /// constants bound by `value` will cause an ICE.
-    pub fn replace_late_bound_regions<T, F>(
+    pub fn instantiate_bound_regions<T, F>(
         self,
         value: Binder<'tcx, T>,
         mut fld_r: F,
@@ -263,11 +261,11 @@ impl<'tcx> TyCtxt<'tcx> {
     {
         let mut region_map = BTreeMap::new();
         let real_fld_r = |br: ty::BoundRegion| *region_map.entry(br).or_insert_with(|| fld_r(br));
-        let value = self.replace_late_bound_regions_uncached(value, real_fld_r);
+        let value = self.instantiate_bound_regions_uncached(value, real_fld_r);
         (value, region_map)
     }
 
-    pub fn replace_late_bound_regions_uncached<T, F>(
+    pub fn instantiate_bound_regions_uncached<T, F>(
         self,
         value: Binder<'tcx, T>,
         mut replace_regions: F,
@@ -327,8 +325,8 @@ impl<'tcx> TyCtxt<'tcx> {
     where
         T: TypeFoldable<TyCtxt<'tcx>>,
     {
-        self.replace_late_bound_regions_uncached(value, |br| {
-            ty::Region::new_free(self, all_outlive_scope, br.kind)
+        self.instantiate_bound_regions_uncached(value, |br| {
+            ty::Region::new_late_param(self, all_outlive_scope, br.kind)
         })
     }
 
@@ -341,7 +339,7 @@ impl<'tcx> TyCtxt<'tcx> {
             value,
             FnMutDelegate {
                 regions: &mut |r: ty::BoundRegion| {
-                    ty::Region::new_late_bound(
+                    ty::Region::new_bound(
                         self,
                         ty::INNERMOST,
                         ty::BoundRegion { var: shift_bv(r.var), kind: r.kind },
@@ -363,11 +361,11 @@ impl<'tcx> TyCtxt<'tcx> {
 
     /// Replaces any late-bound regions bound in `value` with `'erased`. Useful in codegen but also
     /// method lookup and a few other places where precise region relationships are not required.
-    pub fn erase_late_bound_regions<T>(self, value: Binder<'tcx, T>) -> T
+    pub fn instantiate_bound_regions_with_erased<T>(self, value: Binder<'tcx, T>) -> T
     where
         T: TypeFoldable<TyCtxt<'tcx>>,
     {
-        self.replace_late_bound_regions(value, |_| self.lifetimes.re_erased).0
+        self.instantiate_bound_regions(value, |_| self.lifetimes.re_erased).0
     }
 
     /// Anonymize all bound variables in `value`, this is mostly used to improve caching.
@@ -388,7 +386,7 @@ impl<'tcx> TyCtxt<'tcx> {
                     .or_insert_with(|| ty::BoundVariableKind::Region(ty::BrAnon))
                     .expect_region();
                 let br = ty::BoundRegion { var, kind };
-                ty::Region::new_late_bound(self.tcx, ty::INNERMOST, br)
+                ty::Region::new_bound(self.tcx, ty::INNERMOST, br)
             }
             fn replace_ty(&mut self, bt: ty::BoundTy) -> Ty<'tcx> {
                 let entry = self.map.entry(bt.var);
@@ -454,9 +452,9 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for Shifter<'tcx> {
 
     fn fold_region(&mut self, r: ty::Region<'tcx>) -> ty::Region<'tcx> {
         match *r {
-            ty::ReLateBound(debruijn, br) if debruijn >= self.current_index => {
+            ty::ReBound(debruijn, br) if debruijn >= self.current_index => {
                 let debruijn = debruijn.shifted_in(self.amount);
-                ty::Region::new_late_bound(self.tcx, debruijn, br)
+                ty::Region::new_bound(self.tcx, debruijn, br)
             }
             _ => r,
         }
@@ -496,8 +494,8 @@ pub fn shift_region<'tcx>(
     amount: u32,
 ) -> ty::Region<'tcx> {
     match *region {
-        ty::ReLateBound(debruijn, br) if amount > 0 => {
-            ty::Region::new_late_bound(tcx, debruijn.shifted_in(amount), br)
+        ty::ReBound(debruijn, br) if amount > 0 => {
+            ty::Region::new_bound(tcx, debruijn.shifted_in(amount), br)
         }
         _ => region,
     }
diff --git a/compiler/rustc_middle/src/ty/generic_args.rs b/compiler/rustc_middle/src/ty/generic_args.rs
index 41a1bf04e..63f4bab79 100644
--- a/compiler/rustc_middle/src/ty/generic_args.rs
+++ b/compiler/rustc_middle/src/ty/generic_args.rs
@@ -10,7 +10,7 @@ use rustc_data_structures::intern::Interned;
 use rustc_errors::{DiagnosticArgValue, IntoDiagnosticArg};
 use rustc_hir::def_id::DefId;
 use rustc_macros::HashStable;
-use rustc_serialize::{self, Decodable, Encodable};
+use rustc_serialize::{Decodable, Encodable};
 use rustc_type_ir::WithCachedTypeInfo;
 use smallvec::SmallVec;
 
@@ -70,7 +70,7 @@ impl<'tcx> GenericArgKind<'tcx> {
             GenericArgKind::Const(ct) => {
                 // Ensure we can use the tag bits.
                 assert_eq!(mem::align_of_val(&*ct.0.0) & TAG_MASK, 0);
-                (CONST_TAG, ct.0.0 as *const ty::ConstData<'tcx> as usize)
+                (CONST_TAG, ct.0.0 as *const WithCachedTypeInfo<ty::ConstData<'tcx>> as usize)
             }
         };
 
@@ -86,7 +86,7 @@ impl<'tcx> Ord for GenericArg<'tcx> {
 
 impl<'tcx> PartialOrd for GenericArg<'tcx> {
     fn partial_cmp(&self, other: &GenericArg<'tcx>) -> Option<Ordering> {
-        Some(self.cmp(&other))
+        Some(self.cmp(other))
     }
 }
 
@@ -136,7 +136,7 @@ impl<'tcx> GenericArg<'tcx> {
                     &*((ptr & !TAG_MASK) as *const WithCachedTypeInfo<ty::TyKind<'tcx>>),
                 ))),
                 CONST_TAG => GenericArgKind::Const(ty::Const(Interned::new_unchecked(
-                    &*((ptr & !TAG_MASK) as *const ty::ConstData<'tcx>),
+                    &*((ptr & !TAG_MASK) as *const WithCachedTypeInfo<ty::ConstData<'tcx>>),
                 ))),
                 _ => intrinsics::unreachable(),
             }
@@ -604,13 +604,6 @@ impl<T> EarlyBinder<Option<T>> {
     }
 }
 
-impl<T, U> EarlyBinder<(T, U)> {
-    pub fn transpose_tuple2(self) -> (EarlyBinder<T>, EarlyBinder<U>) {
-        let EarlyBinder { value: (lhs, rhs) } = self;
-        (EarlyBinder { value: lhs }, EarlyBinder { value: rhs })
-    }
-}
-
 impl<'tcx, 's, I: IntoIterator> EarlyBinder<I>
 where
     I::Item: TypeFoldable<TyCtxt<'tcx>>,
@@ -809,7 +802,7 @@ impl<'a, 'tcx> TypeFolder<TyCtxt<'tcx>> for ArgFolder<'a, 'tcx> {
     fn fold_region(&mut self, r: ty::Region<'tcx>) -> ty::Region<'tcx> {
         #[cold]
         #[inline(never)]
-        fn region_param_out_of_range(data: ty::EarlyBoundRegion, args: &[GenericArg<'_>]) -> ! {
+        fn region_param_out_of_range(data: ty::EarlyParamRegion, args: &[GenericArg<'_>]) -> ! {
             bug!(
                 "Region parameter out of range when substituting in region {} (index={}, args = {:?})",
                 data.name,
@@ -820,7 +813,7 @@ impl<'a, 'tcx> TypeFolder<TyCtxt<'tcx>> for ArgFolder<'a, 'tcx> {
 
         #[cold]
         #[inline(never)]
-        fn region_param_invalid(data: ty::EarlyBoundRegion, other: GenericArgKind<'_>) -> ! {
+        fn region_param_invalid(data: ty::EarlyParamRegion, other: GenericArgKind<'_>) -> ! {
             bug!(
                 "Unexpected parameter {:?} when substituting in region {} (index={})",
                 other,
@@ -835,7 +828,7 @@ impl<'a, 'tcx> TypeFolder<TyCtxt<'tcx>> for ArgFolder<'a, 'tcx> {
         // regions that appear in a function signature is done using
         // the specialized routine `ty::replace_late_regions()`.
         match *r {
-            ty::ReEarlyBound(data) => {
+            ty::ReEarlyParam(data) => {
                 let rk = self.args.get(data.index as usize).map(|k| k.unpack());
                 match rk {
                     Some(GenericArgKind::Lifetime(lt)) => self.shift_region_through_binders(lt),
@@ -843,8 +836,8 @@ impl<'a, 'tcx> TypeFolder<TyCtxt<'tcx>> for ArgFolder<'a, 'tcx> {
                     None => region_param_out_of_range(data, self.args),
                 }
             }
-            ty::ReLateBound(..)
-            | ty::ReFree(_)
+            ty::ReBound(..)
+            | ty::ReLateParam(_)
             | ty::ReStatic
             | ty::RePlaceholder(_)
             | ty::ReErased
diff --git a/compiler/rustc_middle/src/ty/generics.rs b/compiler/rustc_middle/src/ty/generics.rs
index 888ee1d23..c3699b114 100644
--- a/compiler/rustc_middle/src/ty/generics.rs
+++ b/compiler/rustc_middle/src/ty/generics.rs
@@ -6,7 +6,7 @@ use rustc_hir::def_id::DefId;
 use rustc_span::symbol::{kw, Symbol};
 use rustc_span::Span;
 
-use super::{Clause, EarlyBoundRegion, InstantiatedPredicates, ParamConst, ParamTy, Ty, TyCtxt};
+use super::{Clause, InstantiatedPredicates, ParamConst, ParamTy, Ty, TyCtxt};
 
 #[derive(Clone, Debug, TyEncodable, TyDecodable, HashStable)]
 pub enum GenericParamDefKind {
@@ -62,9 +62,9 @@ pub struct GenericParamDef {
 }
 
 impl GenericParamDef {
-    pub fn to_early_bound_region_data(&self) -> ty::EarlyBoundRegion {
+    pub fn to_early_bound_region_data(&self) -> ty::EarlyParamRegion {
         if let GenericParamDefKind::Lifetime = self.kind {
-            ty::EarlyBoundRegion { def_id: self.def_id, index: self.index, name: self.name }
+            ty::EarlyParamRegion { def_id: self.def_id, index: self.index, name: self.name }
         } else {
             bug!("cannot convert a non-lifetime parameter def to an early bound region")
         }
@@ -260,10 +260,10 @@ impl<'tcx> Generics {
         }
     }
 
-    /// Returns the `GenericParamDef` associated with this `EarlyBoundRegion`.
+    /// Returns the `GenericParamDef` associated with this `EarlyParamRegion`.
     pub fn region_param(
         &'tcx self,
-        param: &EarlyBoundRegion,
+        param: &ty::EarlyParamRegion,
         tcx: TyCtxt<'tcx>,
     ) -> &'tcx GenericParamDef {
         let param = self.param_at(param.index as usize, tcx);
@@ -326,6 +326,8 @@ impl<'tcx> Generics {
             own_params.start = 1;
         }
 
+        let verbose = tcx.sess.verbose();
+
         // Filter the default arguments.
         //
         // This currently uses structural equality instead
@@ -340,6 +342,8 @@ impl<'tcx> Generics {
                 param.default_value(tcx).is_some_and(|default| {
                     default.instantiate(tcx, args) == args[param.index as usize]
                 })
+                // filter out trailing effect params, if we're not in `-Zverbose`.
+                || (!verbose && matches!(param.kind, GenericParamDefKind::Const { is_host_effect: true, .. }))
             })
             .count();
 
@@ -354,7 +358,7 @@ impl<'tcx> Generics {
         args: &'tcx [ty::GenericArg<'tcx>],
     ) -> &'tcx [ty::GenericArg<'tcx>] {
         let own = &args[self.parent_count..][..self.params.len()];
-        if self.has_self && self.parent.is_none() { &own[1..] } else { &own }
+        if self.has_self && self.parent.is_none() { &own[1..] } else { own }
     }
 }
 
diff --git a/compiler/rustc_middle/src/ty/impls_ty.rs b/compiler/rustc_middle/src/ty/impls_ty.rs
index b03874a90..129d94769 100644
--- a/compiler/rustc_middle/src/ty/impls_ty.rs
+++ b/compiler/rustc_middle/src/ty/impls_ty.rs
@@ -29,7 +29,7 @@ where
             }
 
             let mut hasher = StableHasher::new();
-            (&self[..]).hash_stable(hcx, &mut hasher);
+            self[..].hash_stable(hcx, &mut hasher);
 
             let hash: Fingerprint = hasher.finish();
             cache.borrow_mut().insert(key, hash);
@@ -84,6 +84,14 @@ impl<'a> HashStable<StableHashingContext<'a>> for mir::interpret::AllocId {
     }
 }
 
+// CtfeProvenance is an AllocId and a bool.
+impl<'a> HashStable<StableHashingContext<'a>> for mir::interpret::CtfeProvenance {
+    fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
+        self.alloc_id().hash_stable(hcx, hasher);
+        self.immutable().hash_stable(hcx, hasher);
+    }
+}
+
 impl<'a> ToStableHashKey<StableHashingContext<'a>> for region::Scope {
     type KeyType = region::Scope;
 
diff --git a/compiler/rustc_middle/src/ty/inhabitedness/inhabited_predicate.rs b/compiler/rustc_middle/src/ty/inhabitedness/inhabited_predicate.rs
index f278cace9..ae1794278 100644
--- a/compiler/rustc_middle/src/ty/inhabitedness/inhabited_predicate.rs
+++ b/compiler/rustc_middle/src/ty/inhabitedness/inhabited_predicate.rs
@@ -1,3 +1,5 @@
+use smallvec::SmallVec;
+
 use crate::ty::context::TyCtxt;
 use crate::ty::{self, DefId, ParamEnv, Ty};
 
@@ -31,27 +33,31 @@ impl<'tcx> InhabitedPredicate<'tcx> {
     /// Returns true if the corresponding type is inhabited in the given
     /// `ParamEnv` and module
     pub fn apply(self, tcx: TyCtxt<'tcx>, param_env: ParamEnv<'tcx>, module_def_id: DefId) -> bool {
-        let Ok(result) = self
-            .apply_inner::<!>(tcx, param_env, &|id| Ok(tcx.is_descendant_of(module_def_id, id)));
+        let Ok(result) = self.apply_inner::<!>(tcx, param_env, &mut Default::default(), &|id| {
+            Ok(tcx.is_descendant_of(module_def_id, id))
+        });
         result
     }
 
     /// Same as `apply`, but returns `None` if self contains a module predicate
     pub fn apply_any_module(self, tcx: TyCtxt<'tcx>, param_env: ParamEnv<'tcx>) -> Option<bool> {
-        self.apply_inner(tcx, param_env, &|_| Err(())).ok()
+        self.apply_inner(tcx, param_env, &mut Default::default(), &|_| Err(())).ok()
     }
 
     /// Same as `apply`, but `NotInModule(_)` predicates yield `false`. That is,
     /// privately uninhabited types are considered always uninhabited.
     pub fn apply_ignore_module(self, tcx: TyCtxt<'tcx>, param_env: ParamEnv<'tcx>) -> bool {
-        let Ok(result) = self.apply_inner::<!>(tcx, param_env, &|_| Ok(true));
+        let Ok(result) =
+            self.apply_inner::<!>(tcx, param_env, &mut Default::default(), &|_| Ok(true));
         result
     }
 
-    fn apply_inner<E>(
+    #[instrument(level = "debug", skip(tcx, param_env, in_module), ret)]
+    fn apply_inner<E: std::fmt::Debug>(
         self,
         tcx: TyCtxt<'tcx>,
         param_env: ParamEnv<'tcx>,
+        eval_stack: &mut SmallVec<[Ty<'tcx>; 1]>, // for cycle detection
         in_module: &impl Fn(DefId) -> Result<bool, E>,
     ) -> Result<bool, E> {
         match self {
@@ -71,11 +77,25 @@ impl<'tcx> InhabitedPredicate<'tcx> {
                 match normalized_pred {
                     // We don't have more information than we started with, so consider inhabited.
                     Self::GenericType(_) => Ok(true),
-                    pred => pred.apply_inner(tcx, param_env, in_module),
+                    pred => {
+                        // A type which is cyclic when monomorphized can happen here since the
+                        // layout error would only trigger later. See e.g. `tests/ui/sized/recursive-type-2.rs`.
+                        if eval_stack.contains(&t) {
+                            return Ok(true); // Recover; this will error later.
+                        }
+                        eval_stack.push(t);
+                        let ret = pred.apply_inner(tcx, param_env, eval_stack, in_module);
+                        eval_stack.pop();
+                        ret
+                    }
                 }
             }
-            Self::And([a, b]) => try_and(a, b, |x| x.apply_inner(tcx, param_env, in_module)),
-            Self::Or([a, b]) => try_or(a, b, |x| x.apply_inner(tcx, param_env, in_module)),
+            Self::And([a, b]) => {
+                try_and(a, b, |x| x.apply_inner(tcx, param_env, eval_stack, in_module))
+            }
+            Self::Or([a, b]) => {
+                try_or(a, b, |x| x.apply_inner(tcx, param_env, eval_stack, in_module))
+            }
         }
     }
 
@@ -197,7 +217,7 @@ impl<'tcx> InhabitedPredicate<'tcx> {
 
 // this is basically like `f(a)? && f(b)?` but different in the case of
 // `Ok(false) && Err(_) -> Ok(false)`
-fn try_and<T, E>(a: T, b: T, f: impl Fn(T) -> Result<bool, E>) -> Result<bool, E> {
+fn try_and<T, E>(a: T, b: T, mut f: impl FnMut(T) -> Result<bool, E>) -> Result<bool, E> {
     let a = f(a);
     if matches!(a, Ok(false)) {
         return Ok(false);
@@ -209,7 +229,7 @@ fn try_and<T, E>(a: T, b: T, f: impl Fn(T) -> Result<bool, E>) -> Result<bool, E
     }
 }
 
-fn try_or<T, E>(a: T, b: T, f: impl Fn(T) -> Result<bool, E>) -> Result<bool, E> {
+fn try_or<T, E>(a: T, b: T, mut f: impl FnMut(T) -> Result<bool, E>) -> Result<bool, E> {
     let a = f(a);
     if matches!(a, Ok(true)) {
         return Ok(true);
diff --git a/compiler/rustc_middle/src/ty/inhabitedness/mod.rs b/compiler/rustc_middle/src/ty/inhabitedness/mod.rs
index 68ac54e89..d67fb9fd0 100644
--- a/compiler/rustc_middle/src/ty/inhabitedness/mod.rs
+++ b/compiler/rustc_middle/src/ty/inhabitedness/mod.rs
@@ -103,6 +103,7 @@ impl<'tcx> VariantDef {
 }
 
 impl<'tcx> Ty<'tcx> {
+    #[instrument(level = "debug", skip(tcx), ret)]
     pub fn inhabited_predicate(self, tcx: TyCtxt<'tcx>) -> InhabitedPredicate<'tcx> {
         match self.kind() {
             // For now, unions are always considered inhabited
diff --git a/compiler/rustc_middle/src/ty/instance.rs b/compiler/rustc_middle/src/ty/instance.rs
index cebefbccc..1c7a7545e 100644
--- a/compiler/rustc_middle/src/ty/instance.rs
+++ b/compiler/rustc_middle/src/ty/instance.rs
@@ -215,7 +215,7 @@ impl<'tcx> InstanceDef<'tcx> {
         };
         matches!(
             tcx.def_key(def_id).disambiguated_data.data,
-            DefPathData::Ctor | DefPathData::ClosureExpr
+            DefPathData::Ctor | DefPathData::Closure
         )
     }
 
diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs
index 4223e503f..225dd2178 100644
--- a/compiler/rustc_middle/src/ty/layout.rs
+++ b/compiler/rustc_middle/src/ty/layout.rs
@@ -2,10 +2,10 @@ use crate::error::UnsupportedFnAbi;
 use crate::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use crate::query::TyCtxtAt;
 use crate::ty::normalize_erasing_regions::NormalizationError;
-use crate::ty::{self, ConstKind, ReprOptions, Ty, TyCtxt, TypeVisitableExt};
+use crate::ty::{self, ConstKind, Ty, TyCtxt, TypeVisitableExt};
 use rustc_error_messages::DiagnosticMessage;
 use rustc_errors::{
-    DiagnosticArgValue, DiagnosticBuilder, Handler, IntoDiagnostic, IntoDiagnosticArg,
+    DiagCtxt, DiagnosticArgValue, DiagnosticBuilder, IntoDiagnostic, IntoDiagnosticArg,
 };
 use rustc_hir as hir;
 use rustc_hir::def_id::DefId;
@@ -215,7 +215,7 @@ pub enum LayoutError<'tcx> {
     SizeOverflow(Ty<'tcx>),
     NormalizationFailure(Ty<'tcx>, NormalizationError<'tcx>),
     ReferencesError(ErrorGuaranteed),
-    Cycle,
+    Cycle(ErrorGuaranteed),
 }
 
 impl<'tcx> LayoutError<'tcx> {
@@ -226,7 +226,7 @@ impl<'tcx> LayoutError<'tcx> {
             Unknown(_) => middle_unknown_layout,
             SizeOverflow(_) => middle_values_too_big,
             NormalizationFailure(_, _) => middle_cannot_be_normalized,
-            Cycle => middle_cycle,
+            Cycle(_) => middle_cycle,
             ReferencesError(_) => middle_layout_references_error,
         }
     }
@@ -240,7 +240,7 @@ impl<'tcx> LayoutError<'tcx> {
             NormalizationFailure(ty, e) => {
                 E::NormalizationFailure { ty, failure_ty: e.get_type_for_failure() }
             }
-            Cycle => E::Cycle,
+            Cycle(_) => E::Cycle,
             ReferencesError(_) => E::ReferencesError,
         }
     }
@@ -261,7 +261,7 @@ impl<'tcx> fmt::Display for LayoutError<'tcx> {
                 t,
                 e.get_type_for_failure()
             ),
-            LayoutError::Cycle => write!(f, "a cycle occurred during layout computation"),
+            LayoutError::Cycle(_) => write!(f, "a cycle occurred during layout computation"),
             LayoutError::ReferencesError(_) => write!(f, "the type has an unknown layout"),
         }
     }
@@ -282,8 +282,8 @@ pub struct LayoutCx<'tcx, C> {
 impl<'tcx> LayoutCalculator for LayoutCx<'tcx, TyCtxt<'tcx>> {
     type TargetDataLayoutRef = &'tcx TargetDataLayout;
 
-    fn delay_bug(&self, txt: String) {
-        self.tcx.sess.delay_span_bug(DUMMY_SP, txt);
+    fn delayed_bug(&self, txt: String) {
+        self.tcx.sess.span_delayed_bug(DUMMY_SP, txt);
     }
 
     fn current_data_layout(&self) -> Self::TargetDataLayoutRef {
@@ -333,7 +333,7 @@ impl<'tcx> SizeSkeleton<'tcx> {
             Err(err @ LayoutError::Unknown(_)) => err,
             // We can't extract SizeSkeleton info from other layout errors
             Err(
-                e @ LayoutError::Cycle
+                e @ LayoutError::Cycle(_)
                 | e @ LayoutError::SizeOverflow(_)
                 | e @ LayoutError::NormalizationFailure(..)
                 | e @ LayoutError::ReferencesError(_),
@@ -899,13 +899,13 @@ where
                 ty::Str => TyMaybeWithLayout::Ty(tcx.types.u8),
 
                 // Tuples, coroutines and closures.
-                ty::Closure(_, ref args) => field_ty_or_layout(
+                ty::Closure(_, args) => field_ty_or_layout(
                     TyAndLayout { ty: args.as_closure().tupled_upvars_ty(), ..this },
                     cx,
                     i,
                 ),
 
-                ty::Coroutine(def_id, ref args, _) => match this.variants {
+                ty::Coroutine(def_id, args, _) => match this.variants {
                     Variants::Single { index } => TyMaybeWithLayout::Ty(
                         args.as_coroutine()
                             .state_tys(def_id, tcx)
@@ -1273,13 +1273,13 @@ pub enum FnAbiError<'tcx> {
 }
 
 impl<'a, 'b> IntoDiagnostic<'a, !> for FnAbiError<'b> {
-    fn into_diagnostic(self, handler: &'a Handler) -> DiagnosticBuilder<'a, !> {
+    fn into_diagnostic(self, dcx: &'a DiagCtxt) -> DiagnosticBuilder<'a, !> {
         match self {
-            Self::Layout(e) => e.into_diagnostic().into_diagnostic(handler),
+            Self::Layout(e) => e.into_diagnostic().into_diagnostic(dcx),
             Self::AdjustForForeignAbi(call::AdjustForForeignAbiError::Unsupported {
                 arch,
                 abi,
-            }) => UnsupportedFnAbi { arch, abi: abi.name() }.into_diagnostic(handler),
+            }) => UnsupportedFnAbi { arch, abi: abi.name() }.into_diagnostic(dcx),
         }
     }
 }
diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs
index e1c616ba0..35c135830 100644
--- a/compiler/rustc_middle/src/ty/mod.rs
+++ b/compiler/rustc_middle/src/ty/mod.rs
@@ -42,7 +42,6 @@ use rustc_errors::{DiagnosticBuilder, ErrorGuaranteed, StashKey};
 use rustc_hir as hir;
 use rustc_hir::def::{CtorKind, CtorOf, DefKind, DocLinkResMap, LifetimeRes, Res};
 use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LocalDefId, LocalDefIdMap};
-use rustc_hir::Node;
 use rustc_index::IndexVec;
 use rustc_macros::HashStable;
 use rustc_query_system::ich::StableHashingContext;
@@ -66,27 +65,20 @@ use std::ops::ControlFlow;
 use std::{fmt, str};
 
 pub use crate::ty::diagnostics::*;
-pub use rustc_type_ir::AliasKind::*;
 pub use rustc_type_ir::ConstKind::{
     Bound as BoundCt, Error as ErrorCt, Expr as ExprCt, Infer as InferCt, Param as ParamCt,
     Placeholder as PlaceholderCt, Unevaluated, Value,
 };
-pub use rustc_type_ir::DynKind::*;
-pub use rustc_type_ir::InferTy::*;
-pub use rustc_type_ir::RegionKind::*;
-pub use rustc_type_ir::TyKind::*;
 pub use rustc_type_ir::*;
 
 pub use self::binding::BindingMode;
 pub use self::binding::BindingMode::*;
 pub use self::closure::{
     is_ancestor_or_same_capture, place_to_string_for_capture, BorrowKind, CaptureInfo,
-    CapturedPlace, ClosureKind, ClosureTypeInfo, MinCaptureInformationMap, MinCaptureList,
+    CapturedPlace, ClosureTypeInfo, MinCaptureInformationMap, MinCaptureList,
     RootVariableMinCaptureList, UpvarCapture, UpvarId, UpvarPath, CAPTURE_STRUCT_LOCAL,
 };
-pub use self::consts::{
-    Const, ConstData, ConstInt, Expr, InferConst, ScalarInt, UnevaluatedConst, ValTree,
-};
+pub use self::consts::{Const, ConstData, ConstInt, Expr, ScalarInt, UnevaluatedConst, ValTree};
 pub use self::context::{
     tls, CtxtInterners, DeducedParamAttrs, FreeRegionInfo, GlobalCtxt, Lift, TyCtxt, TyCtxtFeed,
 };
@@ -96,13 +88,13 @@ pub use self::parameterized::ParameterizedOverTcx;
 pub use self::rvalue_scopes::RvalueScopes;
 pub use self::sty::BoundRegionKind::*;
 pub use self::sty::{
-    AliasTy, Article, Binder, BoundRegion, BoundRegionKind, BoundTy, BoundTyKind, BoundVar,
+    AliasTy, Article, Binder, BoundRegion, BoundRegionKind, BoundTy, BoundTyKind,
     BoundVariableKind, CanonicalPolyFnSig, ClauseKind, ClosureArgs, ClosureArgsParts, ConstKind,
-    ConstVid, CoroutineArgs, CoroutineArgsParts, EarlyBoundRegion, EffectVid, ExistentialPredicate,
-    ExistentialProjection, ExistentialTraitRef, FnSig, FreeRegion, GenSig, InlineConstArgs,
-    InlineConstArgsParts, ParamConst, ParamTy, PolyExistentialPredicate, PolyExistentialProjection,
-    PolyExistentialTraitRef, PolyFnSig, PolyGenSig, PolyTraitRef, PredicateKind, Region,
-    RegionKind, RegionVid, TraitRef, TyKind, TypeAndMut, UpvarArgs, VarianceDiagInfo,
+    CoroutineArgs, CoroutineArgsParts, EarlyParamRegion, ExistentialPredicate,
+    ExistentialProjection, ExistentialTraitRef, FnSig, GenSig, InlineConstArgs,
+    InlineConstArgsParts, LateParamRegion, ParamConst, ParamTy, PolyExistentialPredicate,
+    PolyExistentialProjection, PolyExistentialTraitRef, PolyFnSig, PolyTraitRef, PredicateKind,
+    Region, RegionKind, RegionVid, TraitRef, TyKind, TypeAndMut, UpvarArgs, VarianceDiagInfo,
 };
 pub use self::trait_def::TraitDef;
 pub use self::typeck_results::{
@@ -160,7 +152,7 @@ pub struct ResolverOutputs {
 
 #[derive(Debug)]
 pub struct ResolverGlobalCtxt {
-    pub visibilities: FxHashMap<LocalDefId, Visibility>,
+    pub visibilities_for_hashing: Vec<(LocalDefId, Visibility)>,
     /// Item with a given `LocalDefId` was defined during macro expansion with ID `ExpnId`.
     pub expn_that_defined: FxHashMap<LocalDefId, ExpnId>,
     pub effective_visibilities: EffectiveVisibilities,
@@ -200,13 +192,10 @@ pub struct ResolverAstLowering {
 
     pub next_node_id: ast::NodeId,
 
-    pub node_id_to_def_id: FxHashMap<ast::NodeId, LocalDefId>,
+    pub node_id_to_def_id: NodeMap<LocalDefId>,
     pub def_id_to_node_id: IndexVec<LocalDefId, ast::NodeId>,
 
     pub trait_map: NodeMap<Vec<hir::TraitCandidate>>,
-    /// A small map keeping true kinds of built-in macros that appear to be fn-like on
-    /// the surface (`macro` items in libcore), but are actually attributes or derives.
-    pub builtin_macro_kinds: FxHashMap<LocalDefId, MacroKind>,
     /// List functions and methods for which lifetime elision was successful.
     pub lifetime_elision_allowed: FxHashSet<ast::NodeId>,
 
@@ -301,6 +290,23 @@ pub enum Visibility<Id = LocalDefId> {
     Restricted(Id),
 }
 
+impl Visibility {
+    pub fn to_string(self, def_id: LocalDefId, tcx: TyCtxt<'_>) -> String {
+        match self {
+            ty::Visibility::Restricted(restricted_id) => {
+                if restricted_id.is_top_level_module() {
+                    "pub(crate)".to_string()
+                } else if restricted_id == tcx.parent_module_from_def_id(def_id).to_local_def_id() {
+                    "pub(self)".to_string()
+                } else {
+                    format!("pub({})", tcx.item_name(restricted_id.to_def_id()))
+                }
+            }
+            ty::Visibility::Public => "pub".to_string(),
+        }
+    }
+}
+
 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, HashStable, TyEncodable, TyDecodable)]
 pub enum BoundConstness {
     /// `T: Trait`
@@ -463,7 +469,15 @@ pub struct CReaderCacheKey {
 #[rustc_pass_by_value]
 pub struct Ty<'tcx>(Interned<'tcx, WithCachedTypeInfo<TyKind<'tcx>>>);
 
-impl ty::EarlyBoundRegion {
+impl<'tcx> IntoKind for Ty<'tcx> {
+    type Kind = TyKind<'tcx>;
+
+    fn kind(self) -> TyKind<'tcx> {
+        *self.kind()
+    }
+}
+
+impl EarlyParamRegion {
     /// Does this early bound region have a name? Early bound regions normally
     /// always have names except when using anonymous lifetimes (`'_`).
     pub fn has_name(&self) -> bool {
@@ -542,6 +556,10 @@ impl<'tcx> Predicate<'tcx> {
     pub fn allow_normalization(self) -> bool {
         match self.kind().skip_binder() {
             PredicateKind::Clause(ClauseKind::WellFormed(_)) => false,
+            // `NormalizesTo` is only used in the new solver, so this shouldn't
+            // matter. Normalizing `term` would be 'wrong' however, as it changes whether
+            // `normalizes-to(<T as Trait>::Assoc, <T as Trait>::Assoc)` holds.
+            PredicateKind::NormalizesTo(..) => false,
             PredicateKind::Clause(ClauseKind::Trait(_))
             | PredicateKind::Clause(ClauseKind::RegionOutlives(_))
             | PredicateKind::Clause(ClauseKind::TypeOutlives(_))
@@ -549,7 +567,6 @@ impl<'tcx> Predicate<'tcx> {
             | PredicateKind::Clause(ClauseKind::ConstArgHasType(..))
             | PredicateKind::AliasRelate(..)
             | PredicateKind::ObjectSafe(_)
-            | PredicateKind::ClosureKind(_, _, _)
             | PredicateKind::Subtype(_)
             | PredicateKind::Coerce(_)
             | PredicateKind::Clause(ClauseKind::ConstEvaluatable(_))
@@ -902,7 +919,7 @@ impl<'tcx> Term<'tcx> {
                     &*((ptr & !TAG_MASK) as *const WithCachedTypeInfo<ty::TyKind<'tcx>>),
                 ))),
                 CONST_TAG => TermKind::Const(ty::Const(Interned::new_unchecked(
-                    &*((ptr & !TAG_MASK) as *const ty::ConstData<'tcx>),
+                    &*((ptr & !TAG_MASK) as *const WithCachedTypeInfo<ty::ConstData<'tcx>>),
                 ))),
                 _ => core::intrinsics::unreachable(),
             }
@@ -969,7 +986,7 @@ impl<'tcx> TermKind<'tcx> {
             TermKind::Const(ct) => {
                 // Ensure we can use the tag bits.
                 assert_eq!(mem::align_of_val(&*ct.0.0) & TAG_MASK, 0);
-                (CONST_TAG, ct.0.0 as *const ty::ConstData<'tcx> as usize)
+                (CONST_TAG, ct.0.0 as *const WithCachedTypeInfo<ty::ConstData<'tcx>> as usize)
             }
         };
 
@@ -1083,6 +1100,33 @@ impl<'tcx> PolyProjectionPredicate<'tcx> {
     }
 }
 
+/// Used by the new solver. Unlike a `ProjectionPredicate` this can only be
+/// proven by actually normalizing `alias`.
+#[derive(Copy, Clone, PartialEq, Eq, Hash, TyEncodable, TyDecodable)]
+#[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
+pub struct NormalizesTo<'tcx> {
+    pub alias: AliasTy<'tcx>,
+    pub term: Term<'tcx>,
+}
+
+impl<'tcx> NormalizesTo<'tcx> {
+    pub fn self_ty(self) -> Ty<'tcx> {
+        self.alias.self_ty()
+    }
+
+    pub fn with_self_ty(self, tcx: TyCtxt<'tcx>, self_ty: Ty<'tcx>) -> NormalizesTo<'tcx> {
+        Self { alias: self.alias.with_self_ty(tcx, self_ty), ..self }
+    }
+
+    pub fn trait_def_id(self, tcx: TyCtxt<'tcx>) -> DefId {
+        self.alias.trait_def_id(tcx)
+    }
+
+    pub fn def_id(self) -> DefId {
+        self.alias.def_id
+    }
+}
+
 pub trait ToPolyTraitRef<'tcx> {
     fn to_poly_trait_ref(&self) -> PolyTraitRef<'tcx>;
 }
@@ -1264,6 +1308,12 @@ impl<'tcx> ToPredicate<'tcx, Clause<'tcx>> for PolyProjectionPredicate<'tcx> {
     }
 }
 
+impl<'tcx> ToPredicate<'tcx> for NormalizesTo<'tcx> {
+    fn to_predicate(self, tcx: TyCtxt<'tcx>) -> Predicate<'tcx> {
+        PredicateKind::NormalizesTo(self).to_predicate(tcx)
+    }
+}
+
 impl<'tcx> Predicate<'tcx> {
     pub fn to_opt_poly_trait_pred(self) -> Option<PolyTraitPredicate<'tcx>> {
         let predicate = self.kind();
@@ -1271,13 +1321,13 @@ impl<'tcx> Predicate<'tcx> {
             PredicateKind::Clause(ClauseKind::Trait(t)) => Some(predicate.rebind(t)),
             PredicateKind::Clause(ClauseKind::Projection(..))
             | PredicateKind::Clause(ClauseKind::ConstArgHasType(..))
+            | PredicateKind::NormalizesTo(..)
             | PredicateKind::AliasRelate(..)
             | PredicateKind::Subtype(..)
             | PredicateKind::Coerce(..)
             | PredicateKind::Clause(ClauseKind::RegionOutlives(..))
             | PredicateKind::Clause(ClauseKind::WellFormed(..))
             | PredicateKind::ObjectSafe(..)
-            | PredicateKind::ClosureKind(..)
             | PredicateKind::Clause(ClauseKind::TypeOutlives(..))
             | PredicateKind::Clause(ClauseKind::ConstEvaluatable(..))
             | PredicateKind::ConstEquate(..)
@@ -1291,13 +1341,13 @@ impl<'tcx> Predicate<'tcx> {
             PredicateKind::Clause(ClauseKind::Projection(t)) => Some(predicate.rebind(t)),
             PredicateKind::Clause(ClauseKind::Trait(..))
             | PredicateKind::Clause(ClauseKind::ConstArgHasType(..))
+            | PredicateKind::NormalizesTo(..)
             | PredicateKind::AliasRelate(..)
             | PredicateKind::Subtype(..)
             | PredicateKind::Coerce(..)
             | PredicateKind::Clause(ClauseKind::RegionOutlives(..))
             | PredicateKind::Clause(ClauseKind::WellFormed(..))
             | PredicateKind::ObjectSafe(..)
-            | PredicateKind::ClosureKind(..)
             | PredicateKind::Clause(ClauseKind::TypeOutlives(..))
             | PredicateKind::Clause(ClauseKind::ConstEvaluatable(..))
             | PredicateKind::ConstEquate(..)
@@ -1305,26 +1355,6 @@ impl<'tcx> Predicate<'tcx> {
         }
     }
 
-    pub fn to_opt_type_outlives(self) -> Option<PolyTypeOutlivesPredicate<'tcx>> {
-        let predicate = self.kind();
-        match predicate.skip_binder() {
-            PredicateKind::Clause(ClauseKind::TypeOutlives(data)) => Some(predicate.rebind(data)),
-            PredicateKind::Clause(ClauseKind::Trait(..))
-            | PredicateKind::Clause(ClauseKind::ConstArgHasType(..))
-            | PredicateKind::Clause(ClauseKind::Projection(..))
-            | PredicateKind::AliasRelate(..)
-            | PredicateKind::Subtype(..)
-            | PredicateKind::Coerce(..)
-            | PredicateKind::Clause(ClauseKind::RegionOutlives(..))
-            | PredicateKind::Clause(ClauseKind::WellFormed(..))
-            | PredicateKind::ObjectSafe(..)
-            | PredicateKind::ClosureKind(..)
-            | PredicateKind::Clause(ClauseKind::ConstEvaluatable(..))
-            | PredicateKind::ConstEquate(..)
-            | PredicateKind::Ambiguous => None,
-        }
-    }
-
     /// Matches a `PredicateKind::Clause` and turns it into a `Clause`, otherwise returns `None`.
     pub fn as_clause(self) -> Option<Clause<'tcx>> {
         match self.kind().skip_binder() {
@@ -1377,7 +1407,7 @@ impl<'tcx> InstantiatedPredicates<'tcx> {
     }
 
     pub fn iter(&self) -> <&Self as IntoIterator>::IntoIter {
-        (&self).into_iter()
+        self.into_iter()
     }
 }
 
@@ -1454,7 +1484,7 @@ impl<'tcx> OpaqueHiddenType<'tcx> {
     ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
         if let Some(diag) = tcx
             .sess
-            .diagnostic()
+            .dcx()
             .steal_diagnostic(tcx.def_span(opaque_def_id), StashKey::OpaqueHiddenTypeMismatch)
         {
             diag.cancel();
@@ -1518,8 +1548,44 @@ pub struct Placeholder<T> {
 
 pub type PlaceholderRegion = Placeholder<BoundRegion>;
 
+impl PlaceholderLike for PlaceholderRegion {
+    fn universe(self) -> UniverseIndex {
+        self.universe
+    }
+
+    fn var(self) -> BoundVar {
+        self.bound.var
+    }
+
+    fn with_updated_universe(self, ui: UniverseIndex) -> Self {
+        Placeholder { universe: ui, ..self }
+    }
+
+    fn new(ui: UniverseIndex, var: BoundVar) -> Self {
+        Placeholder { universe: ui, bound: BoundRegion { var, kind: BoundRegionKind::BrAnon } }
+    }
+}
+
 pub type PlaceholderType = Placeholder<BoundTy>;
 
+impl PlaceholderLike for PlaceholderType {
+    fn universe(self) -> UniverseIndex {
+        self.universe
+    }
+
+    fn var(self) -> BoundVar {
+        self.bound.var
+    }
+
+    fn with_updated_universe(self, ui: UniverseIndex) -> Self {
+        Placeholder { universe: ui, ..self }
+    }
+
+    fn new(ui: UniverseIndex, var: BoundVar) -> Self {
+        Placeholder { universe: ui, bound: BoundTy { var, kind: BoundTyKind::Anon } }
+    }
+}
+
 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, HashStable)]
 #[derive(TyEncodable, TyDecodable, PartialOrd, Ord)]
 pub struct BoundConst<'tcx> {
@@ -1529,6 +1595,24 @@ pub struct BoundConst<'tcx> {
 
 pub type PlaceholderConst = Placeholder<BoundVar>;
 
+impl PlaceholderLike for PlaceholderConst {
+    fn universe(self) -> UniverseIndex {
+        self.universe
+    }
+
+    fn var(self) -> BoundVar {
+        self.bound
+    }
+
+    fn with_updated_universe(self, ui: UniverseIndex) -> Self {
+        Placeholder { universe: ui, ..self }
+    }
+
+    fn new(ui: UniverseIndex, var: BoundVar) -> Self {
+        Placeholder { universe: ui, bound: var }
+    }
+}
+
 /// When type checking, we use the `ParamEnv` to track
 /// details about the set of where-clauses that are in scope at this
 /// particular point.
@@ -2021,7 +2105,7 @@ impl<'tcx> TyCtxt<'tcx> {
         }
 
         for attr in self.get_attrs(did, sym::repr) {
-            for r in attr::parse_repr_attr(&self.sess, attr) {
+            for r in attr::parse_repr_attr(self.sess, attr) {
                 flags.insert(match r {
                     attr::ReprRust => ReprFlags::empty(),
                     attr::ReprC => ReprFlags::IS_C,
@@ -2256,7 +2340,7 @@ impl<'tcx> TyCtxt<'tcx> {
     // FIXME(@lcnr): Remove this function.
     pub fn get_attrs_unchecked(self, did: DefId) -> &'tcx [ast::Attribute] {
         if let Some(did) = did.as_local() {
-            self.hir().attrs(self.hir().local_def_id_to_hir_id(did))
+            self.hir().attrs(self.local_def_id_to_hir_id(did))
         } else {
             self.item_attrs(did)
         }
@@ -2271,7 +2355,7 @@ impl<'tcx> TyCtxt<'tcx> {
         let did: DefId = did.into();
         let filter_fn = move |a: &&ast::Attribute| a.has_name(attr);
         if let Some(did) = did.as_local() {
-            self.hir().attrs(self.hir().local_def_id_to_hir_id(did)).iter().filter(filter_fn)
+            self.hir().attrs(self.local_def_id_to_hir_id(did)).iter().filter(filter_fn)
         } else if cfg!(debug_assertions) && rustc_feature::is_builtin_only_local(attr) {
             bug!("tried to access the `only_local` attribute `{}` from an extern crate", attr);
         } else {
@@ -2287,9 +2371,9 @@ impl<'tcx> TyCtxt<'tcx> {
     where
         'tcx: 'attr,
     {
-        let filter_fn = move |a: &&ast::Attribute| a.path_matches(&attr);
+        let filter_fn = move |a: &&ast::Attribute| a.path_matches(attr);
         if let Some(did) = did.as_local() {
-            self.hir().attrs(self.hir().local_def_id_to_hir_id(did)).iter().filter(filter_fn)
+            self.hir().attrs(self.local_def_id_to_hir_id(did)).iter().filter(filter_fn)
         } else {
             self.item_attrs(did).iter().filter(filter_fn)
         }
@@ -2495,22 +2579,6 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 }
 
-/// Yields the parent function's `LocalDefId` if `def_id` is an `impl Trait` definition.
-pub fn is_impl_trait_defn(tcx: TyCtxt<'_>, def_id: DefId) -> Option<LocalDefId> {
-    let def_id = def_id.as_local()?;
-    if let Node::Item(item) = tcx.hir().get_by_def_id(def_id) {
-        if let hir::ItemKind::OpaqueTy(ref opaque_ty) = item.kind {
-            return match opaque_ty.origin {
-                hir::OpaqueTyOrigin::FnReturn(parent) | hir::OpaqueTyOrigin::AsyncFn(parent) => {
-                    Some(parent)
-                }
-                hir::OpaqueTyOrigin::TyAlias { .. } => None,
-            };
-        }
-    }
-    None
-}
-
 pub fn int_ty(ity: ast::IntTy) -> IntTy {
     match ity {
         ast::IntTy::Isize => IntTy::Isize,
@@ -2599,9 +2667,7 @@ pub struct SymbolName<'tcx> {
 
 impl<'tcx> SymbolName<'tcx> {
     pub fn new(tcx: TyCtxt<'tcx>, name: &str) -> SymbolName<'tcx> {
-        SymbolName {
-            name: unsafe { str::from_utf8_unchecked(tcx.arena.alloc_slice(name.as_bytes())) },
-        }
+        SymbolName { name: tcx.arena.alloc_str(name) }
     }
 }
 
diff --git a/compiler/rustc_middle/src/ty/normalize_erasing_regions.rs b/compiler/rustc_middle/src/ty/normalize_erasing_regions.rs
index fd125af20..27c436c82 100644
--- a/compiler/rustc_middle/src/ty/normalize_erasing_regions.rs
+++ b/compiler/rustc_middle/src/ty/normalize_erasing_regions.rs
@@ -97,6 +97,10 @@ impl<'tcx> TyCtxt<'tcx> {
     /// N.B., currently, higher-ranked type bounds inhibit
     /// normalization. Therefore, each time we erase them in
     /// codegen, we need to normalize the contents.
+    // FIXME(@lcnr): This method should not be necessary, we now normalize
+    // inside of binders. We should be able to only use
+    // `tcx.instantiate_bound_regions_with_erased`. Same for the `try_X`
+    // variant.
     #[tracing::instrument(level = "debug", skip(self, param_env))]
     pub fn normalize_erasing_late_bound_regions<T>(
         self,
@@ -106,7 +110,7 @@ impl<'tcx> TyCtxt<'tcx> {
     where
         T: TypeFoldable<TyCtxt<'tcx>>,
     {
-        let value = self.erase_late_bound_regions(value);
+        let value = self.instantiate_bound_regions_with_erased(value);
         self.normalize_erasing_regions(param_env, value)
     }
 
@@ -126,7 +130,7 @@ impl<'tcx> TyCtxt<'tcx> {
     where
         T: TypeFoldable<TyCtxt<'tcx>>,
     {
-        let value = self.erase_late_bound_regions(value);
+        let value = self.instantiate_bound_regions_with_erased(value);
         self.try_normalize_erasing_regions(param_env, value)
     }
 
diff --git a/compiler/rustc_middle/src/ty/opaque_types.rs b/compiler/rustc_middle/src/ty/opaque_types.rs
index 8d895732d..1305f63bd 100644
--- a/compiler/rustc_middle/src/ty/opaque_types.rs
+++ b/compiler/rustc_middle/src/ty/opaque_types.rs
@@ -102,8 +102,9 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for ReverseMapper<'tcx> {
             // Ignore bound regions and `'static` regions that appear in the
             // type, we only need to remap regions that reference lifetimes
             // from the function declaration.
-            // This would ignore `'r` in a type like `for<'r> fn(&'r u32)`.
-            ty::ReLateBound(..) | ty::ReStatic => return r,
+            //
+            // E.g. We ignore `'r` in a type like `for<'r> fn(&'r u32)`.
+            ty::ReBound(..) | ty::ReStatic => return r,
 
             // If regions have been erased (by writeback), don't try to unerase
             // them.
@@ -112,7 +113,7 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for ReverseMapper<'tcx> {
             ty::ReError(_) => return r,
 
             // The regions that we expect from borrow checking.
-            ty::ReEarlyBound(_) | ty::ReFree(_) => {}
+            ty::ReEarlyParam(_) | ty::ReLateParam(_) => {}
 
             ty::RePlaceholder(_) | ty::ReVar(_) => {
                 // All of the regions in the type should either have been
diff --git a/compiler/rustc_middle/src/ty/parameterized.rs b/compiler/rustc_middle/src/ty/parameterized.rs
index 9afa50cf5..a63a4eff5 100644
--- a/compiler/rustc_middle/src/ty/parameterized.rs
+++ b/compiler/rustc_middle/src/ty/parameterized.rs
@@ -59,6 +59,7 @@ trivially_parameterized_over_tcx! {
     crate::middle::codegen_fn_attrs::CodegenFnAttrs,
     crate::middle::debugger_visualizer::DebuggerVisualizerFile,
     crate::middle::exported_symbols::SymbolExportInfo,
+    crate::middle::lib_features::FeatureStability,
     crate::middle::resolve_bound_vars::ObjectLifetimeDefault,
     crate::mir::ConstQualifs,
     ty::AssocItemContainer,
diff --git a/compiler/rustc_middle/src/ty/print/mod.rs b/compiler/rustc_middle/src/ty/print/mod.rs
index 6bbc8f70f..5e0915478 100644
--- a/compiler/rustc_middle/src/ty/print/mod.rs
+++ b/compiler/rustc_middle/src/ty/print/mod.rs
@@ -131,7 +131,7 @@ pub trait Printer<'tcx>: Sized {
 
                     match key.disambiguated_data.data {
                         // Closures' own generics are only captures, don't print them.
-                        DefPathData::ClosureExpr => {}
+                        DefPathData::Closure => {}
                         // This covers both `DefKind::AnonConst` and `DefKind::InlineConst`.
                         // Anon consts doesn't have their own generics, and inline consts' own
                         // generics are their inferred types, so don't print them.
@@ -250,7 +250,7 @@ fn characteristic_def_id_of_type_cached<'a>(
 
         ty::Ref(_, ty, _) => characteristic_def_id_of_type_cached(ty, visited),
 
-        ty::Tuple(ref tys) => tys.iter().find_map(|ty| {
+        ty::Tuple(tys) => tys.iter().find_map(|ty| {
             if visited.insert(ty) {
                 return characteristic_def_id_of_type_cached(ty, visited);
             }
diff --git a/compiler/rustc_middle/src/ty/print/pretty.rs b/compiler/rustc_middle/src/ty/print/pretty.rs
index baf160bcc..8e045397b 100644
--- a/compiler/rustc_middle/src/ty/print/pretty.rs
+++ b/compiler/rustc_middle/src/ty/print/pretty.rs
@@ -2,23 +2,22 @@ use crate::mir::interpret::{AllocRange, GlobalAlloc, Pointer, Provenance, Scalar
 use crate::query::IntoQueryParam;
 use crate::query::Providers;
 use crate::traits::util::supertraits_for_pretty_printing;
+use crate::ty::GenericArgKind;
 use crate::ty::{
-    self, ConstInt, ParamConst, ScalarInt, Term, TermKind, Ty, TyCtxt, TypeFoldable,
-    TypeSuperFoldable, TypeSuperVisitable, TypeVisitable, TypeVisitableExt,
+    ConstInt, ParamConst, ScalarInt, Term, TermKind, TypeFoldable, TypeSuperFoldable,
+    TypeSuperVisitable, TypeVisitable, TypeVisitableExt,
 };
-use crate::ty::{GenericArg, GenericArgKind};
 use rustc_apfloat::ieee::{Double, Single};
+use rustc_apfloat::Float;
 use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
-use rustc_data_structures::sso::SsoHashSet;
 use rustc_hir as hir;
 use rustc_hir::def::{self, CtorKind, DefKind, Namespace};
-use rustc_hir::def_id::{DefId, DefIdSet, ModDefId, CRATE_DEF_ID, LOCAL_CRATE};
-use rustc_hir::definitions::{DefKey, DefPathData, DefPathDataName, DisambiguatedDefPathData};
+use rustc_hir::def_id::{DefIdSet, ModDefId, CRATE_DEF_ID, LOCAL_CRATE};
+use rustc_hir::definitions::{DefKey, DefPathDataName};
 use rustc_hir::LangItem;
 use rustc_session::config::TrimmedDefPaths;
 use rustc_session::cstore::{ExternCrate, ExternCrateSource};
 use rustc_session::Limit;
-use rustc_span::sym;
 use rustc_span::symbol::{kw, Ident, Symbol};
 use rustc_span::FileNameDisplayPreference;
 use rustc_target::abi::Size;
@@ -282,7 +281,7 @@ pub trait PrettyPrinter<'tcx>: Printer<'tcx> + fmt::Write {
     /// from at least one local module, and returns `true`. If the crate defining `def_id` is
     /// declared with an `extern crate`, the path is guaranteed to use the `extern crate`.
     fn try_print_visible_def_path(&mut self, def_id: DefId) -> Result<bool, PrintError> {
-        if NO_VISIBLE_PATH.with(|flag| flag.get()) {
+        if with_no_visible_paths() {
             return Ok(false);
         }
 
@@ -322,7 +321,7 @@ pub trait PrettyPrinter<'tcx>: Printer<'tcx> + fmt::Write {
             && let Some(symbol) = self.tcx().trimmed_def_paths(()).get(&def_id)
         {
             // If `Assoc` is unique, we don't want to talk about `Trait::Assoc`.
-            self.write_str(get_local_name(&self, *symbol, def_id, key).as_str())?;
+            self.write_str(get_local_name(self, *symbol, def_id, key).as_str())?;
             return Ok(true);
         }
         if let Some(symbol) = key.get_opt_name() {
@@ -332,7 +331,7 @@ pub trait PrettyPrinter<'tcx>: Printer<'tcx> + fmt::Write {
                 && let Some(symbol) = parent_key.get_opt_name()
             {
                 // Trait
-                self.write_str(get_local_name(&self, symbol, parent, parent_key).as_str())?;
+                self.write_str(get_local_name(self, symbol, parent, parent_key).as_str())?;
                 self.write_str("::")?;
             } else if let DefKind::Variant = kind
                 && let Some(parent) = self.tcx().opt_parent(def_id)
@@ -343,7 +342,7 @@ pub trait PrettyPrinter<'tcx>: Printer<'tcx> + fmt::Write {
 
                 // For associated items and variants, we want the "full" path, namely, include
                 // the parent type in the path. For example, `Iterator::Item`.
-                self.write_str(get_local_name(&self, symbol, parent, parent_key).as_str())?;
+                self.write_str(get_local_name(self, symbol, parent, parent_key).as_str())?;
                 self.write_str("::")?;
             } else if let DefKind::Struct
             | DefKind::Union
@@ -358,7 +357,7 @@ pub trait PrettyPrinter<'tcx>: Printer<'tcx> + fmt::Write {
                 // If not covered above, like for example items out of `impl` blocks, fallback.
                 return Ok(false);
             }
-            self.write_str(get_local_name(&self, symbol, def_id, key).as_str())?;
+            self.write_str(get_local_name(self, symbol, def_id, key).as_str())?;
             return Ok(true);
         }
         Ok(false)
@@ -366,7 +365,7 @@ pub trait PrettyPrinter<'tcx>: Printer<'tcx> + fmt::Write {
 
     /// Try to see if this path can be trimmed to a unique symbol name.
     fn try_print_trimmed_def_path(&mut self, def_id: DefId) -> Result<bool, PrintError> {
-        if FORCE_TRIMMED_PATH.with(|flag| flag.get()) {
+        if with_forced_trimmed_paths() {
             let trimmed = self.force_print_trimmed_def_path(def_id)?;
             if trimmed {
                 return Ok(true);
@@ -374,8 +373,8 @@ pub trait PrettyPrinter<'tcx>: Printer<'tcx> + fmt::Write {
         }
         if !self.tcx().sess.opts.unstable_opts.trim_diagnostic_paths
             || matches!(self.tcx().sess.opts.trimmed_def_paths, TrimmedDefPaths::Never)
-            || NO_TRIMMED_PATH.with(|flag| flag.get())
-            || SHOULD_PREFIX_WITH_CRATE.with(|flag| flag.get())
+            || with_no_trimmed_paths()
+            || with_crate_prefix()
         {
             return Ok(false);
         }
@@ -660,7 +659,7 @@ pub trait PrettyPrinter<'tcx>: Printer<'tcx> + fmt::Write {
                 p!(print(ty::TypeAndMut { ty, mutbl }))
             }
             ty::Never => p!("!"),
-            ty::Tuple(ref tys) => {
+            ty::Tuple(tys) => {
                 p!("(", comma_sep(tys.iter()));
                 if tys.len() == 1 {
                     p!(",");
@@ -860,7 +859,7 @@ pub trait PrettyPrinter<'tcx>: Printer<'tcx> + fmt::Write {
                             p!("@", print_def_path(did.to_def_id(), args));
                         } else {
                             let span = self.tcx().def_span(did);
-                            let preference = if FORCE_TRIMMED_PATH.with(|flag| flag.get()) {
+                            let preference = if with_forced_trimmed_paths() {
                                 FileNameDisplayPreference::Short
                             } else {
                                 FileNameDisplayPreference::Remapped
@@ -1101,7 +1100,7 @@ pub trait PrettyPrinter<'tcx>: Printer<'tcx> + fmt::Write {
             write!(self, "Sized")?;
         }
 
-        if !FORCE_TRIMMED_PATH.with(|flag| flag.get()) {
+        if !with_forced_trimmed_paths() {
             for re in lifetimes {
                 write!(self, " + ")?;
                 self.print_region(re)?;
@@ -1409,14 +1408,14 @@ pub trait PrettyPrinter<'tcx>: Printer<'tcx> + fmt::Write {
     ) -> Result<(), PrintError> {
         define_scoped_cx!(self);
 
-        let (alloc_id, offset) = ptr.into_parts();
+        let (prov, offset) = ptr.into_parts();
         match ty.kind() {
             // Byte strings (&[u8; N])
             ty::Ref(_, inner, _) => {
                 if let ty::Array(elem, len) = inner.kind() {
                     if let ty::Uint(ty::UintTy::U8) = elem.kind() {
                         if let ty::ConstKind::Value(ty::ValTree::Leaf(int)) = len.kind() {
-                            match self.tcx().try_get_global_alloc(alloc_id) {
+                            match self.tcx().try_get_global_alloc(prov.alloc_id()) {
                                 Some(GlobalAlloc::Memory(alloc)) => {
                                     let len = int.assert_bits(self.tcx().data_layout.pointer_size);
                                     let range =
@@ -1446,7 +1445,7 @@ pub trait PrettyPrinter<'tcx>: Printer<'tcx> + fmt::Write {
                 // FIXME: We should probably have a helper method to share code with the "Byte strings"
                 // printing above (which also has to handle pointers to all sorts of things).
                 if let Some(GlobalAlloc::Function(instance)) =
-                    self.tcx().try_get_global_alloc(alloc_id)
+                    self.tcx().try_get_global_alloc(prov.alloc_id())
                 {
                     self.typed_value(
                         |this| this.print_value_path(instance.def_id(), instance.args),
@@ -1477,10 +1476,12 @@ pub trait PrettyPrinter<'tcx>: Printer<'tcx> + fmt::Write {
             ty::Bool if int == ScalarInt::TRUE => p!("true"),
             // Float
             ty::Float(ty::FloatTy::F32) => {
-                p!(write("{}f32", Single::try_from(int).unwrap()))
+                let val = Single::try_from(int).unwrap();
+                p!(write("{}{}f32", val, if val.is_finite() { "" } else { "_" }))
             }
             ty::Float(ty::FloatTy::F64) => {
-                p!(write("{}f64", Double::try_from(int).unwrap()))
+                let val = Double::try_from(int).unwrap();
+                p!(write("{}{}f64", val, if val.is_finite() { "" } else { "_" }))
             }
             // Int
             ty::Uint(_) | ty::Int(_) => {
@@ -1679,7 +1680,7 @@ pub trait PrettyPrinter<'tcx>: Printer<'tcx> + fmt::Write {
         self.wrap_binder(&sig, |sig, cx| {
             define_scoped_cx!(cx);
 
-            p!(print(kind), "(");
+            p!(write("{kind}("));
             for (i, arg) in sig.inputs()[0].tuple_fields().iter().enumerate() {
                 if i > 0 {
                     p!(", ");
@@ -1801,13 +1802,13 @@ impl<'a, 'tcx> FmtPrinter<'a, 'tcx> {
 // (but also some things just print a `DefId` generally so maybe we need this?)
 fn guess_def_namespace(tcx: TyCtxt<'_>, def_id: DefId) -> Namespace {
     match tcx.def_key(def_id).disambiguated_data.data {
-        DefPathData::TypeNs(..) | DefPathData::CrateRoot | DefPathData::ImplTrait => {
+        DefPathData::TypeNs(..) | DefPathData::CrateRoot | DefPathData::OpaqueTy => {
             Namespace::TypeNS
         }
 
         DefPathData::ValueNs(..)
         | DefPathData::AnonConst
-        | DefPathData::ClosureExpr
+        | DefPathData::Closure
         | DefPathData::Ctor => Namespace::ValueNS,
 
         DefPathData::MacroNs(..) => Namespace::MacroNS,
@@ -1883,7 +1884,7 @@ impl<'tcx> Printer<'tcx> for FmtPrinter<'_, 'tcx> {
             // available, and filename/line-number is mostly uninteresting.
             let use_types = !def_id.is_local() || {
                 // Otherwise, use filename/line-number if forced.
-                let force_no_types = FORCE_IMPL_FILENAME_LINE.with(|f| f.get());
+                let force_no_types = with_forced_impl_filename_line();
                 !force_no_types
             };
 
@@ -1948,7 +1949,7 @@ impl<'tcx> Printer<'tcx> for FmtPrinter<'_, 'tcx> {
         if cnum == LOCAL_CRATE {
             if self.tcx.sess.at_least_rust_2018() {
                 // We add the `crate::` keyword on Rust 2018, only when desired.
-                if SHOULD_PREFIX_WITH_CRATE.with(|flag| flag.get()) {
+                if with_crate_prefix() {
                     write!(self, "{}", kw::Crate)?;
                     self.empty_path = false;
                 }
@@ -2031,37 +2032,11 @@ impl<'tcx> Printer<'tcx> for FmtPrinter<'_, 'tcx> {
     ) -> Result<(), PrintError> {
         print_prefix(self)?;
 
-        let tcx = self.tcx;
-
-        let args = args.iter().copied();
-
-        let args: Vec<_> = if !tcx.sess.verbose() {
-            // skip host param as those are printed as `~const`
-            args.filter(|arg| match arg.unpack() {
-                // FIXME(effects) there should be a better way than just matching the name
-                GenericArgKind::Const(c)
-                    if tcx.features().effects
-                        && matches!(
-                            c.kind(),
-                            ty::ConstKind::Param(ty::ParamConst { name: sym::host, .. })
-                        ) =>
-                {
-                    false
-                }
-                _ => true,
-            })
-            .collect()
-        } else {
-            // If -Zverbose is passed, we should print the host parameter instead
-            // of eating it.
-            args.collect()
-        };
-
         if !args.is_empty() {
             if self.in_value {
                 write!(self, "::")?;
             }
-            self.generic_delimiters(|cx| cx.comma_sep(args.into_iter()))
+            self.generic_delimiters(|cx| cx.comma_sep(args.iter().copied()))
         } else {
             Ok(())
         }
@@ -2151,17 +2126,17 @@ impl<'tcx> PrettyPrinter<'tcx> for FmtPrinter<'_, 'tcx> {
             return true;
         }
 
-        if FORCE_TRIMMED_PATH.with(|flag| flag.get()) {
+        if with_forced_trimmed_paths() {
             return false;
         }
 
         let identify_regions = self.tcx.sess.opts.unstable_opts.identify_regions;
 
         match *region {
-            ty::ReEarlyBound(ref data) => data.has_name(),
+            ty::ReEarlyParam(ref data) => data.has_name(),
 
-            ty::ReLateBound(_, ty::BoundRegion { kind: br, .. })
-            | ty::ReFree(ty::FreeRegion { bound_region: br, .. })
+            ty::ReBound(_, ty::BoundRegion { kind: br, .. })
+            | ty::ReLateParam(ty::LateParamRegion { bound_region: br, .. })
             | ty::RePlaceholder(ty::Placeholder {
                 bound: ty::BoundRegion { kind: br, .. }, ..
             }) => {
@@ -2228,14 +2203,14 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
         // to fit that into a short string. Hence the recommendation to use
         // `explain_region()` or `note_and_explain_region()`.
         match *region {
-            ty::ReEarlyBound(ref data) => {
+            ty::ReEarlyParam(ref data) => {
                 if data.name != kw::Empty {
                     p!(write("{}", data.name));
                     return Ok(());
                 }
             }
-            ty::ReLateBound(_, ty::BoundRegion { kind: br, .. })
-            | ty::ReFree(ty::FreeRegion { bound_region: br, .. })
+            ty::ReBound(_, ty::BoundRegion { kind: br, .. })
+            | ty::ReLateParam(ty::LateParamRegion { bound_region: br, .. })
             | ty::RePlaceholder(ty::Placeholder {
                 bound: ty::BoundRegion { kind: br, .. }, ..
             }) => {
@@ -2315,7 +2290,7 @@ impl<'a, 'tcx> ty::TypeFolder<TyCtxt<'tcx>> for RegionFolder<'a, 'tcx> {
     fn fold_region(&mut self, r: ty::Region<'tcx>) -> ty::Region<'tcx> {
         let name = &mut self.name;
         let region = match *r {
-            ty::ReLateBound(db, br) if db >= self.current_index => {
+            ty::ReBound(db, br) if db >= self.current_index => {
                 *self.region_map.entry(br).or_insert_with(|| name(Some(db), self.current_index, br))
             }
             ty::RePlaceholder(ty::PlaceholderRegion {
@@ -2338,9 +2313,9 @@ impl<'a, 'tcx> ty::TypeFolder<TyCtxt<'tcx>> for RegionFolder<'a, 'tcx> {
             }
             _ => return r,
         };
-        if let ty::ReLateBound(debruijn1, br) = *region {
+        if let ty::ReBound(debruijn1, br) = *region {
             assert_eq!(debruijn1, ty::INNERMOST);
-            ty::Region::new_late_bound(self.tcx, self.current_index, br)
+            ty::Region::new_bound(self.tcx, self.current_index, br)
         } else {
             region
         }
@@ -2399,7 +2374,7 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
         let possible_names = ('a'..='z').rev().map(|s| Symbol::intern(&format!("'{s}")));
 
         let mut available_names = possible_names
-            .filter(|name| !self.used_region_names.contains(&name))
+            .filter(|name| !self.used_region_names.contains(name))
             .collect::<Vec<_>>();
         debug!(?available_names);
         let num_available = available_names.len();
@@ -2434,7 +2409,7 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
         } else {
             let tcx = self.tcx;
 
-            let trim_path = FORCE_TRIMMED_PATH.with(|flag| flag.get());
+            let trim_path = with_forced_trimmed_paths();
             // Closure used in `RegionFolder` to create names for anonymous late-bound
             // regions. We use two `DebruijnIndex`es (one for the currently folded
             // late-bound region and the other for the binder level) to determine
@@ -2445,12 +2420,12 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
                             br: ty::BoundRegion| {
                 let (name, kind) = match br.kind {
                     ty::BrAnon | ty::BrEnv => {
-                        let name = next_name(&self);
+                        let name = next_name(self);
 
                         if let Some(lt_idx) = lifetime_idx {
                             if lt_idx > binder_level_idx {
                                 let kind = ty::BrNamed(CRATE_DEF_ID.to_def_id(), name);
-                                return ty::Region::new_late_bound(
+                                return ty::Region::new_bound(
                                     tcx,
                                     ty::INNERMOST,
                                     ty::BoundRegion { var: br.var, kind },
@@ -2461,12 +2436,12 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
                         (name, ty::BrNamed(CRATE_DEF_ID.to_def_id(), name))
                     }
                     ty::BrNamed(def_id, kw::UnderscoreLifetime | kw::Empty) => {
-                        let name = next_name(&self);
+                        let name = next_name(self);
 
                         if let Some(lt_idx) = lifetime_idx {
                             if lt_idx > binder_level_idx {
                                 let kind = ty::BrNamed(def_id, name);
-                                return ty::Region::new_late_bound(
+                                return ty::Region::new_bound(
                                     tcx,
                                     ty::INNERMOST,
                                     ty::BoundRegion { var: br.var, kind },
@@ -2480,7 +2455,7 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
                         if let Some(lt_idx) = lifetime_idx {
                             if lt_idx > binder_level_idx {
                                 let kind = br.kind;
-                                return ty::Region::new_late_bound(
+                                return ty::Region::new_bound(
                                     tcx,
                                     ty::INNERMOST,
                                     ty::BoundRegion { var: br.var, kind },
@@ -2496,11 +2471,7 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
                     start_or_continue(self, "for<", ", ");
                     do_continue(self, name);
                 }
-                ty::Region::new_late_bound(
-                    tcx,
-                    ty::INNERMOST,
-                    ty::BoundRegion { var: br.var, kind },
-                )
+                ty::Region::new_bound(tcx, ty::INNERMOST, ty::BoundRegion { var: br.var, kind })
             };
             let mut folder = RegionFolder {
                 tcx,
@@ -2642,6 +2613,23 @@ impl<'tcx> fmt::Debug for TraitRefPrintOnlyTraitPath<'tcx> {
 }
 
 /// Wrapper type for `ty::TraitRef` which opts-in to pretty printing only
+/// the trait path, and additionally tries to "sugar" `Fn(...)` trait bounds.
+#[derive(Copy, Clone, TypeFoldable, TypeVisitable, Lift)]
+pub struct TraitRefPrintSugared<'tcx>(ty::TraitRef<'tcx>);
+
+impl<'tcx> rustc_errors::IntoDiagnosticArg for TraitRefPrintSugared<'tcx> {
+    fn into_diagnostic_arg(self) -> rustc_errors::DiagnosticArgValue<'static> {
+        self.to_string().into_diagnostic_arg()
+    }
+}
+
+impl<'tcx> fmt::Debug for TraitRefPrintSugared<'tcx> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        fmt::Display::fmt(self, f)
+    }
+}
+
+/// Wrapper type for `ty::TraitRef` which opts-in to pretty printing only
 /// the trait name. That is, it will print `Trait` instead of
 /// `<T as Trait<U>>`.
 #[derive(Copy, Clone, TypeFoldable, TypeVisitable, Lift)]
@@ -2658,6 +2646,10 @@ impl<'tcx> ty::TraitRef<'tcx> {
         TraitRefPrintOnlyTraitPath(self)
     }
 
+    pub fn print_trait_sugared(self) -> TraitRefPrintSugared<'tcx> {
+        TraitRefPrintSugared(self)
+    }
+
     pub fn print_only_trait_name(self) -> TraitRefPrintOnlyTraitName<'tcx> {
         TraitRefPrintOnlyTraitName(self)
     }
@@ -2667,6 +2659,10 @@ impl<'tcx> ty::Binder<'tcx, ty::TraitRef<'tcx>> {
     pub fn print_only_trait_path(self) -> ty::Binder<'tcx, TraitRefPrintOnlyTraitPath<'tcx>> {
         self.map_bound(|tr| tr.print_only_trait_path())
     }
+
+    pub fn print_trait_sugared(self) -> ty::Binder<'tcx, TraitRefPrintSugared<'tcx>> {
+        self.map_bound(|tr| tr.print_trait_sugared())
+    }
 }
 
 #[derive(Copy, Clone, TypeFoldable, TypeVisitable, Lift)]
@@ -2746,6 +2742,7 @@ forward_display_to_print! {
     ty::PolyExistentialTraitRef<'tcx>,
     ty::Binder<'tcx, ty::TraitRef<'tcx>>,
     ty::Binder<'tcx, TraitRefPrintOnlyTraitPath<'tcx>>,
+    ty::Binder<'tcx, TraitRefPrintSugared<'tcx>>,
     ty::Binder<'tcx, ty::FnSig<'tcx>>,
     ty::Binder<'tcx, ty::TraitPredicate<'tcx>>,
     ty::Binder<'tcx, TraitPredPrintModifiersAndPath<'tcx>>,
@@ -2757,6 +2754,10 @@ forward_display_to_print! {
 define_print! {
     (self, cx):
 
+    ty::TypeAndMut<'tcx> {
+        p!(write("{}", self.mutbl.prefix_str()), print(self.ty))
+    }
+
     ty::ClauseKind<'tcx> {
         match *self {
             ty::ClauseKind::Trait(ref data) => {
@@ -2785,15 +2786,11 @@ define_print! {
             ty::PredicateKind::ObjectSafe(trait_def_id) => {
                 p!("the trait `", print_def_path(trait_def_id, &[]), "` is object-safe")
             }
-            ty::PredicateKind::ClosureKind(closure_def_id, _closure_args, kind) => p!(
-                "the closure `",
-                print_value_path(closure_def_id, &[]),
-                write("` implements the trait `{}`", kind)
-            ),
             ty::PredicateKind::ConstEquate(c1, c2) => {
                 p!("the constant `", print(c1), "` equals `", print(c2), "`")
             }
             ty::PredicateKind::Ambiguous => p!("ambiguous"),
+            ty::PredicateKind::NormalizesTo(data) => p!(print(data)),
             ty::PredicateKind::AliasRelate(t1, t2, dir) => p!(print(t1), write(" {} ", dir), print(t2)),
         }
     }
@@ -2806,10 +2803,6 @@ define_print_and_forward_display! {
         p!("{{", comma_sep(self.iter()), "}}")
     }
 
-    ty::TypeAndMut<'tcx> {
-        p!(write("{}", self.mutbl.prefix_str()), print(self.ty))
-    }
-
     ty::ExistentialTraitRef<'tcx> {
         // Use a type that can't appear in defaults of type parameters.
         let dummy_self = Ty::new_fresh(cx.tcx(),0);
@@ -2850,16 +2843,39 @@ define_print_and_forward_display! {
         p!(print_def_path(self.0.def_id, self.0.args));
     }
 
+    TraitRefPrintSugared<'tcx> {
+        if !with_no_queries()
+            && let Some(kind) = cx.tcx().fn_trait_kind_from_def_id(self.0.def_id)
+            && let ty::Tuple(args) = self.0.args.type_at(1).kind()
+        {
+            p!(write("{}", kind.as_str()), "(");
+            for (i, arg) in args.iter().enumerate() {
+                if i > 0 {
+                    p!(", ");
+                }
+                p!(print(arg));
+            }
+            p!(")");
+        } else {
+            p!(print_def_path(self.0.def_id, self.0.args));
+        }
+    }
+
     TraitRefPrintOnlyTraitName<'tcx> {
         p!(print_def_path(self.0.def_id, &[]));
     }
 
     TraitPredPrintModifiersAndPath<'tcx> {
-        // FIXME(effects) print `~const` here
+        if let Some(idx) = cx.tcx().generics_of(self.0.trait_ref.def_id).host_effect_index
+        {
+            let arg = self.0.trait_ref.args.const_at(idx);
+            if arg != cx.tcx().consts.true_ && !arg.has_infer() {
+                p!("~const ");
+            }
+        }
         if let ty::ImplPolarity::Negative = self.0.polarity {
             p!("!")
         }
-
         p!(print(self.0.trait_ref.print_only_trait_path()));
     }
 
@@ -2894,11 +2910,10 @@ define_print_and_forward_display! {
                 p!("~const ");
             }
         }
-        // FIXME(effects) print `~const` here
         if let ty::ImplPolarity::Negative = self.polarity {
             p!("!");
         }
-        p!(print(self.trait_ref.print_only_trait_path()))
+        p!(print(self.trait_ref.print_trait_sugared()))
     }
 
     ty::ProjectionPredicate<'tcx> {
@@ -2907,6 +2922,12 @@ define_print_and_forward_display! {
         p!(print(self.term))
     }
 
+    ty::NormalizesTo<'tcx> {
+        p!(print(self.alias), " normalizes-to ");
+        cx.reset_type_limit();
+        p!(print(self.term))
+    }
+
     ty::Term<'tcx> {
       match self.unpack() {
         ty::TermKind::Ty(ty) => p!(print(ty)),
@@ -2922,10 +2943,6 @@ define_print_and_forward_display! {
         }
     }
 
-    ty::ClosureKind {
-        p!(write("{}", self.as_str()))
-    }
-
     ty::Predicate<'tcx> {
         p!(print(self.kind()))
     }
@@ -3022,7 +3039,8 @@ fn for_each_def(tcx: TyCtxt<'_>, mut collect_fn: impl for<'b> FnMut(&'b Ident, N
 /// The implementation uses similar import discovery logic to that of 'use' suggestions.
 ///
 /// See also [`DelayDm`](rustc_error_messages::DelayDm) and [`with_no_trimmed_paths!`].
-fn trimmed_def_paths(tcx: TyCtxt<'_>, (): ()) -> FxHashMap<DefId, Symbol> {
+// this is pub to be able to intra-doc-link it
+pub fn trimmed_def_paths(tcx: TyCtxt<'_>, (): ()) -> FxHashMap<DefId, Symbol> {
     let mut map: FxHashMap<DefId, Symbol> = FxHashMap::default();
 
     if let TrimmedDefPaths::GoodPath = tcx.sess.opts.trimmed_def_paths {
@@ -3030,7 +3048,7 @@ fn trimmed_def_paths(tcx: TyCtxt<'_>, (): ()) -> FxHashMap<DefId, Symbol> {
         //
         // For good paths causing this bug, the `rustc_middle::ty::print::with_no_trimmed_paths`
         // wrapper can be used to suppress this query, in exchange for full paths being formatted.
-        tcx.sess.delay_good_path_bug(
+        tcx.sess.good_path_delayed_bug(
             "trimmed_def_paths constructed but no error emitted; use `DelayDm` for lints or `with_no_trimmed_paths` for debugging",
         );
     }
diff --git a/compiler/rustc_middle/src/ty/relate.rs b/compiler/rustc_middle/src/ty/relate.rs
index 27e9be37f..9d92f81db 100644
--- a/compiler/rustc_middle/src/ty/relate.rs
+++ b/compiler/rustc_middle/src/ty/relate.rs
@@ -23,8 +23,6 @@ pub enum Cause {
 pub trait TypeRelation<'tcx>: Sized {
     fn tcx(&self) -> TyCtxt<'tcx>;
 
-    fn param_env(&self) -> ty::ParamEnv<'tcx>;
-
     /// Returns a static string we can use for printouts.
     fn tag(&self) -> &'static str;
 
@@ -482,7 +480,7 @@ pub fn structurally_relate_tys<'tcx, R: TypeRelation<'tcx>>(
             // the (anonymous) type of the same closure expression. So
             // all of their regions should be equated.
             let args = relate_args_invariantly(relation, a_args, b_args)?;
-            Ok(Ty::new_closure(tcx, a_id, &args))
+            Ok(Ty::new_closure(tcx, a_id, args))
         }
 
         (&ty::RawPtr(a_mt), &ty::RawPtr(b_mt)) => {
@@ -505,13 +503,9 @@ pub fn structurally_relate_tys<'tcx, R: TypeRelation<'tcx>>(
                 Err(err) => {
                     // Check whether the lengths are both concrete/known values,
                     // but are unequal, for better diagnostics.
-                    //
-                    // It might seem dubious to eagerly evaluate these constants here,
-                    // we however cannot end up with errors in `Relate` during both
-                    // `type_of` and `predicates_of`. This means that evaluating the
-                    // constants should not cause cycle errors here.
-                    let sz_a = sz_a.try_eval_target_usize(tcx, relation.param_env());
-                    let sz_b = sz_b.try_eval_target_usize(tcx, relation.param_env());
+                    let sz_a = sz_a.try_to_target_usize(tcx);
+                    let sz_b = sz_b.try_to_target_usize(tcx);
+
                     match (sz_a, sz_b) {
                         (Some(sz_a_val), Some(sz_b_val)) if sz_a_val != sz_b_val => Err(
                             TypeError::FixedArraySize(expected_found(relation, sz_a_val, sz_b_val)),
diff --git a/compiler/rustc_middle/src/ty/structural_impls.rs b/compiler/rustc_middle/src/ty/structural_impls.rs
index 6af68bc5d..1c75d73e5 100644
--- a/compiler/rustc_middle/src/ty/structural_impls.rs
+++ b/compiler/rustc_middle/src/ty/structural_impls.rs
@@ -79,9 +79,9 @@ impl fmt::Debug for ty::BoundRegionKind {
     }
 }
 
-impl fmt::Debug for ty::FreeRegion {
+impl fmt::Debug for ty::LateParamRegion {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "ReFree({:?}, {:?})", self.scope, self.bound_region)
+        write!(f, "ReLateParam({:?}, {:?})", self.scope, self.bound_region)
     }
 }
 
@@ -173,6 +173,12 @@ impl<'tcx> fmt::Debug for ty::ProjectionPredicate<'tcx> {
     }
 }
 
+impl<'tcx> fmt::Debug for ty::NormalizesTo<'tcx> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "NormalizesTo({:?}, {:?})", self.alias, self.term)
+    }
+}
+
 impl<'tcx> fmt::Debug for ty::Predicate<'tcx> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(f, "{:?}", self.kind())
@@ -202,34 +208,6 @@ impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for AliasTy<'tcx> {
     }
 }
 
-impl fmt::Debug for ty::InferConst {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            InferConst::Var(var) => write!(f, "{var:?}"),
-            InferConst::EffectVar(var) => write!(f, "{var:?}"),
-            InferConst::Fresh(var) => write!(f, "Fresh({var:?})"),
-        }
-    }
-}
-impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for ty::InferConst {
-    fn fmt<Infcx: InferCtxtLike<Interner = TyCtxt<'tcx>>>(
-        this: WithInfcx<'_, Infcx, &Self>,
-        f: &mut core::fmt::Formatter<'_>,
-    ) -> core::fmt::Result {
-        use ty::InferConst::*;
-        match this.infcx.universe_of_ct(*this.data) {
-            None => write!(f, "{:?}", this.data),
-            Some(universe) => match *this.data {
-                Var(vid) => write!(f, "?{}_{}c", vid.index(), universe.index()),
-                EffectVar(vid) => write!(f, "?{}_{}e", vid.index(), universe.index()),
-                Fresh(_) => {
-                    unreachable!()
-                }
-            },
-        }
-    }
-}
-
 impl<'tcx> fmt::Debug for ty::consts::Expr<'tcx> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         WithInfcx::with_no_infcx(self).fmt(f)
@@ -444,7 +422,7 @@ TrivialTypeTraversalImpls! {
     crate::ty::Placeholder<crate::ty::BoundRegion>,
     crate::ty::Placeholder<crate::ty::BoundTy>,
     crate::ty::Placeholder<ty::BoundVar>,
-    crate::ty::FreeRegion,
+    crate::ty::LateParamRegion,
     crate::ty::InferTy,
     crate::ty::IntVarValue,
     crate::ty::adjustment::PointerCoercion,
@@ -462,14 +440,14 @@ TrivialTypeTraversalImpls! {
 // interners).
 TrivialTypeTraversalAndLiftImpls! {
     ::rustc_hir::def_id::DefId,
-    ::rustc_hir::Mutability,
     ::rustc_hir::Unsafety,
     ::rustc_target::spec::abi::Abi,
     crate::ty::ClosureKind,
     crate::ty::ParamConst,
     crate::ty::ParamTy,
-    interpret::Scalar,
     interpret::AllocId,
+    interpret::CtfeProvenance,
+    interpret::Scalar,
     rustc_target::abi::Size,
 }
 
diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs
index 44592b10d..a0debc8a1 100644
--- a/compiler/rustc_middle/src/ty/sty.rs
+++ b/compiler/rustc_middle/src/ty/sty.rs
@@ -6,7 +6,7 @@ use crate::infer::canonical::Canonical;
 use crate::ty::visit::ValidateBoundVars;
 use crate::ty::InferTy::*;
 use crate::ty::{
-    self, AdtDef, Discr, Term, Ty, TyCtxt, TypeFlags, TypeSuperVisitable, TypeVisitable,
+    self, AdtDef, Discr, IntoKind, Term, Ty, TyCtxt, TypeFlags, TypeSuperVisitable, TypeVisitable,
     TypeVisitableExt, TypeVisitor,
 };
 use crate::ty::{GenericArg, GenericArgs, GenericArgsRef};
@@ -15,7 +15,9 @@ use hir::def::DefKind;
 use polonius_engine::Atom;
 use rustc_data_structures::captures::Captures;
 use rustc_data_structures::intern::Interned;
-use rustc_errors::{DiagnosticArgValue, ErrorGuaranteed, IntoDiagnosticArg, MultiSpan};
+use rustc_errors::{
+    DiagnosticArgValue, DiagnosticMessage, ErrorGuaranteed, IntoDiagnosticArg, MultiSpan,
+};
 use rustc_hir as hir;
 use rustc_hir::def_id::DefId;
 use rustc_hir::LangItem;
@@ -32,6 +34,7 @@ use std::fmt;
 use std::ops::{ControlFlow, Deref, Range};
 use ty::util::IntTypeExt;
 
+use rustc_type_ir::BoundVar;
 use rustc_type_ir::ClauseKind as IrClauseKind;
 use rustc_type_ir::CollectAndApply;
 use rustc_type_ir::ConstKind as IrConstKind;
@@ -41,29 +44,24 @@ use rustc_type_ir::PredicateKind as IrPredicateKind;
 use rustc_type_ir::RegionKind as IrRegionKind;
 use rustc_type_ir::TyKind as IrTyKind;
 use rustc_type_ir::TyKind::*;
+use rustc_type_ir::TypeAndMut as IrTypeAndMut;
 
 use super::GenericParamDefKind;
 
-// Re-export the `TyKind` from `rustc_type_ir` here for convenience
+// Re-export and re-parameterize some `I = TyCtxt<'tcx>` types here
 #[rustc_diagnostic_item = "TyKind"]
 pub type TyKind<'tcx> = IrTyKind<TyCtxt<'tcx>>;
 pub type RegionKind<'tcx> = IrRegionKind<TyCtxt<'tcx>>;
 pub type ConstKind<'tcx> = IrConstKind<TyCtxt<'tcx>>;
 pub type PredicateKind<'tcx> = IrPredicateKind<TyCtxt<'tcx>>;
 pub type ClauseKind<'tcx> = IrClauseKind<TyCtxt<'tcx>>;
-
-#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, TyEncodable, TyDecodable)]
-#[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
-pub struct TypeAndMut<'tcx> {
-    pub ty: Ty<'tcx>,
-    pub mutbl: hir::Mutability,
-}
+pub type TypeAndMut<'tcx> = IrTypeAndMut<TyCtxt<'tcx>>;
 
 #[derive(Clone, PartialEq, PartialOrd, Eq, Ord, Hash, TyEncodable, TyDecodable, Copy)]
 #[derive(HashStable)]
-/// A "free" region `fr` can be interpreted as "some region
+/// The parameter representation of late-bound function parameters, "some region
 /// at least as big as the scope `fr.scope`".
-pub struct FreeRegion {
+pub struct LateParamRegion {
     pub scope: DefId,
     pub bound_region: BoundRegionKind,
 }
@@ -467,16 +465,6 @@ impl<'tcx> CoroutineArgs<'tcx> {
         self.split().return_ty.expect_ty()
     }
 
-    /// Returns the "coroutine signature", which consists of its yield
-    /// and return types.
-    ///
-    /// N.B., some bits of the code prefers to see this wrapped in a
-    /// binder, but it never contains bound regions. Probably this
-    /// function should be removed.
-    pub fn poly_sig(self) -> PolyGenSig<'tcx> {
-        ty::Binder::dummy(self.sig())
-    }
-
     /// Returns the "coroutine signature", which consists of its resume, yield
     /// and return types.
     pub fn sig(self) -> GenSig<'tcx> {
@@ -1036,7 +1024,7 @@ impl<'tcx, T> Binder<'tcx, T> {
     /// risky thing to do because it's easy to get confused about
     /// De Bruijn indices and the like. It is usually better to
     /// discharge the binder using `no_bound_vars` or
-    /// `replace_late_bound_regions` or something like
+    /// `instantiate_bound_regions` or something like
     /// that. `skip_binder` is only valid when you are either
     /// extracting data that has nothing to do with bound vars, you
     /// are doing some sort of test that does not involve bound
@@ -1245,6 +1233,28 @@ impl<'tcx> AliasTy<'tcx> {
         }
     }
 
+    /// Whether this alias type is an opaque.
+    pub fn is_opaque(self, tcx: TyCtxt<'tcx>) -> bool {
+        matches!(self.opt_kind(tcx), Some(ty::AliasKind::Opaque))
+    }
+
+    /// FIXME: rename `AliasTy` to `AliasTerm` and always handle
+    /// constants. This function can then be removed.
+    pub fn opt_kind(self, tcx: TyCtxt<'tcx>) -> Option<ty::AliasKind> {
+        match tcx.def_kind(self.def_id) {
+            DefKind::AssocTy
+                if let DefKind::Impl { of_trait: false } =
+                    tcx.def_kind(tcx.parent(self.def_id)) =>
+            {
+                Some(ty::Inherent)
+            }
+            DefKind::AssocTy => Some(ty::Projection),
+            DefKind::OpaqueTy => Some(ty::Opaque),
+            DefKind::TyAlias => Some(ty::Weak),
+            _ => None,
+        }
+    }
+
     pub fn to_ty(self, tcx: TyCtxt<'tcx>) -> Ty<'tcx> {
         Ty::new_alias(tcx, self.kind(tcx), self)
     }
@@ -1329,8 +1339,6 @@ pub struct GenSig<'tcx> {
     pub return_ty: Ty<'tcx>,
 }
 
-pub type PolyGenSig<'tcx> = Binder<'tcx, GenSig<'tcx>>;
-
 /// Signature of a function type, which we have arbitrarily
 /// decided to use to refer to the input/output types.
 ///
@@ -1466,17 +1474,25 @@ impl ParamConst {
 #[rustc_pass_by_value]
 pub struct Region<'tcx>(pub Interned<'tcx, RegionKind<'tcx>>);
 
+impl<'tcx> IntoKind for Region<'tcx> {
+    type Kind = RegionKind<'tcx>;
+
+    fn kind(self) -> RegionKind<'tcx> {
+        *self
+    }
+}
+
 impl<'tcx> Region<'tcx> {
     #[inline]
-    pub fn new_early_bound(
+    pub fn new_early_param(
         tcx: TyCtxt<'tcx>,
-        early_bound_region: ty::EarlyBoundRegion,
+        early_bound_region: ty::EarlyParamRegion,
     ) -> Region<'tcx> {
-        tcx.intern_region(ty::ReEarlyBound(early_bound_region))
+        tcx.intern_region(ty::ReEarlyParam(early_bound_region))
     }
 
     #[inline]
-    pub fn new_late_bound(
+    pub fn new_bound(
         tcx: TyCtxt<'tcx>,
         debruijn: ty::DebruijnIndex,
         bound_region: ty::BoundRegion,
@@ -1488,17 +1504,17 @@ impl<'tcx> Region<'tcx> {
         {
             re
         } else {
-            tcx.intern_region(ty::ReLateBound(debruijn, bound_region))
+            tcx.intern_region(ty::ReBound(debruijn, bound_region))
         }
     }
 
     #[inline]
-    pub fn new_free(
+    pub fn new_late_param(
         tcx: TyCtxt<'tcx>,
         scope: DefId,
         bound_region: ty::BoundRegionKind,
     ) -> Region<'tcx> {
-        tcx.intern_region(ty::ReFree(ty::FreeRegion { scope, bound_region }))
+        tcx.intern_region(ty::ReLateParam(ty::LateParamRegion { scope, bound_region }))
     }
 
     #[inline]
@@ -1522,7 +1538,7 @@ impl<'tcx> Region<'tcx> {
         tcx.intern_region(ty::ReError(reported))
     }
 
-    /// Constructs a `RegionKind::ReError` region and registers a `delay_span_bug` to ensure it
+    /// Constructs a `RegionKind::ReError` region and registers a `span_delayed_bug` to ensure it
     /// gets used.
     #[track_caller]
     pub fn new_error_misc(tcx: TyCtxt<'tcx>) -> Region<'tcx> {
@@ -1533,7 +1549,7 @@ impl<'tcx> Region<'tcx> {
         )
     }
 
-    /// Constructs a `RegionKind::ReError` region and registers a `delay_span_bug` with the given
+    /// Constructs a `RegionKind::ReError` region and registers a `span_delayed_bug` with the given
     /// `msg` to ensure it gets used.
     #[track_caller]
     pub fn new_error_with_message<S: Into<MultiSpan>>(
@@ -1541,7 +1557,7 @@ impl<'tcx> Region<'tcx> {
         span: S,
         msg: &'static str,
     ) -> Region<'tcx> {
-        let reported = tcx.sess.delay_span_bug(span, msg);
+        let reported = tcx.sess.span_delayed_bug(span, msg);
         Region::new_error(tcx, reported)
     }
 
@@ -1549,10 +1565,10 @@ impl<'tcx> Region<'tcx> {
     /// to avoid the cost of the `match`.
     pub fn new_from_kind(tcx: TyCtxt<'tcx>, kind: RegionKind<'tcx>) -> Region<'tcx> {
         match kind {
-            ty::ReEarlyBound(region) => Region::new_early_bound(tcx, region),
-            ty::ReLateBound(debruijn, region) => Region::new_late_bound(tcx, debruijn, region),
-            ty::ReFree(ty::FreeRegion { scope, bound_region }) => {
-                Region::new_free(tcx, scope, bound_region)
+            ty::ReEarlyParam(region) => Region::new_early_param(tcx, region),
+            ty::ReBound(debruijn, region) => Region::new_bound(tcx, debruijn, region),
+            ty::ReLateParam(ty::LateParamRegion { scope, bound_region }) => {
+                Region::new_late_param(tcx, scope, bound_region)
             }
             ty::ReStatic => tcx.lifetimes.re_static,
             ty::ReVar(vid) => Region::new_var(tcx, vid),
@@ -1568,45 +1584,29 @@ impl<'tcx> Deref for Region<'tcx> {
 
     #[inline]
     fn deref(&self) -> &RegionKind<'tcx> {
-        &self.0.0
+        self.0.0
     }
 }
 
 #[derive(Copy, Clone, PartialEq, Eq, Hash, TyEncodable, TyDecodable, PartialOrd, Ord)]
 #[derive(HashStable)]
-pub struct EarlyBoundRegion {
+pub struct EarlyParamRegion {
     pub def_id: DefId,
     pub index: u32,
     pub name: Symbol,
 }
 
-impl fmt::Debug for EarlyBoundRegion {
+impl fmt::Debug for EarlyParamRegion {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(f, "{:?}, {}, {}", self.def_id, self.index, self.name)
     }
 }
 
 rustc_index::newtype_index! {
-    /// A **`const`** **v**ariable **ID**.
-    #[debug_format = "?{}c"]
-    pub struct ConstVid {}
-}
-
-rustc_index::newtype_index! {
-    /// An **effect** **v**ariable **ID**.
-    ///
-    /// Handling effect infer variables happens separately from const infer variables
-    /// because we do not want to reuse any of the const infer machinery. If we try to
-    /// relate an effect variable with a normal one, we would ICE, which can catch bugs
-    /// where we are not correctly using the effect var for an effect param. Fallback
-    /// is also implemented on top of having separate effect and normal const variables.
-    #[debug_format = "?{}e"]
-    pub struct EffectVid {}
-}
-
-rustc_index::newtype_index! {
     /// A **region** (lifetime) **v**ariable **ID**.
     #[derive(HashStable)]
+    #[encodable]
+    #[orderable]
     #[debug_format = "'?{}"]
     pub struct RegionVid {}
 }
@@ -1617,12 +1617,6 @@ impl Atom for RegionVid {
     }
 }
 
-rustc_index::newtype_index! {
-    #[derive(HashStable)]
-    #[debug_format = "{}"]
-    pub struct BoundVar {}
-}
-
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, TyEncodable, TyDecodable)]
 #[derive(HashStable)]
 pub struct BoundTy {
@@ -1722,9 +1716,9 @@ impl<'tcx> Region<'tcx> {
     pub fn get_name(self) -> Option<Symbol> {
         if self.has_name() {
             match *self {
-                ty::ReEarlyBound(ebr) => Some(ebr.name),
-                ty::ReLateBound(_, br) => br.kind.get_name(),
-                ty::ReFree(fr) => fr.bound_region.get_name(),
+                ty::ReEarlyParam(ebr) => Some(ebr.name),
+                ty::ReBound(_, br) => br.kind.get_name(),
+                ty::ReLateParam(fr) => fr.bound_region.get_name(),
                 ty::ReStatic => Some(kw::StaticLifetime),
                 ty::RePlaceholder(placeholder) => placeholder.bound.kind.get_name(),
                 _ => None,
@@ -1744,9 +1738,9 @@ impl<'tcx> Region<'tcx> {
     /// Is this region named by the user?
     pub fn has_name(self) -> bool {
         match *self {
-            ty::ReEarlyBound(ebr) => ebr.has_name(),
-            ty::ReLateBound(_, br) => br.kind.is_named(),
-            ty::ReFree(fr) => fr.bound_region.is_named(),
+            ty::ReEarlyParam(ebr) => ebr.has_name(),
+            ty::ReBound(_, br) => br.kind.is_named(),
+            ty::ReLateParam(fr) => fr.bound_region.is_named(),
             ty::ReStatic => true,
             ty::ReVar(..) => false,
             ty::RePlaceholder(placeholder) => placeholder.bound.kind.is_named(),
@@ -1771,8 +1765,8 @@ impl<'tcx> Region<'tcx> {
     }
 
     #[inline]
-    pub fn is_late_bound(self) -> bool {
-        matches!(*self, ty::ReLateBound(..))
+    pub fn is_bound(self) -> bool {
+        matches!(*self, ty::ReBound(..))
     }
 
     #[inline]
@@ -1783,7 +1777,7 @@ impl<'tcx> Region<'tcx> {
     #[inline]
     pub fn bound_at_or_above_binder(self, index: ty::DebruijnIndex) -> bool {
         match *self {
-            ty::ReLateBound(debruijn, _) => debruijn >= index,
+            ty::ReBound(debruijn, _) => debruijn >= index,
             _ => false,
         }
     }
@@ -1802,20 +1796,20 @@ impl<'tcx> Region<'tcx> {
                 flags = flags | TypeFlags::HAS_FREE_LOCAL_REGIONS;
                 flags = flags | TypeFlags::HAS_RE_PLACEHOLDER;
             }
-            ty::ReEarlyBound(..) => {
+            ty::ReEarlyParam(..) => {
                 flags = flags | TypeFlags::HAS_FREE_REGIONS;
                 flags = flags | TypeFlags::HAS_FREE_LOCAL_REGIONS;
                 flags = flags | TypeFlags::HAS_RE_PARAM;
             }
-            ty::ReFree { .. } => {
+            ty::ReLateParam { .. } => {
                 flags = flags | TypeFlags::HAS_FREE_REGIONS;
                 flags = flags | TypeFlags::HAS_FREE_LOCAL_REGIONS;
             }
             ty::ReStatic => {
                 flags = flags | TypeFlags::HAS_FREE_REGIONS;
             }
-            ty::ReLateBound(..) => {
-                flags = flags | TypeFlags::HAS_RE_LATE_BOUND;
+            ty::ReBound(..) => {
+                flags = flags | TypeFlags::HAS_RE_BOUND;
             }
             ty::ReErased => {
                 flags = flags | TypeFlags::HAS_RE_ERASED;
@@ -1851,22 +1845,28 @@ impl<'tcx> Region<'tcx> {
     /// function might return the `DefId` of a closure.
     pub fn free_region_binding_scope(self, tcx: TyCtxt<'_>) -> DefId {
         match *self {
-            ty::ReEarlyBound(br) => tcx.parent(br.def_id),
-            ty::ReFree(fr) => fr.scope,
+            ty::ReEarlyParam(br) => tcx.parent(br.def_id),
+            ty::ReLateParam(fr) => fr.scope,
             _ => bug!("free_region_binding_scope invoked on inappropriate region: {:?}", self),
         }
     }
 
     /// True for free regions other than `'static`.
-    pub fn is_free(self) -> bool {
-        matches!(*self, ty::ReEarlyBound(_) | ty::ReFree(_))
+    pub fn is_param(self) -> bool {
+        matches!(*self, ty::ReEarlyParam(_) | ty::ReLateParam(_))
     }
 
-    /// True if `self` is a free region or static.
-    pub fn is_free_or_static(self) -> bool {
+    /// True for free region in the current context.
+    ///
+    /// This is the case for `'static` and param regions.
+    pub fn is_free(self) -> bool {
         match *self {
-            ty::ReStatic => true,
-            _ => self.is_free(),
+            ty::ReStatic | ty::ReEarlyParam(..) | ty::ReLateParam(..) => true,
+            ty::ReVar(..)
+            | ty::RePlaceholder(..)
+            | ty::ReBound(..)
+            | ty::ReErased
+            | ty::ReError(..) => false,
         }
     }
 
@@ -1990,21 +1990,21 @@ impl<'tcx> Ty<'tcx> {
         Ty::new(tcx, Error(reported))
     }
 
-    /// Constructs a `TyKind::Error` type and registers a `delay_span_bug` to ensure it gets used.
+    /// Constructs a `TyKind::Error` type and registers a `span_delayed_bug` to ensure it gets used.
     #[track_caller]
     pub fn new_misc_error(tcx: TyCtxt<'tcx>) -> Ty<'tcx> {
         Ty::new_error_with_message(tcx, DUMMY_SP, "TyKind::Error constructed but no error reported")
     }
 
-    /// Constructs a `TyKind::Error` type and registers a `delay_span_bug` with the given `msg` to
+    /// Constructs a `TyKind::Error` type and registers a `span_delayed_bug` with the given `msg` to
     /// ensure it gets used.
     #[track_caller]
     pub fn new_error_with_message<S: Into<MultiSpan>>(
         tcx: TyCtxt<'tcx>,
         span: S,
-        msg: impl Into<String>,
+        msg: impl Into<DiagnosticMessage>,
     ) -> Ty<'tcx> {
-        let reported = tcx.sess.delay_span_bug(span, msg);
+        let reported = tcx.sess.span_delayed_bug(span, msg);
         Ty::new(tcx, Error(reported))
     }
 
@@ -2104,7 +2104,7 @@ impl<'tcx> Ty<'tcx> {
 
     #[inline]
     pub fn new_tup(tcx: TyCtxt<'tcx>, ts: &[Ty<'tcx>]) -> Ty<'tcx> {
-        if ts.is_empty() { tcx.types.unit } else { Ty::new(tcx, Tuple(tcx.mk_type_list(&ts))) }
+        if ts.is_empty() { tcx.types.unit } else { Ty::new(tcx, Tuple(tcx.mk_type_list(ts))) }
     }
 
     pub fn new_tup_from_iter<I, T>(tcx: TyCtxt<'tcx>, iter: I) -> T::Output
@@ -2260,7 +2260,7 @@ impl<'tcx> Ty<'tcx> {
 impl<'tcx> Ty<'tcx> {
     #[inline(always)]
     pub fn kind(self) -> &'tcx TyKind<'tcx> {
-        &self.0.0
+        self.0.0
     }
 
     #[inline(always)]
@@ -2271,7 +2271,7 @@ impl<'tcx> Ty<'tcx> {
     #[inline]
     pub fn is_unit(self) -> bool {
         match self.kind() {
-            Tuple(ref tys) => tys.is_empty(),
+            Tuple(tys) => tys.is_empty(),
             _ => false,
         }
     }
@@ -2813,6 +2813,15 @@ impl<'tcx> Ty<'tcx> {
         }
     }
 
+    /// Inverse of [`Ty::to_opt_closure_kind`].
+    pub fn from_closure_kind(tcx: TyCtxt<'tcx>, kind: ty::ClosureKind) -> Ty<'tcx> {
+        match kind {
+            ty::ClosureKind::Fn => tcx.types.i8,
+            ty::ClosureKind::FnMut => tcx.types.i16,
+            ty::ClosureKind::FnOnce => tcx.types.i32,
+        }
+    }
+
     /// Fast path helper for testing if a type is `Sized`.
     ///
     /// Returning true means the type is known to be sized. Returning
diff --git a/compiler/rustc_middle/src/ty/typeck_results.rs b/compiler/rustc_middle/src/ty/typeck_results.rs
index e9240d1b2..d372c1cd6 100644
--- a/compiler/rustc_middle/src/ty/typeck_results.rs
+++ b/compiler/rustc_middle/src/ty/typeck_results.rs
@@ -126,7 +126,7 @@ pub struct TypeckResults<'tcx> {
     /// fn(&'a u32) -> u32
     /// ```
     ///
-    /// Note that `'a` is not bound (it would be an `ReFree`) and
+    /// Note that `'a` is not bound (it would be an `ReLateParam`) and
     /// that the `Foo` opaque type is replaced by its hidden type.
     liberated_fn_sigs: ItemLocalMap<ty::FnSig<'tcx>>,
 
@@ -241,7 +241,7 @@ impl<'tcx> TypeckResults<'tcx> {
     /// Returns the final resolution of a `QPath` in an `Expr` or `Pat` node.
     pub fn qpath_res(&self, qpath: &hir::QPath<'_>, id: hir::HirId) -> Res {
         match *qpath {
-            hir::QPath::Resolved(_, ref path) => path.res,
+            hir::QPath::Resolved(_, path) => path.res,
             hir::QPath::TypeRelative(..) | hir::QPath::LangItem(..) => self
                 .type_dependent_def(id)
                 .map_or(Res::Err, |(kind, def_id)| Res::Def(kind, def_id)),
@@ -391,7 +391,7 @@ impl<'tcx> TypeckResults<'tcx> {
 
     pub fn extract_binding_mode(&self, s: &Session, id: HirId, sp: Span) -> Option<BindingMode> {
         self.pat_binding_modes().get(id).copied().or_else(|| {
-            s.delay_span_bug(sp, "missing binding mode");
+            s.span_delayed_bug(sp, "missing binding mode");
             None
         })
     }
@@ -578,6 +578,7 @@ impl<'a, V> LocalTableInContextMut<'a, V> {
 
 rustc_index::newtype_index! {
     #[derive(HashStable)]
+    #[encodable]
     #[debug_format = "UserType({})"]
     pub struct UserTypeAnnotationIndex {
         const START_INDEX = 0;
@@ -638,7 +639,7 @@ impl<'tcx> IsIdentity for CanonicalUserType<'tcx> {
                         },
 
                         GenericArgKind::Lifetime(r) => match *r {
-                            ty::ReLateBound(debruijn, br) => {
+                            ty::ReBound(debruijn, br) => {
                                 // We only allow a `ty::INNERMOST` index in substitutions.
                                 assert_eq!(debruijn, ty::INNERMOST);
                                 cvar == br.var
diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs
index a251518d1..8b2b76764 100644
--- a/compiler/rustc_middle/src/ty/util.rs
+++ b/compiler/rustc_middle/src/ty/util.rs
@@ -35,12 +35,14 @@ pub struct Discr<'tcx> {
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
 pub enum CheckRegions {
     No,
-    /// Only permit early bound regions. This is useful for Adts which
-    /// can never have late bound regions.
-    OnlyEarlyBound,
-    /// Permit both late bound and early bound regions. Use this for functions,
-    /// which frequently have late bound regions.
-    Bound,
+    /// Only permit parameter regions. This should be used
+    /// for everything apart from functions, which may use
+    /// `ReBound` to represent late-bound regions.
+    OnlyParam,
+    /// Check region parameters from a function definition.
+    /// Allows `ReEarlyParam` and `ReBound` to handle early
+    /// and late-bound region parameters.
+    FromFunction,
 }
 
 #[derive(Copy, Clone, Debug)]
@@ -359,7 +361,7 @@ impl<'tcx> TyCtxt<'tcx> {
 
             let Some(item_id) = self.associated_item_def_ids(impl_did).first() else {
                 self.sess
-                    .delay_span_bug(self.def_span(impl_did), "Drop impl without drop function");
+                    .span_delayed_bug(self.def_span(impl_did), "Drop impl without drop function");
                 return;
             };
 
@@ -419,19 +421,16 @@ impl<'tcx> TyCtxt<'tcx> {
 
         let impl_args = match *self.type_of(impl_def_id).instantiate_identity().kind() {
             ty::Adt(def_, args) if def_ == def => args,
-            _ => bug!(),
+            _ => span_bug!(self.def_span(impl_def_id), "expected ADT for self type of `Drop` impl"),
         };
 
-        let item_args = match *self.type_of(def.did()).instantiate_identity().kind() {
-            ty::Adt(def_, args) if def_ == def => args,
-            _ => bug!(),
-        };
+        let item_args = ty::GenericArgs::identity_for_item(self, def.did());
 
         let result = iter::zip(item_args, impl_args)
             .filter(|&(_, k)| {
                 match k.unpack() {
                     GenericArgKind::Lifetime(region) => match region.kind() {
-                        ty::ReEarlyBound(ref ebr) => {
+                        ty::ReEarlyParam(ref ebr) => {
                             !impl_generics.region_param(ebr, self).pure_wrt_drop
                         }
                         // Error: not a region param
@@ -468,17 +467,17 @@ impl<'tcx> TyCtxt<'tcx> {
         for arg in args {
             match arg.unpack() {
                 GenericArgKind::Lifetime(lt) => match (ignore_regions, lt.kind()) {
-                    (CheckRegions::Bound, ty::ReLateBound(di, reg)) => {
+                    (CheckRegions::FromFunction, ty::ReBound(di, reg)) => {
                         if !seen_late.insert((di, reg)) {
                             return Err(NotUniqueParam::DuplicateParam(lt.into()));
                         }
                     }
-                    (CheckRegions::OnlyEarlyBound | CheckRegions::Bound, ty::ReEarlyBound(p)) => {
+                    (CheckRegions::OnlyParam | CheckRegions::FromFunction, ty::ReEarlyParam(p)) => {
                         if !seen.insert(p.index) {
                             return Err(NotUniqueParam::DuplicateParam(lt.into()));
                         }
                     }
-                    (CheckRegions::OnlyEarlyBound | CheckRegions::Bound, _) => {
+                    (CheckRegions::OnlyParam | CheckRegions::FromFunction, _) => {
                         return Err(NotUniqueParam::NotParam(lt.into()));
                     }
                     (CheckRegions::No, _) => {}
@@ -548,16 +547,13 @@ impl<'tcx> TyCtxt<'tcx> {
     /// those are not yet phased out). The parent of the closure's
     /// `DefId` will also be the context where it appears.
     pub fn is_closure(self, def_id: DefId) -> bool {
-        matches!(self.def_kind(def_id), DefKind::Closure | DefKind::Coroutine)
+        matches!(self.def_kind(def_id), DefKind::Closure)
     }
 
     /// Returns `true` if `def_id` refers to a definition that does not have its own
     /// type-checking context, i.e. closure, coroutine or inline const.
     pub fn is_typeck_child(self, def_id: DefId) -> bool {
-        matches!(
-            self.def_kind(def_id),
-            DefKind::Closure | DefKind::Coroutine | DefKind::InlineConst
-        )
+        matches!(self.def_kind(def_id), DefKind::Closure | DefKind::InlineConst)
     }
 
     /// Returns `true` if `def_id` refers to a trait (i.e., `trait Foo { ... }`).
@@ -699,22 +695,6 @@ impl<'tcx> TyCtxt<'tcx> {
             .map(|decl| ty::EarlyBinder::bind(decl.ty))
     }
 
-    /// Normalizes all opaque types in the given value, replacing them
-    /// with their underlying types.
-    pub fn expand_opaque_types(self, val: Ty<'tcx>) -> Ty<'tcx> {
-        let mut visitor = OpaqueTypeExpander {
-            seen_opaque_tys: FxHashSet::default(),
-            expanded_cache: FxHashMap::default(),
-            primary_def_id: None,
-            found_recursion: false,
-            found_any_recursion: false,
-            check_recursion: false,
-            expand_coroutines: false,
-            tcx: self,
-        };
-        val.fold_with(&mut visitor)
-    }
-
     /// Expands the given impl trait type, stopping if the type is recursive.
     #[instrument(skip(self), level = "debug", ret)]
     pub fn try_expand_impl_trait_type(
@@ -746,11 +726,14 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn def_kind_descr(self, def_kind: DefKind, def_id: DefId) -> &'static str {
         match def_kind {
             DefKind::AssocFn if self.associated_item(def_id).fn_has_self_parameter => "method",
-            DefKind::Coroutine => match self.coroutine_kind(def_id).unwrap() {
-                rustc_hir::CoroutineKind::Async(..) => "async closure",
-                rustc_hir::CoroutineKind::Coroutine => "coroutine",
-                rustc_hir::CoroutineKind::Gen(..) => "gen closure",
-            },
+            DefKind::Closure if let Some(coroutine_kind) = self.coroutine_kind(def_id) => {
+                match coroutine_kind {
+                    rustc_hir::CoroutineKind::Async(..) => "async closure",
+                    rustc_hir::CoroutineKind::AsyncGen(..) => "async gen closure",
+                    rustc_hir::CoroutineKind::Coroutine => "coroutine",
+                    rustc_hir::CoroutineKind::Gen(..) => "gen closure",
+                }
+            }
             _ => def_kind.descr(def_id),
         }
     }
@@ -764,11 +747,14 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn def_kind_descr_article(self, def_kind: DefKind, def_id: DefId) -> &'static str {
         match def_kind {
             DefKind::AssocFn if self.associated_item(def_id).fn_has_self_parameter => "a",
-            DefKind::Coroutine => match self.coroutine_kind(def_id).unwrap() {
-                rustc_hir::CoroutineKind::Async(..) => "an",
-                rustc_hir::CoroutineKind::Coroutine => "a",
-                rustc_hir::CoroutineKind::Gen(..) => "a",
-            },
+            DefKind::Closure if let Some(coroutine_kind) = self.coroutine_kind(def_id) => {
+                match coroutine_kind {
+                    rustc_hir::CoroutineKind::Async(..) => "an",
+                    rustc_hir::CoroutineKind::AsyncGen(..) => "an",
+                    rustc_hir::CoroutineKind::Coroutine => "a",
+                    rustc_hir::CoroutineKind::Gen(..) => "a",
+                }
+            }
             _ => def_kind.article(),
         }
     }
@@ -790,7 +776,60 @@ impl<'tcx> TyCtxt<'tcx> {
             // If `extern_crate` is `None`, then the crate was injected (e.g., by the allocator).
             // Treat that kind of crate as "indirect", since it's an implementation detail of
             // the language.
-            || self.extern_crate(key.as_def_id()).map_or(false, |e| e.is_direct())
+            || self.extern_crate(key.as_def_id()).is_some_and(|e| e.is_direct())
+    }
+
+    pub fn expected_host_effect_param_for_body(self, def_id: impl Into<DefId>) -> ty::Const<'tcx> {
+        let def_id = def_id.into();
+        // FIXME(effects): This is suspicious and should probably not be done,
+        // especially now that we enforce host effects and then properly handle
+        // effect vars during fallback.
+        let mut host_always_on =
+            !self.features().effects || self.sess.opts.unstable_opts.unleash_the_miri_inside_of_you;
+
+        // Compute the constness required by the context.
+        let const_context = self.hir().body_const_context(def_id);
+
+        let kind = self.def_kind(def_id);
+        debug_assert_ne!(kind, DefKind::ConstParam);
+
+        if self.has_attr(def_id, sym::rustc_do_not_const_check) {
+            trace!("do not const check this context");
+            host_always_on = true;
+        }
+
+        match const_context {
+            _ if host_always_on => self.consts.true_,
+            Some(hir::ConstContext::Static(_) | hir::ConstContext::Const { .. }) => {
+                self.consts.false_
+            }
+            Some(hir::ConstContext::ConstFn) => {
+                let host_idx = self
+                    .generics_of(def_id)
+                    .host_effect_index
+                    .expect("ConstContext::Maybe must have host effect param");
+                ty::GenericArgs::identity_for_item(self, def_id).const_at(host_idx)
+            }
+            None => self.consts.true_,
+        }
+    }
+
+    /// Constructs generic args for an item, optionally appending a const effect param type
+    pub fn with_opt_host_effect_param(
+        self,
+        caller_def_id: LocalDefId,
+        callee_def_id: DefId,
+        args: impl IntoIterator<Item: Into<ty::GenericArg<'tcx>>>,
+    ) -> ty::GenericArgsRef<'tcx> {
+        let generics = self.generics_of(callee_def_id);
+        assert_eq!(generics.parent, None);
+
+        let opt_const_param = generics
+            .host_effect_index
+            .is_some()
+            .then(|| ty::GenericArg::from(self.expected_host_effect_param_for_body(caller_def_id)));
+
+        self.mk_args_from_iter(args.into_iter().map(|arg| arg.into()).chain(opt_const_param))
     }
 }
 
diff --git a/compiler/rustc_middle/src/ty/visit.rs b/compiler/rustc_middle/src/ty/visit.rs
index 8fc5c0302..e1ce94125 100644
--- a/compiler/rustc_middle/src/ty/visit.rs
+++ b/compiler/rustc_middle/src/ty/visit.rs
@@ -1,4 +1,4 @@
-use crate::ty::{self, flags::FlagComputation, Binder, Ty, TyCtxt, TypeFlags};
+use crate::ty::{self, Binder, Ty, TyCtxt, TypeFlags};
 use rustc_errors::ErrorGuaranteed;
 
 use rustc_data_structures::fx::FxHashSet;
@@ -111,16 +111,16 @@ pub trait TypeVisitableExt<'tcx>: TypeVisitable<TyCtxt<'tcx>> {
     }
 
     /// True if there are any late-bound regions
-    fn has_late_bound_regions(&self) -> bool {
-        self.has_type_flags(TypeFlags::HAS_RE_LATE_BOUND)
+    fn has_bound_regions(&self) -> bool {
+        self.has_type_flags(TypeFlags::HAS_RE_BOUND)
     }
     /// True if there are any late-bound non-region variables
-    fn has_non_region_late_bound(&self) -> bool {
-        self.has_type_flags(TypeFlags::HAS_LATE_BOUND - TypeFlags::HAS_RE_LATE_BOUND)
+    fn has_non_region_bound_vars(&self) -> bool {
+        self.has_type_flags(TypeFlags::HAS_BOUND_VARS - TypeFlags::HAS_RE_BOUND)
     }
-    /// True if there are any late-bound variables
-    fn has_late_bound_vars(&self) -> bool {
-        self.has_type_flags(TypeFlags::HAS_LATE_BOUND)
+    /// True if there are any bound variables
+    fn has_bound_vars(&self) -> bool {
+        self.has_type_flags(TypeFlags::HAS_BOUND_VARS)
     }
 
     /// Indicates whether this value still has parameters/placeholders/inference variables
@@ -204,7 +204,7 @@ impl<'tcx> TyCtxt<'tcx> {
 
             fn visit_region(&mut self, r: ty::Region<'tcx>) -> ControlFlow<Self::BreakTy> {
                 match *r {
-                    ty::ReLateBound(debruijn, _) if debruijn < self.outer_index => {
+                    ty::ReBound(debruijn, _) if debruijn < self.outer_index => {
                         ControlFlow::Continue(())
                     }
                     _ => {
@@ -337,7 +337,7 @@ impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for ValidateBoundVars<'tcx> {
 
     fn visit_region(&mut self, r: ty::Region<'tcx>) -> ControlFlow<Self::BreakTy> {
         match *r {
-            ty::ReLateBound(index, br) if index == self.binder_index => {
+            ty::ReBound(index, br) if index == self.binder_index => {
                 if self.bound_vars.len() <= br.var.as_usize() {
                     bug!("Not enough bound vars: {:?} not found in {:?}", br, self.bound_vars);
                 }
@@ -440,16 +440,15 @@ impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for HasEscapingVarsVisitor {
     }
 
     fn visit_const(&mut self, ct: ty::Const<'tcx>) -> ControlFlow<Self::BreakTy> {
-        // we don't have a `visit_infer_const` callback, so we have to
-        // hook in here to catch this case (annoying...), but
-        // otherwise we do want to remember to visit the rest of the
-        // const, as it has types/regions embedded in a lot of other
-        // places.
-        match ct.kind() {
-            ty::ConstKind::Bound(debruijn, _) if debruijn >= self.outer_index => {
-                ControlFlow::Break(FoundEscapingVars)
-            }
-            _ => ct.super_visit_with(self),
+        // If the outer-exclusive-binder is *strictly greater* than
+        // `outer_index`, that means that `ct` contains some content
+        // bound at `outer_index` or above (because
+        // `outer_exclusive_binder` is always 1 higher than the
+        // content in `t`). Therefore, `t` has some escaping vars.
+        if ct.outer_exclusive_binder() > self.outer_index {
+            ControlFlow::Break(FoundEscapingVars)
+        } else {
+            ControlFlow::Continue(())
         }
     }
 
@@ -529,9 +528,7 @@ impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for HasTypeFlagsVisitor {
     #[inline]
     fn visit_const(&mut self, c: ty::Const<'tcx>) -> ControlFlow<Self::BreakTy> {
         // Note: no `super_visit_with` call.
-        let flags = FlagComputation::for_const(c);
-        trace!(r.flags=?flags);
-        if flags.intersects(self.flags) {
+        if c.flags().intersects(self.flags) {
             ControlFlow::Break(FoundFlags)
         } else {
             ControlFlow::Continue(())
@@ -613,7 +610,7 @@ impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for LateBoundRegionsCollector {
     }
 
     fn visit_region(&mut self, r: ty::Region<'tcx>) -> ControlFlow<Self::BreakTy> {
-        if let ty::ReLateBound(debruijn, br) = *r {
+        if let ty::ReBound(debruijn, br) = *r {
             if debruijn == self.current_index {
                 self.regions.insert(br.kind);
             }
diff --git a/compiler/rustc_middle/src/util/bug.rs b/compiler/rustc_middle/src/util/bug.rs
index 634ed5ec5..5c9dd18ae 100644
--- a/compiler/rustc_middle/src/util/bug.rs
+++ b/compiler/rustc_middle/src/util/bug.rs
@@ -31,23 +31,23 @@ fn opt_span_bug_fmt<S: Into<MultiSpan>>(
     tls::with_opt(move |tcx| {
         let msg = format!("{location}: {args}");
         match (tcx, span) {
-            (Some(tcx), Some(span)) => tcx.sess.diagnostic().span_bug(span, msg),
-            (Some(tcx), None) => tcx.sess.diagnostic().bug(msg),
+            (Some(tcx), Some(span)) => tcx.sess.dcx().span_bug(span, msg),
+            (Some(tcx), None) => tcx.sess.dcx().bug(msg),
             (None, _) => panic_any(msg),
         }
     })
 }
 
-/// A query to trigger a `delay_span_bug`. Clearly, if one has a `tcx` one can already trigger a
-/// `delay_span_bug`, so what is the point of this? It exists to help us test `delay_span_bug`'s
+/// A query to trigger a `span_delayed_bug`. Clearly, if one has a `tcx` one can already trigger a
+/// `span_delayed_bug`, so what is the point of this? It exists to help us test `span_delayed_bug`'s
 /// interactions with the query system and incremental.
-pub fn trigger_delay_span_bug(tcx: TyCtxt<'_>, key: rustc_hir::def_id::DefId) {
-    tcx.sess.delay_span_bug(
+pub fn trigger_span_delayed_bug(tcx: TyCtxt<'_>, key: rustc_hir::def_id::DefId) {
+    tcx.sess.span_delayed_bug(
         tcx.def_span(key),
-        "delayed span bug triggered by #[rustc_error(delay_span_bug_from_inside_query)]",
+        "delayed span bug triggered by #[rustc_error(span_delayed_bug_from_inside_query)]",
     );
 }
 
 pub fn provide(providers: &mut crate::query::Providers) {
-    *providers = crate::query::Providers { trigger_delay_span_bug, ..*providers };
+    *providers = crate::query::Providers { trigger_span_delayed_bug, ..*providers };
 }
diff --git a/compiler/rustc_middle/src/values.rs b/compiler/rustc_middle/src/values.rs
index f30993c9a..3224f2f26 100644
--- a/compiler/rustc_middle/src/values.rs
+++ b/compiler/rustc_middle/src/values.rs
@@ -114,12 +114,11 @@ impl<'tcx> Value<TyCtxt<'tcx>> for ty::EarlyBinder<ty::Binder<'_, ty::FnSig<'_>>
 }
 
 impl<'tcx, T> Value<TyCtxt<'tcx>> for Result<T, &'_ ty::layout::LayoutError<'_>> {
-    fn from_cycle_error(_tcx: TyCtxt<'tcx>, _cycle: &[QueryInfo], _guar: ErrorGuaranteed) -> Self {
+    fn from_cycle_error(_tcx: TyCtxt<'tcx>, _cycle: &[QueryInfo], guar: ErrorGuaranteed) -> Self {
         // tcx.arena.alloc cannot be used because we are not allowed to use &'tcx LayoutError under
         // min_specialization. Since this is an error path anyways, leaking doesn't matter (and really,
         // tcx.arena.alloc is pretty much equal to leaking).
-        // FIXME: `Cycle` should carry the ErrorGuaranteed
-        Err(Box::leak(Box::new(ty::layout::LayoutError::Cycle)))
+        Err(Box::leak(Box::new(ty::layout::LayoutError::Cycle(guar))))
     }
 }
 
@@ -155,8 +154,9 @@ pub fn recursive_type_error(
         let (_, field_id) = item_and_field_ids[i];
         let (next_item_id, _) = item_and_field_ids[(i + 1) % cycle_len];
         // Find the span(s) that contain the next item in the cycle
-        let hir_id = tcx.hir().local_def_id_to_hir_id(field_id);
-        let hir::Node::Field(field) = tcx.hir().get(hir_id) else { bug!("expected field") };
+        let hir::Node::Field(field) = tcx.hir_node_by_def_id(field_id) else {
+            bug!("expected field")
+        };
         let mut found = Vec::new();
         find_item_ty_spans(tcx, field.ty, next_item_id, &mut found, representable_ids);
 
diff --git a/compiler/rustc_mir_build/Cargo.toml b/compiler/rustc_mir_build/Cargo.toml
index 6dceacd75..6d681dc29 100644
--- a/compiler/rustc_mir_build/Cargo.toml
+++ b/compiler/rustc_mir_build/Cargo.toml
@@ -17,7 +17,7 @@ rustc_index = { path = "../rustc_index" }
 rustc_infer = { path = "../rustc_infer" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_middle = { path = "../rustc_middle" }
-rustc_serialize = { path = "../rustc_serialize" }
+rustc_pattern_analysis = { path = "../rustc_pattern_analysis" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
diff --git a/compiler/rustc_mir_build/messages.ftl b/compiler/rustc_mir_build/messages.ftl
index 7dd0e7d4b..615b55343 100644
--- a/compiler/rustc_mir_build/messages.ftl
+++ b/compiler/rustc_mir_build/messages.ftl
@@ -30,12 +30,32 @@ mir_build_borrow_of_moved_value = borrow of moved value
 
 mir_build_call_to_fn_with_requires_unsafe =
     call to function `{$function}` with `#[target_feature]` is unsafe and requires unsafe block
-    .note = can only be called if the required target features are available
+    .help = in order for the call to be safe, the context requires the following additional target {$missing_target_features_count ->
+        [1] feature
+        *[count] features
+        }: {$missing_target_features}
+    .note = the {$build_target_features} target {$build_target_features_count ->
+        [1] feature
+        *[count] features
+        } being enabled in the build configuration does not remove the requirement to list {$build_target_features_count ->
+        [1] it
+        *[count] them
+        } in `#[target_feature]`
     .label = call to function with `#[target_feature]`
 
 mir_build_call_to_fn_with_requires_unsafe_unsafe_op_in_unsafe_fn_allowed =
     call to function `{$function}` with `#[target_feature]` is unsafe and requires unsafe function or block
-    .note = can only be called if the required target features are available
+    .help = in order for the call to be safe, the context requires the following additional target {$missing_target_features_count ->
+        [1] feature
+        *[count] features
+        }: {$missing_target_features}
+    .note = the {$build_target_features} target {$build_target_features_count ->
+        [1] feature
+        *[count] features
+        } being enabled in the build configuration does not remove the requirement to list {$build_target_features_count ->
+        [1] it
+        *[count] them
+        } in `#[target_feature]`
     .label = call to function with `#[target_feature]`
 
 mir_build_call_to_unsafe_fn_requires_unsafe =
@@ -217,15 +237,6 @@ mir_build_non_const_path = runtime values cannot be referenced in patterns
 mir_build_non_exhaustive_match_all_arms_guarded =
     match arms with guards don't count towards exhaustivity
 
-mir_build_non_exhaustive_omitted_pattern = some variants are not matched explicitly
-    .help = ensure that all variants are matched explicitly by adding the suggested match arms
-    .note = the matched value is of type `{$scrut_ty}` and the `non_exhaustive_omitted_patterns` attribute was found
-
-mir_build_non_exhaustive_omitted_pattern_lint_on_arm = the lint level must be set on the whole match
-    .help = it no longer has any effect to set the lint level on an individual match arm
-    .label = remove this attribute
-    .suggestion = set the lint level on the whole match
-
 mir_build_non_exhaustive_patterns_type_not_empty = non-exhaustive patterns: type `{$ty}` is non-empty
     .def_note = `{$peeled_ty}` defined here
     .type_note = the matched value is of type `{$ty}`
@@ -240,10 +251,6 @@ mir_build_non_partial_eq_match =
 mir_build_nontrivial_structural_match =
     to use a constant of type `{$non_sm_ty}` in a pattern, the constant's initializer must be trivial or `{$non_sm_ty}` must be annotated with `#[derive(PartialEq, Eq)]`
 
-mir_build_overlapping_range_endpoints = multiple patterns overlap on their endpoints
-    .range = ... with this range
-    .note = you likely meant to write mutually exclusive ranges
-
 mir_build_pattern_not_covered = refutable pattern in {$origin}
     .pattern_ty = the matched value is of type `{$pattern_ty}`
 
@@ -297,13 +304,6 @@ mir_build_unconditional_recursion = function cannot return without recursing
 
 mir_build_unconditional_recursion_call_site_label = recursive call site
 
-mir_build_uncovered = {$count ->
-        [1] pattern `{$witness_1}`
-        [2] patterns `{$witness_1}` and `{$witness_2}`
-        [3] patterns `{$witness_1}`, `{$witness_2}` and `{$witness_3}`
-        *[other] patterns `{$witness_1}`, `{$witness_2}`, `{$witness_3}` and {$remainder} more
-    } not covered
-
 mir_build_union_field_requires_unsafe =
     access to union field is unsafe and requires unsafe block
     .note = the field may not be properly initialized: using uninitialized data will cause undefined behavior
@@ -330,7 +330,17 @@ mir_build_unsafe_op_in_unsafe_fn_borrow_of_layout_constrained_field_requires_uns
 
 mir_build_unsafe_op_in_unsafe_fn_call_to_fn_with_requires_unsafe =
     call to function `{$function}` with `#[target_feature]` is unsafe and requires unsafe block (error E0133)
-    .note = can only be called if the required target features are available
+    .help = in order for the call to be safe, the context requires the following additional target {$missing_target_features_count ->
+        [1] feature
+        *[count] features
+        }: {$missing_target_features}
+    .note = the {$build_target_features} target {$build_target_features_count ->
+        [1] feature
+        *[count] features
+        } being enabled in the build configuration does not remove the requirement to list {$build_target_features_count ->
+        [1] it
+        *[count] them
+        } in `#[target_feature]`
     .label = call to function with `#[target_feature]`
 
 mir_build_unsafe_op_in_unsafe_fn_call_to_unsafe_fn_requires_unsafe =
diff --git a/compiler/rustc_mir_build/src/build/block.rs b/compiler/rustc_mir_build/src/build/block.rs
index ab4cd2488..8cad6976c 100644
--- a/compiler/rustc_mir_build/src/build/block.rs
+++ b/compiler/rustc_mir_build/src/build/block.rs
@@ -13,42 +13,33 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         ast_block: BlockId,
         source_info: SourceInfo,
     ) -> BlockAnd<()> {
-        let Block {
-            region_scope,
-            opt_destruction_scope,
-            span,
-            ref stmts,
-            expr,
-            targeted_by_break,
-            safety_mode,
-        } = self.thir[ast_block];
+        let Block { region_scope, span, ref stmts, expr, targeted_by_break, safety_mode } =
+            self.thir[ast_block];
         let expr = expr.map(|expr| &self.thir[expr]);
-        self.in_opt_scope(opt_destruction_scope.map(|de| (de, source_info)), move |this| {
-            this.in_scope((region_scope, source_info), LintLevel::Inherited, move |this| {
-                if targeted_by_break {
-                    this.in_breakable_scope(None, destination, span, |this| {
-                        Some(this.ast_block_stmts(
-                            destination,
-                            block,
-                            span,
-                            &stmts,
-                            expr,
-                            safety_mode,
-                            region_scope,
-                        ))
-                    })
-                } else {
-                    this.ast_block_stmts(
+        self.in_scope((region_scope, source_info), LintLevel::Inherited, move |this| {
+            if targeted_by_break {
+                this.in_breakable_scope(None, destination, span, |this| {
+                    Some(this.ast_block_stmts(
                         destination,
                         block,
                         span,
-                        &stmts,
+                        stmts,
                         expr,
                         safety_mode,
                         region_scope,
-                    )
-                }
-            })
+                    ))
+                })
+            } else {
+                this.ast_block_stmts(
+                    destination,
+                    block,
+                    span,
+                    stmts,
+                    expr,
+                    safety_mode,
+                    region_scope,
+                )
+            }
         })
     }
 
@@ -92,20 +83,15 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
         let source_info = this.source_info(span);
         for stmt in stmts {
-            let Stmt { ref kind, opt_destruction_scope } = this.thir[*stmt];
+            let Stmt { ref kind } = this.thir[*stmt];
             match kind {
                 StmtKind::Expr { scope, expr } => {
                     this.block_context.push(BlockFrame::Statement { ignores_expr_result: true });
+                    let si = (*scope, source_info);
                     unpack!(
-                        block = this.in_opt_scope(
-                            opt_destruction_scope.map(|de| (de, source_info)),
-                            |this| {
-                                let si = (*scope, source_info);
-                                this.in_scope(si, LintLevel::Inherited, |this| {
-                                    this.stmt_expr(block, &this.thir[*expr], Some(*scope))
-                                })
-                            }
-                        )
+                        block = this.in_scope(si, LintLevel::Inherited, |this| {
+                            this.stmt_expr(block, &this.thir[*expr], Some(*scope))
+                        })
                     );
                 }
                 StmtKind::Let {
@@ -221,43 +207,38 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
                     let init = &this.thir[*initializer];
                     let initializer_span = init.span;
+                    let scope = (*init_scope, source_info);
                     let failure = unpack!(
-                        block = this.in_opt_scope(
-                            opt_destruction_scope.map(|de| (de, source_info)),
-                            |this| {
-                                let scope = (*init_scope, source_info);
-                                this.in_scope(scope, *lint_level, |this| {
-                                    this.declare_bindings(
-                                        visibility_scope,
-                                        remainder_span,
-                                        pattern,
-                                        None,
-                                        Some((Some(&destination), initializer_span)),
-                                    );
-                                    this.visit_primary_bindings(
-                                        pattern,
-                                        UserTypeProjections::none(),
-                                        &mut |this, _, _, _, node, span, _, _| {
-                                            this.storage_live_binding(
-                                                block,
-                                                node,
-                                                span,
-                                                OutsideGuard,
-                                                true,
-                                            );
-                                        },
-                                    );
-                                    this.ast_let_else(
+                        block = this.in_scope(scope, *lint_level, |this| {
+                            this.declare_bindings(
+                                visibility_scope,
+                                remainder_span,
+                                pattern,
+                                None,
+                                Some((Some(&destination), initializer_span)),
+                            );
+                            this.visit_primary_bindings(
+                                pattern,
+                                UserTypeProjections::none(),
+                                &mut |this, _, _, _, node, span, _, _| {
+                                    this.storage_live_binding(
                                         block,
-                                        init,
-                                        initializer_span,
-                                        *else_block,
-                                        &last_remainder_scope,
-                                        pattern,
-                                    )
-                                })
-                            }
-                        )
+                                        node,
+                                        span,
+                                        OutsideGuard,
+                                        true,
+                                    );
+                                },
+                            );
+                            this.ast_let_else(
+                                block,
+                                init,
+                                initializer_span,
+                                *else_block,
+                                &last_remainder_scope,
+                                pattern,
+                            )
+                        })
                     );
                     this.cfg.goto(failure, source_info, failure_entry);
 
@@ -298,25 +279,20 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     if let Some(init) = initializer {
                         let init = &this.thir[*init];
                         let initializer_span = init.span;
+                        let scope = (*init_scope, source_info);
 
                         unpack!(
-                            block = this.in_opt_scope(
-                                opt_destruction_scope.map(|de| (de, source_info)),
-                                |this| {
-                                    let scope = (*init_scope, source_info);
-                                    this.in_scope(scope, *lint_level, |this| {
-                                        this.declare_bindings(
-                                            visibility_scope,
-                                            remainder_span,
-                                            pattern,
-                                            None,
-                                            Some((None, initializer_span)),
-                                        );
-                                        this.expr_into_pattern(block, &pattern, init)
-                                        // irrefutable pattern
-                                    })
-                                },
-                            )
+                            block = this.in_scope(scope, *lint_level, |this| {
+                                this.declare_bindings(
+                                    visibility_scope,
+                                    remainder_span,
+                                    pattern,
+                                    None,
+                                    Some((None, initializer_span)),
+                                );
+                                this.expr_into_pattern(block, &pattern, init)
+                                // irrefutable pattern
+                            })
                         )
                     } else {
                         let scope = (*init_scope, source_info);
diff --git a/compiler/rustc_mir_build/src/build/cfg.rs b/compiler/rustc_mir_build/src/build/cfg.rs
index fddcf9de7..2bd0e2897 100644
--- a/compiler/rustc_mir_build/src/build/cfg.rs
+++ b/compiler/rustc_mir_build/src/build/cfg.rs
@@ -101,6 +101,19 @@ impl<'tcx> CFG<'tcx> {
         self.push(block, stmt);
     }
 
+    /// Adds a dummy statement whose only role is to associate a span with its
+    /// enclosing block for the purposes of coverage instrumentation.
+    ///
+    /// This results in more accurate coverage reports for certain kinds of
+    /// syntax (e.g. `continue` or `if !`) that would otherwise not appear in MIR.
+    pub(crate) fn push_coverage_span_marker(&mut self, block: BasicBlock, source_info: SourceInfo) {
+        let kind = StatementKind::Coverage(Box::new(Coverage {
+            kind: coverage::CoverageKind::SpanMarker,
+        }));
+        let stmt = Statement { source_info, kind };
+        self.push(block, stmt);
+    }
+
     pub(crate) fn terminate(
         &mut self,
         block: BasicBlock,
diff --git a/compiler/rustc_mir_build/src/build/custom/mod.rs b/compiler/rustc_mir_build/src/build/custom/mod.rs
index 3de2f45ad..ead20539e 100644
--- a/compiler/rustc_mir_build/src/build/custom/mod.rs
+++ b/compiler/rustc_mir_build/src/build/custom/mod.rs
@@ -88,11 +88,11 @@ pub(super) fn build_custom_mir<'tcx>(
     };
 
     let res: PResult<_> = try {
-        pctxt.parse_args(&params)?;
+        pctxt.parse_args(params)?;
         pctxt.parse_body(expr)?;
     };
     if let Err(err) = res {
-        tcx.sess.diagnostic().span_fatal(
+        tcx.sess.dcx().span_fatal(
             err.span,
             format!("Could not parse {}, found: {:?}", err.expected, err.item_description),
         )
@@ -162,6 +162,19 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
             expected: expected.to_string(),
         }
     }
+
+    fn stmt_error(&self, stmt: StmtId, expected: &'static str) -> ParseError {
+        let stmt = &self.thir[stmt];
+        let span = match stmt.kind {
+            StmtKind::Expr { expr, .. } => self.thir[expr].span,
+            StmtKind::Let { span, .. } => span,
+        };
+        ParseError {
+            span,
+            item_description: format!("{:?}", stmt.kind),
+            expected: expected.to_string(),
+        }
+    }
 }
 
 type PResult<T> = Result<T, ParseError>;
diff --git a/compiler/rustc_mir_build/src/build/custom/parse.rs b/compiler/rustc_mir_build/src/build/custom/parse.rs
index e2ab2cb90..a6f9caada 100644
--- a/compiler/rustc_mir_build/src/build/custom/parse.rs
+++ b/compiler/rustc_mir_build/src/build/custom/parse.rs
@@ -27,10 +27,13 @@ macro_rules! parse_by_kind {
         $expr_name:pat,
         $expected:literal,
         $(
-            @call($name:literal, $args:ident) => $call_expr:expr,
+            @call($name:ident, $args:ident) => $call_expr:expr,
         )*
         $(
-            $pat:pat => $expr:expr,
+            @variant($adt:ident, $variant:ident) => $variant_expr:expr,
+        )*
+        $(
+            $pat:pat $(if $guard:expr)? => $expr:expr,
         )*
     ) => {{
         let expr_id = $self.preparse($expr_id);
@@ -42,14 +45,20 @@ macro_rules! parse_by_kind {
                 ExprKind::Call { ty, fun: _, args: $args, .. } if {
                     match ty.kind() {
                         ty::FnDef(did, _) => {
-                            $self.tcx.is_diagnostic_item(rustc_span::Symbol::intern($name), *did)
+                            $self.tcx.is_diagnostic_item(rustc_span::sym::$name, *did)
                         }
                         _ => false,
                     }
                 } => $call_expr,
             )*
             $(
-                $pat => $expr,
+                ExprKind::Adt(box AdtExpr { adt_def, variant_index, .. }) if {
+                    $self.tcx.is_diagnostic_item(rustc_span::sym::$adt, adt_def.did()) &&
+                    adt_def.variants()[*variant_index].name == rustc_span::sym::$variant
+                } => $variant_expr,
+            )*
+            $(
+                $pat $(if $guard)? => $expr,
             )*
             #[allow(unreachable_patterns)]
             _ => return Err($self.expr_error(expr_id, $expected))
@@ -172,7 +181,8 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
             ExprKind::Block { block } => &self.thir[*block].stmts,
         );
         for (i, block_def) in block_defs.iter().enumerate() {
-            let block = self.parse_block_def(self.statement_as_expr(*block_def)?)?;
+            let is_cleanup = self.body.basic_blocks_mut()[BasicBlock::from_usize(i)].is_cleanup;
+            let block = self.parse_block_def(self.statement_as_expr(*block_def)?, is_cleanup)?;
             self.body.basic_blocks_mut()[BasicBlock::from_usize(i)] = block;
         }
 
@@ -181,15 +191,28 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
 
     fn parse_block_decls(&mut self, stmts: impl Iterator<Item = StmtId>) -> PResult<()> {
         for stmt in stmts {
-            let (var, _, _) = self.parse_let_statement(stmt)?;
-            let data = BasicBlockData::new(None);
-            let block = self.body.basic_blocks_mut().push(data);
-            self.block_map.insert(var, block);
+            self.parse_basic_block_decl(stmt)?;
         }
-
         Ok(())
     }
 
+    fn parse_basic_block_decl(&mut self, stmt: StmtId) -> PResult<()> {
+        match &self.thir[stmt].kind {
+            StmtKind::Let { pattern, initializer: Some(initializer), .. } => {
+                let (var, ..) = self.parse_var(pattern)?;
+                let mut data = BasicBlockData::new(None);
+                data.is_cleanup = parse_by_kind!(self, *initializer, _, "basic block declaration",
+                    @variant(mir_basic_block, Normal) => false,
+                    @variant(mir_basic_block, Cleanup) => true,
+                );
+                let block = self.body.basic_blocks_mut().push(data);
+                self.block_map.insert(var, block);
+                Ok(())
+            }
+            _ => Err(self.stmt_error(stmt, "let statement with an initializer")),
+        }
+    }
+
     fn parse_local_decls(&mut self, mut stmts: impl Iterator<Item = StmtId>) -> PResult<()> {
         let (ret_var, ..) = self.parse_let_statement(stmts.next().unwrap())?;
         self.local_map.insert(ret_var, Local::from_u32(0));
@@ -219,7 +242,7 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
             };
             let span = self.thir[expr].span;
             let (name, operand) = parse_by_kind!(self, expr, _, "debuginfo",
-                @call("mir_debuginfo", args) => {
+                @call(mir_debuginfo, args) => {
                     (args[0], args[1])
                 },
             );
@@ -281,12 +304,13 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
         }
     }
 
-    fn parse_block_def(&self, expr_id: ExprId) -> PResult<BasicBlockData<'tcx>> {
+    fn parse_block_def(&self, expr_id: ExprId, is_cleanup: bool) -> PResult<BasicBlockData<'tcx>> {
         let block = parse_by_kind!(self, expr_id, _, "basic block",
             ExprKind::Block { block } => &self.thir[*block],
         );
 
         let mut data = BasicBlockData::new(None);
+        data.is_cleanup = is_cleanup;
         for stmt_id in &*block.stmts {
             let stmt = self.statement_as_expr(*stmt_id)?;
             let span = self.thir[stmt].span;
diff --git a/compiler/rustc_mir_build/src/build/custom/parse/instruction.rs b/compiler/rustc_mir_build/src/build/custom/parse/instruction.rs
index fd2c57a0a..4ce7f831c 100644
--- a/compiler/rustc_mir_build/src/build/custom/parse/instruction.rs
+++ b/compiler/rustc_mir_build/src/build/custom/parse/instruction.rs
@@ -13,19 +13,19 @@ use super::{parse_by_kind, PResult, ParseCtxt};
 impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
     pub fn parse_statement(&self, expr_id: ExprId) -> PResult<StatementKind<'tcx>> {
         parse_by_kind!(self, expr_id, _, "statement",
-            @call("mir_storage_live", args) => {
+            @call(mir_storage_live, args) => {
                 Ok(StatementKind::StorageLive(self.parse_local(args[0])?))
             },
-            @call("mir_storage_dead", args) => {
+            @call(mir_storage_dead, args) => {
                 Ok(StatementKind::StorageDead(self.parse_local(args[0])?))
             },
-            @call("mir_deinit", args) => {
+            @call(mir_deinit, args) => {
                 Ok(StatementKind::Deinit(Box::new(self.parse_place(args[0])?)))
             },
-            @call("mir_retag", args) => {
+            @call(mir_retag, args) => {
                 Ok(StatementKind::Retag(RetagKind::Default, Box::new(self.parse_place(args[0])?)))
             },
-            @call("mir_set_discriminant", args) => {
+            @call(mir_set_discriminant, args) => {
                 let place = self.parse_place(args[0])?;
                 let var = self.parse_integer_literal(args[1])? as u32;
                 Ok(StatementKind::SetDiscriminant {
@@ -43,24 +43,30 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
 
     pub fn parse_terminator(&self, expr_id: ExprId) -> PResult<TerminatorKind<'tcx>> {
         parse_by_kind!(self, expr_id, expr, "terminator",
-            @call("mir_return", _args) => {
+            @call(mir_return, _args) => {
                 Ok(TerminatorKind::Return)
             },
-            @call("mir_goto", args) => {
+            @call(mir_goto, args) => {
                 Ok(TerminatorKind::Goto { target: self.parse_block(args[0])? } )
             },
-            @call("mir_unreachable", _args) => {
+            @call(mir_unreachable, _args) => {
                 Ok(TerminatorKind::Unreachable)
             },
-            @call("mir_drop", args) => {
+            @call(mir_unwind_resume, _args) => {
+                Ok(TerminatorKind::UnwindResume)
+            },
+            @call(mir_unwind_terminate, args) => {
+                Ok(TerminatorKind::UnwindTerminate(self.parse_unwind_terminate_reason(args[0])?))
+            },
+            @call(mir_drop, args) => {
                 Ok(TerminatorKind::Drop {
                     place: self.parse_place(args[0])?,
                     target: self.parse_block(args[1])?,
-                    unwind: UnwindAction::Continue,
+                    unwind: self.parse_unwind_action(args[2])?,
                     replace: false,
                 })
             },
-            @call("mir_call", args) => {
+            @call(mir_call, args) => {
                 self.parse_call(args)
             },
             ExprKind::Match { scrutinee, arms, .. } => {
@@ -70,6 +76,34 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
         )
     }
 
+    fn parse_unwind_terminate_reason(&self, expr_id: ExprId) -> PResult<UnwindTerminateReason> {
+        parse_by_kind!(self, expr_id, _, "unwind terminate reason",
+            @variant(mir_unwind_terminate_reason, Abi) => {
+                Ok(UnwindTerminateReason::Abi)
+            },
+            @variant(mir_unwind_terminate_reason, InCleanup) => {
+                Ok(UnwindTerminateReason::InCleanup)
+            },
+        )
+    }
+
+    fn parse_unwind_action(&self, expr_id: ExprId) -> PResult<UnwindAction> {
+        parse_by_kind!(self, expr_id, _, "unwind action",
+            @call(mir_unwind_continue, _args) => {
+                Ok(UnwindAction::Continue)
+            },
+            @call(mir_unwind_unreachable, _args) => {
+                Ok(UnwindAction::Unreachable)
+            },
+            @call(mir_unwind_terminate, args) => {
+                Ok(UnwindAction::Terminate(self.parse_unwind_terminate_reason(args[0])?))
+            },
+            @call(mir_unwind_cleanup, args) => {
+                Ok(UnwindAction::Cleanup(self.parse_block(args[0])?))
+            },
+        )
+    }
+
     fn parse_match(&self, arms: &[ArmId], span: Span) -> PResult<SwitchTargets> {
         let Some((otherwise, rest)) = arms.split_last() else {
             return Err(ParseError {
@@ -113,6 +147,7 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
         );
         let destination = self.parse_place(destination)?;
         let target = self.parse_block(args[1])?;
+        let unwind = self.parse_unwind_action(args[2])?;
 
         parse_by_kind!(self, call, _, "function call",
             ExprKind::Call { fun, args, from_hir_call, fn_span, .. } => {
@@ -126,7 +161,7 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
                     args,
                     destination,
                     target: Some(target),
-                    unwind: UnwindAction::Continue,
+                    unwind,
                     call_source: if *from_hir_call { CallSource::Normal } else {
                         CallSource::OverloadedOperator
                     },
@@ -138,25 +173,25 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
 
     fn parse_rvalue(&self, expr_id: ExprId) -> PResult<Rvalue<'tcx>> {
         parse_by_kind!(self, expr_id, expr, "rvalue",
-            @call("mir_discriminant", args) => self.parse_place(args[0]).map(Rvalue::Discriminant),
-            @call("mir_cast_transmute", args) => {
+            @call(mir_discriminant, args) => self.parse_place(args[0]).map(Rvalue::Discriminant),
+            @call(mir_cast_transmute, args) => {
                 let source = self.parse_operand(args[0])?;
                 Ok(Rvalue::Cast(CastKind::Transmute, source, expr.ty))
             },
-            @call("mir_checked", args) => {
+            @call(mir_checked, args) => {
                 parse_by_kind!(self, args[0], _, "binary op",
                     ExprKind::Binary { op, lhs, rhs } => Ok(Rvalue::CheckedBinaryOp(
                         *op, Box::new((self.parse_operand(*lhs)?, self.parse_operand(*rhs)?))
                     )),
                 )
             },
-            @call("mir_offset", args) => {
+            @call(mir_offset, args) => {
                 let ptr = self.parse_operand(args[0])?;
                 let offset = self.parse_operand(args[1])?;
                 Ok(Rvalue::BinaryOp(BinOp::Offset, Box::new((ptr, offset))))
             },
-            @call("mir_len", args) => Ok(Rvalue::Len(self.parse_place(args[0])?)),
-            @call("mir_copy_for_deref", args) => Ok(Rvalue::CopyForDeref(self.parse_place(args[0])?)),
+            @call(mir_len, args) => Ok(Rvalue::Len(self.parse_place(args[0])?)),
+            @call(mir_copy_for_deref, args) => Ok(Rvalue::CopyForDeref(self.parse_place(args[0])?)),
             ExprKind::Borrow { borrow_kind, arg } => Ok(
                 Rvalue::Ref(self.tcx.lifetimes.re_erased, *borrow_kind, self.parse_place(*arg)?)
             ),
@@ -206,9 +241,9 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
 
     pub fn parse_operand(&self, expr_id: ExprId) -> PResult<Operand<'tcx>> {
         parse_by_kind!(self, expr_id, expr, "operand",
-            @call("mir_move", args) => self.parse_place(args[0]).map(Operand::Move),
-            @call("mir_static", args) => self.parse_static(args[0]),
-            @call("mir_static_mut", args) => self.parse_static(args[0]),
+            @call(mir_move, args) => self.parse_place(args[0]).map(Operand::Move),
+            @call(mir_static, args) => self.parse_static(args[0]),
+            @call(mir_static_mut, args) => self.parse_static(args[0]),
             ExprKind::Literal { .. }
             | ExprKind::NamedConst { .. }
             | ExprKind::NonHirLiteral { .. }
@@ -229,7 +264,7 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
 
     fn parse_place_inner(&self, expr_id: ExprId) -> PResult<(Place<'tcx>, PlaceTy<'tcx>)> {
         let (parent, proj) = parse_by_kind!(self, expr_id, expr, "place",
-            @call("mir_field", args) => {
+            @call(mir_field, args) => {
                 let (parent, ty) = self.parse_place_inner(args[0])?;
                 let field = FieldIdx::from_u32(self.parse_integer_literal(args[1])? as u32);
                 let field_ty = ty.field_ty(self.tcx, field);
@@ -237,7 +272,7 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
                 let place = parent.project_deeper(&[proj], self.tcx);
                 return Ok((place, PlaceTy::from_ty(field_ty)));
             },
-            @call("mir_variant", args) => {
+            @call(mir_variant, args) => {
                 (args[0], PlaceElem::Downcast(
                     None,
                     VariantIdx::from_u32(self.parse_integer_literal(args[1])? as u32)
@@ -245,7 +280,7 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
             },
             ExprKind::Deref { arg } => {
                 parse_by_kind!(self, *arg, _, "does not matter",
-                    @call("mir_make_place", args) => return self.parse_place_inner(args[0]),
+                    @call(mir_make_place, args) => return self.parse_place_inner(args[0]),
                     _ => (*arg, PlaceElem::Deref),
                 )
             },
diff --git a/compiler/rustc_mir_build/src/build/expr/as_constant.rs b/compiler/rustc_mir_build/src/build/expr/as_constant.rs
index 4ed49e787..3bfffa735 100644
--- a/compiler/rustc_mir_build/src/build/expr/as_constant.rs
+++ b/compiler/rustc_mir_build/src/build/expr/as_constant.rs
@@ -114,7 +114,7 @@ fn lit_to_mir_constant<'tcx>(
         let width = tcx
             .layout_of(param_ty)
             .map_err(|_| {
-                LitToConstError::Reported(tcx.sess.delay_span_bug(
+                LitToConstError::Reported(tcx.sess.span_delayed_bug(
                     DUMMY_SP,
                     format!("couldn't compute width of literal: {:?}", lit_input.lit),
                 ))
@@ -158,7 +158,7 @@ fn lit_to_mir_constant<'tcx>(
         }
         (ast::LitKind::Float(n, _), ty::Float(fty)) => parse_float_into_constval(*n, *fty, neg)
             .ok_or_else(|| {
-                LitToConstError::Reported(tcx.sess.delay_span_bug(
+                LitToConstError::Reported(tcx.sess.span_delayed_bug(
                     DUMMY_SP,
                     format!("couldn't parse float literal: {:?}", lit_input.lit),
                 ))
@@ -167,7 +167,7 @@ fn lit_to_mir_constant<'tcx>(
         (ast::LitKind::Char(c), ty::Char) => ConstValue::Scalar(Scalar::from_char(*c)),
         (ast::LitKind::Err, _) => {
             return Err(LitToConstError::Reported(
-                tcx.sess.delay_span_bug(DUMMY_SP, "encountered LitKind::Err during mir build"),
+                tcx.sess.span_delayed_bug(DUMMY_SP, "encountered LitKind::Err during mir build"),
             ));
         }
         _ => return Err(LitToConstError::TypeError),
diff --git a/compiler/rustc_mir_build/src/build/expr/as_place.rs b/compiler/rustc_mir_build/src/build/expr/as_place.rs
index 7e9191a37..43e834890 100644
--- a/compiler/rustc_mir_build/src/build/expr/as_place.rs
+++ b/compiler/rustc_mir_build/src/build/expr/as_place.rs
@@ -175,7 +175,7 @@ fn to_upvars_resolved_place_builder<'tcx>(
     projection: &[PlaceElem<'tcx>],
 ) -> Option<PlaceBuilder<'tcx>> {
     let Some((capture_index, capture)) =
-        find_capture_matching_projections(&cx.upvars, var_hir_id, &projection)
+        find_capture_matching_projections(&cx.upvars, var_hir_id, projection)
     else {
         let closure_span = cx.tcx.def_span(closure_def_id);
         if !enable_precise_capture(closure_span) {
@@ -683,7 +683,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                             Ty::new_imm_ref(tcx, tcx.lifetimes.re_erased, fake_borrow_deref_ty);
                         let fake_borrow_temp =
                             self.local_decls.push(LocalDecl::new(fake_borrow_ty, expr_span));
-                        let projection = tcx.mk_place_elems(&base_place.projection);
+                        let projection = tcx.mk_place_elems(base_place.projection);
                         self.cfg.push_assign(
                             block,
                             source_info,
diff --git a/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs b/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs
index eece8684e..a5f6bb12e 100644
--- a/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs
+++ b/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs
@@ -10,8 +10,6 @@ use crate::build::{BlockAnd, BlockAndExtension, Builder, NeedsTemporary};
 use rustc_hir::lang_items::LangItem;
 use rustc_middle::middle::region;
 use rustc_middle::mir::interpret::Scalar;
-use rustc_middle::mir::AssertKind;
-use rustc_middle::mir::Place;
 use rustc_middle::mir::*;
 use rustc_middle::thir::*;
 use rustc_middle::ty::cast::{mir_cast_kind, CastTy};
@@ -600,10 +598,12 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             BinOp::Shl | BinOp::Shr if self.check_overflow && ty.is_integral() => {
                 // For an unsigned RHS, the shift is in-range for `rhs < bits`.
                 // For a signed RHS, `IntToInt` cast to the equivalent unsigned
-                // type and do that same comparison. Because the type is the
-                // same size, there's no negative shift amount that ends up
-                // overlapping with valid ones, thus it catches negatives too.
+                // type and do that same comparison.
+                // A negative value will be *at least* 128 after the cast (that's i8::MIN),
+                // and 128 is an overflowing shift amount for all our currently existing types,
+                // so this cast can never make us miss an overflow.
                 let (lhs_size, _) = ty.int_size_and_signed(self.tcx);
+                assert!(lhs_size.bits() <= 128);
                 let rhs_ty = rhs.ty(&self.local_decls, self.tcx);
                 let (rhs_size, _) = rhs_ty.int_size_and_signed(self.tcx);
 
@@ -625,7 +625,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
                 // This can't overflow because the largest shiftable types are 128-bit,
                 // which fits in `u8`, the smallest possible `unsigned_ty`.
-                // (And `from_uint` will `bug!` if that's ever no longer true.)
                 let lhs_bits = Operand::const_from_scalar(
                     self.tcx,
                     unsigned_ty,
diff --git a/compiler/rustc_mir_build/src/build/matches/mod.rs b/compiler/rustc_mir_build/src/build/matches/mod.rs
index 304870274..487b1f44b 100644
--- a/compiler/rustc_mir_build/src/build/matches/mod.rs
+++ b/compiler/rustc_mir_build/src/build/matches/mod.rs
@@ -90,6 +90,11 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 let local_scope = this.local_scope();
                 let (success_block, failure_block) =
                     this.in_if_then_scope(local_scope, expr_span, |this| {
+                        // Help out coverage instrumentation by injecting a dummy statement with
+                        // the original condition's span (including `!`). This fixes #115468.
+                        if this.tcx.sess.instrument_coverage() {
+                            this.cfg.push_coverage_span_marker(block, this.source_info(expr_span));
+                        }
                         this.then_else_break(
                             block,
                             &this.thir[arg],
@@ -212,7 +217,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         let scrutinee_place =
             unpack!(block = self.lower_scrutinee(block, scrutinee, scrutinee_span,));
 
-        let mut arm_candidates = self.create_match_candidates(&scrutinee_place, &arms);
+        let mut arm_candidates = self.create_match_candidates(&scrutinee_place, arms);
 
         let match_has_guard = arm_candidates.iter().any(|(_, candidate)| candidate.has_guard);
         let mut candidates =
@@ -424,7 +429,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         this.source_scope = source_scope;
                     }
 
-                    this.expr_into_dest(destination, arm_block, &&this.thir[arm.body])
+                    this.expr_into_dest(destination, arm_block, &this.thir[arm.body])
                 })
             })
             .collect();
@@ -505,7 +510,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     let binding_end = self.bind_and_guard_matched_candidate(
                         leaf_candidate,
                         parent_bindings,
-                        &fake_borrow_temps,
+                        fake_borrow_temps,
                         scrutinee_span,
                         arm_match_scope,
                         schedule_drops,
@@ -613,7 +618,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             _ => {
                 let place_builder =
                     unpack!(block = self.lower_scrutinee(block, initializer, initializer.span));
-                self.place_into_pattern(block, &irrefutable_pat, place_builder, true)
+                self.place_into_pattern(block, irrefutable_pat, place_builder, true)
             }
         }
     }
@@ -625,7 +630,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         initializer: PlaceBuilder<'tcx>,
         set_match_place: bool,
     ) -> BlockAnd<()> {
-        let mut candidate = Candidate::new(initializer.clone(), &irrefutable_pat, false, self);
+        let mut candidate = Candidate::new(initializer.clone(), irrefutable_pat, false, self);
         let fake_borrow_temps = self.lower_match_tree(
             block,
             irrefutable_pat.span,
@@ -700,7 +705,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         opt_match_place: Option<(Option<&Place<'tcx>>, Span)>,
     ) -> Option<SourceScope> {
         self.visit_primary_bindings(
-            &pattern,
+            pattern,
             UserTypeProjections::none(),
             &mut |this, mutability, name, mode, var, span, ty, user_ty| {
                 if visibility_scope.is_none() {
@@ -827,6 +832,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             PatKind::Constant { .. }
             | PatKind::Range { .. }
             | PatKind::Wild
+            | PatKind::Never
             | PatKind::Error(_) => {}
 
             PatKind::Deref { ref subpattern } => {
@@ -1693,59 +1699,51 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         debug!("tested_candidates: {}", total_candidate_count - candidates.len());
         debug!("untested_candidates: {}", candidates.len());
 
-        // HACK(matthewjasper) This is a closure so that we can let the test
-        // create its blocks before the rest of the match. This currently
-        // improves the speed of llvm when optimizing long string literal
-        // matches
-        let make_target_blocks = move |this: &mut Self| -> Vec<BasicBlock> {
-            // The block that we should branch to if none of the
-            // `target_candidates` match. This is either the block where we
-            // start matching the untested candidates if there are any,
-            // otherwise it's the `otherwise_block`.
-            let remainder_start = &mut None;
-            let remainder_start =
-                if candidates.is_empty() { &mut *otherwise_block } else { remainder_start };
-
-            // For each outcome of test, process the candidates that still
-            // apply. Collect a list of blocks where control flow will
-            // branch if one of the `target_candidate` sets is not
-            // exhaustive.
-            let target_blocks: Vec<_> = target_candidates
-                .into_iter()
-                .map(|mut candidates| {
-                    if !candidates.is_empty() {
-                        let candidate_start = this.cfg.start_new_block();
-                        this.match_candidates(
-                            span,
-                            scrutinee_span,
-                            candidate_start,
-                            remainder_start,
-                            &mut *candidates,
-                            fake_borrows,
-                        );
-                        candidate_start
-                    } else {
-                        *remainder_start.get_or_insert_with(|| this.cfg.start_new_block())
-                    }
-                })
-                .collect();
-
-            if !candidates.is_empty() {
-                let remainder_start = remainder_start.unwrap_or_else(|| this.cfg.start_new_block());
-                this.match_candidates(
-                    span,
-                    scrutinee_span,
-                    remainder_start,
-                    otherwise_block,
-                    candidates,
-                    fake_borrows,
-                );
-            };
+        // The block that we should branch to if none of the
+        // `target_candidates` match. This is either the block where we
+        // start matching the untested candidates if there are any,
+        // otherwise it's the `otherwise_block`.
+        let remainder_start = &mut None;
+        let remainder_start =
+            if candidates.is_empty() { &mut *otherwise_block } else { remainder_start };
+
+        // For each outcome of test, process the candidates that still
+        // apply. Collect a list of blocks where control flow will
+        // branch if one of the `target_candidate` sets is not
+        // exhaustive.
+        let target_blocks: Vec<_> = target_candidates
+            .into_iter()
+            .map(|mut candidates| {
+                if !candidates.is_empty() {
+                    let candidate_start = self.cfg.start_new_block();
+                    self.match_candidates(
+                        span,
+                        scrutinee_span,
+                        candidate_start,
+                        remainder_start,
+                        &mut *candidates,
+                        fake_borrows,
+                    );
+                    candidate_start
+                } else {
+                    *remainder_start.get_or_insert_with(|| self.cfg.start_new_block())
+                }
+            })
+            .collect();
 
-            target_blocks
-        };
+        if !candidates.is_empty() {
+            let remainder_start = remainder_start.unwrap_or_else(|| self.cfg.start_new_block());
+            self.match_candidates(
+                span,
+                scrutinee_span,
+                remainder_start,
+                otherwise_block,
+                candidates,
+                fake_borrows,
+            );
+        }
 
-        self.perform_test(span, scrutinee_span, block, &match_place, &test, make_target_blocks);
+        self.perform_test(span, scrutinee_span, block, &match_place, &test, target_blocks);
     }
 
     /// Determine the fake borrows that are needed from a set of places that
@@ -1843,7 +1841,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         let expr_span = expr.span;
         let expr_place_builder = unpack!(block = self.lower_scrutinee(block, expr, expr_span));
         let wildcard = Pat::wildcard_from_ty(pat.ty);
-        let mut guard_candidate = Candidate::new(expr_place_builder.clone(), &pat, false, self);
+        let mut guard_candidate = Candidate::new(expr_place_builder.clone(), pat, false, self);
         let mut otherwise_candidate =
             Candidate::new(expr_place_builder.clone(), &wildcard, false, self);
         let fake_borrow_temps = self.lower_match_tree(
diff --git a/compiler/rustc_mir_build/src/build/matches/simplify.rs b/compiler/rustc_mir_build/src/build/matches/simplify.rs
index 6a40c8d84..a7f6f4873 100644
--- a/compiler/rustc_mir_build/src/build/matches/simplify.rs
+++ b/compiler/rustc_mir_build/src/build/matches/simplify.rs
@@ -69,7 +69,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             {
                 existing_bindings.extend_from_slice(&new_bindings);
                 mem::swap(&mut candidate.bindings, &mut existing_bindings);
-                candidate.subcandidates = self.create_or_subcandidates(candidate, &place, pats);
+                candidate.subcandidates = self.create_or_subcandidates(candidate, place, pats);
                 return true;
             }
 
@@ -194,6 +194,12 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 Ok(())
             }
 
+            PatKind::Never => {
+                // A never pattern acts like a load from the place.
+                // FIXME(never_patterns): load from the place
+                Ok(())
+            }
+
             PatKind::Constant { .. } => {
                 // FIXME normalize patterns when possible
                 Err(match_pair)
diff --git a/compiler/rustc_mir_build/src/build/matches/test.rs b/compiler/rustc_mir_build/src/build/matches/test.rs
index bdd4f2011..53e5d70f9 100644
--- a/compiler/rustc_mir_build/src/build/matches/test.rs
+++ b/compiler/rustc_mir_build/src/build/matches/test.rs
@@ -75,6 +75,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             | PatKind::Array { .. }
             | PatKind::Wild
             | PatKind::Binding { .. }
+            | PatKind::Never
             | PatKind::Leaf { .. }
             | PatKind::Deref { .. }
             | PatKind::Error(_) => self.error_simplifiable(match_pair),
@@ -107,6 +108,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             PatKind::Slice { .. }
             | PatKind::Array { .. }
             | PatKind::Wild
+            | PatKind::Never
             | PatKind::Or { .. }
             | PatKind::Binding { .. }
             | PatKind::AscribeUserType { .. }
@@ -145,7 +147,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         }
     }
 
-    #[instrument(skip(self, make_target_blocks, place_builder), level = "debug")]
+    #[instrument(skip(self, target_blocks, place_builder), level = "debug")]
     pub(super) fn perform_test(
         &mut self,
         match_start_span: Span,
@@ -153,7 +155,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         block: BasicBlock,
         place_builder: &PlaceBuilder<'tcx>,
         test: &Test<'tcx>,
-        make_target_blocks: impl FnOnce(&mut Self) -> Vec<BasicBlock>,
+        target_blocks: Vec<BasicBlock>,
     ) {
         let place = place_builder.to_place(self);
         let place_ty = place.ty(&self.local_decls, self.tcx);
@@ -162,7 +164,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         let source_info = self.source_info(test.span);
         match test.kind {
             TestKind::Switch { adt_def, ref variants } => {
-                let target_blocks = make_target_blocks(self);
                 // Variants is a BitVec of indexes into adt_def.variants.
                 let num_enum_variants = adt_def.variants().len();
                 debug_assert_eq!(target_blocks.len(), num_enum_variants + 1);
@@ -208,7 +209,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             }
 
             TestKind::SwitchInt { switch_ty, ref options } => {
-                let target_blocks = make_target_blocks(self);
                 let terminator = if *switch_ty.kind() == ty::Bool {
                     assert!(!options.is_empty() && options.len() <= 2);
                     let [first_bb, second_bb] = *target_blocks else {
@@ -238,6 +238,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
             TestKind::Eq { value, ty } => {
                 let tcx = self.tcx;
+                let [success_block, fail_block] = *target_blocks else {
+                    bug!("`TestKind::Eq` should have two target blocks")
+                };
                 if let ty::Adt(def, _) = ty.kind()
                     && Some(def.did()) == tcx.lang_items().string()
                 {
@@ -278,38 +281,43 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     );
                     self.non_scalar_compare(
                         eq_block,
-                        make_target_blocks,
+                        success_block,
+                        fail_block,
                         source_info,
                         value,
                         ref_str,
                         ref_str_ty,
                     );
-                    return;
-                }
-                if !ty.is_scalar() {
+                } else if !ty.is_scalar() {
                     // Use `PartialEq::eq` instead of `BinOp::Eq`
                     // (the binop can only handle primitives)
                     self.non_scalar_compare(
                         block,
-                        make_target_blocks,
+                        success_block,
+                        fail_block,
                         source_info,
                         value,
                         place,
                         ty,
                     );
-                } else if let [success, fail] = *make_target_blocks(self) {
+                } else {
                     assert_eq!(value.ty(), ty);
                     let expect = self.literal_operand(test.span, value);
                     let val = Operand::Copy(place);
-                    self.compare(block, success, fail, source_info, BinOp::Eq, expect, val);
-                } else {
-                    bug!("`TestKind::Eq` should have two target blocks");
+                    self.compare(
+                        block,
+                        success_block,
+                        fail_block,
+                        source_info,
+                        BinOp::Eq,
+                        expect,
+                        val,
+                    );
                 }
             }
 
             TestKind::Range(ref range) => {
                 let lower_bound_success = self.cfg.start_new_block();
-                let target_blocks = make_target_blocks(self);
 
                 // Test `val` by computing `lo <= val && val <= hi`, using primitive comparisons.
                 // FIXME: skip useless comparison when the range is half-open.
@@ -339,8 +347,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             }
 
             TestKind::Len { len, op } => {
-                let target_blocks = make_target_blocks(self);
-
                 let usize_ty = self.tcx.types.usize;
                 let actual = self.temp(usize_ty, test.span);
 
@@ -404,7 +410,8 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     fn non_scalar_compare(
         &mut self,
         block: BasicBlock,
-        make_target_blocks: impl FnOnce(&mut Self) -> Vec<BasicBlock>,
+        success_block: BasicBlock,
+        fail_block: BasicBlock,
         source_info: SourceInfo,
         value: Const<'tcx>,
         mut val: Place<'tcx>,
@@ -494,7 +501,12 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         }
 
         let eq_def_id = self.tcx.require_lang_item(LangItem::PartialEq, Some(source_info.span));
-        let method = trait_method(self.tcx, eq_def_id, sym::eq, [ty, ty]);
+        let method = trait_method(
+            self.tcx,
+            eq_def_id,
+            sym::eq,
+            self.tcx.with_opt_host_effect_param(self.def_id, eq_def_id, [ty, ty]),
+        );
 
         let bool_ty = self.tcx.types.bool;
         let eq_result = self.temp(bool_ty, source_info.span);
@@ -524,9 +536,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         );
         self.diverge_from(block);
 
-        let [success_block, fail_block] = *make_target_blocks(self) else {
-            bug!("`TestKind::Eq` should have two target blocks")
-        };
         // check the result
         self.cfg.terminate(
             eq_block,
@@ -736,7 +745,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 // These are all binary tests.
                 //
                 // FIXME(#29623) we can be more clever here
-                let pattern_test = self.test(&match_pair);
+                let pattern_test = self.test(match_pair);
                 if pattern_test.kind == test.kind {
                     self.candidate_without_match_pair(match_pair_index, candidate);
                     Some(0)
diff --git a/compiler/rustc_mir_build/src/build/mod.rs b/compiler/rustc_mir_build/src/build/mod.rs
index 886d80545..dae83d4b4 100644
--- a/compiler/rustc_mir_build/src/build/mod.rs
+++ b/compiler/rustc_mir_build/src/build/mod.rs
@@ -451,7 +451,7 @@ fn construct_fn<'tcx>(
     fn_sig: ty::FnSig<'tcx>,
 ) -> Body<'tcx> {
     let span = tcx.def_span(fn_def);
-    let fn_id = tcx.hir().local_def_id_to_hir_id(fn_def);
+    let fn_id = tcx.local_def_id_to_hir_id(fn_def);
     let coroutine_kind = tcx.coroutine_kind(fn_def);
 
     // The representation of thir for `-Zunpretty=thir-tree` relies on
@@ -569,10 +569,10 @@ fn construct_const<'a, 'tcx>(
     expr: ExprId,
     const_ty: Ty<'tcx>,
 ) -> Body<'tcx> {
-    let hir_id = tcx.hir().local_def_id_to_hir_id(def);
+    let hir_id = tcx.local_def_id_to_hir_id(def);
 
     // Figure out what primary body this item has.
-    let (span, const_ty_span) = match tcx.hir().get(hir_id) {
+    let (span, const_ty_span) = match tcx.hir_node(hir_id) {
         Node::Item(hir::Item {
             kind: hir::ItemKind::Static(ty, _, _) | hir::ItemKind::Const(ty, _, _),
             span,
@@ -622,7 +622,7 @@ fn construct_const<'a, 'tcx>(
 /// with type errors, but normal MIR construction can't handle that in general.
 fn construct_error(tcx: TyCtxt<'_>, def_id: LocalDefId, guar: ErrorGuaranteed) -> Body<'_> {
     let span = tcx.def_span(def_id);
-    let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
+    let hir_id = tcx.local_def_id_to_hir_id(def_id);
     let coroutine_kind = tcx.coroutine_kind(def_id);
 
     let (inputs, output, yield_ty) = match tcx.def_kind(def_id) {
@@ -638,9 +638,21 @@ fn construct_error(tcx: TyCtxt<'_>, def_id: LocalDefId, guar: ErrorGuaranteed) -
             );
             (sig.inputs().to_vec(), sig.output(), None)
         }
+        DefKind::Closure if coroutine_kind.is_some() => {
+            let coroutine_ty = tcx.type_of(def_id).instantiate_identity();
+            let ty::Coroutine(_, args, _) = coroutine_ty.kind() else {
+                bug!("expected type of coroutine-like closure to be a coroutine")
+            };
+            let args = args.as_coroutine();
+            let yield_ty = args.yield_ty();
+            let return_ty = args.return_ty();
+            (vec![coroutine_ty, args.resume_ty()], return_ty, Some(yield_ty))
+        }
         DefKind::Closure => {
             let closure_ty = tcx.type_of(def_id).instantiate_identity();
-            let ty::Closure(_, args) = closure_ty.kind() else { bug!() };
+            let ty::Closure(_, args) = closure_ty.kind() else {
+                bug!("expected type of closure to be a closure")
+            };
             let args = args.as_closure();
             let sig = tcx.liberate_late_bound_regions(def_id.to_def_id(), args.sig());
             let self_ty = match args.kind() {
@@ -650,14 +662,6 @@ fn construct_error(tcx: TyCtxt<'_>, def_id: LocalDefId, guar: ErrorGuaranteed) -
             };
             ([self_ty].into_iter().chain(sig.inputs().to_vec()).collect(), sig.output(), None)
         }
-        DefKind::Coroutine => {
-            let coroutine_ty = tcx.type_of(def_id).instantiate_identity();
-            let ty::Coroutine(_, args, _) = coroutine_ty.kind() else { bug!() };
-            let args = args.as_coroutine();
-            let yield_ty = args.yield_ty();
-            let return_ty = args.return_ty();
-            (vec![coroutine_ty, args.resume_ty()], return_ty, Some(yield_ty))
-        }
         dk => bug!("{:?} is not a body: {:?}", def_id, dk),
     };
 
@@ -938,12 +942,12 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     scope = self.declare_bindings(
                         scope,
                         expr.span,
-                        &pat,
+                        pat,
                         None,
                         Some((Some(&place), span)),
                     );
                     let place_builder = PlaceBuilder::from(local);
-                    unpack!(block = self.place_into_pattern(block, &pat, place_builder, false));
+                    unpack!(block = self.place_into_pattern(block, pat, place_builder, false));
                 }
             }
             self.source_scope = original_source_scope;
@@ -954,7 +958,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             self.source_scope = source_scope;
         }
 
-        self.expr_into_dest(Place::return_place(), block, &expr)
+        self.expr_into_dest(Place::return_place(), block, expr)
     }
 
     fn set_correct_source_scope_for_arg(
diff --git a/compiler/rustc_mir_build/src/build/scope.rs b/compiler/rustc_mir_build/src/build/scope.rs
index b3d3863b5..25b79e6a5 100644
--- a/compiler/rustc_mir_build/src/build/scope.rs
+++ b/compiler/rustc_mir_build/src/build/scope.rs
@@ -90,7 +90,6 @@ use rustc_index::{IndexSlice, IndexVec};
 use rustc_middle::middle::region;
 use rustc_middle::mir::*;
 use rustc_middle::thir::{Expr, LintLevel};
-use rustc_middle::ty::Ty;
 use rustc_session::lint::Level;
 use rustc_span::{Span, DUMMY_SP};
 
@@ -186,6 +185,7 @@ pub(crate) enum BreakableTarget {
 }
 
 rustc_index::newtype_index! {
+    #[orderable]
     struct DropIdx {}
 }
 
@@ -536,27 +536,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         (then_block, else_block)
     }
 
-    pub(crate) fn in_opt_scope<F, R>(
-        &mut self,
-        opt_scope: Option<(region::Scope, SourceInfo)>,
-        f: F,
-    ) -> BlockAnd<R>
-    where
-        F: FnOnce(&mut Builder<'a, 'tcx>) -> BlockAnd<R>,
-    {
-        debug!("in_opt_scope(opt_scope={:?})", opt_scope);
-        if let Some(region_scope) = opt_scope {
-            self.push_scope(region_scope);
-        }
-        let mut block;
-        let rv = unpack!(block = f(self));
-        if let Some(region_scope) = opt_scope {
-            unpack!(block = self.pop_scope(region_scope, block));
-        }
-        debug!("in_scope: exiting opt_scope={:?} block={:?}", opt_scope, block);
-        block.and(rv)
-    }
-
     /// Convenience wrapper that pushes a scope and then executes `f`
     /// to build its contents, popping the scope afterwards.
     #[instrument(skip(self, f), level = "debug")]
@@ -659,14 +638,15 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             (None, Some(_)) => {
                 panic!("`return`, `become` and `break` with value and must have a destination")
             }
-            (None, None) if self.tcx.sess.instrument_coverage() => {
-                // Unlike `break` and `return`, which push an `Assign` statement to MIR, from which
-                // a Coverage code region can be generated, `continue` needs no `Assign`; but
-                // without one, the `InstrumentCoverage` MIR pass cannot generate a code region for
-                // `continue`. Coverage will be missing unless we add a dummy `Assign` to MIR.
-                self.add_dummy_assignment(span, block, source_info);
+            (None, None) => {
+                if self.tcx.sess.instrument_coverage() {
+                    // Normally we wouldn't build any MIR in this case, but that makes it
+                    // harder for coverage instrumentation to extract a relevant span for
+                    // `continue` expressions. So here we inject a dummy statement with the
+                    // desired span.
+                    self.cfg.push_coverage_span_marker(block, source_info);
+                }
             }
-            (None, None) => {}
         }
 
         let region_scope = self.scopes.breakable_scopes[break_index].region_scope;
@@ -722,14 +702,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         self.cfg.terminate(block, source_info, TerminatorKind::UnwindResume);
     }
 
-    // Add a dummy `Assign` statement to the CFG, with the span for the source code's `continue`
-    // statement.
-    fn add_dummy_assignment(&mut self, span: Span, block: BasicBlock, source_info: SourceInfo) {
-        let local_decl = LocalDecl::new(Ty::new_unit(self.tcx), span);
-        let temp_place = Place::from(self.local_decls.push(local_decl));
-        self.cfg.push_assign_unit(block, source_info, temp_place, self.tcx);
-    }
-
     fn leave_top_scope(&mut self, block: BasicBlock) -> BasicBlock {
         // If we are emitting a `drop` statement, we need to have the cached
         // diverge cleanup pads ready in case that drop panics.
@@ -995,7 +967,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             }
 
             if scope.region_scope == region_scope {
-                let region_scope_span = region_scope.span(self.tcx, &self.region_scope_tree);
+                let region_scope_span = region_scope.span(self.tcx, self.region_scope_tree);
                 // Attribute scope exit drops to scope's closing brace.
                 let scope_end = self.tcx.sess.source_map().end_point(region_scope_span);
 
diff --git a/compiler/rustc_mir_build/src/check_unsafety.rs b/compiler/rustc_mir_build/src/check_unsafety.rs
index fcb563607..62190848d 100644
--- a/compiler/rustc_mir_build/src/check_unsafety.rs
+++ b/compiler/rustc_mir_build/src/check_unsafety.rs
@@ -1,7 +1,10 @@
+use std::borrow::Cow;
+
 use crate::build::ExprCategory;
 use crate::errors::*;
-use rustc_middle::thir::visit::{self, Visitor};
+use rustc_middle::thir::visit::Visitor;
 
+use rustc_errors::DiagnosticArgValue;
 use rustc_hir as hir;
 use rustc_middle::mir::BorrowKind;
 use rustc_middle::thir::*;
@@ -138,7 +141,7 @@ impl<'tcx> UnsafetyVisitor<'_, 'tcx> {
             // Runs all other queries that depend on THIR.
             self.tcx.ensure_with_value().mir_built(def);
             let inner_thir = &inner_thir.steal();
-            let hir_context = self.tcx.hir().local_def_id_to_hir_id(def);
+            let hir_context = self.tcx.local_def_id_to_hir_id(def);
             let safety_context = mem::replace(&mut self.safety_context, SafetyContext::Safe);
             let mut inner_visitor = UnsafetyVisitor {
                 thir: inner_thir,
@@ -200,7 +203,7 @@ impl<'a, 'tcx> Visitor<'a, 'tcx> for LayoutConstrainedPlaceVisitor<'a, 'tcx> {
 
 impl<'a, 'tcx> Visitor<'a, 'tcx> for UnsafetyVisitor<'a, 'tcx> {
     fn thir(&self) -> &'a Thir<'tcx> {
-        &self.thir
+        self.thir
     }
 
     fn visit_block(&mut self, block: &Block) {
@@ -247,8 +250,9 @@ impl<'a, 'tcx> Visitor<'a, 'tcx> for UnsafetyVisitor<'a, 'tcx> {
                     self.requires_unsafe(pat.span, AccessToUnionField);
                     return; // we can return here since this already requires unsafe
                 }
-                // wildcard doesn't take anything
+                // wildcard/never don't take anything
                 PatKind::Wild |
+                PatKind::Never |
                 // these just wrap other patterns
                 PatKind::Or { .. } |
                 PatKind::InlineConstant { .. } |
@@ -392,15 +396,29 @@ impl<'a, 'tcx> Visitor<'a, 'tcx> for UnsafetyVisitor<'a, 'tcx> {
                     // the call requires `unsafe`. Don't check this on wasm
                     // targets, though. For more information on wasm see the
                     // is_like_wasm check in hir_analysis/src/collect.rs
+                    let callee_features = &self.tcx.codegen_fn_attrs(func_did).target_features;
                     if !self.tcx.sess.target.options.is_like_wasm
-                        && !self
-                            .tcx
-                            .codegen_fn_attrs(func_did)
-                            .target_features
+                        && !callee_features
                             .iter()
                             .all(|feature| self.body_target_features.contains(feature))
                     {
-                        self.requires_unsafe(expr.span, CallToFunctionWith(func_did));
+                        let missing: Vec<_> = callee_features
+                            .iter()
+                            .copied()
+                            .filter(|feature| !self.body_target_features.contains(feature))
+                            .collect();
+                        let build_enabled = self
+                            .tcx
+                            .sess
+                            .target_features
+                            .iter()
+                            .copied()
+                            .filter(|feature| missing.contains(feature))
+                            .collect();
+                        self.requires_unsafe(
+                            expr.span,
+                            CallToFunctionWith { function: func_did, missing, build_enabled },
+                        );
                     }
                 }
             }
@@ -452,7 +470,7 @@ impl<'a, 'tcx> Visitor<'a, 'tcx> for UnsafetyVisitor<'a, 'tcx> {
                     if let Some((assigned_ty, assignment_span)) = self.assignment_info {
                         if assigned_ty.needs_drop(self.tcx, self.param_env) {
                             // This would be unsafe, but should be outright impossible since we reject such unions.
-                            self.tcx.sess.delay_span_bug(assignment_span, format!("union fields that need dropping should be impossible: {assigned_ty}"));
+                            self.tcx.sess.span_delayed_bug(assignment_span, format!("union fields that need dropping should be impossible: {assigned_ty}"));
                         }
                     } else {
                         self.requires_unsafe(expr.span, AccessToUnionField);
@@ -526,7 +544,7 @@ struct UnusedUnsafeWarning {
     enclosing_unsafe: Option<UnusedUnsafeEnclosing>,
 }
 
-#[derive(Clone, Copy, PartialEq)]
+#[derive(Clone, PartialEq)]
 enum UnsafeOpKind {
     CallToUnsafeFunction(Option<DefId>),
     UseOfInlineAssembly,
@@ -537,7 +555,15 @@ enum UnsafeOpKind {
     AccessToUnionField,
     MutationOfLayoutConstrainedField,
     BorrowOfLayoutConstrainedField,
-    CallToFunctionWith(DefId),
+    CallToFunctionWith {
+        function: DefId,
+        /// Target features enabled in callee's `#[target_feature]` but missing in
+        /// caller's `#[target_feature]`.
+        missing: Vec<Symbol>,
+        /// Target features in `missing` that are enabled at compile time
+        /// (e.g., with `-C target-feature`).
+        build_enabled: Vec<Symbol>,
+    },
 }
 
 use UnsafeOpKind::*;
@@ -552,7 +578,7 @@ impl UnsafeOpKind {
     ) {
         let parent_id = tcx.hir().get_parent_item(hir_id);
         let parent_owner = tcx.hir().owner(parent_id);
-        let should_suggest = parent_owner.fn_sig().map_or(false, |sig| sig.header.is_unsafe());
+        let should_suggest = parent_owner.fn_sig().is_some_and(|sig| sig.header.is_unsafe());
         let unsafe_not_inherited_note = if should_suggest {
             suggest_unsafe_block.then(|| {
                 let body_span = tcx.hir().body(parent_owner.body_id().unwrap()).value.span;
@@ -658,13 +684,22 @@ impl UnsafeOpKind {
                     unsafe_not_inherited_note,
                 },
             ),
-            CallToFunctionWith(did) => tcx.emit_spanned_lint(
+            CallToFunctionWith { function, missing, build_enabled } => tcx.emit_spanned_lint(
                 UNSAFE_OP_IN_UNSAFE_FN,
                 hir_id,
                 span,
                 UnsafeOpInUnsafeFnCallToFunctionWithRequiresUnsafe {
                     span,
-                    function: &with_no_trimmed_paths!(tcx.def_path_str(*did)),
+                    function: &with_no_trimmed_paths!(tcx.def_path_str(*function)),
+                    missing_target_features: DiagnosticArgValue::StrListSepByAnd(
+                        missing.iter().map(|feature| Cow::from(feature.as_str())).collect(),
+                    ),
+                    missing_target_features_count: missing.len(),
+                    note: if build_enabled.is_empty() { None } else { Some(()) },
+                    build_target_features: DiagnosticArgValue::StrListSepByAnd(
+                        build_enabled.iter().map(|feature| Cow::from(feature.as_str())).collect(),
+                    ),
+                    build_target_features_count: build_enabled.len(),
                     unsafe_not_inherited_note,
                 },
             ),
@@ -821,18 +856,38 @@ impl UnsafeOpKind {
                     unsafe_not_inherited_note,
                 });
             }
-            CallToFunctionWith(did) if unsafe_op_in_unsafe_fn_allowed => {
+            CallToFunctionWith { function, missing, build_enabled }
+                if unsafe_op_in_unsafe_fn_allowed =>
+            {
                 tcx.sess.emit_err(CallToFunctionWithRequiresUnsafeUnsafeOpInUnsafeFnAllowed {
                     span,
+                    missing_target_features: DiagnosticArgValue::StrListSepByAnd(
+                        missing.iter().map(|feature| Cow::from(feature.as_str())).collect(),
+                    ),
+                    missing_target_features_count: missing.len(),
+                    note: if build_enabled.is_empty() { None } else { Some(()) },
+                    build_target_features: DiagnosticArgValue::StrListSepByAnd(
+                        build_enabled.iter().map(|feature| Cow::from(feature.as_str())).collect(),
+                    ),
+                    build_target_features_count: build_enabled.len(),
                     unsafe_not_inherited_note,
-                    function: &tcx.def_path_str(*did),
+                    function: &tcx.def_path_str(*function),
                 });
             }
-            CallToFunctionWith(did) => {
+            CallToFunctionWith { function, missing, build_enabled } => {
                 tcx.sess.emit_err(CallToFunctionWithRequiresUnsafe {
                     span,
+                    missing_target_features: DiagnosticArgValue::StrListSepByAnd(
+                        missing.iter().map(|feature| Cow::from(feature.as_str())).collect(),
+                    ),
+                    missing_target_features_count: missing.len(),
+                    note: if build_enabled.is_empty() { None } else { Some(()) },
+                    build_target_features: DiagnosticArgValue::StrListSepByAnd(
+                        build_enabled.iter().map(|feature| Cow::from(feature.as_str())).collect(),
+                    ),
+                    build_target_features_count: build_enabled.len(),
                     unsafe_not_inherited_note,
-                    function: &tcx.def_path_str(*did),
+                    function: &tcx.def_path_str(*function),
                 });
             }
         }
@@ -859,7 +914,7 @@ pub fn thir_check_unsafety(tcx: TyCtxt<'_>, def: LocalDefId) {
         return;
     }
 
-    let hir_id = tcx.hir().local_def_id_to_hir_id(def);
+    let hir_id = tcx.local_def_id_to_hir_id(def);
     let safety_context = tcx.hir().fn_sig_by_hir_id(hir_id).map_or(SafetyContext::Safe, |fn_sig| {
         if fn_sig.header.unsafety == hir::Unsafety::Unsafe {
             SafetyContext::UnsafeFn
diff --git a/compiler/rustc_mir_build/src/errors.rs b/compiler/rustc_mir_build/src/errors.rs
index 418f9bb9d..c66687330 100644
--- a/compiler/rustc_mir_build/src/errors.rs
+++ b/compiler/rustc_mir_build/src/errors.rs
@@ -1,14 +1,12 @@
-use crate::{
-    fluent_generated as fluent,
-    thir::pattern::{deconstruct_pat::WitnessPat, MatchCheckCtxt},
-};
+use crate::fluent_generated as fluent;
+use rustc_errors::DiagnosticArgValue;
 use rustc_errors::{
-    error_code, AddToDiagnostic, Applicability, Diagnostic, DiagnosticBuilder, ErrorGuaranteed,
-    Handler, IntoDiagnostic, MultiSpan, SubdiagnosticMessage,
+    error_code, AddToDiagnostic, Applicability, DiagCtxt, Diagnostic, DiagnosticBuilder,
+    ErrorGuaranteed, IntoDiagnostic, MultiSpan, SubdiagnosticMessage,
 };
 use rustc_macros::{Diagnostic, LintDiagnostic, Subdiagnostic};
-use rustc_middle::thir::Pat;
 use rustc_middle::ty::{self, Ty};
+use rustc_pattern_analysis::{errors::Uncovered, rustc::RustcMatchCheckCtxt};
 use rustc_span::symbol::Symbol;
 use rustc_span::Span;
 
@@ -124,11 +122,17 @@ pub struct UnsafeOpInUnsafeFnBorrowOfLayoutConstrainedFieldRequiresUnsafe {
 
 #[derive(LintDiagnostic)]
 #[diag(mir_build_unsafe_op_in_unsafe_fn_call_to_fn_with_requires_unsafe)]
-#[note]
+#[help]
 pub struct UnsafeOpInUnsafeFnCallToFunctionWithRequiresUnsafe<'a> {
     #[label]
     pub span: Span,
     pub function: &'a str,
+    pub missing_target_features: DiagnosticArgValue<'a>,
+    pub missing_target_features_count: usize,
+    #[note]
+    pub note: Option<()>,
+    pub build_target_features: DiagnosticArgValue<'a>,
+    pub build_target_features_count: usize,
     #[subdiagnostic]
     pub unsafe_not_inherited_note: Option<UnsafeNotInheritedLintNote>,
 }
@@ -369,24 +373,36 @@ pub struct BorrowOfLayoutConstrainedFieldRequiresUnsafeUnsafeOpInUnsafeFnAllowed
 
 #[derive(Diagnostic)]
 #[diag(mir_build_call_to_fn_with_requires_unsafe, code = "E0133")]
-#[note]
+#[help]
 pub struct CallToFunctionWithRequiresUnsafe<'a> {
     #[primary_span]
     #[label]
     pub span: Span,
     pub function: &'a str,
+    pub missing_target_features: DiagnosticArgValue<'a>,
+    pub missing_target_features_count: usize,
+    #[note]
+    pub note: Option<()>,
+    pub build_target_features: DiagnosticArgValue<'a>,
+    pub build_target_features_count: usize,
     #[subdiagnostic]
     pub unsafe_not_inherited_note: Option<UnsafeNotInheritedNote>,
 }
 
 #[derive(Diagnostic)]
 #[diag(mir_build_call_to_fn_with_requires_unsafe_unsafe_op_in_unsafe_fn_allowed, code = "E0133")]
-#[note]
+#[help]
 pub struct CallToFunctionWithRequiresUnsafeUnsafeOpInUnsafeFnAllowed<'a> {
     #[primary_span]
     #[label]
     pub span: Span,
     pub function: &'a str,
+    pub missing_target_features: DiagnosticArgValue<'a>,
+    pub missing_target_features_count: usize,
+    #[note]
+    pub note: Option<()>,
+    pub build_target_features: DiagnosticArgValue<'a>,
+    pub build_target_features_count: usize,
     #[subdiagnostic]
     pub unsafe_not_inherited_note: Option<UnsafeNotInheritedNote>,
 }
@@ -438,15 +454,15 @@ pub enum UnusedUnsafeEnclosing {
 }
 
 pub(crate) struct NonExhaustivePatternsTypeNotEmpty<'p, 'tcx, 'm> {
-    pub cx: &'m MatchCheckCtxt<'p, 'tcx>,
+    pub cx: &'m RustcMatchCheckCtxt<'p, 'tcx>,
     pub expr_span: Span,
     pub span: Span,
     pub ty: Ty<'tcx>,
 }
 
 impl<'a> IntoDiagnostic<'a> for NonExhaustivePatternsTypeNotEmpty<'_, '_, '_> {
-    fn into_diagnostic(self, handler: &'a Handler) -> DiagnosticBuilder<'_, ErrorGuaranteed> {
-        let mut diag = handler.struct_span_err_with_code(
+    fn into_diagnostic(self, dcx: &'a DiagCtxt) -> DiagnosticBuilder<'_, ErrorGuaranteed> {
+        let mut diag = dcx.struct_span_err_with_code(
             self.span,
             fluent::mir_build_non_exhaustive_patterns_type_not_empty,
             error_code!(E0004),
@@ -793,94 +809,6 @@ pub struct NonPartialEqMatch<'tcx> {
     pub non_peq_ty: Ty<'tcx>,
 }
 
-#[derive(LintDiagnostic)]
-#[diag(mir_build_overlapping_range_endpoints)]
-#[note]
-pub struct OverlappingRangeEndpoints<'tcx> {
-    #[label(mir_build_range)]
-    pub range: Span,
-    #[subdiagnostic]
-    pub overlap: Vec<Overlap<'tcx>>,
-}
-
-pub struct Overlap<'tcx> {
-    pub span: Span,
-    pub range: Pat<'tcx>,
-}
-
-impl<'tcx> AddToDiagnostic for Overlap<'tcx> {
-    fn add_to_diagnostic_with<F>(self, diag: &mut Diagnostic, _: F)
-    where
-        F: Fn(&mut Diagnostic, SubdiagnosticMessage) -> SubdiagnosticMessage,
-    {
-        let Overlap { span, range } = self;
-
-        // FIXME(mejrs) unfortunately `#[derive(LintDiagnostic)]`
-        // does not support `#[subdiagnostic(eager)]`...
-        let message = format!("this range overlaps on `{range}`...");
-        diag.span_label(span, message);
-    }
-}
-
-#[derive(LintDiagnostic)]
-#[diag(mir_build_non_exhaustive_omitted_pattern)]
-#[help]
-#[note]
-pub(crate) struct NonExhaustiveOmittedPattern<'tcx> {
-    pub scrut_ty: Ty<'tcx>,
-    #[subdiagnostic]
-    pub uncovered: Uncovered<'tcx>,
-}
-
-#[derive(LintDiagnostic)]
-#[diag(mir_build_non_exhaustive_omitted_pattern_lint_on_arm)]
-#[help]
-pub(crate) struct NonExhaustiveOmittedPatternLintOnArm {
-    #[label]
-    pub lint_span: Span,
-    #[suggestion(code = "#[{lint_level}({lint_name})]\n", applicability = "maybe-incorrect")]
-    pub suggest_lint_on_match: Option<Span>,
-    pub lint_level: &'static str,
-    pub lint_name: &'static str,
-}
-
-#[derive(Subdiagnostic)]
-#[label(mir_build_uncovered)]
-pub(crate) struct Uncovered<'tcx> {
-    #[primary_span]
-    span: Span,
-    count: usize,
-    witness_1: Pat<'tcx>,
-    witness_2: Pat<'tcx>,
-    witness_3: Pat<'tcx>,
-    remainder: usize,
-}
-
-impl<'tcx> Uncovered<'tcx> {
-    pub fn new<'p>(
-        span: Span,
-        cx: &MatchCheckCtxt<'p, 'tcx>,
-        witnesses: Vec<WitnessPat<'tcx>>,
-    ) -> Self {
-        let witness_1 = witnesses.get(0).unwrap().to_diagnostic_pat(cx);
-        Self {
-            span,
-            count: witnesses.len(),
-            // Substitute dummy values if witnesses is smaller than 3. These will never be read.
-            witness_2: witnesses
-                .get(1)
-                .map(|w| w.to_diagnostic_pat(cx))
-                .unwrap_or_else(|| witness_1.clone()),
-            witness_3: witnesses
-                .get(2)
-                .map(|w| w.to_diagnostic_pat(cx))
-                .unwrap_or_else(|| witness_1.clone()),
-            witness_1,
-            remainder: witnesses.len().saturating_sub(3),
-        }
-    }
-}
-
 #[derive(Diagnostic)]
 #[diag(mir_build_pattern_not_covered, code = "E0005")]
 pub(crate) struct PatternNotCovered<'s, 'tcx> {
diff --git a/compiler/rustc_mir_build/src/lib.rs b/compiler/rustc_mir_build/src/lib.rs
index 745c3046d..a776e917d 100644
--- a/compiler/rustc_mir_build/src/lib.rs
+++ b/compiler/rustc_mir_build/src/lib.rs
@@ -1,7 +1,6 @@
 //! Construction of MIR from HIR.
 //!
 //! This crate also contains the match exhaustiveness and usefulness checking.
-#![allow(rustc::potential_query_instability)]
 #![feature(assert_matches)]
 #![feature(associated_type_bounds)]
 #![feature(box_patterns)]
@@ -24,10 +23,7 @@ mod thir;
 
 use rustc_middle::query::Providers;
 
-use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
-use rustc_fluent_macro::fluent_messages;
-
-fluent_messages! { "../messages.ftl" }
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
 
 pub fn provide(providers: &mut Providers) {
     providers.check_match = thir::pattern::check_match;
diff --git a/compiler/rustc_mir_build/src/lints.rs b/compiler/rustc_mir_build/src/lints.rs
index acf4d6bc2..167b65328 100644
--- a/compiler/rustc_mir_build/src/lints.rs
+++ b/compiler/rustc_mir_build/src/lints.rs
@@ -52,7 +52,7 @@ fn check_recursion<'tcx>(
         vis.reachable_recursive_calls.sort();
 
         let sp = tcx.def_span(def_id);
-        let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
+        let hir_id = tcx.local_def_id_to_hir_id(def_id);
         tcx.emit_spanned_lint(
             UNCONDITIONAL_RECURSION,
             hir_id,
diff --git a/compiler/rustc_mir_build/src/thir/constant.rs b/compiler/rustc_mir_build/src/thir/constant.rs
index fbb74650f..240aa10ac 100644
--- a/compiler/rustc_mir_build/src/thir/constant.rs
+++ b/compiler/rustc_mir_build/src/thir/constant.rs
@@ -16,7 +16,7 @@ pub(crate) fn lit_to_const<'tcx>(
         let width = tcx
             .layout_of(param_ty)
             .map_err(|_| {
-                LitToConstError::Reported(tcx.sess.delay_span_bug(
+                LitToConstError::Reported(tcx.sess.span_delayed_bug(
                     DUMMY_SP,
                     format!("couldn't compute width of literal: {:?}", lit_input.lit),
                 ))
@@ -62,7 +62,7 @@ pub(crate) fn lit_to_const<'tcx>(
         (ast::LitKind::Float(n, _), ty::Float(fty)) => {
             let bits = parse_float_into_scalar(*n, *fty, neg)
                 .ok_or_else(|| {
-                    LitToConstError::Reported(tcx.sess.delay_span_bug(
+                    LitToConstError::Reported(tcx.sess.span_delayed_bug(
                         DUMMY_SP,
                         format!("couldn't parse float literal: {:?}", lit_input.lit),
                     ))
@@ -73,7 +73,7 @@ pub(crate) fn lit_to_const<'tcx>(
         (ast::LitKind::Char(c), ty::Char) => ty::ValTree::from_scalar_int((*c).into()),
         (ast::LitKind::Err, _) => {
             return Err(LitToConstError::Reported(
-                tcx.sess.delay_span_bug(DUMMY_SP, "encountered LitKind::Err during mir build"),
+                tcx.sess.span_delayed_bug(DUMMY_SP, "encountered LitKind::Err during mir build"),
             ));
         }
         _ => return Err(LitToConstError::TypeError),
diff --git a/compiler/rustc_mir_build/src/thir/cx/block.rs b/compiler/rustc_mir_build/src/thir/cx/block.rs
index a46ad6423..1e93e126b 100644
--- a/compiler/rustc_mir_build/src/thir/cx/block.rs
+++ b/compiler/rustc_mir_build/src/thir/cx/block.rs
@@ -13,15 +13,12 @@ impl<'tcx> Cx<'tcx> {
         // We have to eagerly lower the "spine" of the statements
         // in order to get the lexical scoping correctly.
         let stmts = self.mirror_stmts(block.hir_id.local_id, block.stmts);
-        let opt_destruction_scope =
-            self.region_scope_tree.opt_destruction_scope(block.hir_id.local_id);
         let block = Block {
             targeted_by_break: block.targeted_by_break,
             region_scope: region::Scope {
                 id: block.hir_id.local_id,
                 data: region::ScopeData::Node,
             },
-            opt_destruction_scope,
             span: block.span,
             stmts,
             expr: block.expr.map(|expr| self.mirror_expr(expr)),
@@ -49,9 +46,8 @@ impl<'tcx> Cx<'tcx> {
             .enumerate()
             .filter_map(|(index, stmt)| {
                 let hir_id = stmt.hir_id;
-                let opt_dxn_ext = self.region_scope_tree.opt_destruction_scope(hir_id.local_id);
                 match stmt.kind {
-                    hir::StmtKind::Expr(ref expr) | hir::StmtKind::Semi(ref expr) => {
+                    hir::StmtKind::Expr(expr) | hir::StmtKind::Semi(expr) => {
                         let stmt = Stmt {
                             kind: StmtKind::Expr {
                                 scope: region::Scope {
@@ -60,7 +56,6 @@ impl<'tcx> Cx<'tcx> {
                                 },
                                 expr: self.mirror_expr(expr),
                             },
-                            opt_destruction_scope: opt_dxn_ext,
                         };
                         Some(self.thir.stmts.push(stmt))
                     }
@@ -68,7 +63,7 @@ impl<'tcx> Cx<'tcx> {
                         // ignore for purposes of the MIR
                         None
                     }
-                    hir::StmtKind::Local(ref local) => {
+                    hir::StmtKind::Local(local) => {
                         let remainder_scope = region::Scope {
                             id: block_id,
                             data: region::ScopeData::Remainder(region::FirstStatementIndex::new(
@@ -122,7 +117,6 @@ impl<'tcx> Cx<'tcx> {
                                 lint_level: LintLevel::Explicit(local.hir_id),
                                 span,
                             },
-                            opt_destruction_scope: opt_dxn_ext,
                         };
                         Some(self.thir.stmts.push(stmt))
                     }
diff --git a/compiler/rustc_mir_build/src/thir/cx/expr.rs b/compiler/rustc_mir_build/src/thir/cx/expr.rs
index dfd39b512..ab8846382 100644
--- a/compiler/rustc_mir_build/src/thir/cx/expr.rs
+++ b/compiler/rustc_mir_build/src/thir/cx/expr.rs
@@ -54,7 +54,7 @@ impl<'tcx> Cx<'tcx> {
 
         trace!(?expr.ty, "after adjustments");
 
-        // Next, wrap this up in the expr's scope.
+        // Finally, wrap this up in the expr's scope.
         expr = Expr {
             temp_lifetime: expr.temp_lifetime,
             ty: expr.ty,
@@ -66,22 +66,6 @@ impl<'tcx> Cx<'tcx> {
             },
         };
 
-        // Finally, create a destruction scope, if any.
-        if let Some(region_scope) =
-            self.region_scope_tree.opt_destruction_scope(hir_expr.hir_id.local_id)
-        {
-            expr = Expr {
-                temp_lifetime: expr.temp_lifetime,
-                ty: expr.ty,
-                span: hir_expr.span,
-                kind: ExprKind::Scope {
-                    region_scope,
-                    value: self.thir.exprs.push(expr),
-                    lint_level: LintLevel::Inherited,
-                },
-            };
-        }
-
         // OK, all done!
         self.thir.exprs.push(expr)
     }
@@ -192,10 +176,10 @@ impl<'tcx> Cx<'tcx> {
                 cast: PointerCoercion::ArrayToPointer,
             }
         } else if let hir::ExprKind::Path(ref qpath) = source.kind
-           && let res = self.typeck_results().qpath_res(qpath, source.hir_id)
-           && let ty = self.typeck_results().node_type(source.hir_id)
-           && let ty::Adt(adt_def, args) = ty.kind()
-           && let Res::Def(DefKind::Ctor(CtorOf::Variant, CtorKind::Const), variant_ctor_id) = res
+            && let res = self.typeck_results().qpath_res(qpath, source.hir_id)
+            && let ty = self.typeck_results().node_type(source.hir_id)
+            && let ty::Adt(adt_def, args) = ty.kind()
+            && let Res::Def(DefKind::Ctor(CtorOf::Variant, CtorKind::Const), variant_ctor_id) = res
         {
             // Check whether this is casting an enum variant discriminant.
             // To prevent cycles, we refer to the discriminant initializer,
@@ -261,7 +245,7 @@ impl<'tcx> Cx<'tcx> {
 
         let kind = match expr.kind {
             // Here comes the interesting stuff:
-            hir::ExprKind::MethodCall(segment, receiver, ref args, fn_span) => {
+            hir::ExprKind::MethodCall(segment, receiver, args, fn_span) => {
                 // Rewrite a.b(c) into UFCS form like Trait::b(a, c)
                 let expr = self.method_callee(expr, segment.ident.span, None);
                 info!("Using method span: {:?}", expr.span);
@@ -278,7 +262,7 @@ impl<'tcx> Cx<'tcx> {
                 }
             }
 
-            hir::ExprKind::Call(ref fun, ref args) => {
+            hir::ExprKind::Call(fun, ref args) => {
                 if self.typeck_results().is_method_call(expr) {
                     // The callee is something implementing Fn, FnMut, or FnOnce.
                     // Find the actual method implementation being called and
@@ -346,7 +330,7 @@ impl<'tcx> Cx<'tcx> {
                         && let Some(adt_def) = expr_ty.ty_adt_def()
                     {
                         match qpath {
-                            hir::QPath::Resolved(_, ref path) => match path.res {
+                            hir::QPath::Resolved(_, path) => match path.res {
                                 Res::Def(DefKind::Ctor(_, CtorKind::Fn), ctor_id) => {
                                     Some((adt_def, adt_def.variant_index_with_ctor_id(ctor_id)))
                                 }
@@ -407,21 +391,21 @@ impl<'tcx> Cx<'tcx> {
                 }
             }
 
-            hir::ExprKind::AddrOf(hir::BorrowKind::Ref, mutbl, ref arg) => {
+            hir::ExprKind::AddrOf(hir::BorrowKind::Ref, mutbl, arg) => {
                 ExprKind::Borrow { borrow_kind: mutbl.to_borrow_kind(), arg: self.mirror_expr(arg) }
             }
 
-            hir::ExprKind::AddrOf(hir::BorrowKind::Raw, mutability, ref arg) => {
+            hir::ExprKind::AddrOf(hir::BorrowKind::Raw, mutability, arg) => {
                 ExprKind::AddressOf { mutability, arg: self.mirror_expr(arg) }
             }
 
-            hir::ExprKind::Block(ref blk, _) => ExprKind::Block { block: self.mirror_block(blk) },
+            hir::ExprKind::Block(blk, _) => ExprKind::Block { block: self.mirror_block(blk) },
 
-            hir::ExprKind::Assign(ref lhs, ref rhs, _) => {
+            hir::ExprKind::Assign(lhs, rhs, _) => {
                 ExprKind::Assign { lhs: self.mirror_expr(lhs), rhs: self.mirror_expr(rhs) }
             }
 
-            hir::ExprKind::AssignOp(op, ref lhs, ref rhs) => {
+            hir::ExprKind::AssignOp(op, lhs, rhs) => {
                 if self.typeck_results().is_method_call(expr) {
                     let lhs = self.mirror_expr(lhs);
                     let rhs = self.mirror_expr(rhs);
@@ -435,9 +419,9 @@ impl<'tcx> Cx<'tcx> {
                 }
             }
 
-            hir::ExprKind::Lit(ref lit) => ExprKind::Literal { lit, neg: false },
+            hir::ExprKind::Lit(lit) => ExprKind::Literal { lit, neg: false },
 
-            hir::ExprKind::Binary(op, ref lhs, ref rhs) => {
+            hir::ExprKind::Binary(op, lhs, rhs) => {
                 if self.typeck_results().is_method_call(expr) {
                     let lhs = self.mirror_expr(lhs);
                     let rhs = self.mirror_expr(rhs);
@@ -466,7 +450,7 @@ impl<'tcx> Cx<'tcx> {
                 }
             }
 
-            hir::ExprKind::Index(ref lhs, ref index, brackets_span) => {
+            hir::ExprKind::Index(lhs, index, brackets_span) => {
                 if self.typeck_results().is_method_call(expr) {
                     let lhs = self.mirror_expr(lhs);
                     let index = self.mirror_expr(index);
@@ -482,7 +466,7 @@ impl<'tcx> Cx<'tcx> {
                 }
             }
 
-            hir::ExprKind::Unary(hir::UnOp::Deref, ref arg) => {
+            hir::ExprKind::Unary(hir::UnOp::Deref, arg) => {
                 if self.typeck_results().is_method_call(expr) {
                     let arg = self.mirror_expr(arg);
                     self.overloaded_place(expr, expr_ty, None, Box::new([arg]), expr.span)
@@ -491,7 +475,7 @@ impl<'tcx> Cx<'tcx> {
                 }
             }
 
-            hir::ExprKind::Unary(hir::UnOp::Not, ref arg) => {
+            hir::ExprKind::Unary(hir::UnOp::Not, arg) => {
                 if self.typeck_results().is_method_call(expr) {
                     let arg = self.mirror_expr(arg);
                     self.overloaded_operator(expr, Box::new([arg]))
@@ -500,18 +484,18 @@ impl<'tcx> Cx<'tcx> {
                 }
             }
 
-            hir::ExprKind::Unary(hir::UnOp::Neg, ref arg) => {
+            hir::ExprKind::Unary(hir::UnOp::Neg, arg) => {
                 if self.typeck_results().is_method_call(expr) {
                     let arg = self.mirror_expr(arg);
                     self.overloaded_operator(expr, Box::new([arg]))
-                } else if let hir::ExprKind::Lit(ref lit) = arg.kind {
+                } else if let hir::ExprKind::Lit(lit) = arg.kind {
                     ExprKind::Literal { lit, neg: true }
                 } else {
                     ExprKind::Unary { op: UnOp::Neg, arg: self.mirror_expr(arg) }
                 }
             }
 
-            hir::ExprKind::Struct(ref qpath, ref fields, ref base) => match expr_ty.kind() {
+            hir::ExprKind::Struct(qpath, fields, ref base) => match expr_ty.kind() {
                 ty::Adt(adt, args) => match adt.adt_kind() {
                     AdtKind::Struct | AdtKind::Union => {
                         let user_provided_types = self.typeck_results().user_provided_types();
@@ -614,13 +598,13 @@ impl<'tcx> Cx<'tcx> {
                 self.convert_path_expr(expr, res)
             }
 
-            hir::ExprKind::InlineAsm(ref asm) => ExprKind::InlineAsm(Box::new(InlineAsmExpr {
+            hir::ExprKind::InlineAsm(asm) => ExprKind::InlineAsm(Box::new(InlineAsmExpr {
                 template: asm.template,
                 operands: asm
                     .operands
                     .iter()
                     .map(|(op, _op_sp)| match *op {
-                        hir::InlineAsmOperand::In { reg, ref expr } => {
+                        hir::InlineAsmOperand::In { reg, expr } => {
                             InlineAsmOperand::In { reg, expr: self.mirror_expr(expr) }
                         }
                         hir::InlineAsmOperand::Out { reg, late, ref expr } => {
@@ -630,30 +614,35 @@ impl<'tcx> Cx<'tcx> {
                                 expr: expr.map(|expr| self.mirror_expr(expr)),
                             }
                         }
-                        hir::InlineAsmOperand::InOut { reg, late, ref expr } => {
+                        hir::InlineAsmOperand::InOut { reg, late, expr } => {
                             InlineAsmOperand::InOut { reg, late, expr: self.mirror_expr(expr) }
                         }
-                        hir::InlineAsmOperand::SplitInOut {
-                            reg,
-                            late,
-                            ref in_expr,
-                            ref out_expr,
-                        } => InlineAsmOperand::SplitInOut {
-                            reg,
-                            late,
-                            in_expr: self.mirror_expr(in_expr),
-                            out_expr: out_expr.map(|expr| self.mirror_expr(expr)),
-                        },
+                        hir::InlineAsmOperand::SplitInOut { reg, late, in_expr, ref out_expr } => {
+                            InlineAsmOperand::SplitInOut {
+                                reg,
+                                late,
+                                in_expr: self.mirror_expr(in_expr),
+                                out_expr: out_expr.map(|expr| self.mirror_expr(expr)),
+                            }
+                        }
                         hir::InlineAsmOperand::Const { ref anon_const } => {
-                            let value =
-                                mir::Const::from_anon_const(tcx, anon_const.def_id, self.param_env);
+                            let value = mir::Const::identity_unevaluated(
+                                tcx,
+                                anon_const.def_id.to_def_id(),
+                            )
+                            .instantiate_identity()
+                            .normalize(tcx, self.param_env);
                             let span = tcx.def_span(anon_const.def_id);
 
                             InlineAsmOperand::Const { value, span }
                         }
                         hir::InlineAsmOperand::SymFn { ref anon_const } => {
-                            let value =
-                                mir::Const::from_anon_const(tcx, anon_const.def_id, self.param_env);
+                            let value = mir::Const::identity_unevaluated(
+                                tcx,
+                                anon_const.def_id.to_def_id(),
+                            )
+                            .instantiate_identity()
+                            .normalize(tcx, self.param_env);
                             let span = tcx.def_span(anon_const.def_id);
 
                             InlineAsmOperand::SymFn { value, span }
@@ -686,7 +675,7 @@ impl<'tcx> Cx<'tcx> {
                 ExprKind::ConstBlock { did, args }
             }
             // Now comes the rote stuff:
-            hir::ExprKind::Repeat(ref v, _) => {
+            hir::ExprKind::Repeat(v, _) => {
                 let ty = self.typeck_results().expr_ty(expr);
                 let ty::Array(_, count) = ty.kind() else {
                     span_bug!(expr.span, "unexpected repeat expr ty: {:?}", ty);
@@ -722,12 +711,12 @@ impl<'tcx> Cx<'tcx> {
                 then: self.mirror_expr(then),
                 else_opt: else_opt.map(|el| self.mirror_expr(el)),
             },
-            hir::ExprKind::Match(ref discr, ref arms, _) => ExprKind::Match {
+            hir::ExprKind::Match(discr, arms, _) => ExprKind::Match {
                 scrutinee: self.mirror_expr(discr),
                 scrutinee_hir_id: discr.hir_id,
                 arms: arms.iter().map(|a| self.convert_arm(a)).collect(),
             },
-            hir::ExprKind::Loop(ref body, ..) => {
+            hir::ExprKind::Loop(body, ..) => {
                 let block_ty = self.typeck_results().node_type(body.hir_id);
                 let temp_lifetime = self
                     .rvalue_scopes
@@ -741,12 +730,12 @@ impl<'tcx> Cx<'tcx> {
                 });
                 ExprKind::Loop { body }
             }
-            hir::ExprKind::Field(ref source, ..) => ExprKind::Field {
+            hir::ExprKind::Field(source, ..) => ExprKind::Field {
                 lhs: self.mirror_expr(source),
                 variant_index: FIRST_VARIANT,
                 name: self.typeck_results.field_index(expr.hir_id),
             },
-            hir::ExprKind::Cast(ref source, ref cast_ty) => {
+            hir::ExprKind::Cast(source, cast_ty) => {
                 // Check for a user-given type annotation on this `cast`
                 let user_provided_types = self.typeck_results.user_provided_types();
                 let user_ty = user_provided_types.get(cast_ty.hir_id);
@@ -756,7 +745,7 @@ impl<'tcx> Cx<'tcx> {
                     expr, cast_ty.hir_id, user_ty,
                 );
 
-                let cast = self.mirror_expr_cast(*source, temp_lifetime, expr.span);
+                let cast = self.mirror_expr_cast(source, temp_lifetime, expr.span);
 
                 if let Some(user_ty) = user_ty {
                     // NOTE: Creating a new Expr and wrapping a Cast inside of it may be
@@ -777,7 +766,7 @@ impl<'tcx> Cx<'tcx> {
                     cast
                 }
             }
-            hir::ExprKind::Type(ref source, ref ty) => {
+            hir::ExprKind::Type(source, ty) => {
                 let user_provided_types = self.typeck_results.user_provided_types();
                 let user_ty = user_provided_types.get(ty.hir_id).copied().map(Box::new);
                 debug!("make_mirror_unadjusted: (type) user_ty={:?}", user_ty);
@@ -788,16 +777,12 @@ impl<'tcx> Cx<'tcx> {
                     ExprKind::ValueTypeAscription { source: mirrored, user_ty }
                 }
             }
-            hir::ExprKind::DropTemps(ref source) => {
-                ExprKind::Use { source: self.mirror_expr(source) }
-            }
-            hir::ExprKind::Array(ref fields) => {
-                ExprKind::Array { fields: self.mirror_exprs(fields) }
-            }
-            hir::ExprKind::Tup(ref fields) => ExprKind::Tuple { fields: self.mirror_exprs(fields) },
+            hir::ExprKind::DropTemps(source) => ExprKind::Use { source: self.mirror_expr(source) },
+            hir::ExprKind::Array(fields) => ExprKind::Array { fields: self.mirror_exprs(fields) },
+            hir::ExprKind::Tup(fields) => ExprKind::Tuple { fields: self.mirror_exprs(fields) },
 
-            hir::ExprKind::Yield(ref v, _) => ExprKind::Yield { value: self.mirror_expr(v) },
-            hir::ExprKind::Err(_) => unreachable!(),
+            hir::ExprKind::Yield(v, _) => ExprKind::Yield { value: self.mirror_expr(v) },
+            hir::ExprKind::Err(_) => unreachable!("cannot lower a `hir::ExprKind::Err` to THIR"),
         };
 
         Expr { temp_lifetime, ty: expr_ty, span: expr.span, kind }
@@ -870,10 +855,10 @@ impl<'tcx> Cx<'tcx> {
 
     fn convert_arm(&mut self, arm: &'tcx hir::Arm<'tcx>) -> ArmId {
         let arm = Arm {
-            pattern: self.pattern_from_hir(&arm.pat),
+            pattern: self.pattern_from_hir(arm.pat),
             guard: arm.guard.as_ref().map(|g| match g {
-                hir::Guard::If(ref e) => Guard::If(self.mirror_expr(e)),
-                hir::Guard::IfLet(ref l) => {
+                hir::Guard::If(e) => Guard::If(self.mirror_expr(e)),
+                hir::Guard::IfLet(l) => {
                     Guard::IfLet(self.pattern_from_hir(l.pat), self.mirror_expr(l.init))
                 }
             }),
@@ -898,7 +883,7 @@ impl<'tcx> Cx<'tcx> {
             }
 
             Res::Def(DefKind::ConstParam, def_id) => {
-                let hir_id = self.tcx.hir().local_def_id_to_hir_id(def_id.expect_local());
+                let hir_id = self.tcx.local_def_id_to_hir_id(def_id.expect_local());
                 let generics = self.tcx.generics_of(hir_id.owner);
                 let index = generics.param_def_id_to_index[&def_id];
                 let name = self.tcx.hir().name(hir_id);
diff --git a/compiler/rustc_mir_build/src/thir/cx/mod.rs b/compiler/rustc_mir_build/src/thir/cx/mod.rs
index b6adb383f..fa3dd2afa 100644
--- a/compiler/rustc_mir_build/src/thir/cx/mod.rs
+++ b/compiler/rustc_mir_build/src/thir/cx/mod.rs
@@ -27,17 +27,17 @@ pub(crate) fn thir_body(
     if let Some(reported) = cx.typeck_results.tainted_by_errors {
         return Err(reported);
     }
-    let expr = cx.mirror_expr(&body.value);
+    let expr = cx.mirror_expr(body.value);
 
-    let owner_id = hir.local_def_id_to_hir_id(owner_def);
-    if let Some(ref fn_decl) = hir.fn_decl_by_hir_id(owner_id) {
+    let owner_id = tcx.local_def_id_to_hir_id(owner_def);
+    if let Some(fn_decl) = hir.fn_decl_by_hir_id(owner_id) {
         let closure_env_param = cx.closure_env_param(owner_def, owner_id);
         let explicit_params = cx.explicit_params(owner_id, fn_decl, body);
         cx.thir.params = closure_env_param.into_iter().chain(explicit_params).collect();
 
         // The resume argument may be missing, in that case we need to provide it here.
         // It will always be `()` in this case.
-        if tcx.def_kind(owner_def) == DefKind::Coroutine && body.params.is_empty() {
+        if tcx.is_coroutine(owner_def.to_def_id()) && body.params.is_empty() {
             cx.thir.params.push(Param {
                 ty: Ty::new_unit(tcx),
                 pat: None,
@@ -72,7 +72,7 @@ impl<'tcx> Cx<'tcx> {
     fn new(tcx: TyCtxt<'tcx>, def: LocalDefId) -> Cx<'tcx> {
         let typeck_results = tcx.typeck(def);
         let hir = tcx.hir();
-        let hir_id = hir.local_def_id_to_hir_id(def);
+        let hir_id = tcx.local_def_id_to_hir_id(def);
 
         let body_type = if hir.body_owner_kind(def).is_fn_or_closure() {
             // fetch the fully liberated fn signature (that is, all bound
@@ -110,7 +110,7 @@ impl<'tcx> Cx<'tcx> {
 
     #[instrument(level = "debug", skip(self))]
     fn pattern_from_hir(&mut self, p: &hir::Pat<'_>) -> Box<Pat<'tcx>> {
-        let p = match self.tcx.hir().get(p.hir_id) {
+        let p = match self.tcx.hir_node(p.hir_id) {
             Node::Pat(p) => p,
             node => bug!("pattern became {:?}", node),
         };
@@ -119,6 +119,17 @@ impl<'tcx> Cx<'tcx> {
 
     fn closure_env_param(&self, owner_def: LocalDefId, owner_id: HirId) -> Option<Param<'tcx>> {
         match self.tcx.def_kind(owner_def) {
+            DefKind::Closure if self.tcx.is_coroutine(owner_def.to_def_id()) => {
+                let coroutine_ty = self.typeck_results.node_type(owner_id);
+                let coroutine_param = Param {
+                    ty: coroutine_ty,
+                    pat: None,
+                    ty_span: None,
+                    self_kind: None,
+                    hir_id: None,
+                };
+                Some(coroutine_param)
+            }
             DefKind::Closure => {
                 let closure_ty = self.typeck_results.node_type(owner_id);
 
@@ -132,10 +143,10 @@ impl<'tcx> Cx<'tcx> {
                     var: ty::BoundVar::from_usize(bound_vars.len() - 1),
                     kind: ty::BrEnv,
                 };
-                let env_region = ty::Region::new_late_bound(self.tcx, ty::INNERMOST, br);
+                let env_region = ty::Region::new_bound(self.tcx, ty::INNERMOST, br);
                 let closure_env_ty =
                     self.tcx.closure_env_ty(closure_def_id, closure_args, env_region).unwrap();
-                let liberated_closure_env_ty = self.tcx.erase_late_bound_regions(
+                let liberated_closure_env_ty = self.tcx.instantiate_bound_regions_with_erased(
                     ty::Binder::bind_with_vars(closure_env_ty, bound_vars),
                 );
                 let env_param = Param {
@@ -148,17 +159,6 @@ impl<'tcx> Cx<'tcx> {
 
                 Some(env_param)
             }
-            DefKind::Coroutine => {
-                let coroutine_ty = self.typeck_results.node_type(owner_id);
-                let coroutine_param = Param {
-                    ty: coroutine_ty,
-                    pat: None,
-                    ty_span: None,
-                    self_kind: None,
-                    hir_id: None,
-                };
-                Some(coroutine_param)
-            }
             _ => None,
         }
     }
diff --git a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
index 8c3d09c19..c435f4023 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
@@ -1,20 +1,22 @@
-use super::deconstruct_pat::{Constructor, DeconstructedPat, WitnessPat};
-use super::usefulness::{
-    compute_match_usefulness, MatchArm, MatchCheckCtxt, Reachability, UsefulnessReport,
+use rustc_pattern_analysis::errors::Uncovered;
+use rustc_pattern_analysis::rustc::{
+    Constructor, DeconstructedPat, RustcMatchCheckCtxt as MatchCheckCtxt, Usefulness,
+    UsefulnessReport, WitnessPat,
 };
+use rustc_pattern_analysis::{analyze_match, MatchArm};
 
 use crate::errors::*;
 
-use rustc_arena::TypedArena;
+use rustc_arena::{DroplessArena, TypedArena};
 use rustc_ast::Mutability;
-use rustc_data_structures::fx::FxHashSet;
+use rustc_data_structures::fx::FxIndexSet;
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_errors::{struct_span_err, Applicability, DiagnosticBuilder, ErrorGuaranteed, MultiSpan};
 use rustc_hir as hir;
 use rustc_hir::def::*;
 use rustc_hir::def_id::LocalDefId;
 use rustc_hir::HirId;
-use rustc_middle::thir::visit::{self, Visitor};
+use rustc_middle::thir::visit::Visitor;
 use rustc_middle::thir::*;
 use rustc_middle::ty::print::with_no_trimmed_paths;
 use rustc_middle::ty::{self, AdtDef, Ty, TyCtxt};
@@ -29,20 +31,22 @@ pub(crate) fn check_match(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Result<(), Err
     let (thir, expr) = tcx.thir_body(def_id)?;
     let thir = thir.borrow();
     let pattern_arena = TypedArena::default();
+    let dropless_arena = DroplessArena::default();
     let mut visitor = MatchVisitor {
         tcx,
         thir: &*thir,
         param_env: tcx.param_env(def_id),
-        lint_level: tcx.hir().local_def_id_to_hir_id(def_id),
+        lint_level: tcx.local_def_id_to_hir_id(def_id),
         let_source: LetSource::None,
         pattern_arena: &pattern_arena,
+        dropless_arena: &dropless_arena,
         error: Ok(()),
     };
     visitor.visit_expr(&thir[expr]);
 
     for param in thir.params.iter() {
         if let Some(box ref pattern) = param.pat {
-            visitor.check_binding_is_irrefutable(pattern, "function argument", None);
+            visitor.check_binding_is_irrefutable(pattern, "function argument", None, None);
         }
     }
     visitor.error
@@ -80,6 +84,7 @@ struct MatchVisitor<'thir, 'p, 'tcx> {
     lint_level: HirId,
     let_source: LetSource,
     pattern_arena: &'p TypedArena<DeconstructedPat<'p, 'tcx>>,
+    dropless_arena: &'p DroplessArena,
     /// Tracks if we encountered an error while checking this body. That the first function to
     /// report it stores it here. Some functions return `Result` to allow callers to short-circuit
     /// on error, but callers don't need to store it here again.
@@ -254,10 +259,11 @@ impl<'thir, 'p, 'tcx> MatchVisitor<'thir, 'p, 'tcx> {
                 self.with_lint_level(lint_level, |this| this.visit_land_rhs(&this.thir[value]))
             }
             ExprKind::Let { box ref pat, expr } => {
+                let expr = &self.thir()[expr];
                 self.with_let_source(LetSource::None, |this| {
-                    this.visit_expr(&this.thir()[expr]);
+                    this.visit_expr(expr);
                 });
-                Ok(Some((ex.span, self.is_let_irrefutable(pat)?)))
+                Ok(Some((ex.span, self.is_let_irrefutable(pat, Some(expr))?)))
             }
             _ => {
                 self.with_let_source(LetSource::None, |this| {
@@ -279,38 +285,123 @@ impl<'thir, 'p, 'tcx> MatchVisitor<'thir, 'p, 'tcx> {
         } else {
             // Check the pattern for some things unrelated to exhaustiveness.
             let refutable = if cx.refutable { Refutable } else { Irrefutable };
-            pat.walk_always(|pat| check_borrow_conflicts_in_at_patterns(self, pat));
-            pat.walk_always(|pat| check_for_bindings_named_same_as_variants(self, pat, refutable));
-            Ok(cx.pattern_arena.alloc(DeconstructedPat::from_pat(cx, pat)))
+            pat.walk_always(|pat| {
+                check_borrow_conflicts_in_at_patterns(self, pat);
+                check_for_bindings_named_same_as_variants(self, pat, refutable);
+            });
+            Ok(cx.pattern_arena.alloc(cx.lower_pat(pat)))
+        }
+    }
+
+    /// Inspects the match scrutinee expression to determine whether the place it evaluates to may
+    /// hold invalid data.
+    fn is_known_valid_scrutinee(&self, scrutinee: &Expr<'tcx>) -> bool {
+        use ExprKind::*;
+        match &scrutinee.kind {
+            // Both pointers and references can validly point to a place with invalid data.
+            Deref { .. } => false,
+            // Inherit validity of the parent place, unless the parent is an union.
+            Field { lhs, .. } => {
+                let lhs = &self.thir()[*lhs];
+                match lhs.ty.kind() {
+                    ty::Adt(def, _) if def.is_union() => false,
+                    _ => self.is_known_valid_scrutinee(lhs),
+                }
+            }
+            // Essentially a field access.
+            Index { lhs, .. } => {
+                let lhs = &self.thir()[*lhs];
+                self.is_known_valid_scrutinee(lhs)
+            }
+
+            // No-op.
+            Scope { value, .. } => self.is_known_valid_scrutinee(&self.thir()[*value]),
+
+            // Casts don't cause a load.
+            NeverToAny { source }
+            | Cast { source }
+            | Use { source }
+            | PointerCoercion { source, .. }
+            | PlaceTypeAscription { source, .. }
+            | ValueTypeAscription { source, .. } => {
+                self.is_known_valid_scrutinee(&self.thir()[*source])
+            }
+
+            // These diverge.
+            Become { .. } | Break { .. } | Continue { .. } | Return { .. } => true,
+
+            // These are statements that evaluate to `()`.
+            Assign { .. } | AssignOp { .. } | InlineAsm { .. } | Let { .. } => true,
+
+            // These evaluate to a value.
+            AddressOf { .. }
+            | Adt { .. }
+            | Array { .. }
+            | Binary { .. }
+            | Block { .. }
+            | Borrow { .. }
+            | Box { .. }
+            | Call { .. }
+            | Closure { .. }
+            | ConstBlock { .. }
+            | ConstParam { .. }
+            | If { .. }
+            | Literal { .. }
+            | LogicalOp { .. }
+            | Loop { .. }
+            | Match { .. }
+            | NamedConst { .. }
+            | NonHirLiteral { .. }
+            | OffsetOf { .. }
+            | Repeat { .. }
+            | StaticRef { .. }
+            | ThreadLocalRef { .. }
+            | Tuple { .. }
+            | Unary { .. }
+            | UpvarRef { .. }
+            | VarRef { .. }
+            | ZstLiteral { .. }
+            | Yield { .. } => true,
         }
     }
 
     fn new_cx(
         &self,
         refutability: RefutableFlag,
-        match_span: Option<Span>,
+        whole_match_span: Option<Span>,
+        scrutinee: Option<&Expr<'tcx>>,
+        scrut_span: Span,
     ) -> MatchCheckCtxt<'p, 'tcx> {
         let refutable = match refutability {
             Irrefutable => false,
             Refutable => true,
         };
+        // If we don't have a scrutinee we're either a function parameter or a `let x;`. Both cases
+        // require validity.
+        let known_valid_scrutinee =
+            scrutinee.map(|scrut| self.is_known_valid_scrutinee(scrut)).unwrap_or(true);
         MatchCheckCtxt {
             tcx: self.tcx,
             param_env: self.param_env,
             module: self.tcx.parent_module(self.lint_level).to_def_id(),
-            pattern_arena: &self.pattern_arena,
-            match_span,
+            pattern_arena: self.pattern_arena,
+            dropless_arena: self.dropless_arena,
+            match_lint_level: self.lint_level,
+            whole_match_span,
+            scrut_span,
             refutable,
+            known_valid_scrutinee,
         }
     }
 
     #[instrument(level = "trace", skip(self))]
     fn check_let(&mut self, pat: &Pat<'tcx>, scrutinee: Option<ExprId>, span: Span) {
         assert!(self.let_source != LetSource::None);
+        let scrut = scrutinee.map(|id| &self.thir[id]);
         if let LetSource::PlainLet = self.let_source {
-            self.check_binding_is_irrefutable(pat, "local binding", Some(span))
+            self.check_binding_is_irrefutable(pat, "local binding", scrut, Some(span))
         } else {
-            let Ok(refutability) = self.is_let_irrefutable(pat) else { return };
+            let Ok(refutability) = self.is_let_irrefutable(pat, scrut) else { return };
             if matches!(refutability, Irrefutable) {
                 report_irrefutable_let_patterns(
                     self.tcx,
@@ -330,14 +421,16 @@ impl<'thir, 'p, 'tcx> MatchVisitor<'thir, 'p, 'tcx> {
         source: hir::MatchSource,
         expr_span: Span,
     ) {
-        let cx = self.new_cx(Refutable, Some(expr_span));
+        let scrut = &self.thir[scrut];
+        let cx = self.new_cx(Refutable, Some(expr_span), Some(scrut), scrut.span);
 
         let mut tarms = Vec::with_capacity(arms.len());
         for &arm in arms {
             let arm = &self.thir.arms[arm];
             let got_error = self.with_lint_level(arm.lint_level, |this| {
                 let Ok(pat) = this.lower_pattern(&cx, &arm.pattern) else { return true };
-                let arm = MatchArm { pat, hir_id: this.lint_level, has_guard: arm.guard.is_some() };
+                let arm =
+                    MatchArm { pat, arm_data: this.lint_level, has_guard: arm.guard.is_some() };
                 tarms.push(arm);
                 false
             });
@@ -346,9 +439,8 @@ impl<'thir, 'p, 'tcx> MatchVisitor<'thir, 'p, 'tcx> {
             }
         }
 
-        let scrut = &self.thir[scrut];
         let scrut_ty = scrut.ty;
-        let report = compute_match_usefulness(&cx, &tarms, self.lint_level, scrut_ty, scrut.span);
+        let report = analyze_match(&cx, &tarms, scrut_ty);
 
         match source {
             // Don't report arm reachability of desugared `match $iter.into_iter() { iter => .. }`
@@ -372,7 +464,12 @@ impl<'thir, 'p, 'tcx> MatchVisitor<'thir, 'p, 'tcx> {
                 debug_assert_eq!(pat.span.desugaring_kind(), Some(DesugaringKind::ForLoop));
                 let PatKind::Variant { ref subpatterns, .. } = pat.kind else { bug!() };
                 let [pat_field] = &subpatterns[..] else { bug!() };
-                self.check_binding_is_irrefutable(&pat_field.pattern, "`for` loop binding", None);
+                self.check_binding_is_irrefutable(
+                    &pat_field.pattern,
+                    "`for` loop binding",
+                    None,
+                    None,
+                );
             } else {
                 self.error = Err(report_non_exhaustive_match(
                     &cx, self.thir, scrut_ty, scrut.span, witnesses, arms, expr_span,
@@ -452,16 +549,21 @@ impl<'thir, 'p, 'tcx> MatchVisitor<'thir, 'p, 'tcx> {
         &mut self,
         pat: &Pat<'tcx>,
         refutability: RefutableFlag,
+        scrut: Option<&Expr<'tcx>>,
     ) -> Result<(MatchCheckCtxt<'p, 'tcx>, UsefulnessReport<'p, 'tcx>), ErrorGuaranteed> {
-        let cx = self.new_cx(refutability, None);
+        let cx = self.new_cx(refutability, None, scrut, pat.span);
         let pat = self.lower_pattern(&cx, pat)?;
-        let arms = [MatchArm { pat, hir_id: self.lint_level, has_guard: false }];
-        let report = compute_match_usefulness(&cx, &arms, self.lint_level, pat.ty(), pat.span());
+        let arms = [MatchArm { pat, arm_data: self.lint_level, has_guard: false }];
+        let report = analyze_match(&cx, &arms, pat.ty());
         Ok((cx, report))
     }
 
-    fn is_let_irrefutable(&mut self, pat: &Pat<'tcx>) -> Result<RefutableFlag, ErrorGuaranteed> {
-        let (cx, report) = self.analyze_binding(pat, Refutable)?;
+    fn is_let_irrefutable(
+        &mut self,
+        pat: &Pat<'tcx>,
+        scrut: Option<&Expr<'tcx>>,
+    ) -> Result<RefutableFlag, ErrorGuaranteed> {
+        let (cx, report) = self.analyze_binding(pat, Refutable, scrut)?;
         // Report if the pattern is unreachable, which can only occur when the type is uninhabited.
         // This also reports unreachable sub-patterns.
         report_arm_reachability(&cx, &report);
@@ -471,10 +573,16 @@ impl<'thir, 'p, 'tcx> MatchVisitor<'thir, 'p, 'tcx> {
     }
 
     #[instrument(level = "trace", skip(self))]
-    fn check_binding_is_irrefutable(&mut self, pat: &Pat<'tcx>, origin: &str, sp: Option<Span>) {
+    fn check_binding_is_irrefutable(
+        &mut self,
+        pat: &Pat<'tcx>,
+        origin: &str,
+        scrut: Option<&Expr<'tcx>>,
+        sp: Option<Span>,
+    ) {
         let pattern_ty = pat.ty;
 
-        let Ok((cx, report)) = self.analyze_binding(pat, Irrefutable) else { return };
+        let Ok((cx, report)) = self.analyze_binding(pat, Irrefutable, scrut) else { return };
         let witnesses = report.non_exhaustiveness_witnesses;
         if witnesses.is_empty() {
             // The pattern is irrefutable.
@@ -744,34 +852,34 @@ fn report_arm_reachability<'p, 'tcx>(
         );
     };
 
-    use Reachability::*;
     let mut catchall = None;
     for (arm, is_useful) in report.arm_usefulness.iter() {
         match is_useful {
-            Unreachable => report_unreachable_pattern(arm.pat.span(), arm.hir_id, catchall),
-            Reachable(unreachables) if unreachables.is_empty() => {}
-            // The arm is reachable, but contains unreachable subpatterns (from or-patterns).
-            Reachable(unreachables) => {
-                let mut unreachables = unreachables.clone();
+            Usefulness::Redundant => {
+                report_unreachable_pattern(*arm.pat.data(), arm.arm_data, catchall)
+            }
+            Usefulness::Useful(redundant_subpats) if redundant_subpats.is_empty() => {}
+            // The arm is reachable, but contains redundant subpatterns (from or-patterns).
+            Usefulness::Useful(redundant_subpats) => {
+                let mut redundant_subpats = redundant_subpats.clone();
                 // Emit lints in the order in which they occur in the file.
-                unreachables.sort_unstable();
-                for span in unreachables {
-                    report_unreachable_pattern(span, arm.hir_id, None);
+                redundant_subpats.sort_unstable_by_key(|pat| pat.data());
+                for pat in redundant_subpats {
+                    report_unreachable_pattern(*pat.data(), arm.arm_data, None);
                 }
             }
         }
         if !arm.has_guard && catchall.is_none() && pat_is_catchall(arm.pat) {
-            catchall = Some(arm.pat.span());
+            catchall = Some(*arm.pat.data());
         }
     }
 }
 
 /// Checks for common cases of "catchall" patterns that may not be intended as such.
 fn pat_is_catchall(pat: &DeconstructedPat<'_, '_>) -> bool {
-    use Constructor::*;
     match pat.ctor() {
-        Wildcard => true,
-        Single => pat.iter_fields().all(|pat| pat_is_catchall(pat)),
+        Constructor::Wildcard => true,
+        Constructor::Struct | Constructor::Ref => pat.iter_fields().all(|pat| pat_is_catchall(pat)),
         _ => false,
     }
 }
@@ -782,7 +890,7 @@ fn report_non_exhaustive_match<'p, 'tcx>(
     thir: &Thir<'tcx>,
     scrut_ty: Ty<'tcx>,
     sp: Span,
-    witnesses: Vec<WitnessPat<'tcx>>,
+    witnesses: Vec<WitnessPat<'p, 'tcx>>,
     arms: &[ArmId],
     expr_span: Span,
 ) -> ErrorGuaranteed {
@@ -823,7 +931,7 @@ fn report_non_exhaustive_match<'p, 'tcx>(
         pattern = if witnesses.len() < 4 {
             witnesses
                 .iter()
-                .map(|witness| witness.to_diagnostic_pat(cx).to_string())
+                .map(|witness| cx.hoist_witness_pat(witness).to_string())
                 .collect::<Vec<String>>()
                 .join(" | ")
         } else {
@@ -845,9 +953,9 @@ fn report_non_exhaustive_match<'p, 'tcx>(
     err.note(format!("the matched value is of type `{}`", scrut_ty));
 
     if !is_empty_match {
-        let mut non_exhaustive_tys = FxHashSet::default();
+        let mut non_exhaustive_tys = FxIndexSet::default();
         // Look at the first witness.
-        collect_non_exhaustive_tys(cx.tcx, &witnesses[0], &mut non_exhaustive_tys);
+        collect_non_exhaustive_tys(cx, &witnesses[0], &mut non_exhaustive_tys);
 
         for ty in non_exhaustive_tys {
             if ty.is_ptr_sized_integral() {
@@ -862,12 +970,6 @@ fn report_non_exhaustive_match<'p, 'tcx>(
                              exhaustively",
                     ));
                 }
-                if cx.tcx.sess.is_nightly_build() {
-                    err.help(format!(
-                            "add `#![feature(precise_pointer_size_matching)]` to the crate attributes to \
-                             enable precise `{ty}` matching",
-                        ));
-                }
             } else if ty == cx.tcx.types.str_ {
                 err.note("`&str` cannot be matched exhaustively, so a wildcard `_` is necessary");
             } else if cx.is_foreign_non_exhaustive_enum(ty) {
@@ -985,16 +1087,16 @@ fn report_non_exhaustive_match<'p, 'tcx>(
 
 fn joined_uncovered_patterns<'p, 'tcx>(
     cx: &MatchCheckCtxt<'p, 'tcx>,
-    witnesses: &[WitnessPat<'tcx>],
+    witnesses: &[WitnessPat<'p, 'tcx>],
 ) -> String {
     const LIMIT: usize = 3;
-    let pat_to_str = |pat: &WitnessPat<'tcx>| pat.to_diagnostic_pat(cx).to_string();
+    let pat_to_str = |pat: &WitnessPat<'p, 'tcx>| cx.hoist_witness_pat(pat).to_string();
     match witnesses {
         [] => bug!(),
-        [witness] => format!("`{}`", witness.to_diagnostic_pat(cx)),
+        [witness] => format!("`{}`", cx.hoist_witness_pat(witness)),
         [head @ .., tail] if head.len() < LIMIT => {
             let head: Vec<_> = head.iter().map(pat_to_str).collect();
-            format!("`{}` and `{}`", head.join("`, `"), tail.to_diagnostic_pat(cx))
+            format!("`{}` and `{}`", head.join("`, `"), cx.hoist_witness_pat(tail))
         }
         _ => {
             let (head, tail) = witnesses.split_at(LIMIT);
@@ -1005,27 +1107,27 @@ fn joined_uncovered_patterns<'p, 'tcx>(
 }
 
 fn collect_non_exhaustive_tys<'tcx>(
-    tcx: TyCtxt<'tcx>,
-    pat: &WitnessPat<'tcx>,
-    non_exhaustive_tys: &mut FxHashSet<Ty<'tcx>>,
+    cx: &MatchCheckCtxt<'_, 'tcx>,
+    pat: &WitnessPat<'_, 'tcx>,
+    non_exhaustive_tys: &mut FxIndexSet<Ty<'tcx>>,
 ) {
     if matches!(pat.ctor(), Constructor::NonExhaustive) {
         non_exhaustive_tys.insert(pat.ty());
     }
     if let Constructor::IntRange(range) = pat.ctor() {
-        if range.is_beyond_boundaries(pat.ty(), tcx) {
+        if cx.is_range_beyond_boundaries(range, pat.ty()) {
             // The range denotes the values before `isize::MIN` or the values after `usize::MAX`/`isize::MAX`.
             non_exhaustive_tys.insert(pat.ty());
         }
     }
     pat.iter_fields()
-        .for_each(|field_pat| collect_non_exhaustive_tys(tcx, field_pat, non_exhaustive_tys))
+        .for_each(|field_pat| collect_non_exhaustive_tys(cx, field_pat, non_exhaustive_tys))
 }
 
 fn report_adt_defined_here<'tcx>(
     tcx: TyCtxt<'tcx>,
     ty: Ty<'tcx>,
-    witnesses: &[WitnessPat<'tcx>],
+    witnesses: &[WitnessPat<'_, 'tcx>],
     point_at_non_local_ty: bool,
 ) -> Option<AdtDefinedHere<'tcx>> {
     let ty = ty.peel_refs();
@@ -1047,15 +1149,14 @@ fn report_adt_defined_here<'tcx>(
     Some(AdtDefinedHere { adt_def_span, ty, variants })
 }
 
-fn maybe_point_at_variant<'a, 'tcx: 'a>(
+fn maybe_point_at_variant<'a, 'p: 'a, 'tcx: 'p>(
     tcx: TyCtxt<'tcx>,
     def: AdtDef<'tcx>,
-    patterns: impl Iterator<Item = &'a WitnessPat<'tcx>>,
+    patterns: impl Iterator<Item = &'a WitnessPat<'p, 'tcx>>,
 ) -> Vec<Span> {
-    use Constructor::*;
     let mut covered = vec![];
     for pattern in patterns {
-        if let Variant(variant_index) = pattern.ctor() {
+        if let Constructor::Variant(variant_index) = pattern.ctor() {
             if let ty::Adt(this_def, _) = pattern.ty().kind()
                 && this_def.did() != def.did()
             {
diff --git a/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs
index 48a590f5d..a7f237721 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs
@@ -200,7 +200,9 @@ impl<'tcx> ConstToPat<'tcx> {
                     // We errored. Signal that in the pattern, so that follow up errors can be silenced.
                     let kind = PatKind::Error(e);
                     return Box::new(Pat { span: self.span, ty: cv.ty(), kind });
-                } else if let ty::Adt(..) = cv.ty().kind() && matches!(cv, mir::Const::Val(..)) {
+                } else if let ty::Adt(..) = cv.ty().kind()
+                    && matches!(cv, mir::Const::Val(..))
+                {
                     // This branch is only entered when the current `cv` is `mir::Const::Val`.
                     // This is because `mir::Const::ty` has already been handled by `Self::recur`
                     // and the invalid types may be ignored.
@@ -256,18 +258,26 @@ impl<'tcx> ConstToPat<'tcx> {
 
     #[instrument(level = "trace", skip(self), ret)]
     fn type_has_partial_eq_impl(&self, ty: Ty<'tcx>) -> bool {
+        let tcx = self.tcx();
         // double-check there even *is* a semantic `PartialEq` to dispatch to.
         //
         // (If there isn't, then we can safely issue a hard
         // error, because that's never worked, due to compiler
         // using `PartialEq::eq` in this scenario in the past.)
-        let partial_eq_trait_id =
-            self.tcx().require_lang_item(hir::LangItem::PartialEq, Some(self.span));
+        let partial_eq_trait_id = tcx.require_lang_item(hir::LangItem::PartialEq, Some(self.span));
         let partial_eq_obligation = Obligation::new(
-            self.tcx(),
+            tcx,
             ObligationCause::dummy(),
             self.param_env,
-            ty::TraitRef::new(self.tcx(), partial_eq_trait_id, [ty, ty]),
+            ty::TraitRef::new(
+                tcx,
+                partial_eq_trait_id,
+                tcx.with_opt_host_effect_param(
+                    tcx.hir().enclosing_body_owner(self.id),
+                    partial_eq_trait_id,
+                    [ty, ty],
+                ),
+            ),
         );
 
         // This *could* accept a type that isn't actually `PartialEq`, because region bounds get
@@ -482,8 +492,9 @@ impl<'tcx> ConstToPat<'tcx> {
                 PatKind::Constant { value: mir::Const::Ty(ty::Const::new_value(tcx, cv, ty)) }
             }
             ty::FnPtr(..) => {
-                // Valtree construction would never succeed for these, so this is unreachable.
-                unreachable!()
+                unreachable!(
+                    "Valtree construction would never succeed for FnPtr, so this is unreachable."
+                )
             }
             _ => {
                 let err = InvalidPattern { span, non_sm_ty: ty };
diff --git a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
deleted file mode 100644
index 0c7c2c6f9..000000000
--- a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
+++ /dev/null
@@ -1,1912 +0,0 @@
-//! [`super::usefulness`] explains most of what is happening in this file. As explained there,
-//! values and patterns are made from constructors applied to fields. This file defines a
-//! `Constructor` enum, a `Fields` struct, and various operations to manipulate them and convert
-//! them from/to patterns.
-//!
-//! There's one idea that is not detailed in [`super::usefulness`] because the details are not
-//! needed there: _constructor splitting_.
-//!
-//! # Constructor splitting
-//!
-//! The idea is as follows: given a constructor `c` and a matrix, we want to specialize in turn
-//! with all the value constructors that are covered by `c`, and compute usefulness for each.
-//! Instead of listing all those constructors (which is intractable), we group those value
-//! constructors together as much as possible. Example:
-//!
-//! ```compile_fail,E0004
-//! match (0, false) {
-//!     (0 ..=100, true) => {} // `p_1`
-//!     (50..=150, false) => {} // `p_2`
-//!     (0 ..=200, _) => {} // `q`
-//! }
-//! ```
-//!
-//! The naive approach would try all numbers in the range `0..=200`. But we can be a lot more
-//! clever: `0` and `1` for example will match the exact same rows, and return equivalent
-//! witnesses. In fact all of `0..50` would. We can thus restrict our exploration to 4
-//! constructors: `0..50`, `50..=100`, `101..=150` and `151..=200`. That is enough and infinitely
-//! more tractable.
-//!
-//! We capture this idea in a function `split(p_1 ... p_n, c)` which returns a list of constructors
-//! `c'` covered by `c`. Given such a `c'`, we require that all value ctors `c''` covered by `c'`
-//! return an equivalent set of witnesses after specializing and computing usefulness.
-//! In the example above, witnesses for specializing by `c''` covered by `0..50` will only differ
-//! in their first element.
-//!
-//! We usually also ask that the `c'` together cover all of the original `c`. However we allow
-//! skipping some constructors as long as it doesn't change whether the resulting list of witnesses
-//! is empty of not. We use this in the wildcard `_` case.
-//!
-//! Splitting is implemented in the [`Constructor::split`] function. We don't do splitting for
-//! or-patterns; instead we just try the alternatives one-by-one. For details on splitting
-//! wildcards, see [`Constructor::split`]; for integer ranges, see
-//! [`IntRange::split`]; for slices, see [`Slice::split`].
-
-use std::cell::Cell;
-use std::cmp::{self, max, min, Ordering};
-use std::fmt;
-use std::iter::once;
-
-use smallvec::{smallvec, SmallVec};
-
-use rustc_apfloat::ieee::{DoubleS, IeeeFloat, SingleS};
-use rustc_data_structures::captures::Captures;
-use rustc_data_structures::fx::FxHashSet;
-use rustc_hir::RangeEnd;
-use rustc_index::Idx;
-use rustc_middle::middle::stability::EvalResult;
-use rustc_middle::mir;
-use rustc_middle::mir::interpret::Scalar;
-use rustc_middle::thir::{FieldPat, Pat, PatKind, PatRange, PatRangeBoundary};
-use rustc_middle::ty::layout::IntegerExt;
-use rustc_middle::ty::{self, Ty, TyCtxt, VariantDef};
-use rustc_span::{Span, DUMMY_SP};
-use rustc_target::abi::{FieldIdx, Integer, VariantIdx, FIRST_VARIANT};
-
-use self::Constructor::*;
-use self::MaybeInfiniteInt::*;
-use self::SliceKind::*;
-
-use super::usefulness::{MatchCheckCtxt, PatCtxt};
-
-/// Recursively expand this pattern into its subpatterns. Only useful for or-patterns.
-fn expand_or_pat<'p, 'tcx>(pat: &'p Pat<'tcx>) -> Vec<&'p Pat<'tcx>> {
-    fn expand<'p, 'tcx>(pat: &'p Pat<'tcx>, vec: &mut Vec<&'p Pat<'tcx>>) {
-        if let PatKind::Or { pats } = &pat.kind {
-            for pat in pats.iter() {
-                expand(&pat, vec);
-            }
-        } else {
-            vec.push(pat)
-        }
-    }
-
-    let mut pats = Vec::new();
-    expand(pat, &mut pats);
-    pats
-}
-
-/// Whether we have seen a constructor in the column or not.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
-enum Presence {
-    Unseen,
-    Seen,
-}
-
-/// A possibly infinite integer. Values are encoded such that the ordering on `u128` matches the
-/// natural order on the original type. For example, `-128i8` is encoded as `0` and `127i8` as
-/// `255`. See `signed_bias` for details.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
-pub(crate) enum MaybeInfiniteInt {
-    NegInfinity,
-    /// Encoded value. DO NOT CONSTRUCT BY HAND; use `new_finite`.
-    Finite(u128),
-    /// The integer after `u128::MAX`. We need it to represent `x..=u128::MAX` as an exclusive range.
-    JustAfterMax,
-    PosInfinity,
-}
-
-impl MaybeInfiniteInt {
-    // The return value of `signed_bias` should be XORed with a value to encode/decode it.
-    fn signed_bias(tcx: TyCtxt<'_>, ty: Ty<'_>) -> u128 {
-        match *ty.kind() {
-            ty::Int(ity) => {
-                let bits = Integer::from_int_ty(&tcx, ity).size().bits() as u128;
-                1u128 << (bits - 1)
-            }
-            _ => 0,
-        }
-    }
-
-    fn new_finite(tcx: TyCtxt<'_>, ty: Ty<'_>, bits: u128) -> Self {
-        let bias = Self::signed_bias(tcx, ty);
-        // Perform a shift if the underlying types are signed, which makes the interval arithmetic
-        // type-independent.
-        let x = bits ^ bias;
-        Finite(x)
-    }
-    fn from_pat_range_bdy<'tcx>(
-        bdy: PatRangeBoundary<'tcx>,
-        ty: Ty<'tcx>,
-        tcx: TyCtxt<'tcx>,
-        param_env: ty::ParamEnv<'tcx>,
-    ) -> Self {
-        match bdy {
-            PatRangeBoundary::NegInfinity => NegInfinity,
-            PatRangeBoundary::Finite(value) => {
-                let bits = value.eval_bits(tcx, param_env);
-                Self::new_finite(tcx, ty, bits)
-            }
-            PatRangeBoundary::PosInfinity => PosInfinity,
-        }
-    }
-
-    /// Used only for diagnostics.
-    /// Note: it is possible to get `isize/usize::MAX+1` here, as explained in the doc for
-    /// [`IntRange::split`]. This cannot be represented as a `Const`, so we represent it with
-    /// `PosInfinity`.
-    fn to_diagnostic_pat_range_bdy<'tcx>(
-        self,
-        ty: Ty<'tcx>,
-        tcx: TyCtxt<'tcx>,
-    ) -> PatRangeBoundary<'tcx> {
-        match self {
-            NegInfinity => PatRangeBoundary::NegInfinity,
-            Finite(x) => {
-                let bias = Self::signed_bias(tcx, ty);
-                let bits = x ^ bias;
-                let size = ty.primitive_size(tcx);
-                match Scalar::try_from_uint(bits, size) {
-                    Some(scalar) => {
-                        let value = mir::Const::from_scalar(tcx, scalar, ty);
-                        PatRangeBoundary::Finite(value)
-                    }
-                    // The value doesn't fit. Since `x >= 0` and 0 always encodes the minimum value
-                    // for a type, the problem isn't that the value is too small. So it must be too
-                    // large.
-                    None => PatRangeBoundary::PosInfinity,
-                }
-            }
-            JustAfterMax | PosInfinity => PatRangeBoundary::PosInfinity,
-        }
-    }
-
-    /// Note: this will not turn a finite value into an infinite one or vice-versa.
-    pub(crate) fn minus_one(self) -> Self {
-        match self {
-            Finite(n) => match n.checked_sub(1) {
-                Some(m) => Finite(m),
-                None => bug!(),
-            },
-            JustAfterMax => Finite(u128::MAX),
-            x => x,
-        }
-    }
-    /// Note: this will not turn a finite value into an infinite one or vice-versa.
-    pub(crate) fn plus_one(self) -> Self {
-        match self {
-            Finite(n) => match n.checked_add(1) {
-                Some(m) => Finite(m),
-                None => JustAfterMax,
-            },
-            JustAfterMax => bug!(),
-            x => x,
-        }
-    }
-}
-
-/// An exclusive interval, used for precise integer exhaustiveness checking. `IntRange`s always
-/// store a contiguous range.
-///
-/// `IntRange` is never used to encode an empty range or a "range" that wraps around the (offset)
-/// space: i.e., `range.lo < range.hi`.
-#[derive(Clone, Copy, PartialEq, Eq)]
-pub(crate) struct IntRange {
-    pub(crate) lo: MaybeInfiniteInt, // Must not be `PosInfinity`.
-    pub(crate) hi: MaybeInfiniteInt, // Must not be `NegInfinity`.
-}
-
-impl IntRange {
-    #[inline]
-    pub(super) fn is_integral(ty: Ty<'_>) -> bool {
-        matches!(ty.kind(), ty::Char | ty::Int(_) | ty::Uint(_))
-    }
-
-    /// Best effort; will not know that e.g. `255u8..` is a singleton.
-    pub(super) fn is_singleton(&self) -> bool {
-        // Since `lo` and `hi` can't be the same `Infinity` and `plus_one` never changes from finite
-        // to infinite, this correctly only detects ranges that contain exacly one `Finite(x)`.
-        self.lo.plus_one() == self.hi
-    }
-
-    #[inline]
-    fn from_bits<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>, bits: u128) -> IntRange {
-        let x = MaybeInfiniteInt::new_finite(tcx, ty, bits);
-        IntRange { lo: x, hi: x.plus_one() }
-    }
-
-    #[inline]
-    fn from_range(lo: MaybeInfiniteInt, mut hi: MaybeInfiniteInt, end: RangeEnd) -> IntRange {
-        if end == RangeEnd::Included {
-            hi = hi.plus_one();
-        }
-        if lo >= hi {
-            // This should have been caught earlier by E0030.
-            bug!("malformed range pattern: {lo:?}..{hi:?}");
-        }
-        IntRange { lo, hi }
-    }
-
-    fn is_subrange(&self, other: &Self) -> bool {
-        other.lo <= self.lo && self.hi <= other.hi
-    }
-
-    fn intersection(&self, other: &Self) -> Option<Self> {
-        if self.lo < other.hi && other.lo < self.hi {
-            Some(IntRange { lo: max(self.lo, other.lo), hi: min(self.hi, other.hi) })
-        } else {
-            None
-        }
-    }
-
-    /// Partition a range of integers into disjoint subranges. This does constructor splitting for
-    /// integer ranges as explained at the top of the file.
-    ///
-    /// This returns an output that covers `self`. The output is split so that the only
-    /// intersections between an output range and a column range are inclusions. No output range
-    /// straddles the boundary of one of the inputs.
-    ///
-    /// Additionally, we track for each output range whether it is covered by one of the column ranges or not.
-    ///
-    /// The following input:
-    /// ```text
-    ///   (--------------------------) // `self`
-    /// (------) (----------)    (-)
-    ///     (------) (--------)
-    /// ```
-    /// is first intersected with `self`:
-    /// ```text
-    ///   (--------------------------) // `self`
-    ///   (----) (----------)    (-)
-    ///     (------) (--------)
-    /// ```
-    /// and then iterated over as follows:
-    /// ```text
-    ///   (-(--)-(-)-(------)-)--(-)-
-    /// ```
-    /// where each sequence of dashes is an output range, and dashes outside parentheses are marked
-    /// as `Presence::Missing`.
-    ///
-    /// ## `isize`/`usize`
-    ///
-    /// Whereas a wildcard of type `i32` stands for the range `i32::MIN..=i32::MAX`, a `usize`
-    /// wildcard stands for `0..PosInfinity` and a `isize` wildcard stands for
-    /// `NegInfinity..PosInfinity`. In other words, as far as `IntRange` is concerned, there are
-    /// values before `isize::MIN` and after `usize::MAX`/`isize::MAX`.
-    /// This is to avoid e.g. `0..(u32::MAX as usize)` from being exhaustive on one architecture and
-    /// not others. See discussions around the `precise_pointer_size_matching` feature for more
-    /// details.
-    ///
-    /// These infinities affect splitting subtly: it is possible to get `NegInfinity..0` and
-    /// `usize::MAX+1..PosInfinity` in the output. Diagnostics must be careful to handle these
-    /// fictitious ranges sensibly.
-    fn split(
-        &self,
-        column_ranges: impl Iterator<Item = IntRange>,
-    ) -> impl Iterator<Item = (Presence, IntRange)> {
-        // The boundaries of ranges in `column_ranges` intersected with `self`.
-        // We do parenthesis matching for input ranges. A boundary counts as +1 if it starts
-        // a range and -1 if it ends it. When the count is > 0 between two boundaries, we
-        // are within an input range.
-        let mut boundaries: Vec<(MaybeInfiniteInt, isize)> = column_ranges
-            .filter_map(|r| self.intersection(&r))
-            .flat_map(|r| [(r.lo, 1), (r.hi, -1)])
-            .collect();
-        // We sort by boundary, and for each boundary we sort the "closing parentheses" first. The
-        // order of +1/-1 for a same boundary value is actually irrelevant, because we only look at
-        // the accumulated count between distinct boundary values.
-        boundaries.sort_unstable();
-
-        // Accumulate parenthesis counts.
-        let mut paren_counter = 0isize;
-        // Gather pairs of adjacent boundaries.
-        let mut prev_bdy = self.lo;
-        boundaries
-            .into_iter()
-            // End with the end of the range. The count is ignored.
-            .chain(once((self.hi, 0)))
-            // List pairs of adjacent boundaries and the count between them.
-            .map(move |(bdy, delta)| {
-                // `delta` affects the count as we cross `bdy`, so the relevant count between
-                // `prev_bdy` and `bdy` is untouched by `delta`.
-                let ret = (prev_bdy, paren_counter, bdy);
-                prev_bdy = bdy;
-                paren_counter += delta;
-                ret
-            })
-            // Skip empty ranges.
-            .filter(|&(prev_bdy, _, bdy)| prev_bdy != bdy)
-            // Convert back to ranges.
-            .map(move |(prev_bdy, paren_count, bdy)| {
-                use Presence::*;
-                let presence = if paren_count > 0 { Seen } else { Unseen };
-                let range = IntRange { lo: prev_bdy, hi: bdy };
-                (presence, range)
-            })
-    }
-
-    /// Whether the range denotes the fictitious values before `isize::MIN` or after
-    /// `usize::MAX`/`isize::MAX` (see doc of [`IntRange::split`] for why these exist).
-    pub(crate) fn is_beyond_boundaries<'tcx>(&self, ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> bool {
-        ty.is_ptr_sized_integral() && !tcx.features().precise_pointer_size_matching && {
-            // The two invalid ranges are `NegInfinity..isize::MIN` (represented as
-            // `NegInfinity..0`), and `{u,i}size::MAX+1..PosInfinity`. `to_diagnostic_pat_range_bdy`
-            // converts `MAX+1` to `PosInfinity`, and we couldn't have `PosInfinity` in `self.lo`
-            // otherwise.
-            let lo = self.lo.to_diagnostic_pat_range_bdy(ty, tcx);
-            matches!(lo, PatRangeBoundary::PosInfinity)
-                || matches!(self.hi, MaybeInfiniteInt::Finite(0))
-        }
-    }
-    /// Only used for displaying the range.
-    pub(super) fn to_diagnostic_pat<'tcx>(&self, ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> Pat<'tcx> {
-        let kind = if matches!((self.lo, self.hi), (NegInfinity, PosInfinity)) {
-            PatKind::Wild
-        } else if self.is_singleton() {
-            let lo = self.lo.to_diagnostic_pat_range_bdy(ty, tcx);
-            let value = lo.as_finite().unwrap();
-            PatKind::Constant { value }
-        } else {
-            // We convert to an inclusive range for diagnostics.
-            let mut end = RangeEnd::Included;
-            let mut lo = self.lo.to_diagnostic_pat_range_bdy(ty, tcx);
-            if matches!(lo, PatRangeBoundary::PosInfinity) {
-                // The only reason to get `PosInfinity` here is the special case where
-                // `to_diagnostic_pat_range_bdy` found `{u,i}size::MAX+1`. So the range denotes the
-                // fictitious values after `{u,i}size::MAX` (see [`IntRange::split`] for why we do
-                // this). We show this to the user as `usize::MAX..` which is slightly incorrect but
-                // probably clear enough.
-                let c = ty.numeric_max_val(tcx).unwrap();
-                let value = mir::Const::from_ty_const(c, tcx);
-                lo = PatRangeBoundary::Finite(value);
-            }
-            let hi = if matches!(self.hi, MaybeInfiniteInt::Finite(0)) {
-                // The range encodes `..ty::MIN`, so we can't convert it to an inclusive range.
-                end = RangeEnd::Excluded;
-                self.hi
-            } else {
-                self.hi.minus_one()
-            };
-            let hi = hi.to_diagnostic_pat_range_bdy(ty, tcx);
-            PatKind::Range(Box::new(PatRange { lo, hi, end, ty }))
-        };
-
-        Pat { ty, span: DUMMY_SP, kind }
-    }
-}
-
-/// Note: this will render signed ranges incorrectly. To render properly, convert to a pattern
-/// first.
-impl fmt::Debug for IntRange {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        if let Finite(lo) = self.lo {
-            write!(f, "{lo}")?;
-        }
-        write!(f, "{}", RangeEnd::Excluded)?;
-        if let Finite(hi) = self.hi {
-            write!(f, "{hi}")?;
-        }
-        Ok(())
-    }
-}
-
-#[derive(Copy, Clone, Debug, PartialEq, Eq)]
-enum SliceKind {
-    /// Patterns of length `n` (`[x, y]`).
-    FixedLen(usize),
-    /// Patterns using the `..` notation (`[x, .., y]`).
-    /// Captures any array constructor of `length >= i + j`.
-    /// In the case where `array_len` is `Some(_)`,
-    /// this indicates that we only care about the first `i` and the last `j` values of the array,
-    /// and everything in between is a wildcard `_`.
-    VarLen(usize, usize),
-}
-
-impl SliceKind {
-    fn arity(self) -> usize {
-        match self {
-            FixedLen(length) => length,
-            VarLen(prefix, suffix) => prefix + suffix,
-        }
-    }
-
-    /// Whether this pattern includes patterns of length `other_len`.
-    fn covers_length(self, other_len: usize) -> bool {
-        match self {
-            FixedLen(len) => len == other_len,
-            VarLen(prefix, suffix) => prefix + suffix <= other_len,
-        }
-    }
-}
-
-/// A constructor for array and slice patterns.
-#[derive(Copy, Clone, Debug, PartialEq, Eq)]
-pub(super) struct Slice {
-    /// `None` if the matched value is a slice, `Some(n)` if it is an array of size `n`.
-    array_len: Option<usize>,
-    /// The kind of pattern it is: fixed-length `[x, y]` or variable length `[x, .., y]`.
-    kind: SliceKind,
-}
-
-impl Slice {
-    fn new(array_len: Option<usize>, kind: SliceKind) -> Self {
-        let kind = match (array_len, kind) {
-            // If the middle `..` is empty, we effectively have a fixed-length pattern.
-            (Some(len), VarLen(prefix, suffix)) if prefix + suffix >= len => FixedLen(len),
-            _ => kind,
-        };
-        Slice { array_len, kind }
-    }
-
-    fn arity(self) -> usize {
-        self.kind.arity()
-    }
-
-    /// See `Constructor::is_covered_by`
-    fn is_covered_by(self, other: Self) -> bool {
-        other.kind.covers_length(self.arity())
-    }
-
-    /// This computes constructor splitting for variable-length slices, as explained at the top of
-    /// the file.
-    ///
-    /// A slice pattern `[x, .., y]` behaves like the infinite or-pattern `[x, y] | [x, _, y] | [x,
-    /// _, _, y] | etc`. The corresponding value constructors are fixed-length array constructors of
-    /// corresponding lengths. We obviously can't list this infinitude of constructors.
-    /// Thankfully, it turns out that for each finite set of slice patterns, all sufficiently large
-    /// array lengths are equivalent.
-    ///
-    /// Let's look at an example, where we are trying to split the last pattern:
-    /// ```
-    /// # fn foo(x: &[bool]) {
-    /// match x {
-    ///     [true, true, ..] => {}
-    ///     [.., false, false] => {}
-    ///     [..] => {}
-    /// }
-    /// # }
-    /// ```
-    /// Here are the results of specialization for the first few lengths:
-    /// ```
-    /// # fn foo(x: &[bool]) { match x {
-    /// // length 0
-    /// [] => {}
-    /// // length 1
-    /// [_] => {}
-    /// // length 2
-    /// [true, true] => {}
-    /// [false, false] => {}
-    /// [_, _] => {}
-    /// // length 3
-    /// [true, true,  _    ] => {}
-    /// [_,    false, false] => {}
-    /// [_,    _,     _    ] => {}
-    /// // length 4
-    /// [true, true, _,     _    ] => {}
-    /// [_,    _,    false, false] => {}
-    /// [_,    _,    _,     _    ] => {}
-    /// // length 5
-    /// [true, true, _, _,     _    ] => {}
-    /// [_,    _,    _, false, false] => {}
-    /// [_,    _,    _, _,     _    ] => {}
-    /// # _ => {}
-    /// # }}
-    /// ```
-    ///
-    /// We see that above length 4, we are simply inserting columns full of wildcards in the middle.
-    /// This means that specialization and witness computation with slices of length `l >= 4` will
-    /// give equivalent results regardless of `l`. This applies to any set of slice patterns: there
-    /// will be a length `L` above which all lengths behave the same. This is exactly what we need
-    /// for constructor splitting.
-    ///
-    /// A variable-length slice pattern covers all lengths from its arity up to infinity. As we just
-    /// saw, we can split this in two: lengths below `L` are treated individually with a
-    /// fixed-length slice each; lengths above `L` are grouped into a single variable-length slice
-    /// constructor.
-    ///
-    /// For each variable-length slice pattern `p` with a prefix of length `plₚ` and suffix of
-    /// length `slₚ`, only the first `plₚ` and the last `slₚ` elements are examined. Therefore, as
-    /// long as `L` is positive (to avoid concerns about empty types), all elements after the
-    /// maximum prefix length and before the maximum suffix length are not examined by any
-    /// variable-length pattern, and therefore can be ignored. This gives us a way to compute `L`.
-    ///
-    /// Additionally, if fixed-length patterns exist, we must pick an `L` large enough to miss them,
-    /// so we can pick `L = max(max(FIXED_LEN)+1, max(PREFIX_LEN) + max(SUFFIX_LEN))`.
-    /// `max_slice` below will be made to have this arity `L`.
-    ///
-    /// If `self` is fixed-length, it is returned as-is.
-    ///
-    /// Additionally, we track for each output slice whether it is covered by one of the column slices or not.
-    fn split(
-        self,
-        column_slices: impl Iterator<Item = Slice>,
-    ) -> impl Iterator<Item = (Presence, Slice)> {
-        // Range of lengths below `L`.
-        let smaller_lengths;
-        let arity = self.arity();
-        let mut max_slice = self.kind;
-        // Tracks the smallest variable-length slice we've seen. Any slice arity above it is
-        // therefore `Presence::Seen` in the column.
-        let mut min_var_len = usize::MAX;
-        // Tracks the fixed-length slices we've seen, to mark them as `Presence::Seen`.
-        let mut seen_fixed_lens = FxHashSet::default();
-        match &mut max_slice {
-            VarLen(max_prefix_len, max_suffix_len) => {
-                // We grow `max_slice` to be larger than all slices encountered, as described above.
-                // For diagnostics, we keep the prefix and suffix lengths separate, but grow them so that
-                // `L = max_prefix_len + max_suffix_len`.
-                let mut max_fixed_len = 0;
-                for slice in column_slices {
-                    match slice.kind {
-                        FixedLen(len) => {
-                            max_fixed_len = cmp::max(max_fixed_len, len);
-                            if arity <= len {
-                                seen_fixed_lens.insert(len);
-                            }
-                        }
-                        VarLen(prefix, suffix) => {
-                            *max_prefix_len = cmp::max(*max_prefix_len, prefix);
-                            *max_suffix_len = cmp::max(*max_suffix_len, suffix);
-                            min_var_len = cmp::min(min_var_len, prefix + suffix);
-                        }
-                    }
-                }
-                // We want `L = max(L, max_fixed_len + 1)`, modulo the fact that we keep prefix and
-                // suffix separate.
-                if max_fixed_len + 1 >= *max_prefix_len + *max_suffix_len {
-                    // The subtraction can't overflow thanks to the above check.
-                    // The new `max_prefix_len` is larger than its previous value.
-                    *max_prefix_len = max_fixed_len + 1 - *max_suffix_len;
-                }
-
-                // We cap the arity of `max_slice` at the array size.
-                match self.array_len {
-                    Some(len) if max_slice.arity() >= len => max_slice = FixedLen(len),
-                    _ => {}
-                }
-
-                smaller_lengths = match self.array_len {
-                    // The only admissible fixed-length slice is one of the array size. Whether `max_slice`
-                    // is fixed-length or variable-length, it will be the only relevant slice to output
-                    // here.
-                    Some(_) => 0..0, // empty range
-                    // We need to cover all arities in the range `(arity..infinity)`. We split that
-                    // range into two: lengths smaller than `max_slice.arity()` are treated
-                    // independently as fixed-lengths slices, and lengths above are captured by
-                    // `max_slice`.
-                    None => self.arity()..max_slice.arity(),
-                };
-            }
-            FixedLen(_) => {
-                // No need to split here. We only track presence.
-                for slice in column_slices {
-                    match slice.kind {
-                        FixedLen(len) => {
-                            if len == arity {
-                                seen_fixed_lens.insert(len);
-                            }
-                        }
-                        VarLen(prefix, suffix) => {
-                            min_var_len = cmp::min(min_var_len, prefix + suffix);
-                        }
-                    }
-                }
-                smaller_lengths = 0..0;
-            }
-        };
-
-        smaller_lengths.map(FixedLen).chain(once(max_slice)).map(move |kind| {
-            let arity = kind.arity();
-            let seen = if min_var_len <= arity || seen_fixed_lens.contains(&arity) {
-                Presence::Seen
-            } else {
-                Presence::Unseen
-            };
-            (seen, Slice::new(self.array_len, kind))
-        })
-    }
-}
-
-/// A value can be decomposed into a constructor applied to some fields. This struct represents
-/// the constructor. See also `Fields`.
-///
-/// `pat_constructor` retrieves the constructor corresponding to a pattern.
-/// `specialize_constructor` returns the list of fields corresponding to a pattern, given a
-/// constructor. `Constructor::apply` reconstructs the pattern from a pair of `Constructor` and
-/// `Fields`.
-#[derive(Clone, Debug, PartialEq)]
-pub(super) enum Constructor<'tcx> {
-    /// The constructor for patterns that have a single constructor, like tuples, struct patterns
-    /// and fixed-length arrays.
-    Single,
-    /// Enum variants.
-    Variant(VariantIdx),
-    /// Booleans
-    Bool(bool),
-    /// Ranges of integer literal values (`2`, `2..=5` or `2..5`).
-    IntRange(IntRange),
-    /// Ranges of floating-point literal values (`2.0..=5.2`).
-    F32Range(IeeeFloat<SingleS>, IeeeFloat<SingleS>, RangeEnd),
-    F64Range(IeeeFloat<DoubleS>, IeeeFloat<DoubleS>, RangeEnd),
-    /// String literals. Strings are not quite the same as `&[u8]` so we treat them separately.
-    Str(mir::Const<'tcx>),
-    /// Array and slice patterns.
-    Slice(Slice),
-    /// Constants that must not be matched structurally. They are treated as black
-    /// boxes for the purposes of exhaustiveness: we must not inspect them, and they
-    /// don't count towards making a match exhaustive.
-    Opaque,
-    /// Or-pattern.
-    Or,
-    /// Wildcard pattern.
-    Wildcard,
-    /// Fake extra constructor for enums that aren't allowed to be matched exhaustively. Also used
-    /// for those types for which we cannot list constructors explicitly, like `f64` and `str`.
-    NonExhaustive,
-    /// Fake extra constructor for variants that should not be mentioned in diagnostics.
-    /// We use this for variants behind an unstable gate as well as
-    /// `#[doc(hidden)]` ones.
-    Hidden,
-    /// Fake extra constructor for constructors that are not seen in the matrix, as explained in the
-    /// code for [`Constructor::split`].
-    Missing,
-}
-
-impl<'tcx> Constructor<'tcx> {
-    pub(super) fn is_non_exhaustive(&self) -> bool {
-        matches!(self, NonExhaustive)
-    }
-
-    pub(super) fn as_variant(&self) -> Option<VariantIdx> {
-        match self {
-            Variant(i) => Some(*i),
-            _ => None,
-        }
-    }
-    fn as_bool(&self) -> Option<bool> {
-        match self {
-            Bool(b) => Some(*b),
-            _ => None,
-        }
-    }
-    pub(super) fn as_int_range(&self) -> Option<&IntRange> {
-        match self {
-            IntRange(range) => Some(range),
-            _ => None,
-        }
-    }
-    fn as_slice(&self) -> Option<Slice> {
-        match self {
-            Slice(slice) => Some(*slice),
-            _ => None,
-        }
-    }
-
-    fn variant_index_for_adt(&self, adt: ty::AdtDef<'tcx>) -> VariantIdx {
-        match *self {
-            Variant(idx) => idx,
-            Single => {
-                assert!(!adt.is_enum());
-                FIRST_VARIANT
-            }
-            _ => bug!("bad constructor {:?} for adt {:?}", self, adt),
-        }
-    }
-
-    /// The number of fields for this constructor. This must be kept in sync with
-    /// `Fields::wildcards`.
-    pub(super) fn arity(&self, pcx: &PatCtxt<'_, '_, 'tcx>) -> usize {
-        match self {
-            Single | Variant(_) => match pcx.ty.kind() {
-                ty::Tuple(fs) => fs.len(),
-                ty::Ref(..) => 1,
-                ty::Adt(adt, ..) => {
-                    if adt.is_box() {
-                        // The only legal patterns of type `Box` (outside `std`) are `_` and box
-                        // patterns. If we're here we can assume this is a box pattern.
-                        1
-                    } else {
-                        let variant = &adt.variant(self.variant_index_for_adt(*adt));
-                        Fields::list_variant_nonhidden_fields(pcx.cx, pcx.ty, variant).count()
-                    }
-                }
-                _ => bug!("Unexpected type for `Single` constructor: {:?}", pcx.ty),
-            },
-            Slice(slice) => slice.arity(),
-            Bool(..)
-            | IntRange(..)
-            | F32Range(..)
-            | F64Range(..)
-            | Str(..)
-            | Opaque
-            | NonExhaustive
-            | Hidden
-            | Missing { .. }
-            | Wildcard => 0,
-            Or => bug!("The `Or` constructor doesn't have a fixed arity"),
-        }
-    }
-
-    /// Some constructors (namely `Wildcard`, `IntRange` and `Slice`) actually stand for a set of
-    /// actual constructors (like variants, integers or fixed-sized slices). When specializing for
-    /// these constructors, we want to be specialising for the actual underlying constructors.
-    /// Naively, we would simply return the list of constructors they correspond to. We instead are
-    /// more clever: if there are constructors that we know will behave the same w.r.t. the current
-    /// matrix, we keep them grouped. For example, all slices of a sufficiently large length will
-    /// either be all useful or all non-useful with a given matrix.
-    ///
-    /// See the branches for details on how the splitting is done.
-    ///
-    /// This function may discard some irrelevant constructors if this preserves behavior. Eg. for
-    /// the `_` case, we ignore the constructors already present in the column, unless all of them
-    /// are.
-    pub(super) fn split<'a>(
-        &self,
-        pcx: &PatCtxt<'_, '_, 'tcx>,
-        ctors: impl Iterator<Item = &'a Constructor<'tcx>> + Clone,
-    ) -> SmallVec<[Self; 1]>
-    where
-        'tcx: 'a,
-    {
-        match self {
-            Wildcard => {
-                let split_set = ConstructorSet::for_ty(pcx.cx, pcx.ty).split(pcx, ctors);
-                if !split_set.missing.is_empty() {
-                    // We are splitting a wildcard in order to compute its usefulness. Some constructors are
-                    // not present in the column. The first thing we note is that specializing with any of
-                    // the missing constructors would select exactly the rows with wildcards. Moreover, they
-                    // would all return equivalent results. We can therefore group them all into a
-                    // fictitious `Missing` constructor.
-                    //
-                    // As an important optimization, this function will skip all the present constructors.
-                    // This is correct because specializing with any of the present constructors would
-                    // select a strict superset of the wildcard rows, and thus would only find witnesses
-                    // already found with the `Missing` constructor.
-                    // This does mean that diagnostics are incomplete: in
-                    // ```
-                    // match x {
-                    //   Some(true) => {}
-                    // }
-                    // ```
-                    // we report `None` as missing but not `Some(false)`.
-                    //
-                    // When all the constructors are missing we can equivalently return the `Wildcard`
-                    // constructor on its own. The difference between `Wildcard` and `Missing` will then
-                    // only be in diagnostics.
-
-                    // If some constructors are missing, we typically want to report those constructors,
-                    // e.g.:
-                    // ```
-                    //     enum Direction { N, S, E, W }
-                    //     let Direction::N = ...;
-                    // ```
-                    // we can report 3 witnesses: `S`, `E`, and `W`.
-                    //
-                    // However, if the user didn't actually specify a constructor
-                    // in this arm, e.g., in
-                    // ```
-                    //     let x: (Direction, Direction, bool) = ...;
-                    //     let (_, _, false) = x;
-                    // ```
-                    // we don't want to show all 16 possible witnesses `(<direction-1>, <direction-2>,
-                    // true)` - we are satisfied with `(_, _, true)`. So if all constructors are missing we
-                    // prefer to report just a wildcard `_`.
-                    //
-                    // The exception is: if we are at the top-level, for example in an empty match, we
-                    // usually prefer to report the full list of constructors.
-                    let all_missing = split_set.present.is_empty();
-                    let report_when_all_missing =
-                        pcx.is_top_level && !IntRange::is_integral(pcx.ty);
-                    let ctor =
-                        if all_missing && !report_when_all_missing { Wildcard } else { Missing };
-                    smallvec![ctor]
-                } else {
-                    split_set.present
-                }
-            }
-            // Fast-track if the range is trivial.
-            IntRange(this_range) if !this_range.is_singleton() => {
-                let column_ranges = ctors.filter_map(|ctor| ctor.as_int_range()).cloned();
-                this_range.split(column_ranges).map(|(_, range)| IntRange(range)).collect()
-            }
-            Slice(this_slice @ Slice { kind: VarLen(..), .. }) => {
-                let column_slices = ctors.filter_map(|c| c.as_slice());
-                this_slice.split(column_slices).map(|(_, slice)| Slice(slice)).collect()
-            }
-            // Any other constructor can be used unchanged.
-            _ => smallvec![self.clone()],
-        }
-    }
-
-    /// Returns whether `self` is covered by `other`, i.e. whether `self` is a subset of `other`.
-    /// For the simple cases, this is simply checking for equality. For the "grouped" constructors,
-    /// this checks for inclusion.
-    // We inline because this has a single call site in `Matrix::specialize_constructor`.
-    #[inline]
-    pub(super) fn is_covered_by<'p>(&self, pcx: &PatCtxt<'_, 'p, 'tcx>, other: &Self) -> bool {
-        // This must be kept in sync with `is_covered_by_any`.
-        match (self, other) {
-            // Wildcards cover anything
-            (_, Wildcard) => true,
-            // Only a wildcard pattern can match these special constructors.
-            (Wildcard | Missing { .. } | NonExhaustive | Hidden, _) => false,
-
-            (Single, Single) => true,
-            (Variant(self_id), Variant(other_id)) => self_id == other_id,
-            (Bool(self_b), Bool(other_b)) => self_b == other_b,
-
-            (IntRange(self_range), IntRange(other_range)) => self_range.is_subrange(other_range),
-            (F32Range(self_from, self_to, self_end), F32Range(other_from, other_to, other_end)) => {
-                self_from.ge(other_from)
-                    && match self_to.partial_cmp(other_to) {
-                        Some(Ordering::Less) => true,
-                        Some(Ordering::Equal) => other_end == self_end,
-                        _ => false,
-                    }
-            }
-            (F64Range(self_from, self_to, self_end), F64Range(other_from, other_to, other_end)) => {
-                self_from.ge(other_from)
-                    && match self_to.partial_cmp(other_to) {
-                        Some(Ordering::Less) => true,
-                        Some(Ordering::Equal) => other_end == self_end,
-                        _ => false,
-                    }
-            }
-            (Str(self_val), Str(other_val)) => {
-                // FIXME Once valtrees are available we can directly use the bytes
-                // in the `Str` variant of the valtree for the comparison here.
-                self_val == other_val
-            }
-            (Slice(self_slice), Slice(other_slice)) => self_slice.is_covered_by(*other_slice),
-
-            // We are trying to inspect an opaque constant. Thus we skip the row.
-            (Opaque, _) | (_, Opaque) => false,
-
-            _ => span_bug!(
-                pcx.span,
-                "trying to compare incompatible constructors {:?} and {:?}",
-                self,
-                other
-            ),
-        }
-    }
-}
-
-/// Describes the set of all constructors for a type.
-#[derive(Debug)]
-pub(super) enum ConstructorSet {
-    /// The type has a single constructor, e.g. `&T` or a struct.
-    Single,
-    /// This type has the following list of constructors.
-    /// Some variants are hidden, which means they won't be mentioned in diagnostics unless the user
-    /// mentioned them first. We use this for variants behind an unstable gate as well as
-    /// `#[doc(hidden)]` ones.
-    Variants {
-        visible_variants: Vec<VariantIdx>,
-        hidden_variants: Vec<VariantIdx>,
-        non_exhaustive: bool,
-    },
-    /// Booleans.
-    Bool,
-    /// The type is spanned by integer values. The range or ranges give the set of allowed values.
-    /// The second range is only useful for `char`.
-    Integers { range_1: IntRange, range_2: Option<IntRange> },
-    /// The type is matched by slices. The usize is the compile-time length of the array, if known.
-    Slice(Option<usize>),
-    /// The type is matched by slices whose elements are uninhabited.
-    SliceOfEmpty,
-    /// The constructors cannot be listed, and the type cannot be matched exhaustively. E.g. `str`,
-    /// floats.
-    Unlistable,
-    /// The type has no inhabitants.
-    Uninhabited,
-}
-
-/// Describes the result of analyzing the constructors in a column of a match.
-///
-/// `present` is morally the set of constructors present in the column, and `missing` is the set of
-/// constructors that exist in the type but are not present in the column.
-///
-/// More formally, they respect the following constraints:
-/// - the union of `present` and `missing` covers the whole type
-/// - `present` and `missing` are disjoint
-/// - neither contains wildcards
-/// - each constructor in `present` is covered by some non-wildcard constructor in the column
-/// - together, the constructors in `present` cover all the non-wildcard constructor in the column
-/// - non-wildcards in the column do no cover anything in `missing`
-/// - constructors in `present` and `missing` are split for the column; in other words, they are
-///     either fully included in or disjoint from each constructor in the column. This avoids
-///     non-trivial intersections like between `0..10` and `5..15`.
-#[derive(Debug)]
-pub(super) struct SplitConstructorSet<'tcx> {
-    pub(super) present: SmallVec<[Constructor<'tcx>; 1]>,
-    pub(super) missing: Vec<Constructor<'tcx>>,
-}
-
-impl ConstructorSet {
-    #[instrument(level = "debug", skip(cx), ret)]
-    pub(super) fn for_ty<'p, 'tcx>(cx: &MatchCheckCtxt<'p, 'tcx>, ty: Ty<'tcx>) -> Self {
-        let make_range = |start, end| {
-            IntRange::from_range(
-                MaybeInfiniteInt::new_finite(cx.tcx, ty, start),
-                MaybeInfiniteInt::new_finite(cx.tcx, ty, end),
-                RangeEnd::Included,
-            )
-        };
-        // This determines the set of all possible constructors for the type `ty`. For numbers,
-        // arrays and slices we use ranges and variable-length slices when appropriate.
-        //
-        // If the `exhaustive_patterns` feature is enabled, we make sure to omit constructors that
-        // are statically impossible. E.g., for `Option<!>`, we do not include `Some(_)` in the
-        // returned list of constructors.
-        // Invariant: this is `Uninhabited` if and only if the type is uninhabited (as determined by
-        // `cx.is_uninhabited()`).
-        match ty.kind() {
-            ty::Bool => Self::Bool,
-            ty::Char => {
-                // The valid Unicode Scalar Value ranges.
-                Self::Integers {
-                    range_1: make_range('\u{0000}' as u128, '\u{D7FF}' as u128),
-                    range_2: Some(make_range('\u{E000}' as u128, '\u{10FFFF}' as u128)),
-                }
-            }
-            &ty::Int(ity) => {
-                let range = if ty.is_ptr_sized_integral()
-                    && !cx.tcx.features().precise_pointer_size_matching
-                {
-                    // The min/max values of `isize` are not allowed to be observed unless the
-                    // `precise_pointer_size_matching` feature is enabled.
-                    IntRange { lo: NegInfinity, hi: PosInfinity }
-                } else {
-                    let bits = Integer::from_int_ty(&cx.tcx, ity).size().bits() as u128;
-                    let min = 1u128 << (bits - 1);
-                    let max = min - 1;
-                    make_range(min, max)
-                };
-                Self::Integers { range_1: range, range_2: None }
-            }
-            &ty::Uint(uty) => {
-                let range = if ty.is_ptr_sized_integral()
-                    && !cx.tcx.features().precise_pointer_size_matching
-                {
-                    // The max value of `usize` is not allowed to be observed unless the
-                    // `precise_pointer_size_matching` feature is enabled.
-                    let lo = MaybeInfiniteInt::new_finite(cx.tcx, ty, 0);
-                    IntRange { lo, hi: PosInfinity }
-                } else {
-                    let size = Integer::from_uint_ty(&cx.tcx, uty).size();
-                    let max = size.truncate(u128::MAX);
-                    make_range(0, max)
-                };
-                Self::Integers { range_1: range, range_2: None }
-            }
-            ty::Array(sub_ty, len) if len.try_eval_target_usize(cx.tcx, cx.param_env).is_some() => {
-                let len = len.eval_target_usize(cx.tcx, cx.param_env) as usize;
-                if len != 0 && cx.is_uninhabited(*sub_ty) {
-                    Self::Uninhabited
-                } else {
-                    Self::Slice(Some(len))
-                }
-            }
-            // Treat arrays of a constant but unknown length like slices.
-            ty::Array(sub_ty, _) | ty::Slice(sub_ty) => {
-                if cx.is_uninhabited(*sub_ty) {
-                    Self::SliceOfEmpty
-                } else {
-                    Self::Slice(None)
-                }
-            }
-            ty::Adt(def, args) if def.is_enum() => {
-                // If the enum is declared as `#[non_exhaustive]`, we treat it as if it had an
-                // additional "unknown" constructor.
-                // There is no point in enumerating all possible variants, because the user can't
-                // actually match against them all themselves. So we always return only the fictitious
-                // constructor.
-                // E.g., in an example like:
-                //
-                // ```
-                //     let err: io::ErrorKind = ...;
-                //     match err {
-                //         io::ErrorKind::NotFound => {},
-                //     }
-                // ```
-                //
-                // we don't want to show every possible IO error, but instead have only `_` as the
-                // witness.
-                let is_declared_nonexhaustive = cx.is_foreign_non_exhaustive_enum(ty);
-
-                if def.variants().is_empty() && !is_declared_nonexhaustive {
-                    Self::Uninhabited
-                } else {
-                    let is_exhaustive_pat_feature = cx.tcx.features().exhaustive_patterns;
-                    let (hidden_variants, visible_variants) = def
-                        .variants()
-                        .iter_enumerated()
-                        .filter(|(_, v)| {
-                            // If `exhaustive_patterns` is enabled, we exclude variants known to be
-                            // uninhabited.
-                            !is_exhaustive_pat_feature
-                                || v.inhabited_predicate(cx.tcx, *def)
-                                    .instantiate(cx.tcx, args)
-                                    .apply(cx.tcx, cx.param_env, cx.module)
-                        })
-                        .map(|(idx, _)| idx)
-                        .partition(|idx| {
-                            let variant_def_id = def.variant(*idx).def_id;
-                            // Filter variants that depend on a disabled unstable feature.
-                            let is_unstable = matches!(
-                                cx.tcx.eval_stability(variant_def_id, None, DUMMY_SP, None),
-                                EvalResult::Deny { .. }
-                            );
-                            // Filter foreign `#[doc(hidden)]` variants.
-                            let is_doc_hidden =
-                                cx.tcx.is_doc_hidden(variant_def_id) && !variant_def_id.is_local();
-                            is_unstable || is_doc_hidden
-                        });
-
-                    Self::Variants {
-                        visible_variants,
-                        hidden_variants,
-                        non_exhaustive: is_declared_nonexhaustive,
-                    }
-                }
-            }
-            ty::Never => Self::Uninhabited,
-            _ if cx.is_uninhabited(ty) => Self::Uninhabited,
-            ty::Adt(..) | ty::Tuple(..) | ty::Ref(..) => Self::Single,
-            // This type is one for which we cannot list constructors, like `str` or `f64`.
-            _ => Self::Unlistable,
-        }
-    }
-
-    /// This is the core logical operation of exhaustiveness checking. This analyzes a column a
-    /// constructors to 1/ determine which constructors of the type (if any) are missing; 2/ split
-    /// constructors to handle non-trivial intersections e.g. on ranges or slices.
-    #[instrument(level = "debug", skip(self, pcx, ctors), ret)]
-    pub(super) fn split<'a, 'tcx>(
-        &self,
-        pcx: &PatCtxt<'_, '_, 'tcx>,
-        ctors: impl Iterator<Item = &'a Constructor<'tcx>> + Clone,
-    ) -> SplitConstructorSet<'tcx>
-    where
-        'tcx: 'a,
-    {
-        let mut present: SmallVec<[_; 1]> = SmallVec::new();
-        let mut missing = Vec::new();
-        // Constructors in `ctors`, except wildcards.
-        let mut seen = ctors.filter(|c| !(matches!(c, Opaque | Wildcard)));
-        match self {
-            ConstructorSet::Single => {
-                if seen.next().is_none() {
-                    missing.push(Single);
-                } else {
-                    present.push(Single);
-                }
-            }
-            ConstructorSet::Variants { visible_variants, hidden_variants, non_exhaustive } => {
-                let seen_set: FxHashSet<_> = seen.map(|c| c.as_variant().unwrap()).collect();
-                let mut skipped_a_hidden_variant = false;
-
-                for variant in visible_variants {
-                    let ctor = Variant(*variant);
-                    if seen_set.contains(&variant) {
-                        present.push(ctor);
-                    } else {
-                        missing.push(ctor);
-                    }
-                }
-
-                for variant in hidden_variants {
-                    let ctor = Variant(*variant);
-                    if seen_set.contains(&variant) {
-                        present.push(ctor);
-                    } else {
-                        skipped_a_hidden_variant = true;
-                    }
-                }
-                if skipped_a_hidden_variant {
-                    missing.push(Hidden);
-                }
-
-                if *non_exhaustive {
-                    missing.push(NonExhaustive);
-                }
-            }
-            ConstructorSet::Bool => {
-                let mut seen_false = false;
-                let mut seen_true = false;
-                for b in seen.map(|ctor| ctor.as_bool().unwrap()) {
-                    if b {
-                        seen_true = true;
-                    } else {
-                        seen_false = true;
-                    }
-                }
-                if seen_false {
-                    present.push(Bool(false));
-                } else {
-                    missing.push(Bool(false));
-                }
-                if seen_true {
-                    present.push(Bool(true));
-                } else {
-                    missing.push(Bool(true));
-                }
-            }
-            ConstructorSet::Integers { range_1, range_2 } => {
-                let seen_ranges: Vec<_> =
-                    seen.map(|ctor| ctor.as_int_range().unwrap().clone()).collect();
-                for (seen, splitted_range) in range_1.split(seen_ranges.iter().cloned()) {
-                    match seen {
-                        Presence::Unseen => missing.push(IntRange(splitted_range)),
-                        Presence::Seen => present.push(IntRange(splitted_range)),
-                    }
-                }
-                if let Some(range_2) = range_2 {
-                    for (seen, splitted_range) in range_2.split(seen_ranges.into_iter()) {
-                        match seen {
-                            Presence::Unseen => missing.push(IntRange(splitted_range)),
-                            Presence::Seen => present.push(IntRange(splitted_range)),
-                        }
-                    }
-                }
-            }
-            &ConstructorSet::Slice(array_len) => {
-                let seen_slices = seen.map(|c| c.as_slice().unwrap());
-                let base_slice = Slice::new(array_len, VarLen(0, 0));
-                for (seen, splitted_slice) in base_slice.split(seen_slices) {
-                    let ctor = Slice(splitted_slice);
-                    match seen {
-                        Presence::Unseen => missing.push(ctor),
-                        Presence::Seen => present.push(ctor),
-                    }
-                }
-            }
-            ConstructorSet::SliceOfEmpty => {
-                // This one is tricky because even though there's only one possible value of this
-                // type (namely `[]`), slice patterns of all lengths are allowed, they're just
-                // unreachable if length != 0.
-                // We still gather the seen constructors in `present`, but the only slice that can
-                // go in `missing` is `[]`.
-                let seen_slices = seen.map(|c| c.as_slice().unwrap());
-                let base_slice = Slice::new(None, VarLen(0, 0));
-                for (seen, splitted_slice) in base_slice.split(seen_slices) {
-                    let ctor = Slice(splitted_slice);
-                    match seen {
-                        Presence::Seen => present.push(ctor),
-                        Presence::Unseen if splitted_slice.arity() == 0 => {
-                            missing.push(Slice(Slice::new(None, FixedLen(0))))
-                        }
-                        Presence::Unseen => {}
-                    }
-                }
-            }
-            ConstructorSet::Unlistable => {
-                // Since we can't list constructors, we take the ones in the column. This might list
-                // some constructors several times but there's not much we can do.
-                present.extend(seen.cloned());
-                missing.push(NonExhaustive);
-            }
-            // If `exhaustive_patterns` is disabled and our scrutinee is an empty type, we cannot
-            // expose its emptiness. The exception is if the pattern is at the top level, because we
-            // want empty matches to be considered exhaustive.
-            ConstructorSet::Uninhabited
-                if !pcx.cx.tcx.features().exhaustive_patterns && !pcx.is_top_level =>
-            {
-                missing.push(NonExhaustive);
-            }
-            ConstructorSet::Uninhabited => {}
-        }
-
-        SplitConstructorSet { present, missing }
-    }
-
-    /// Compute the set of constructors missing from this column.
-    /// This is only used for reporting to the user.
-    pub(super) fn compute_missing<'a, 'tcx>(
-        &self,
-        pcx: &PatCtxt<'_, '_, 'tcx>,
-        ctors: impl Iterator<Item = &'a Constructor<'tcx>> + Clone,
-    ) -> Vec<Constructor<'tcx>>
-    where
-        'tcx: 'a,
-    {
-        self.split(pcx, ctors).missing
-    }
-}
-
-/// A value can be decomposed into a constructor applied to some fields. This struct represents
-/// those fields, generalized to allow patterns in each field. See also `Constructor`.
-///
-/// This is constructed for a constructor using [`Fields::wildcards()`]. The idea is that
-/// [`Fields::wildcards()`] constructs a list of fields where all entries are wildcards, and then
-/// given a pattern we fill some of the fields with its subpatterns.
-/// In the following example `Fields::wildcards` returns `[_, _, _, _]`. Then in
-/// `extract_pattern_arguments` we fill some of the entries, and the result is
-/// `[Some(0), _, _, _]`.
-/// ```compile_fail,E0004
-/// # fn foo() -> [Option<u8>; 4] { [None; 4] }
-/// let x: [Option<u8>; 4] = foo();
-/// match x {
-///     [Some(0), ..] => {}
-/// }
-/// ```
-///
-/// Note that the number of fields of a constructor may not match the fields declared in the
-/// original struct/variant. This happens if a private or `non_exhaustive` field is uninhabited,
-/// because the code mustn't observe that it is uninhabited. In that case that field is not
-/// included in `fields`. For that reason, when you have a `FieldIdx` you must use
-/// `index_with_declared_idx`.
-#[derive(Debug, Clone, Copy)]
-pub(super) struct Fields<'p, 'tcx> {
-    fields: &'p [DeconstructedPat<'p, 'tcx>],
-}
-
-impl<'p, 'tcx> Fields<'p, 'tcx> {
-    fn empty() -> Self {
-        Fields { fields: &[] }
-    }
-
-    fn singleton(cx: &MatchCheckCtxt<'p, 'tcx>, field: DeconstructedPat<'p, 'tcx>) -> Self {
-        let field: &_ = cx.pattern_arena.alloc(field);
-        Fields { fields: std::slice::from_ref(field) }
-    }
-
-    pub(super) fn from_iter(
-        cx: &MatchCheckCtxt<'p, 'tcx>,
-        fields: impl IntoIterator<Item = DeconstructedPat<'p, 'tcx>>,
-    ) -> Self {
-        let fields: &[_] = cx.pattern_arena.alloc_from_iter(fields);
-        Fields { fields }
-    }
-
-    fn wildcards_from_tys(
-        cx: &MatchCheckCtxt<'p, 'tcx>,
-        tys: impl IntoIterator<Item = Ty<'tcx>>,
-        span: Span,
-    ) -> Self {
-        Fields::from_iter(cx, tys.into_iter().map(|ty| DeconstructedPat::wildcard(ty, span)))
-    }
-
-    // In the cases of either a `#[non_exhaustive]` field list or a non-public field, we hide
-    // uninhabited fields in order not to reveal the uninhabitedness of the whole variant.
-    // This lists the fields we keep along with their types.
-    fn list_variant_nonhidden_fields<'a>(
-        cx: &'a MatchCheckCtxt<'p, 'tcx>,
-        ty: Ty<'tcx>,
-        variant: &'a VariantDef,
-    ) -> impl Iterator<Item = (FieldIdx, Ty<'tcx>)> + Captures<'a> + Captures<'p> {
-        let ty::Adt(adt, args) = ty.kind() else { bug!() };
-        // Whether we must not match the fields of this variant exhaustively.
-        let is_non_exhaustive = variant.is_field_list_non_exhaustive() && !adt.did().is_local();
-
-        variant.fields.iter().enumerate().filter_map(move |(i, field)| {
-            let ty = field.ty(cx.tcx, args);
-            // `field.ty()` doesn't normalize after substituting.
-            let ty = cx.tcx.normalize_erasing_regions(cx.param_env, ty);
-            let is_visible = adt.is_enum() || field.vis.is_accessible_from(cx.module, cx.tcx);
-            let is_uninhabited = cx.is_uninhabited(ty);
-
-            if is_uninhabited && (!is_visible || is_non_exhaustive) {
-                None
-            } else {
-                Some((FieldIdx::new(i), ty))
-            }
-        })
-    }
-
-    /// Creates a new list of wildcard fields for a given constructor. The result must have a
-    /// length of `constructor.arity()`.
-    #[instrument(level = "trace")]
-    pub(super) fn wildcards(pcx: &PatCtxt<'_, 'p, 'tcx>, constructor: &Constructor<'tcx>) -> Self {
-        let ret = match constructor {
-            Single | Variant(_) => match pcx.ty.kind() {
-                ty::Tuple(fs) => Fields::wildcards_from_tys(pcx.cx, fs.iter(), pcx.span),
-                ty::Ref(_, rty, _) => Fields::wildcards_from_tys(pcx.cx, once(*rty), pcx.span),
-                ty::Adt(adt, args) => {
-                    if adt.is_box() {
-                        // The only legal patterns of type `Box` (outside `std`) are `_` and box
-                        // patterns. If we're here we can assume this is a box pattern.
-                        Fields::wildcards_from_tys(pcx.cx, once(args.type_at(0)), pcx.span)
-                    } else {
-                        let variant = &adt.variant(constructor.variant_index_for_adt(*adt));
-                        let tys = Fields::list_variant_nonhidden_fields(pcx.cx, pcx.ty, variant)
-                            .map(|(_, ty)| ty);
-                        Fields::wildcards_from_tys(pcx.cx, tys, pcx.span)
-                    }
-                }
-                _ => bug!("Unexpected type for `Single` constructor: {:?}", pcx),
-            },
-            Slice(slice) => match *pcx.ty.kind() {
-                ty::Slice(ty) | ty::Array(ty, _) => {
-                    let arity = slice.arity();
-                    Fields::wildcards_from_tys(pcx.cx, (0..arity).map(|_| ty), pcx.span)
-                }
-                _ => bug!("bad slice pattern {:?} {:?}", constructor, pcx),
-            },
-            Bool(..)
-            | IntRange(..)
-            | F32Range(..)
-            | F64Range(..)
-            | Str(..)
-            | Opaque
-            | NonExhaustive
-            | Hidden
-            | Missing { .. }
-            | Wildcard => Fields::empty(),
-            Or => {
-                bug!("called `Fields::wildcards` on an `Or` ctor")
-            }
-        };
-        debug!(?ret);
-        ret
-    }
-
-    /// Returns the list of patterns.
-    pub(super) fn iter_patterns<'a>(
-        &'a self,
-    ) -> impl Iterator<Item = &'p DeconstructedPat<'p, 'tcx>> + Captures<'a> {
-        self.fields.iter()
-    }
-}
-
-/// Values and patterns can be represented as a constructor applied to some fields. This represents
-/// a pattern in this form.
-/// This also uses interior mutability to keep track of whether the pattern has been found reachable
-/// during analysis. For this reason they cannot be cloned.
-/// A `DeconstructedPat` will almost always come from user input; the only exception are some
-/// `Wildcard`s introduced during specialization.
-pub(crate) struct DeconstructedPat<'p, 'tcx> {
-    ctor: Constructor<'tcx>,
-    fields: Fields<'p, 'tcx>,
-    ty: Ty<'tcx>,
-    span: Span,
-    reachable: Cell<bool>,
-}
-
-impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
-    pub(super) fn wildcard(ty: Ty<'tcx>, span: Span) -> Self {
-        Self::new(Wildcard, Fields::empty(), ty, span)
-    }
-
-    pub(super) fn new(
-        ctor: Constructor<'tcx>,
-        fields: Fields<'p, 'tcx>,
-        ty: Ty<'tcx>,
-        span: Span,
-    ) -> Self {
-        DeconstructedPat { ctor, fields, ty, span, reachable: Cell::new(false) }
-    }
-
-    pub(crate) fn from_pat(cx: &MatchCheckCtxt<'p, 'tcx>, pat: &Pat<'tcx>) -> Self {
-        let mkpat = |pat| DeconstructedPat::from_pat(cx, pat);
-        let ctor;
-        let fields;
-        match &pat.kind {
-            PatKind::AscribeUserType { subpattern, .. }
-            | PatKind::InlineConstant { subpattern, .. } => return mkpat(subpattern),
-            PatKind::Binding { subpattern: Some(subpat), .. } => return mkpat(subpat),
-            PatKind::Binding { subpattern: None, .. } | PatKind::Wild => {
-                ctor = Wildcard;
-                fields = Fields::empty();
-            }
-            PatKind::Deref { subpattern } => {
-                ctor = Single;
-                fields = Fields::singleton(cx, mkpat(subpattern));
-            }
-            PatKind::Leaf { subpatterns } | PatKind::Variant { subpatterns, .. } => {
-                match pat.ty.kind() {
-                    ty::Tuple(fs) => {
-                        ctor = Single;
-                        let mut wilds: SmallVec<[_; 2]> =
-                            fs.iter().map(|ty| DeconstructedPat::wildcard(ty, pat.span)).collect();
-                        for pat in subpatterns {
-                            wilds[pat.field.index()] = mkpat(&pat.pattern);
-                        }
-                        fields = Fields::from_iter(cx, wilds);
-                    }
-                    ty::Adt(adt, args) if adt.is_box() => {
-                        // The only legal patterns of type `Box` (outside `std`) are `_` and box
-                        // patterns. If we're here we can assume this is a box pattern.
-                        // FIXME(Nadrieril): A `Box` can in theory be matched either with `Box(_,
-                        // _)` or a box pattern. As a hack to avoid an ICE with the former, we
-                        // ignore other fields than the first one. This will trigger an error later
-                        // anyway.
-                        // See https://github.com/rust-lang/rust/issues/82772 ,
-                        // explanation: https://github.com/rust-lang/rust/pull/82789#issuecomment-796921977
-                        // The problem is that we can't know from the type whether we'll match
-                        // normally or through box-patterns. We'll have to figure out a proper
-                        // solution when we introduce generalized deref patterns. Also need to
-                        // prevent mixing of those two options.
-                        let pattern = subpatterns.into_iter().find(|pat| pat.field.index() == 0);
-                        let pat = if let Some(pat) = pattern {
-                            mkpat(&pat.pattern)
-                        } else {
-                            DeconstructedPat::wildcard(args.type_at(0), pat.span)
-                        };
-                        ctor = Single;
-                        fields = Fields::singleton(cx, pat);
-                    }
-                    ty::Adt(adt, _) => {
-                        ctor = match pat.kind {
-                            PatKind::Leaf { .. } => Single,
-                            PatKind::Variant { variant_index, .. } => Variant(variant_index),
-                            _ => bug!(),
-                        };
-                        let variant = &adt.variant(ctor.variant_index_for_adt(*adt));
-                        // For each field in the variant, we store the relevant index into `self.fields` if any.
-                        let mut field_id_to_id: Vec<Option<usize>> =
-                            (0..variant.fields.len()).map(|_| None).collect();
-                        let tys = Fields::list_variant_nonhidden_fields(cx, pat.ty, variant)
-                            .enumerate()
-                            .map(|(i, (field, ty))| {
-                                field_id_to_id[field.index()] = Some(i);
-                                ty
-                            });
-                        let mut wilds: SmallVec<[_; 2]> =
-                            tys.map(|ty| DeconstructedPat::wildcard(ty, pat.span)).collect();
-                        for pat in subpatterns {
-                            if let Some(i) = field_id_to_id[pat.field.index()] {
-                                wilds[i] = mkpat(&pat.pattern);
-                            }
-                        }
-                        fields = Fields::from_iter(cx, wilds);
-                    }
-                    _ => bug!("pattern has unexpected type: pat: {:?}, ty: {:?}", pat, pat.ty),
-                }
-            }
-            PatKind::Constant { value } => {
-                match pat.ty.kind() {
-                    ty::Bool => {
-                        ctor = match value.try_eval_bool(cx.tcx, cx.param_env) {
-                            Some(b) => Bool(b),
-                            None => Opaque,
-                        };
-                        fields = Fields::empty();
-                    }
-                    ty::Char | ty::Int(_) | ty::Uint(_) => {
-                        ctor = match value.try_eval_bits(cx.tcx, cx.param_env) {
-                            Some(bits) => IntRange(IntRange::from_bits(cx.tcx, pat.ty, bits)),
-                            None => Opaque,
-                        };
-                        fields = Fields::empty();
-                    }
-                    ty::Float(ty::FloatTy::F32) => {
-                        ctor = match value.try_eval_bits(cx.tcx, cx.param_env) {
-                            Some(bits) => {
-                                use rustc_apfloat::Float;
-                                let value = rustc_apfloat::ieee::Single::from_bits(bits);
-                                F32Range(value, value, RangeEnd::Included)
-                            }
-                            None => Opaque,
-                        };
-                        fields = Fields::empty();
-                    }
-                    ty::Float(ty::FloatTy::F64) => {
-                        ctor = match value.try_eval_bits(cx.tcx, cx.param_env) {
-                            Some(bits) => {
-                                use rustc_apfloat::Float;
-                                let value = rustc_apfloat::ieee::Double::from_bits(bits);
-                                F64Range(value, value, RangeEnd::Included)
-                            }
-                            None => Opaque,
-                        };
-                        fields = Fields::empty();
-                    }
-                    ty::Ref(_, t, _) if t.is_str() => {
-                        // We want a `&str` constant to behave like a `Deref` pattern, to be compatible
-                        // with other `Deref` patterns. This could have been done in `const_to_pat`,
-                        // but that causes issues with the rest of the matching code.
-                        // So here, the constructor for a `"foo"` pattern is `&` (represented by
-                        // `Single`), and has one field. That field has constructor `Str(value)` and no
-                        // fields.
-                        // Note: `t` is `str`, not `&str`.
-                        let subpattern =
-                            DeconstructedPat::new(Str(*value), Fields::empty(), *t, pat.span);
-                        ctor = Single;
-                        fields = Fields::singleton(cx, subpattern)
-                    }
-                    // All constants that can be structurally matched have already been expanded
-                    // into the corresponding `Pat`s by `const_to_pat`. Constants that remain are
-                    // opaque.
-                    _ => {
-                        ctor = Opaque;
-                        fields = Fields::empty();
-                    }
-                }
-            }
-            PatKind::Range(box PatRange { lo, hi, end, .. }) => {
-                let ty = pat.ty;
-                ctor = match ty.kind() {
-                    ty::Char | ty::Int(_) | ty::Uint(_) => {
-                        let lo =
-                            MaybeInfiniteInt::from_pat_range_bdy(*lo, ty, cx.tcx, cx.param_env);
-                        let hi =
-                            MaybeInfiniteInt::from_pat_range_bdy(*hi, ty, cx.tcx, cx.param_env);
-                        IntRange(IntRange::from_range(lo, hi, *end))
-                    }
-                    ty::Float(fty) => {
-                        use rustc_apfloat::Float;
-                        let lo = lo.as_finite().map(|c| c.eval_bits(cx.tcx, cx.param_env));
-                        let hi = hi.as_finite().map(|c| c.eval_bits(cx.tcx, cx.param_env));
-                        match fty {
-                            ty::FloatTy::F32 => {
-                                use rustc_apfloat::ieee::Single;
-                                let lo = lo.map(Single::from_bits).unwrap_or(-Single::INFINITY);
-                                let hi = hi.map(Single::from_bits).unwrap_or(Single::INFINITY);
-                                F32Range(lo, hi, *end)
-                            }
-                            ty::FloatTy::F64 => {
-                                use rustc_apfloat::ieee::Double;
-                                let lo = lo.map(Double::from_bits).unwrap_or(-Double::INFINITY);
-                                let hi = hi.map(Double::from_bits).unwrap_or(Double::INFINITY);
-                                F64Range(lo, hi, *end)
-                            }
-                        }
-                    }
-                    _ => bug!("invalid type for range pattern: {}", ty),
-                };
-                fields = Fields::empty();
-            }
-            PatKind::Array { prefix, slice, suffix } | PatKind::Slice { prefix, slice, suffix } => {
-                let array_len = match pat.ty.kind() {
-                    ty::Array(_, length) => {
-                        Some(length.eval_target_usize(cx.tcx, cx.param_env) as usize)
-                    }
-                    ty::Slice(_) => None,
-                    _ => span_bug!(pat.span, "bad ty {:?} for slice pattern", pat.ty),
-                };
-                let kind = if slice.is_some() {
-                    VarLen(prefix.len(), suffix.len())
-                } else {
-                    FixedLen(prefix.len() + suffix.len())
-                };
-                ctor = Slice(Slice::new(array_len, kind));
-                fields =
-                    Fields::from_iter(cx, prefix.iter().chain(suffix.iter()).map(|p| mkpat(&*p)));
-            }
-            PatKind::Or { .. } => {
-                ctor = Or;
-                let pats = expand_or_pat(pat);
-                fields = Fields::from_iter(cx, pats.into_iter().map(mkpat));
-            }
-            PatKind::Error(_) => {
-                ctor = Opaque;
-                fields = Fields::empty();
-            }
-        }
-        DeconstructedPat::new(ctor, fields, pat.ty, pat.span)
-    }
-
-    pub(super) fn is_or_pat(&self) -> bool {
-        matches!(self.ctor, Or)
-    }
-    pub(super) fn flatten_or_pat(&'p self) -> SmallVec<[&'p Self; 1]> {
-        if self.is_or_pat() {
-            self.iter_fields().flat_map(|p| p.flatten_or_pat()).collect()
-        } else {
-            smallvec![self]
-        }
-    }
-
-    pub(super) fn ctor(&self) -> &Constructor<'tcx> {
-        &self.ctor
-    }
-    pub(super) fn ty(&self) -> Ty<'tcx> {
-        self.ty
-    }
-    pub(super) fn span(&self) -> Span {
-        self.span
-    }
-
-    pub(super) fn iter_fields<'a>(
-        &'a self,
-    ) -> impl Iterator<Item = &'p DeconstructedPat<'p, 'tcx>> + Captures<'a> {
-        self.fields.iter_patterns()
-    }
-
-    /// Specialize this pattern with a constructor.
-    /// `other_ctor` can be different from `self.ctor`, but must be covered by it.
-    pub(super) fn specialize<'a>(
-        &'a self,
-        pcx: &PatCtxt<'_, 'p, 'tcx>,
-        other_ctor: &Constructor<'tcx>,
-    ) -> SmallVec<[&'p DeconstructedPat<'p, 'tcx>; 2]> {
-        match (&self.ctor, other_ctor) {
-            (Wildcard, _) => {
-                // We return a wildcard for each field of `other_ctor`.
-                Fields::wildcards(pcx, other_ctor).iter_patterns().collect()
-            }
-            (Slice(self_slice), Slice(other_slice))
-                if self_slice.arity() != other_slice.arity() =>
-            {
-                // The only tricky case: two slices of different arity. Since `self_slice` covers
-                // `other_slice`, `self_slice` must be `VarLen`, i.e. of the form
-                // `[prefix, .., suffix]`. Moreover `other_slice` is guaranteed to have a larger
-                // arity. So we fill the middle part with enough wildcards to reach the length of
-                // the new, larger slice.
-                match self_slice.kind {
-                    FixedLen(_) => bug!("{:?} doesn't cover {:?}", self_slice, other_slice),
-                    VarLen(prefix, suffix) => {
-                        let (ty::Slice(inner_ty) | ty::Array(inner_ty, _)) = *self.ty.kind() else {
-                            bug!("bad slice pattern {:?} {:?}", self.ctor, self.ty);
-                        };
-                        let prefix = &self.fields.fields[..prefix];
-                        let suffix = &self.fields.fields[self_slice.arity() - suffix..];
-                        let wildcard: &_ = pcx
-                            .cx
-                            .pattern_arena
-                            .alloc(DeconstructedPat::wildcard(inner_ty, pcx.span));
-                        let extra_wildcards = other_slice.arity() - self_slice.arity();
-                        let extra_wildcards = (0..extra_wildcards).map(|_| wildcard);
-                        prefix.iter().chain(extra_wildcards).chain(suffix).collect()
-                    }
-                }
-            }
-            _ => self.fields.iter_patterns().collect(),
-        }
-    }
-
-    /// We keep track for each pattern if it was ever reachable during the analysis. This is used
-    /// with `unreachable_spans` to report unreachable subpatterns arising from or patterns.
-    pub(super) fn set_reachable(&self) {
-        self.reachable.set(true)
-    }
-    pub(super) fn is_reachable(&self) -> bool {
-        self.reachable.get()
-    }
-
-    /// Report the spans of subpatterns that were not reachable, if any.
-    pub(super) fn unreachable_spans(&self) -> Vec<Span> {
-        let mut spans = Vec::new();
-        self.collect_unreachable_spans(&mut spans);
-        spans
-    }
-
-    fn collect_unreachable_spans(&self, spans: &mut Vec<Span>) {
-        // We don't look at subpatterns if we already reported the whole pattern as unreachable.
-        if !self.is_reachable() {
-            spans.push(self.span);
-        } else {
-            for p in self.iter_fields() {
-                p.collect_unreachable_spans(spans);
-            }
-        }
-    }
-}
-
-/// This is mostly copied from the `Pat` impl. This is best effort and not good enough for a
-/// `Display` impl.
-impl<'p, 'tcx> fmt::Debug for DeconstructedPat<'p, 'tcx> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        // Printing lists is a chore.
-        let mut first = true;
-        let mut start_or_continue = |s| {
-            if first {
-                first = false;
-                ""
-            } else {
-                s
-            }
-        };
-        let mut start_or_comma = || start_or_continue(", ");
-
-        match &self.ctor {
-            Single | Variant(_) => match self.ty.kind() {
-                ty::Adt(def, _) if def.is_box() => {
-                    // Without `box_patterns`, the only legal pattern of type `Box` is `_` (outside
-                    // of `std`). So this branch is only reachable when the feature is enabled and
-                    // the pattern is a box pattern.
-                    let subpattern = self.iter_fields().next().unwrap();
-                    write!(f, "box {subpattern:?}")
-                }
-                ty::Adt(..) | ty::Tuple(..) => {
-                    let variant = match self.ty.kind() {
-                        ty::Adt(adt, _) => Some(adt.variant(self.ctor.variant_index_for_adt(*adt))),
-                        ty::Tuple(_) => None,
-                        _ => unreachable!(),
-                    };
-
-                    if let Some(variant) = variant {
-                        write!(f, "{}", variant.name)?;
-                    }
-
-                    // Without `cx`, we can't know which field corresponds to which, so we can't
-                    // get the names of the fields. Instead we just display everything as a tuple
-                    // struct, which should be good enough.
-                    write!(f, "(")?;
-                    for p in self.iter_fields() {
-                        write!(f, "{}", start_or_comma())?;
-                        write!(f, "{p:?}")?;
-                    }
-                    write!(f, ")")
-                }
-                // Note: given the expansion of `&str` patterns done in `expand_pattern`, we should
-                // be careful to detect strings here. However a string literal pattern will never
-                // be reported as a non-exhaustiveness witness, so we can ignore this issue.
-                ty::Ref(_, _, mutbl) => {
-                    let subpattern = self.iter_fields().next().unwrap();
-                    write!(f, "&{}{:?}", mutbl.prefix_str(), subpattern)
-                }
-                _ => write!(f, "_"),
-            },
-            Slice(slice) => {
-                let mut subpatterns = self.fields.iter_patterns();
-                write!(f, "[")?;
-                match slice.kind {
-                    FixedLen(_) => {
-                        for p in subpatterns {
-                            write!(f, "{}{:?}", start_or_comma(), p)?;
-                        }
-                    }
-                    VarLen(prefix_len, _) => {
-                        for p in subpatterns.by_ref().take(prefix_len) {
-                            write!(f, "{}{:?}", start_or_comma(), p)?;
-                        }
-                        write!(f, "{}", start_or_comma())?;
-                        write!(f, "..")?;
-                        for p in subpatterns {
-                            write!(f, "{}{:?}", start_or_comma(), p)?;
-                        }
-                    }
-                }
-                write!(f, "]")
-            }
-            Bool(b) => write!(f, "{b}"),
-            // Best-effort, will render signed ranges incorrectly
-            IntRange(range) => write!(f, "{range:?}"),
-            F32Range(lo, hi, end) => write!(f, "{lo}{end}{hi}"),
-            F64Range(lo, hi, end) => write!(f, "{lo}{end}{hi}"),
-            Str(value) => write!(f, "{value}"),
-            Opaque => write!(f, "<constant pattern>"),
-            Or => {
-                for pat in self.iter_fields() {
-                    write!(f, "{}{:?}", start_or_continue(" | "), pat)?;
-                }
-                Ok(())
-            }
-            Wildcard | Missing { .. } | NonExhaustive | Hidden => write!(f, "_ : {:?}", self.ty),
-        }
-    }
-}
-
-/// Same idea as `DeconstructedPat`, except this is a fictitious pattern built up for diagnostics
-/// purposes. As such they don't use interning and can be cloned.
-#[derive(Debug, Clone)]
-pub(crate) struct WitnessPat<'tcx> {
-    ctor: Constructor<'tcx>,
-    pub(crate) fields: Vec<WitnessPat<'tcx>>,
-    ty: Ty<'tcx>,
-}
-
-impl<'tcx> WitnessPat<'tcx> {
-    pub(super) fn new(ctor: Constructor<'tcx>, fields: Vec<Self>, ty: Ty<'tcx>) -> Self {
-        Self { ctor, fields, ty }
-    }
-    pub(super) fn wildcard(ty: Ty<'tcx>) -> Self {
-        Self::new(Wildcard, Vec::new(), ty)
-    }
-
-    /// Construct a pattern that matches everything that starts with this constructor.
-    /// For example, if `ctor` is a `Constructor::Variant` for `Option::Some`, we get the pattern
-    /// `Some(_)`.
-    pub(super) fn wild_from_ctor(pcx: &PatCtxt<'_, '_, 'tcx>, ctor: Constructor<'tcx>) -> Self {
-        // Reuse `Fields::wildcards` to get the types.
-        let fields = Fields::wildcards(pcx, &ctor)
-            .iter_patterns()
-            .map(|deco_pat| Self::wildcard(deco_pat.ty()))
-            .collect();
-        Self::new(ctor, fields, pcx.ty)
-    }
-
-    pub(super) fn ctor(&self) -> &Constructor<'tcx> {
-        &self.ctor
-    }
-    pub(super) fn ty(&self) -> Ty<'tcx> {
-        self.ty
-    }
-
-    /// Convert back to a `thir::Pat` for diagnostic purposes. This panics for patterns that don't
-    /// appear in diagnostics, like float ranges.
-    pub(crate) fn to_diagnostic_pat(&self, cx: &MatchCheckCtxt<'_, 'tcx>) -> Pat<'tcx> {
-        let is_wildcard = |pat: &Pat<'_>| matches!(pat.kind, PatKind::Wild);
-        let mut subpatterns = self.iter_fields().map(|p| Box::new(p.to_diagnostic_pat(cx)));
-        let kind = match &self.ctor {
-            Bool(b) => PatKind::Constant { value: mir::Const::from_bool(cx.tcx, *b) },
-            IntRange(range) => return range.to_diagnostic_pat(self.ty, cx.tcx),
-            Single | Variant(_) => match self.ty.kind() {
-                ty::Tuple(..) => PatKind::Leaf {
-                    subpatterns: subpatterns
-                        .enumerate()
-                        .map(|(i, pattern)| FieldPat { field: FieldIdx::new(i), pattern })
-                        .collect(),
-                },
-                ty::Adt(adt_def, _) if adt_def.is_box() => {
-                    // Without `box_patterns`, the only legal pattern of type `Box` is `_` (outside
-                    // of `std`). So this branch is only reachable when the feature is enabled and
-                    // the pattern is a box pattern.
-                    PatKind::Deref { subpattern: subpatterns.next().unwrap() }
-                }
-                ty::Adt(adt_def, args) => {
-                    let variant_index = self.ctor.variant_index_for_adt(*adt_def);
-                    let variant = &adt_def.variant(variant_index);
-                    let subpatterns = Fields::list_variant_nonhidden_fields(cx, self.ty, variant)
-                        .zip(subpatterns)
-                        .map(|((field, _ty), pattern)| FieldPat { field, pattern })
-                        .collect();
-
-                    if adt_def.is_enum() {
-                        PatKind::Variant { adt_def: *adt_def, args, variant_index, subpatterns }
-                    } else {
-                        PatKind::Leaf { subpatterns }
-                    }
-                }
-                // Note: given the expansion of `&str` patterns done in `expand_pattern`, we should
-                // be careful to reconstruct the correct constant pattern here. However a string
-                // literal pattern will never be reported as a non-exhaustiveness witness, so we
-                // ignore this issue.
-                ty::Ref(..) => PatKind::Deref { subpattern: subpatterns.next().unwrap() },
-                _ => bug!("unexpected ctor for type {:?} {:?}", self.ctor, self.ty),
-            },
-            Slice(slice) => {
-                match slice.kind {
-                    FixedLen(_) => PatKind::Slice {
-                        prefix: subpatterns.collect(),
-                        slice: None,
-                        suffix: Box::new([]),
-                    },
-                    VarLen(prefix, _) => {
-                        let mut subpatterns = subpatterns.peekable();
-                        let mut prefix: Vec<_> = subpatterns.by_ref().take(prefix).collect();
-                        if slice.array_len.is_some() {
-                            // Improves diagnostics a bit: if the type is a known-size array, instead
-                            // of reporting `[x, _, .., _, y]`, we prefer to report `[x, .., y]`.
-                            // This is incorrect if the size is not known, since `[_, ..]` captures
-                            // arrays of lengths `>= 1` whereas `[..]` captures any length.
-                            while !prefix.is_empty() && is_wildcard(prefix.last().unwrap()) {
-                                prefix.pop();
-                            }
-                            while subpatterns.peek().is_some()
-                                && is_wildcard(subpatterns.peek().unwrap())
-                            {
-                                subpatterns.next();
-                            }
-                        }
-                        let suffix: Box<[_]> = subpatterns.collect();
-                        let wild = Pat::wildcard_from_ty(self.ty);
-                        PatKind::Slice {
-                            prefix: prefix.into_boxed_slice(),
-                            slice: Some(Box::new(wild)),
-                            suffix,
-                        }
-                    }
-                }
-            }
-            &Str(value) => PatKind::Constant { value },
-            Wildcard | NonExhaustive | Hidden => PatKind::Wild,
-            Missing { .. } => bug!(
-                "trying to convert a `Missing` constructor into a `Pat`; this is probably a bug,
-                `Missing` should have been processed in `apply_constructors`"
-            ),
-            F32Range(..) | F64Range(..) | Opaque | Or => {
-                bug!("can't convert to pattern: {:?}", self)
-            }
-        };
-
-        Pat { ty: self.ty, span: DUMMY_SP, kind }
-    }
-
-    pub(super) fn iter_fields<'a>(&'a self) -> impl Iterator<Item = &'a WitnessPat<'tcx>> {
-        self.fields.iter()
-    }
-}
diff --git a/compiler/rustc_mir_build/src/thir/pattern/mod.rs b/compiler/rustc_mir_build/src/thir/pattern/mod.rs
index 0811ab6a0..c7762f8ce 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/mod.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/mod.rs
@@ -2,11 +2,8 @@
 
 mod check_match;
 mod const_to_pat;
-pub(crate) mod deconstruct_pat;
-mod usefulness;
 
 pub(crate) use self::check_match::check_match;
-pub(crate) use self::usefulness::MatchCheckCtxt;
 
 use crate::errors::*;
 use crate::thir::util::UserAnnotatedTyHelpers;
@@ -18,17 +15,14 @@ use rustc_hir::pat_util::EnumerateAndAdjustIterator;
 use rustc_hir::RangeEnd;
 use rustc_index::Idx;
 use rustc_middle::mir::interpret::{ErrorHandled, GlobalId, LitToConstError, LitToConstInput};
-use rustc_middle::mir::{self, BorrowKind, Const, Mutability, UserTypeProjection};
+use rustc_middle::mir::{self, BorrowKind, Const, Mutability};
 use rustc_middle::thir::{
     Ascription, BindingMode, FieldPat, LocalVarId, Pat, PatKind, PatRange, PatRangeBoundary,
 };
 use rustc_middle::ty::layout::IntegerExt;
-use rustc_middle::ty::{
-    self, AdtDef, CanonicalUserTypeAnnotation, GenericArg, GenericArgsRef, Region, Ty, TyCtxt,
-    TypeVisitableExt, UserType,
-};
+use rustc_middle::ty::{self, CanonicalUserTypeAnnotation, Ty, TyCtxt, TypeVisitableExt};
 use rustc_span::def_id::LocalDefId;
-use rustc_span::{ErrorGuaranteed, Span, Symbol};
+use rustc_span::{ErrorGuaranteed, Span};
 use rustc_target::abi::{FieldIdx, Integer};
 
 use std::cmp::Ordering;
@@ -111,7 +105,7 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
                     let msg = format!(
                         "found bad range pattern endpoint `{expr:?}` outside of error recovery"
                     );
-                    return Err(self.tcx.sess.delay_span_bug(expr.span, msg));
+                    return Err(self.tcx.sess.span_delayed_bug(expr.span, msg));
                 };
                 Ok((Some(PatRangeBoundary::Finite(value)), ascr, inline_const))
             }
@@ -180,8 +174,8 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
         span: Span,
     ) -> Result<PatKind<'tcx>, ErrorGuaranteed> {
         if lo_expr.is_none() && hi_expr.is_none() {
-            let msg = format!("found twice-open range pattern (`..`) outside of error recovery");
-            return Err(self.tcx.sess.delay_span_bug(span, msg));
+            let msg = "found twice-open range pattern (`..`) outside of error recovery";
+            return Err(self.tcx.sess.span_delayed_bug(span, msg));
         }
 
         let (lo, lo_ascr, lo_inline) = self.lower_pattern_range_endpoint(lo_expr)?;
@@ -254,6 +248,8 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
         let kind = match pat.kind {
             hir::PatKind::Wild => PatKind::Wild,
 
+            hir::PatKind::Never => PatKind::Never,
+
             hir::PatKind::Lit(value) => self.lower_lit(value),
 
             hir::PatKind::Range(ref lo_expr, ref hi_expr, end) => {
@@ -266,16 +262,16 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
                 return self.lower_path(qpath, pat.hir_id, pat.span);
             }
 
-            hir::PatKind::Ref(ref subpattern, _) | hir::PatKind::Box(ref subpattern) => {
+            hir::PatKind::Ref(subpattern, _) | hir::PatKind::Box(subpattern) => {
                 PatKind::Deref { subpattern: self.lower_pattern(subpattern) }
             }
 
-            hir::PatKind::Slice(ref prefix, ref slice, ref suffix) => {
+            hir::PatKind::Slice(prefix, ref slice, suffix) => {
                 self.slice_or_array_pattern(pat.span, ty, prefix, slice, suffix)
             }
 
-            hir::PatKind::Tuple(ref pats, ddpos) => {
-                let ty::Tuple(ref tys) = ty.kind() else {
+            hir::PatKind::Tuple(pats, ddpos) => {
+                let ty::Tuple(tys) = ty.kind() else {
                     span_bug!(pat.span, "unexpected type for tuple pattern: {:?}", ty);
                 };
                 let subpatterns = self.lower_tuple_subpats(pats, tys.len(), ddpos);
@@ -325,7 +321,7 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
                 }
             }
 
-            hir::PatKind::TupleStruct(ref qpath, ref pats, ddpos) => {
+            hir::PatKind::TupleStruct(ref qpath, pats, ddpos) => {
                 let res = self.typeck_results.qpath_res(qpath, pat.hir_id);
                 let ty::Adt(adt_def, _) = ty.kind() else {
                     span_bug!(pat.span, "tuple struct pattern not applied to an ADT {:?}", ty);
@@ -335,20 +331,20 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
                 self.lower_variant_or_leaf(res, pat.hir_id, pat.span, ty, subpatterns)
             }
 
-            hir::PatKind::Struct(ref qpath, ref fields, _) => {
+            hir::PatKind::Struct(ref qpath, fields, _) => {
                 let res = self.typeck_results.qpath_res(qpath, pat.hir_id);
                 let subpatterns = fields
                     .iter()
                     .map(|field| FieldPat {
                         field: self.typeck_results.field_index(field.hir_id),
-                        pattern: self.lower_pattern(&field.pat),
+                        pattern: self.lower_pattern(field.pat),
                     })
                     .collect();
 
                 self.lower_variant_or_leaf(res, pat.hir_id, pat.span, ty, subpatterns)
             }
 
-            hir::PatKind::Or(ref pats) => PatKind::Or { pats: self.lower_patterns(pats) },
+            hir::PatKind::Or(pats) => PatKind::Or { pats: self.lower_patterns(pats) },
         };
 
         Box::new(Pat { span, ty, kind })
@@ -701,146 +697,3 @@ impl<'tcx> UserAnnotatedTyHelpers<'tcx> for PatCtxt<'_, 'tcx> {
         self.typeck_results
     }
 }
-
-trait PatternFoldable<'tcx>: Sized {
-    fn fold_with<F: PatternFolder<'tcx>>(&self, folder: &mut F) -> Self {
-        self.super_fold_with(folder)
-    }
-
-    fn super_fold_with<F: PatternFolder<'tcx>>(&self, folder: &mut F) -> Self;
-}
-
-trait PatternFolder<'tcx>: Sized {
-    fn fold_pattern(&mut self, pattern: &Pat<'tcx>) -> Pat<'tcx> {
-        pattern.super_fold_with(self)
-    }
-
-    fn fold_pattern_kind(&mut self, kind: &PatKind<'tcx>) -> PatKind<'tcx> {
-        kind.super_fold_with(self)
-    }
-}
-
-impl<'tcx, T: PatternFoldable<'tcx>> PatternFoldable<'tcx> for Box<T> {
-    fn super_fold_with<F: PatternFolder<'tcx>>(&self, folder: &mut F) -> Self {
-        let content: T = (**self).fold_with(folder);
-        Box::new(content)
-    }
-}
-
-impl<'tcx, T: PatternFoldable<'tcx>> PatternFoldable<'tcx> for Vec<T> {
-    fn super_fold_with<F: PatternFolder<'tcx>>(&self, folder: &mut F) -> Self {
-        self.iter().map(|t| t.fold_with(folder)).collect()
-    }
-}
-
-impl<'tcx, T: PatternFoldable<'tcx>> PatternFoldable<'tcx> for Box<[T]> {
-    fn super_fold_with<F: PatternFolder<'tcx>>(&self, folder: &mut F) -> Self {
-        self.iter().map(|t| t.fold_with(folder)).collect()
-    }
-}
-
-impl<'tcx, T: PatternFoldable<'tcx>> PatternFoldable<'tcx> for Option<T> {
-    fn super_fold_with<F: PatternFolder<'tcx>>(&self, folder: &mut F) -> Self {
-        self.as_ref().map(|t| t.fold_with(folder))
-    }
-}
-
-macro_rules! ClonePatternFoldableImpls {
-    (<$lt_tcx:tt> $($ty:ty),+) => {
-        $(
-            impl<$lt_tcx> PatternFoldable<$lt_tcx> for $ty {
-                fn super_fold_with<F: PatternFolder<$lt_tcx>>(&self, _: &mut F) -> Self {
-                    Clone::clone(self)
-                }
-            }
-        )+
-    }
-}
-
-ClonePatternFoldableImpls! { <'tcx>
-    Span, FieldIdx, Mutability, Symbol, LocalVarId, usize,
-    Region<'tcx>, Ty<'tcx>, BindingMode, AdtDef<'tcx>,
-    GenericArgsRef<'tcx>, &'tcx GenericArg<'tcx>, UserType<'tcx>,
-    UserTypeProjection, CanonicalUserTypeAnnotation<'tcx>
-}
-
-impl<'tcx> PatternFoldable<'tcx> for FieldPat<'tcx> {
-    fn super_fold_with<F: PatternFolder<'tcx>>(&self, folder: &mut F) -> Self {
-        FieldPat { field: self.field.fold_with(folder), pattern: self.pattern.fold_with(folder) }
-    }
-}
-
-impl<'tcx> PatternFoldable<'tcx> for Pat<'tcx> {
-    fn fold_with<F: PatternFolder<'tcx>>(&self, folder: &mut F) -> Self {
-        folder.fold_pattern(self)
-    }
-
-    fn super_fold_with<F: PatternFolder<'tcx>>(&self, folder: &mut F) -> Self {
-        Pat {
-            ty: self.ty.fold_with(folder),
-            span: self.span.fold_with(folder),
-            kind: self.kind.fold_with(folder),
-        }
-    }
-}
-
-impl<'tcx> PatternFoldable<'tcx> for PatKind<'tcx> {
-    fn fold_with<F: PatternFolder<'tcx>>(&self, folder: &mut F) -> Self {
-        folder.fold_pattern_kind(self)
-    }
-
-    fn super_fold_with<F: PatternFolder<'tcx>>(&self, folder: &mut F) -> Self {
-        match *self {
-            PatKind::Wild => PatKind::Wild,
-            PatKind::Error(e) => PatKind::Error(e),
-            PatKind::AscribeUserType {
-                ref subpattern,
-                ascription: Ascription { ref annotation, variance },
-            } => PatKind::AscribeUserType {
-                subpattern: subpattern.fold_with(folder),
-                ascription: Ascription { annotation: annotation.fold_with(folder), variance },
-            },
-            PatKind::Binding { mutability, name, mode, var, ty, ref subpattern, is_primary } => {
-                PatKind::Binding {
-                    mutability: mutability.fold_with(folder),
-                    name: name.fold_with(folder),
-                    mode: mode.fold_with(folder),
-                    var: var.fold_with(folder),
-                    ty: ty.fold_with(folder),
-                    subpattern: subpattern.fold_with(folder),
-                    is_primary,
-                }
-            }
-            PatKind::Variant { adt_def, args, variant_index, ref subpatterns } => {
-                PatKind::Variant {
-                    adt_def: adt_def.fold_with(folder),
-                    args: args.fold_with(folder),
-                    variant_index,
-                    subpatterns: subpatterns.fold_with(folder),
-                }
-            }
-            PatKind::Leaf { ref subpatterns } => {
-                PatKind::Leaf { subpatterns: subpatterns.fold_with(folder) }
-            }
-            PatKind::Deref { ref subpattern } => {
-                PatKind::Deref { subpattern: subpattern.fold_with(folder) }
-            }
-            PatKind::Constant { value } => PatKind::Constant { value },
-            PatKind::InlineConstant { def, subpattern: ref pattern } => {
-                PatKind::InlineConstant { def, subpattern: pattern.fold_with(folder) }
-            }
-            PatKind::Range(ref range) => PatKind::Range(range.clone()),
-            PatKind::Slice { ref prefix, ref slice, ref suffix } => PatKind::Slice {
-                prefix: prefix.fold_with(folder),
-                slice: slice.fold_with(folder),
-                suffix: suffix.fold_with(folder),
-            },
-            PatKind::Array { ref prefix, ref slice, ref suffix } => PatKind::Array {
-                prefix: prefix.fold_with(folder),
-                slice: slice.fold_with(folder),
-                suffix: suffix.fold_with(folder),
-            },
-            PatKind::Or { ref pats } => PatKind::Or { pats: pats.fold_with(folder) },
-        }
-    }
-}
diff --git a/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs b/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
deleted file mode 100644
index da7b6587a..000000000
--- a/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
+++ /dev/null
@@ -1,1205 +0,0 @@
-//! Note: tests specific to this file can be found in:
-//!
-//!   - `ui/pattern/usefulness`
-//!   - `ui/or-patterns`
-//!   - `ui/consts/const_in_pattern`
-//!   - `ui/rfc-2008-non-exhaustive`
-//!   - `ui/half-open-range-patterns`
-//!   - probably many others
-//!
-//! I (Nadrieril) prefer to put new tests in `ui/pattern/usefulness` unless there's a specific
-//! reason not to, for example if they depend on a particular feature like `or_patterns`.
-//!
-//! -----
-//!
-//! This file includes the logic for exhaustiveness and reachability checking for pattern-matching.
-//! Specifically, given a list of patterns for a type, we can tell whether:
-//! (a) each pattern is reachable (reachability)
-//! (b) the patterns cover every possible value for the type (exhaustiveness)
-//!
-//! The algorithm implemented here is a modified version of the one described in [this
-//! paper](http://moscova.inria.fr/~maranget/papers/warn/index.html). We have however generalized
-//! it to accommodate the variety of patterns that Rust supports. We thus explain our version here,
-//! without being as rigorous.
-//!
-//!
-//! # Summary
-//!
-//! The core of the algorithm is the notion of "usefulness". A pattern `q` is said to be *useful*
-//! relative to another pattern `p` of the same type if there is a value that is matched by `q` and
-//! not matched by `p`. This generalizes to many `p`s: `q` is useful w.r.t. a list of patterns
-//! `p_1 .. p_n` if there is a value that is matched by `q` and by none of the `p_i`. We write
-//! `usefulness(p_1 .. p_n, q)` for a function that returns a list of such values. The aim of this
-//! file is to compute it efficiently.
-//!
-//! This is enough to compute reachability: a pattern in a `match` expression is reachable iff it
-//! is useful w.r.t. the patterns above it:
-//! ```rust
-//! # fn foo(x: Option<i32>) {
-//! match x {
-//!     Some(_) => {},
-//!     None => {},    // reachable: `None` is matched by this but not the branch above
-//!     Some(0) => {}, // unreachable: all the values this matches are already matched by
-//!                    // `Some(_)` above
-//! }
-//! # }
-//! ```
-//!
-//! This is also enough to compute exhaustiveness: a match is exhaustive iff the wildcard `_`
-//! pattern is _not_ useful w.r.t. the patterns in the match. The values returned by `usefulness`
-//! are used to tell the user which values are missing.
-//! ```compile_fail,E0004
-//! # fn foo(x: Option<i32>) {
-//! match x {
-//!     Some(0) => {},
-//!     None => {},
-//!     // not exhaustive: `_` is useful because it matches `Some(1)`
-//! }
-//! # }
-//! ```
-//!
-//! The entrypoint of this file is the [`compute_match_usefulness`] function, which computes
-//! reachability for each match branch and exhaustiveness for the whole match.
-//!
-//!
-//! # Constructors and fields
-//!
-//! Note: we will often abbreviate "constructor" as "ctor".
-//!
-//! The idea that powers everything that is done in this file is the following: a (matchable)
-//! value is made from a constructor applied to a number of subvalues. Examples of constructors are
-//! `Some`, `None`, `(,)` (the 2-tuple constructor), `Foo {..}` (the constructor for a struct
-//! `Foo`), and `2` (the constructor for the number `2`). This is natural when we think of
-//! pattern-matching, and this is the basis for what follows.
-//!
-//! Some of the ctors listed above might feel weird: `None` and `2` don't take any arguments.
-//! That's ok: those are ctors that take a list of 0 arguments; they are the simplest case of
-//! ctors. We treat `2` as a ctor because `u64` and other number types behave exactly like a huge
-//! `enum`, with one variant for each number. This allows us to see any matchable value as made up
-//! from a tree of ctors, each having a set number of children. For example: `Foo { bar: None,
-//! baz: Ok(0) }` is made from 4 different ctors, namely `Foo{..}`, `None`, `Ok` and `0`.
-//!
-//! This idea can be extended to patterns: they are also made from constructors applied to fields.
-//! A pattern for a given type is allowed to use all the ctors for values of that type (which we
-//! call "value constructors"), but there are also pattern-only ctors. The most important one is
-//! the wildcard (`_`), and the others are integer ranges (`0..=10`), variable-length slices (`[x,
-//! ..]`), and or-patterns (`Ok(0) | Err(_)`). Examples of valid patterns are `42`, `Some(_)`, `Foo
-//! { bar: Some(0) | None, baz: _ }`. Note that a binder in a pattern (e.g. `Some(x)`) matches the
-//! same values as a wildcard (e.g. `Some(_)`), so we treat both as wildcards.
-//!
-//! From this deconstruction we can compute whether a given value matches a given pattern; we
-//! simply look at ctors one at a time. Given a pattern `p` and a value `v`, we want to compute
-//! `matches!(v, p)`. It's mostly straightforward: we compare the head ctors and when they match
-//! we compare their fields recursively. A few representative examples:
-//!
-//! - `matches!(v, _) := true`
-//! - `matches!((v0,  v1), (p0,  p1)) := matches!(v0, p0) && matches!(v1, p1)`
-//! - `matches!(Foo { bar: v0, baz: v1 }, Foo { bar: p0, baz: p1 }) := matches!(v0, p0) && matches!(v1, p1)`
-//! - `matches!(Ok(v0), Ok(p0)) := matches!(v0, p0)`
-//! - `matches!(Ok(v0), Err(p0)) := false` (incompatible variants)
-//! - `matches!(v, 1..=100) := matches!(v, 1) || ... || matches!(v, 100)`
-//! - `matches!([v0], [p0, .., p1]) := false` (incompatible lengths)
-//! - `matches!([v0, v1, v2], [p0, .., p1]) := matches!(v0, p0) && matches!(v2, p1)`
-//! - `matches!(v, p0 | p1) := matches!(v, p0) || matches!(v, p1)`
-//!
-//! Constructors, fields and relevant operations are defined in the [`super::deconstruct_pat`] module.
-//!
-//! Note: this constructors/fields distinction may not straightforwardly apply to every Rust type.
-//! For example a value of type `Rc<u64>` can't be deconstructed that way, and `&str` has an
-//! infinitude of constructors. There are also subtleties with visibility of fields and
-//! uninhabitedness and various other things. The constructors idea can be extended to handle most
-//! of these subtleties though; caveats are documented where relevant throughout the code.
-//!
-//! Whether constructors cover each other is computed by [`Constructor::is_covered_by`].
-//!
-//!
-//! # Specialization
-//!
-//! Recall that we wish to compute `usefulness(p_1 .. p_n, q)`: given a list of patterns `p_1 ..
-//! p_n` and a pattern `q`, all of the same type, we want to find a list of values (called
-//! "witnesses") that are matched by `q` and by none of the `p_i`. We obviously don't just
-//! enumerate all possible values. From the discussion above we see that we can proceed
-//! ctor-by-ctor: for each value ctor of the given type, we ask "is there a value that starts with
-//! this constructor and matches `q` and none of the `p_i`?". As we saw above, there's a lot we can
-//! say from knowing only the first constructor of our candidate value.
-//!
-//! Let's take the following example:
-//! ```compile_fail,E0004
-//! # enum Enum { Variant1(()), Variant2(Option<bool>, u32)}
-//! # fn foo(x: Enum) {
-//! match x {
-//!     Enum::Variant1(_) => {} // `p1`
-//!     Enum::Variant2(None, 0) => {} // `p2`
-//!     Enum::Variant2(Some(_), 0) => {} // `q`
-//! }
-//! # }
-//! ```
-//!
-//! We can easily see that if our candidate value `v` starts with `Variant1` it will not match `q`.
-//! If `v = Variant2(v0, v1)` however, whether or not it matches `p2` and `q` will depend on `v0`
-//! and `v1`. In fact, such a `v` will be a witness of usefulness of `q` exactly when the tuple
-//! `(v0, v1)` is a witness of usefulness of `q'` in the following reduced match:
-//!
-//! ```compile_fail,E0004
-//! # fn foo(x: (Option<bool>, u32)) {
-//! match x {
-//!     (None, 0) => {} // `p2'`
-//!     (Some(_), 0) => {} // `q'`
-//! }
-//! # }
-//! ```
-//!
-//! This motivates a new step in computing usefulness, that we call _specialization_.
-//! Specialization consist of filtering a list of patterns for those that match a constructor, and
-//! then looking into the constructor's fields. This enables usefulness to be computed recursively.
-//!
-//! Instead of acting on a single pattern in each row, we will consider a list of patterns for each
-//! row, and we call such a list a _pattern-stack_. The idea is that we will specialize the
-//! leftmost pattern, which amounts to popping the constructor and pushing its fields, which feels
-//! like a stack. We note a pattern-stack simply with `[p_1 ... p_n]`.
-//! Here's a sequence of specializations of a list of pattern-stacks, to illustrate what's
-//! happening:
-//! ```ignore (illustrative)
-//! [Enum::Variant1(_)]
-//! [Enum::Variant2(None, 0)]
-//! [Enum::Variant2(Some(_), 0)]
-//! //==>> specialize with `Variant2`
-//! [None, 0]
-//! [Some(_), 0]
-//! //==>> specialize with `Some`
-//! [_, 0]
-//! //==>> specialize with `true` (say the type was `bool`)
-//! [0]
-//! //==>> specialize with `0`
-//! []
-//! ```
-//!
-//! The function `specialize(c, p)` takes a value constructor `c` and a pattern `p`, and returns 0
-//! or more pattern-stacks. If `c` does not match the head constructor of `p`, it returns nothing;
-//! otherwise if returns the fields of the constructor. This only returns more than one
-//! pattern-stack if `p` has a pattern-only constructor.
-//!
-//! - Specializing for the wrong constructor returns nothing
-//!
-//!   `specialize(None, Some(p0)) := []`
-//!
-//! - Specializing for the correct constructor returns a single row with the fields
-//!
-//!   `specialize(Variant1, Variant1(p0, p1, p2)) := [[p0, p1, p2]]`
-//!
-//!   `specialize(Foo{..}, Foo { bar: p0, baz: p1 }) := [[p0, p1]]`
-//!
-//! - For or-patterns, we specialize each branch and concatenate the results
-//!
-//!   `specialize(c, p0 | p1) := specialize(c, p0) ++ specialize(c, p1)`
-//!
-//! - We treat the other pattern constructors as if they were a large or-pattern of all the
-//!   possibilities:
-//!
-//!   `specialize(c, _) := specialize(c, Variant1(_) | Variant2(_, _) | ...)`
-//!
-//!   `specialize(c, 1..=100) := specialize(c, 1 | ... | 100)`
-//!
-//!   `specialize(c, [p0, .., p1]) := specialize(c, [p0, p1] | [p0, _, p1] | [p0, _, _, p1] | ...)`
-//!
-//! - If `c` is a pattern-only constructor, `specialize` is defined on a case-by-case basis. See
-//!   the discussion about constructor splitting in [`super::deconstruct_pat`].
-//!
-//!
-//! We then extend this function to work with pattern-stacks as input, by acting on the first
-//! column and keeping the other columns untouched.
-//!
-//! Specialization for the whole matrix is done in [`Matrix::specialize_constructor`]. Note that
-//! or-patterns in the first column are expanded before being stored in the matrix. Specialization
-//! for a single patstack is done from a combination of [`Constructor::is_covered_by`] and
-//! [`PatStack::pop_head_constructor`]. The internals of how it's done mostly live in the
-//! [`super::deconstruct_pat::Fields`] struct.
-//!
-//!
-//! # Computing usefulness
-//!
-//! We now have all we need to compute usefulness. The inputs to usefulness are a list of
-//! pattern-stacks `p_1 ... p_n` (one per row), and a new pattern_stack `q`. The paper and this
-//! file calls the list of patstacks a _matrix_. They must all have the same number of columns and
-//! the patterns in a given column must all have the same type. `usefulness` returns a (possibly
-//! empty) list of witnesses of usefulness. These witnesses will also be pattern-stacks.
-//!
-//! - base case: `n_columns == 0`.
-//!     Since a pattern-stack functions like a tuple of patterns, an empty one functions like the
-//!     unit type. Thus `q` is useful iff there are no rows above it, i.e. if `n == 0`.
-//!
-//! - inductive case: `n_columns > 0`.
-//!     We need a way to list the constructors we want to try. We will be more clever in the next
-//!     section but for now assume we list all value constructors for the type of the first column.
-//!
-//!     - for each such ctor `c`:
-//!
-//!         - for each `q'` returned by `specialize(c, q)`:
-//!
-//!             - we compute `usefulness(specialize(c, p_1) ... specialize(c, p_n), q')`
-//!
-//!         - for each witness found, we revert specialization by pushing the constructor `c` on top.
-//!
-//!     - We return the concatenation of all the witnesses found, if any.
-//!
-//! Example:
-//! ```ignore (illustrative)
-//! [Some(true)] // p_1
-//! [None] // p_2
-//! [Some(_)] // q
-//! //==>> try `None`: `specialize(None, q)` returns nothing
-//! //==>> try `Some`: `specialize(Some, q)` returns a single row
-//! [true] // p_1'
-//! [_] // q'
-//! //==>> try `true`: `specialize(true, q')` returns a single row
-//! [] // p_1''
-//! [] // q''
-//! //==>> base case; `n != 0` so `q''` is not useful.
-//! //==>> go back up a step
-//! [true] // p_1'
-//! [_] // q'
-//! //==>> try `false`: `specialize(false, q')` returns a single row
-//! [] // q''
-//! //==>> base case; `n == 0` so `q''` is useful. We return the single witness `[]`
-//! witnesses:
-//! []
-//! //==>> undo the specialization with `false`
-//! witnesses:
-//! [false]
-//! //==>> undo the specialization with `Some`
-//! witnesses:
-//! [Some(false)]
-//! //==>> we have tried all the constructors. The output is the single witness `[Some(false)]`.
-//! ```
-//!
-//! This computation is done in [`is_useful`]. In practice we don't care about the list of
-//! witnesses when computing reachability; we only need to know whether any exist. We do keep the
-//! witnesses when computing exhaustiveness to report them to the user.
-//!
-//!
-//! # Making usefulness tractable: constructor splitting
-//!
-//! We're missing one last detail: which constructors do we list? Naively listing all value
-//! constructors cannot work for types like `u64` or `&str`, so we need to be more clever. The
-//! first obvious insight is that we only want to list constructors that are covered by the head
-//! constructor of `q`. If it's a value constructor, we only try that one. If it's a pattern-only
-//! constructor, we use the final clever idea for this algorithm: _constructor splitting_, where we
-//! group together constructors that behave the same.
-//!
-//! The details are not necessary to understand this file, so we explain them in
-//! [`super::deconstruct_pat`]. Splitting is done by the [`Constructor::split`] function.
-//!
-//! # Constants in patterns
-//!
-//! There are two kinds of constants in patterns:
-//!
-//! * literals (`1`, `true`, `"foo"`)
-//! * named or inline consts (`FOO`, `const { 5 + 6 }`)
-//!
-//! The latter are converted into other patterns with literals at the leaves. For example
-//! `const_to_pat(const { [1, 2, 3] })` becomes an `Array(vec![Const(1), Const(2), Const(3)])`
-//! pattern. This gets problematic when comparing the constant via `==` would behave differently
-//! from matching on the constant converted to a pattern. Situations like that can occur, when
-//! the user implements `PartialEq` manually, and thus could make `==` behave arbitrarily different.
-//! In order to honor the `==` implementation, constants of types that implement `PartialEq` manually
-//! stay as a full constant and become an `Opaque` pattern. These `Opaque` patterns do not participate
-//! in exhaustiveness, specialization or overlap checking.
-
-use self::ArmType::*;
-use self::Usefulness::*;
-use super::deconstruct_pat::{
-    Constructor, ConstructorSet, DeconstructedPat, IntRange, MaybeInfiniteInt, SplitConstructorSet,
-    WitnessPat,
-};
-use crate::errors::{
-    NonExhaustiveOmittedPattern, NonExhaustiveOmittedPatternLintOnArm, Overlap,
-    OverlappingRangeEndpoints, Uncovered,
-};
-
-use rustc_data_structures::captures::Captures;
-
-use rustc_arena::TypedArena;
-use rustc_data_structures::stack::ensure_sufficient_stack;
-use rustc_hir::def_id::DefId;
-use rustc_hir::HirId;
-use rustc_middle::ty::{self, Ty, TyCtxt};
-use rustc_session::lint;
-use rustc_session::lint::builtin::NON_EXHAUSTIVE_OMITTED_PATTERNS;
-use rustc_span::{Span, DUMMY_SP};
-
-use smallvec::{smallvec, SmallVec};
-use std::fmt;
-
-pub(crate) struct MatchCheckCtxt<'p, 'tcx> {
-    pub(crate) tcx: TyCtxt<'tcx>,
-    /// The module in which the match occurs. This is necessary for
-    /// checking inhabited-ness of types because whether a type is (visibly)
-    /// inhabited can depend on whether it was defined in the current module or
-    /// not. E.g., `struct Foo { _private: ! }` cannot be seen to be empty
-    /// outside its module and should not be matchable with an empty match statement.
-    pub(crate) module: DefId,
-    pub(crate) param_env: ty::ParamEnv<'tcx>,
-    pub(crate) pattern_arena: &'p TypedArena<DeconstructedPat<'p, 'tcx>>,
-    /// The span of the whole match, if applicable.
-    pub(crate) match_span: Option<Span>,
-    /// Only produce `NON_EXHAUSTIVE_OMITTED_PATTERNS` lint on refutable patterns.
-    pub(crate) refutable: bool,
-}
-
-impl<'a, 'tcx> MatchCheckCtxt<'a, 'tcx> {
-    pub(super) fn is_uninhabited(&self, ty: Ty<'tcx>) -> bool {
-        if self.tcx.features().exhaustive_patterns {
-            !ty.is_inhabited_from(self.tcx, self.module, self.param_env)
-        } else {
-            false
-        }
-    }
-
-    /// Returns whether the given type is an enum from another crate declared `#[non_exhaustive]`.
-    pub(super) fn is_foreign_non_exhaustive_enum(&self, ty: Ty<'tcx>) -> bool {
-        match ty.kind() {
-            ty::Adt(def, ..) => {
-                def.is_enum() && def.is_variant_list_non_exhaustive() && !def.did().is_local()
-            }
-            _ => false,
-        }
-    }
-}
-
-#[derive(Copy, Clone)]
-pub(super) struct PatCtxt<'a, 'p, 'tcx> {
-    pub(super) cx: &'a MatchCheckCtxt<'p, 'tcx>,
-    /// Type of the current column under investigation.
-    pub(super) ty: Ty<'tcx>,
-    /// Span of the current pattern under investigation.
-    pub(super) span: Span,
-    /// Whether the current pattern is the whole pattern as found in a match arm, or if it's a
-    /// subpattern.
-    pub(super) is_top_level: bool,
-}
-
-impl<'a, 'p, 'tcx> fmt::Debug for PatCtxt<'a, 'p, 'tcx> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        f.debug_struct("PatCtxt").field("ty", &self.ty).finish()
-    }
-}
-
-/// A row of a matrix. Rows of len 1 are very common, which is why `SmallVec[_; 2]`
-/// works well.
-#[derive(Clone)]
-pub(crate) struct PatStack<'p, 'tcx> {
-    pub(crate) pats: SmallVec<[&'p DeconstructedPat<'p, 'tcx>; 2]>,
-}
-
-impl<'p, 'tcx> PatStack<'p, 'tcx> {
-    fn from_pattern(pat: &'p DeconstructedPat<'p, 'tcx>) -> Self {
-        Self::from_vec(smallvec![pat])
-    }
-
-    fn from_vec(vec: SmallVec<[&'p DeconstructedPat<'p, 'tcx>; 2]>) -> Self {
-        PatStack { pats: vec }
-    }
-
-    fn is_empty(&self) -> bool {
-        self.pats.is_empty()
-    }
-
-    fn len(&self) -> usize {
-        self.pats.len()
-    }
-
-    fn head(&self) -> &'p DeconstructedPat<'p, 'tcx> {
-        self.pats[0]
-    }
-
-    fn iter(&self) -> impl Iterator<Item = &DeconstructedPat<'p, 'tcx>> {
-        self.pats.iter().copied()
-    }
-
-    // Recursively expand the first pattern into its subpatterns. Only useful if the pattern is an
-    // or-pattern. Panics if `self` is empty.
-    fn expand_or_pat<'a>(&'a self) -> impl Iterator<Item = PatStack<'p, 'tcx>> + Captures<'a> {
-        self.head().iter_fields().map(move |pat| {
-            let mut new_patstack = PatStack::from_pattern(pat);
-            new_patstack.pats.extend_from_slice(&self.pats[1..]);
-            new_patstack
-        })
-    }
-
-    // Recursively expand all patterns into their subpatterns and push each `PatStack` to matrix.
-    fn expand_and_extend<'a>(&'a self, matrix: &mut Matrix<'p, 'tcx>) {
-        if !self.is_empty() && self.head().is_or_pat() {
-            for pat in self.head().iter_fields() {
-                let mut new_patstack = PatStack::from_pattern(pat);
-                new_patstack.pats.extend_from_slice(&self.pats[1..]);
-                if !new_patstack.is_empty() && new_patstack.head().is_or_pat() {
-                    new_patstack.expand_and_extend(matrix);
-                } else if !new_patstack.is_empty() {
-                    matrix.push(new_patstack);
-                }
-            }
-        }
-    }
-
-    /// This computes `S(self.head().ctor(), self)`. See top of the file for explanations.
-    ///
-    /// Structure patterns with a partial wild pattern (Foo { a: 42, .. }) have their missing
-    /// fields filled with wild patterns.
-    ///
-    /// This is roughly the inverse of `Constructor::apply`.
-    fn pop_head_constructor(
-        &self,
-        pcx: &PatCtxt<'_, 'p, 'tcx>,
-        ctor: &Constructor<'tcx>,
-    ) -> PatStack<'p, 'tcx> {
-        // We pop the head pattern and push the new fields extracted from the arguments of
-        // `self.head()`.
-        let mut new_fields: SmallVec<[_; 2]> = self.head().specialize(pcx, ctor);
-        new_fields.extend_from_slice(&self.pats[1..]);
-        PatStack::from_vec(new_fields)
-    }
-}
-
-/// Pretty-printing for matrix row.
-impl<'p, 'tcx> fmt::Debug for PatStack<'p, 'tcx> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "+")?;
-        for pat in self.iter() {
-            write!(f, " {pat:?} +")?;
-        }
-        Ok(())
-    }
-}
-
-/// A 2D matrix.
-#[derive(Clone)]
-pub(super) struct Matrix<'p, 'tcx> {
-    pub patterns: Vec<PatStack<'p, 'tcx>>,
-}
-
-impl<'p, 'tcx> Matrix<'p, 'tcx> {
-    fn empty() -> Self {
-        Matrix { patterns: vec![] }
-    }
-
-    /// Pushes a new row to the matrix. If the row starts with an or-pattern, this recursively
-    /// expands it.
-    fn push(&mut self, row: PatStack<'p, 'tcx>) {
-        if !row.is_empty() && row.head().is_or_pat() {
-            row.expand_and_extend(self);
-        } else {
-            self.patterns.push(row);
-        }
-    }
-
-    /// Iterate over the first component of each row
-    fn heads<'a>(
-        &'a self,
-    ) -> impl Iterator<Item = &'p DeconstructedPat<'p, 'tcx>> + Clone + Captures<'a> {
-        self.patterns.iter().map(|r| r.head())
-    }
-
-    /// This computes `S(constructor, self)`. See top of the file for explanations.
-    fn specialize_constructor(
-        &self,
-        pcx: &PatCtxt<'_, 'p, 'tcx>,
-        ctor: &Constructor<'tcx>,
-    ) -> Matrix<'p, 'tcx> {
-        let mut matrix = Matrix::empty();
-        for row in &self.patterns {
-            if ctor.is_covered_by(pcx, row.head().ctor()) {
-                let new_row = row.pop_head_constructor(pcx, ctor);
-                matrix.push(new_row);
-            }
-        }
-        matrix
-    }
-}
-
-/// Pretty-printer for matrices of patterns, example:
-///
-/// ```text
-/// + _     + []                +
-/// + true  + [First]           +
-/// + true  + [Second(true)]    +
-/// + false + [_]               +
-/// + _     + [_, _, tail @ ..] +
-/// ```
-impl<'p, 'tcx> fmt::Debug for Matrix<'p, 'tcx> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "\n")?;
-
-        let Matrix { patterns: m, .. } = self;
-        let pretty_printed_matrix: Vec<Vec<String>> =
-            m.iter().map(|row| row.iter().map(|pat| format!("{pat:?}")).collect()).collect();
-
-        let column_count = m.iter().map(|row| row.len()).next().unwrap_or(0);
-        assert!(m.iter().all(|row| row.len() == column_count));
-        let column_widths: Vec<usize> = (0..column_count)
-            .map(|col| pretty_printed_matrix.iter().map(|row| row[col].len()).max().unwrap_or(0))
-            .collect();
-
-        for row in pretty_printed_matrix {
-            write!(f, "+")?;
-            for (column, pat_str) in row.into_iter().enumerate() {
-                write!(f, " ")?;
-                write!(f, "{:1$}", pat_str, column_widths[column])?;
-                write!(f, " +")?;
-            }
-            write!(f, "\n")?;
-        }
-        Ok(())
-    }
-}
-
-/// This carries the results of computing usefulness, as described at the top of the file. When
-/// checking usefulness of a match branch, we use the `NoWitnesses` variant, which also keeps track
-/// of potential unreachable sub-patterns (in the presence of or-patterns). When checking
-/// exhaustiveness of a whole match, we use the `WithWitnesses` variant, which carries a list of
-/// witnesses of non-exhaustiveness when there are any.
-/// Which variant to use is dictated by `ArmType`.
-#[derive(Debug, Clone)]
-enum Usefulness<'tcx> {
-    /// If we don't care about witnesses, simply remember if the pattern was useful.
-    NoWitnesses { useful: bool },
-    /// Carries a list of witnesses of non-exhaustiveness. If empty, indicates that the whole
-    /// pattern is unreachable.
-    WithWitnesses(Vec<WitnessStack<'tcx>>),
-}
-
-impl<'tcx> Usefulness<'tcx> {
-    fn new_useful(preference: ArmType) -> Self {
-        match preference {
-            // A single (empty) witness of reachability.
-            FakeExtraWildcard => WithWitnesses(vec![WitnessStack(vec![])]),
-            RealArm => NoWitnesses { useful: true },
-        }
-    }
-
-    fn new_not_useful(preference: ArmType) -> Self {
-        match preference {
-            FakeExtraWildcard => WithWitnesses(vec![]),
-            RealArm => NoWitnesses { useful: false },
-        }
-    }
-
-    fn is_useful(&self) -> bool {
-        match self {
-            Usefulness::NoWitnesses { useful } => *useful,
-            Usefulness::WithWitnesses(witnesses) => !witnesses.is_empty(),
-        }
-    }
-
-    /// Combine usefulnesses from two branches. This is an associative operation.
-    fn extend(&mut self, other: Self) {
-        match (&mut *self, other) {
-            (WithWitnesses(_), WithWitnesses(o)) if o.is_empty() => {}
-            (WithWitnesses(s), WithWitnesses(o)) if s.is_empty() => *self = WithWitnesses(o),
-            (WithWitnesses(s), WithWitnesses(o)) => s.extend(o),
-            (NoWitnesses { useful: s_useful }, NoWitnesses { useful: o_useful }) => {
-                *s_useful = *s_useful || o_useful
-            }
-            _ => unreachable!(),
-        }
-    }
-
-    /// After calculating usefulness after a specialization, call this to reconstruct a usefulness
-    /// that makes sense for the matrix pre-specialization. This new usefulness can then be merged
-    /// with the results of specializing with the other constructors.
-    fn apply_constructor(
-        self,
-        pcx: &PatCtxt<'_, '_, 'tcx>,
-        matrix: &Matrix<'_, 'tcx>, // used to compute missing ctors
-        ctor: &Constructor<'tcx>,
-    ) -> Self {
-        match self {
-            NoWitnesses { .. } => self,
-            WithWitnesses(ref witnesses) if witnesses.is_empty() => self,
-            WithWitnesses(witnesses) => {
-                let new_witnesses = if let Constructor::Missing { .. } = ctor {
-                    let mut missing = ConstructorSet::for_ty(pcx.cx, pcx.ty)
-                        .compute_missing(pcx, matrix.heads().map(DeconstructedPat::ctor));
-                    if missing.iter().any(|c| c.is_non_exhaustive()) {
-                        // We only report `_` here; listing other constructors would be redundant.
-                        missing = vec![Constructor::NonExhaustive];
-                    }
-
-                    // We got the special `Missing` constructor, so each of the missing constructors
-                    // gives a new pattern that is not caught by the match.
-                    // We construct for each missing constructor a version of this constructor with
-                    // wildcards for fields, i.e. that matches everything that can be built with it.
-                    // For example, if `ctor` is a `Constructor::Variant` for `Option::Some`, we get
-                    // the pattern `Some(_)`.
-                    let new_patterns: Vec<WitnessPat<'_>> = missing
-                        .into_iter()
-                        .map(|missing_ctor| WitnessPat::wild_from_ctor(pcx, missing_ctor.clone()))
-                        .collect();
-
-                    witnesses
-                        .into_iter()
-                        .flat_map(|witness| {
-                            new_patterns.iter().map(move |pat| {
-                                let mut stack = witness.clone();
-                                stack.0.push(pat.clone());
-                                stack
-                            })
-                        })
-                        .collect()
-                } else {
-                    witnesses
-                        .into_iter()
-                        .map(|witness| witness.apply_constructor(pcx, &ctor))
-                        .collect()
-                };
-                WithWitnesses(new_witnesses)
-            }
-        }
-    }
-}
-
-#[derive(Copy, Clone, Debug)]
-enum ArmType {
-    FakeExtraWildcard,
-    RealArm,
-}
-
-/// A witness-tuple of non-exhaustiveness for error reporting, represented as a list of patterns (in
-/// reverse order of construction) with wildcards inside to represent elements that can take any
-/// inhabitant of the type as a value.
-///
-/// This mirrors `PatStack`: they function similarly, except `PatStack` contains user patterns we
-/// are inspecting, and `WitnessStack` contains witnesses we are constructing.
-/// FIXME(Nadrieril): use the same order of patterns for both
-///
-/// A `WitnessStack` should have the same types and length as the `PatStacks` we are inspecting
-/// (except we store the patterns in reverse order). Because Rust `match` is always against a single
-/// pattern, at the end the stack will have length 1. In the middle of the algorithm, it can contain
-/// multiple patterns.
-///
-/// For example, if we are constructing a witness for the match against
-///
-/// ```compile_fail,E0004
-/// struct Pair(Option<(u32, u32)>, bool);
-/// # fn foo(p: Pair) {
-/// match p {
-///    Pair(None, _) => {}
-///    Pair(_, false) => {}
-/// }
-/// # }
-/// ```
-///
-/// We'll perform the following steps (among others):
-/// - Start with a matrix representing the match
-///     `PatStack(vec![Pair(None, _)])`
-///     `PatStack(vec![Pair(_, false)])`
-/// - Specialize with `Pair`
-///     `PatStack(vec![None, _])`
-///     `PatStack(vec![_, false])`
-/// - Specialize with `Some`
-///     `PatStack(vec![_, false])`
-/// - Specialize with `_`
-///     `PatStack(vec![false])`
-/// - Specialize with `true`
-///     // no patstacks left
-/// - This is a non-exhaustive match: we have the empty witness stack as a witness.
-///     `WitnessStack(vec![])`
-/// - Apply `true`
-///     `WitnessStack(vec![true])`
-/// - Apply `_`
-///     `WitnessStack(vec![true, _])`
-/// - Apply `Some`
-///     `WitnessStack(vec![true, Some(_)])`
-/// - Apply `Pair`
-///     `WitnessStack(vec![Pair(Some(_), true)])`
-///
-/// The final `Pair(Some(_), true)` is then the resulting witness.
-#[derive(Debug, Clone)]
-pub(crate) struct WitnessStack<'tcx>(Vec<WitnessPat<'tcx>>);
-
-impl<'tcx> WitnessStack<'tcx> {
-    /// Asserts that the witness contains a single pattern, and returns it.
-    fn single_pattern(self) -> WitnessPat<'tcx> {
-        assert_eq!(self.0.len(), 1);
-        self.0.into_iter().next().unwrap()
-    }
-
-    /// Constructs a partial witness for a pattern given a list of
-    /// patterns expanded by the specialization step.
-    ///
-    /// When a pattern P is discovered to be useful, this function is used bottom-up
-    /// to reconstruct a complete witness, e.g., a pattern P' that covers a subset
-    /// of values, V, where each value in that set is not covered by any previously
-    /// used patterns and is covered by the pattern P'. Examples:
-    ///
-    /// left_ty: tuple of 3 elements
-    /// pats: [10, 20, _]           => (10, 20, _)
-    ///
-    /// left_ty: struct X { a: (bool, &'static str), b: usize}
-    /// pats: [(false, "foo"), 42]  => X { a: (false, "foo"), b: 42 }
-    fn apply_constructor(mut self, pcx: &PatCtxt<'_, '_, 'tcx>, ctor: &Constructor<'tcx>) -> Self {
-        let pat = {
-            let len = self.0.len();
-            let arity = ctor.arity(pcx);
-            let fields = self.0.drain((len - arity)..).rev().collect();
-            WitnessPat::new(ctor.clone(), fields, pcx.ty)
-        };
-
-        self.0.push(pat);
-
-        self
-    }
-}
-
-/// Algorithm from <http://moscova.inria.fr/~maranget/papers/warn/index.html>.
-/// The algorithm from the paper has been modified to correctly handle empty
-/// types. The changes are:
-///   (0) We don't exit early if the pattern matrix has zero rows. We just
-///       continue to recurse over columns.
-///   (1) all_constructors will only return constructors that are statically
-///       possible. E.g., it will only return `Ok` for `Result<T, !>`.
-///
-/// This finds whether a (row) vector `v` of patterns is 'useful' in relation
-/// to a set of such vectors `m` - this is defined as there being a set of
-/// inputs that will match `v` but not any of the sets in `m`.
-///
-/// All the patterns at each column of the `matrix ++ v` matrix must have the same type.
-///
-/// This is used both for reachability checking (if a pattern isn't useful in
-/// relation to preceding patterns, it is not reachable) and exhaustiveness
-/// checking (if a wildcard pattern is useful in relation to a matrix, the
-/// matrix isn't exhaustive).
-///
-/// `is_under_guard` is used to inform if the pattern has a guard. If it
-/// has one it must not be inserted into the matrix. This shouldn't be
-/// relied on for soundness.
-#[instrument(level = "debug", skip(cx, matrix, lint_root), ret)]
-fn is_useful<'p, 'tcx>(
-    cx: &MatchCheckCtxt<'p, 'tcx>,
-    matrix: &Matrix<'p, 'tcx>,
-    v: &PatStack<'p, 'tcx>,
-    witness_preference: ArmType,
-    lint_root: HirId,
-    is_under_guard: bool,
-    is_top_level: bool,
-) -> Usefulness<'tcx> {
-    debug!(?matrix, ?v);
-    let Matrix { patterns: rows, .. } = matrix;
-
-    // The base case. We are pattern-matching on () and the return value is
-    // based on whether our matrix has a row or not.
-    // NOTE: This could potentially be optimized by checking rows.is_empty()
-    // first and then, if v is non-empty, the return value is based on whether
-    // the type of the tuple we're checking is inhabited or not.
-    if v.is_empty() {
-        let ret = if rows.is_empty() {
-            Usefulness::new_useful(witness_preference)
-        } else {
-            Usefulness::new_not_useful(witness_preference)
-        };
-        debug!(?ret);
-        return ret;
-    }
-
-    debug_assert!(rows.iter().all(|r| r.len() == v.len()));
-
-    // If the first pattern is an or-pattern, expand it.
-    let mut ret = Usefulness::new_not_useful(witness_preference);
-    if v.head().is_or_pat() {
-        debug!("expanding or-pattern");
-        // We try each or-pattern branch in turn.
-        let mut matrix = matrix.clone();
-        for v in v.expand_or_pat() {
-            debug!(?v);
-            let usefulness = ensure_sufficient_stack(|| {
-                is_useful(cx, &matrix, &v, witness_preference, lint_root, is_under_guard, false)
-            });
-            debug!(?usefulness);
-            ret.extend(usefulness);
-            // If pattern has a guard don't add it to the matrix.
-            if !is_under_guard {
-                // We push the already-seen patterns into the matrix in order to detect redundant
-                // branches like `Some(_) | Some(0)`.
-                matrix.push(v);
-            }
-        }
-    } else {
-        let mut ty = v.head().ty();
-
-        // Opaque types can't get destructured/split, but the patterns can
-        // actually hint at hidden types, so we use the patterns' types instead.
-        if let ty::Alias(ty::Opaque, ..) = ty.kind() {
-            if let Some(row) = rows.first() {
-                ty = row.head().ty();
-            }
-        }
-        debug!("v.head: {:?}, v.span: {:?}", v.head(), v.head().span());
-        let pcx = &PatCtxt { cx, ty, span: v.head().span(), is_top_level };
-
-        let v_ctor = v.head().ctor();
-        debug!(?v_ctor);
-        // We split the head constructor of `v`.
-        let split_ctors = v_ctor.split(pcx, matrix.heads().map(DeconstructedPat::ctor));
-        // For each constructor, we compute whether there's a value that starts with it that would
-        // witness the usefulness of `v`.
-        let start_matrix = &matrix;
-        for ctor in split_ctors {
-            debug!("specialize({:?})", ctor);
-            // We cache the result of `Fields::wildcards` because it is used a lot.
-            let spec_matrix = start_matrix.specialize_constructor(pcx, &ctor);
-            let v = v.pop_head_constructor(pcx, &ctor);
-            let usefulness = ensure_sufficient_stack(|| {
-                is_useful(
-                    cx,
-                    &spec_matrix,
-                    &v,
-                    witness_preference,
-                    lint_root,
-                    is_under_guard,
-                    false,
-                )
-            });
-            let usefulness = usefulness.apply_constructor(pcx, start_matrix, &ctor);
-            ret.extend(usefulness);
-        }
-    }
-
-    if ret.is_useful() {
-        v.head().set_reachable();
-    }
-
-    ret
-}
-
-/// A column of patterns in the matrix, where a column is the intuitive notion of "subpatterns that
-/// inspect the same subvalue".
-/// This is used to traverse patterns column-by-column for lints. Despite similarities with
-/// `is_useful`, this is a different traversal. Notably this is linear in the depth of patterns,
-/// whereas `is_useful` is worst-case exponential (exhaustiveness is NP-complete).
-#[derive(Debug)]
-struct PatternColumn<'p, 'tcx> {
-    patterns: Vec<&'p DeconstructedPat<'p, 'tcx>>,
-}
-
-impl<'p, 'tcx> PatternColumn<'p, 'tcx> {
-    fn new(patterns: Vec<&'p DeconstructedPat<'p, 'tcx>>) -> Self {
-        Self { patterns }
-    }
-
-    fn is_empty(&self) -> bool {
-        self.patterns.is_empty()
-    }
-    fn head_ty(&self) -> Option<Ty<'tcx>> {
-        if self.patterns.len() == 0 {
-            return None;
-        }
-        // If the type is opaque and it is revealed anywhere in the column, we take the revealed
-        // version. Otherwise we could encounter constructors for the revealed type and crash.
-        let is_opaque = |ty: Ty<'tcx>| matches!(ty.kind(), ty::Alias(ty::Opaque, ..));
-        let first_ty = self.patterns[0].ty();
-        if is_opaque(first_ty) {
-            for pat in &self.patterns {
-                let ty = pat.ty();
-                if !is_opaque(ty) {
-                    return Some(ty);
-                }
-            }
-        }
-        Some(first_ty)
-    }
-
-    fn analyze_ctors(&self, pcx: &PatCtxt<'_, 'p, 'tcx>) -> SplitConstructorSet<'tcx> {
-        let column_ctors = self.patterns.iter().map(|p| p.ctor());
-        ConstructorSet::for_ty(pcx.cx, pcx.ty).split(pcx, column_ctors)
-    }
-    fn iter<'a>(&'a self) -> impl Iterator<Item = &'p DeconstructedPat<'p, 'tcx>> + Captures<'a> {
-        self.patterns.iter().copied()
-    }
-
-    /// Does specialization: given a constructor, this takes the patterns from the column that match
-    /// the constructor, and outputs their fields.
-    /// This returns one column per field of the constructor. The normally all have the same length
-    /// (the number of patterns in `self` that matched `ctor`), except that we expand or-patterns
-    /// which may change the lengths.
-    fn specialize(&self, pcx: &PatCtxt<'_, 'p, 'tcx>, ctor: &Constructor<'tcx>) -> Vec<Self> {
-        let arity = ctor.arity(pcx);
-        if arity == 0 {
-            return Vec::new();
-        }
-
-        // We specialize the column by `ctor`. This gives us `arity`-many columns of patterns. These
-        // columns may have different lengths in the presence of or-patterns (this is why we can't
-        // reuse `Matrix`).
-        let mut specialized_columns: Vec<_> =
-            (0..arity).map(|_| Self { patterns: Vec::new() }).collect();
-        let relevant_patterns =
-            self.patterns.iter().filter(|pat| ctor.is_covered_by(pcx, pat.ctor()));
-        for pat in relevant_patterns {
-            let specialized = pat.specialize(pcx, &ctor);
-            for (subpat, column) in specialized.iter().zip(&mut specialized_columns) {
-                if subpat.is_or_pat() {
-                    column.patterns.extend(subpat.flatten_or_pat())
-                } else {
-                    column.patterns.push(subpat)
-                }
-            }
-        }
-
-        assert!(
-            !specialized_columns[0].is_empty(),
-            "ctor {ctor:?} was listed as present but isn't;
-            there is an inconsistency between `Constructor::is_covered_by` and `ConstructorSet::split`"
-        );
-        specialized_columns
-    }
-}
-
-/// Traverse the patterns to collect any variants of a non_exhaustive enum that fail to be mentioned
-/// in a given column.
-#[instrument(level = "debug", skip(cx), ret)]
-fn collect_nonexhaustive_missing_variants<'p, 'tcx>(
-    cx: &MatchCheckCtxt<'p, 'tcx>,
-    column: &PatternColumn<'p, 'tcx>,
-) -> Vec<WitnessPat<'tcx>> {
-    let Some(ty) = column.head_ty() else {
-        return Vec::new();
-    };
-    let pcx = &PatCtxt { cx, ty, span: DUMMY_SP, is_top_level: false };
-
-    let set = column.analyze_ctors(pcx);
-    if set.present.is_empty() {
-        // We can't consistently handle the case where no constructors are present (since this would
-        // require digging deep through any type in case there's a non_exhaustive enum somewhere),
-        // so for consistency we refuse to handle the top-level case, where we could handle it.
-        return vec![];
-    }
-
-    let mut witnesses = Vec::new();
-    if cx.is_foreign_non_exhaustive_enum(ty) {
-        witnesses.extend(
-            set.missing
-                .into_iter()
-                // This will list missing visible variants.
-                .filter(|c| !matches!(c, Constructor::Hidden | Constructor::NonExhaustive))
-                .map(|missing_ctor| WitnessPat::wild_from_ctor(pcx, missing_ctor)),
-        )
-    }
-
-    // Recurse into the fields.
-    for ctor in set.present {
-        let specialized_columns = column.specialize(pcx, &ctor);
-        let wild_pat = WitnessPat::wild_from_ctor(pcx, ctor);
-        for (i, col_i) in specialized_columns.iter().enumerate() {
-            // Compute witnesses for each column.
-            let wits_for_col_i = collect_nonexhaustive_missing_variants(cx, col_i);
-            // For each witness, we build a new pattern in the shape of `ctor(_, _, wit, _, _)`,
-            // adding enough wildcards to match `arity`.
-            for wit in wits_for_col_i {
-                let mut pat = wild_pat.clone();
-                pat.fields[i] = wit;
-                witnesses.push(pat);
-            }
-        }
-    }
-    witnesses
-}
-
-/// Traverse the patterns to warn the user about ranges that overlap on their endpoints.
-#[instrument(level = "debug", skip(cx, lint_root))]
-fn lint_overlapping_range_endpoints<'p, 'tcx>(
-    cx: &MatchCheckCtxt<'p, 'tcx>,
-    column: &PatternColumn<'p, 'tcx>,
-    lint_root: HirId,
-) {
-    let Some(ty) = column.head_ty() else {
-        return;
-    };
-    let pcx = &PatCtxt { cx, ty, span: DUMMY_SP, is_top_level: false };
-
-    let set = column.analyze_ctors(pcx);
-
-    if IntRange::is_integral(ty) {
-        let emit_lint = |overlap: &IntRange, this_span: Span, overlapped_spans: &[Span]| {
-            let overlap_as_pat = overlap.to_diagnostic_pat(ty, cx.tcx);
-            let overlaps: Vec<_> = overlapped_spans
-                .iter()
-                .copied()
-                .map(|span| Overlap { range: overlap_as_pat.clone(), span })
-                .collect();
-            cx.tcx.emit_spanned_lint(
-                lint::builtin::OVERLAPPING_RANGE_ENDPOINTS,
-                lint_root,
-                this_span,
-                OverlappingRangeEndpoints { overlap: overlaps, range: this_span },
-            );
-        };
-
-        // If two ranges overlapped, the split set will contain their intersection as a singleton.
-        let split_int_ranges = set.present.iter().filter_map(|c| c.as_int_range());
-        for overlap_range in split_int_ranges.clone() {
-            if overlap_range.is_singleton() {
-                let overlap: MaybeInfiniteInt = overlap_range.lo;
-                // Ranges that look like `lo..=overlap`.
-                let mut prefixes: SmallVec<[_; 1]> = Default::default();
-                // Ranges that look like `overlap..=hi`.
-                let mut suffixes: SmallVec<[_; 1]> = Default::default();
-                // Iterate on patterns that contained `overlap`.
-                for pat in column.iter() {
-                    let this_span = pat.span();
-                    let Constructor::IntRange(this_range) = pat.ctor() else { continue };
-                    if this_range.is_singleton() {
-                        // Don't lint when one of the ranges is a singleton.
-                        continue;
-                    }
-                    if this_range.lo == overlap {
-                        // `this_range` looks like `overlap..=this_range.hi`; it overlaps with any
-                        // ranges that look like `lo..=overlap`.
-                        if !prefixes.is_empty() {
-                            emit_lint(overlap_range, this_span, &prefixes);
-                        }
-                        suffixes.push(this_span)
-                    } else if this_range.hi == overlap.plus_one() {
-                        // `this_range` looks like `this_range.lo..=overlap`; it overlaps with any
-                        // ranges that look like `overlap..=hi`.
-                        if !suffixes.is_empty() {
-                            emit_lint(overlap_range, this_span, &suffixes);
-                        }
-                        prefixes.push(this_span)
-                    }
-                }
-            }
-        }
-    } else {
-        // Recurse into the fields.
-        for ctor in set.present {
-            for col in column.specialize(pcx, &ctor) {
-                lint_overlapping_range_endpoints(cx, &col, lint_root);
-            }
-        }
-    }
-}
-
-/// The arm of a match expression.
-#[derive(Clone, Copy, Debug)]
-pub(crate) struct MatchArm<'p, 'tcx> {
-    /// The pattern must have been lowered through `check_match::MatchVisitor::lower_pattern`.
-    pub(crate) pat: &'p DeconstructedPat<'p, 'tcx>,
-    pub(crate) hir_id: HirId,
-    pub(crate) has_guard: bool,
-}
-
-/// Indicates whether or not a given arm is reachable.
-#[derive(Clone, Debug)]
-pub(crate) enum Reachability {
-    /// The arm is reachable. This additionally carries a set of or-pattern branches that have been
-    /// found to be unreachable despite the overall arm being reachable. Used only in the presence
-    /// of or-patterns, otherwise it stays empty.
-    Reachable(Vec<Span>),
-    /// The arm is unreachable.
-    Unreachable,
-}
-
-/// The output of checking a match for exhaustiveness and arm reachability.
-pub(crate) struct UsefulnessReport<'p, 'tcx> {
-    /// For each arm of the input, whether that arm is reachable after the arms above it.
-    pub(crate) arm_usefulness: Vec<(MatchArm<'p, 'tcx>, Reachability)>,
-    /// If the match is exhaustive, this is empty. If not, this contains witnesses for the lack of
-    /// exhaustiveness.
-    pub(crate) non_exhaustiveness_witnesses: Vec<WitnessPat<'tcx>>,
-}
-
-/// The entrypoint for the usefulness algorithm. Computes whether a match is exhaustive and which
-/// of its arms are reachable.
-///
-/// Note: the input patterns must have been lowered through
-/// `check_match::MatchVisitor::lower_pattern`.
-#[instrument(skip(cx, arms), level = "debug")]
-pub(crate) fn compute_match_usefulness<'p, 'tcx>(
-    cx: &MatchCheckCtxt<'p, 'tcx>,
-    arms: &[MatchArm<'p, 'tcx>],
-    lint_root: HirId,
-    scrut_ty: Ty<'tcx>,
-    scrut_span: Span,
-) -> UsefulnessReport<'p, 'tcx> {
-    let mut matrix = Matrix::empty();
-    let arm_usefulness: Vec<_> = arms
-        .iter()
-        .copied()
-        .map(|arm| {
-            debug!(?arm);
-            let v = PatStack::from_pattern(arm.pat);
-            is_useful(cx, &matrix, &v, RealArm, arm.hir_id, arm.has_guard, true);
-            if !arm.has_guard {
-                matrix.push(v);
-            }
-            let reachability = if arm.pat.is_reachable() {
-                Reachability::Reachable(arm.pat.unreachable_spans())
-            } else {
-                Reachability::Unreachable
-            };
-            (arm, reachability)
-        })
-        .collect();
-
-    let wild_pattern = cx.pattern_arena.alloc(DeconstructedPat::wildcard(scrut_ty, DUMMY_SP));
-    let v = PatStack::from_pattern(wild_pattern);
-    let usefulness = is_useful(cx, &matrix, &v, FakeExtraWildcard, lint_root, false, true);
-    let non_exhaustiveness_witnesses: Vec<_> = match usefulness {
-        WithWitnesses(pats) => pats.into_iter().map(|w| w.single_pattern()).collect(),
-        NoWitnesses { .. } => bug!(),
-    };
-
-    let pat_column = arms.iter().flat_map(|arm| arm.pat.flatten_or_pat()).collect::<Vec<_>>();
-    let pat_column = PatternColumn::new(pat_column);
-    lint_overlapping_range_endpoints(cx, &pat_column, lint_root);
-
-    // Run the non_exhaustive_omitted_patterns lint. Only run on refutable patterns to avoid hitting
-    // `if let`s. Only run if the match is exhaustive otherwise the error is redundant.
-    if cx.refutable && non_exhaustiveness_witnesses.is_empty() {
-        if !matches!(
-            cx.tcx.lint_level_at_node(NON_EXHAUSTIVE_OMITTED_PATTERNS, lint_root).0,
-            rustc_session::lint::Level::Allow
-        ) {
-            let witnesses = collect_nonexhaustive_missing_variants(cx, &pat_column);
-
-            if !witnesses.is_empty() {
-                // Report that a match of a `non_exhaustive` enum marked with `non_exhaustive_omitted_patterns`
-                // is not exhaustive enough.
-                //
-                // NB: The partner lint for structs lives in `compiler/rustc_hir_analysis/src/check/pat.rs`.
-                cx.tcx.emit_spanned_lint(
-                    NON_EXHAUSTIVE_OMITTED_PATTERNS,
-                    lint_root,
-                    scrut_span,
-                    NonExhaustiveOmittedPattern {
-                        scrut_ty,
-                        uncovered: Uncovered::new(scrut_span, cx, witnesses),
-                    },
-                );
-            }
-        } else {
-            // We used to allow putting the `#[allow(non_exhaustive_omitted_patterns)]` on a match
-            // arm. This no longer makes sense so we warn users, to avoid silently breaking their
-            // usage of the lint.
-            for arm in arms {
-                let (lint_level, lint_level_source) =
-                    cx.tcx.lint_level_at_node(NON_EXHAUSTIVE_OMITTED_PATTERNS, arm.hir_id);
-                if !matches!(lint_level, rustc_session::lint::Level::Allow) {
-                    let decorator = NonExhaustiveOmittedPatternLintOnArm {
-                        lint_span: lint_level_source.span(),
-                        suggest_lint_on_match: cx.match_span.map(|span| span.shrink_to_lo()),
-                        lint_level: lint_level.as_str(),
-                        lint_name: "non_exhaustive_omitted_patterns",
-                    };
-
-                    use rustc_errors::DecorateLint;
-                    let mut err = cx.tcx.sess.struct_span_warn(arm.pat.span(), "");
-                    err.set_primary_message(decorator.msg());
-                    decorator.decorate_lint(&mut err);
-                    err.emit();
-                }
-            }
-        }
-    }
-
-    UsefulnessReport { arm_usefulness, non_exhaustiveness_witnesses }
-}
diff --git a/compiler/rustc_mir_build/src/thir/print.rs b/compiler/rustc_mir_build/src/thir/print.rs
index c3b2309b7..28be31399 100644
--- a/compiler/rustc_mir_build/src/thir/print.rs
+++ b/compiler/rustc_mir_build/src/thir/print.rs
@@ -31,8 +31,8 @@ const INDENT: &str = "    ";
 
 macro_rules! print_indented {
     ($writer:ident, $s:expr, $indent_lvl:expr) => {
-        let indent = (0..$indent_lvl).map(|_| INDENT).collect::<Vec<_>>().concat();
-        writeln!($writer, "{}{}", indent, $s).expect("unable to write to ThirPrinter");
+        $writer.indent($indent_lvl);
+        writeln!($writer, "{}", $s).expect("unable to write to ThirPrinter");
     };
 }
 
@@ -48,6 +48,12 @@ impl<'a, 'tcx> ThirPrinter<'a, 'tcx> {
         Self { thir, fmt: String::new() }
     }
 
+    fn indent(&mut self, level: usize) {
+        for _ in 0..level {
+            self.fmt.push_str(INDENT);
+        }
+    }
+
     fn print(&mut self) {
         print_indented!(self, "params: [", 0);
         for param in self.thir.params.iter() {
@@ -85,23 +91,11 @@ impl<'a, 'tcx> ThirPrinter<'a, 'tcx> {
     }
 
     fn print_block(&mut self, block_id: BlockId, depth_lvl: usize) {
-        let Block {
-            targeted_by_break,
-            opt_destruction_scope,
-            span,
-            region_scope,
-            stmts,
-            expr,
-            safety_mode,
-        } = &self.thir.blocks[block_id];
+        let Block { targeted_by_break, span, region_scope, stmts, expr, safety_mode } =
+            &self.thir.blocks[block_id];
 
         print_indented!(self, "Block {", depth_lvl);
         print_indented!(self, format!("targeted_by_break: {}", targeted_by_break), depth_lvl + 1);
-        print_indented!(
-            self,
-            format!("opt_destruction_scope: {:?}", opt_destruction_scope),
-            depth_lvl + 1
-        );
         print_indented!(self, format!("span: {:?}", span), depth_lvl + 1);
         print_indented!(self, format!("region_scope: {:?}", region_scope), depth_lvl + 1);
         print_indented!(self, format!("safety_mode: {:?}", safety_mode), depth_lvl + 1);
@@ -127,14 +121,9 @@ impl<'a, 'tcx> ThirPrinter<'a, 'tcx> {
     }
 
     fn print_stmt(&mut self, stmt_id: StmtId, depth_lvl: usize) {
-        let Stmt { kind, opt_destruction_scope } = &self.thir.stmts[stmt_id];
+        let Stmt { kind } = &self.thir.stmts[stmt_id];
 
         print_indented!(self, "Stmt {", depth_lvl);
-        print_indented!(
-            self,
-            format!("opt_destruction_scope: {:?}", opt_destruction_scope),
-            depth_lvl + 1
-        );
 
         match kind {
             StmtKind::Expr { scope, expr } => {
@@ -636,6 +625,9 @@ impl<'a, 'tcx> ThirPrinter<'a, 'tcx> {
             PatKind::Wild => {
                 print_indented!(self, "Wild", depth_lvl + 1);
             }
+            PatKind::Never => {
+                print_indented!(self, "Never", depth_lvl + 1);
+            }
             PatKind::AscribeUserType { ascription, subpattern } => {
                 print_indented!(self, "AscribeUserType: {", depth_lvl + 1);
                 print_indented!(self, format!("ascription: {:?}", ascription), depth_lvl + 2);
diff --git a/compiler/rustc_mir_dataflow/Cargo.toml b/compiler/rustc_mir_dataflow/Cargo.toml
index 61664eb2a..7199db677 100644
--- a/compiler/rustc_mir_dataflow/Cargo.toml
+++ b/compiler/rustc_mir_dataflow/Cargo.toml
@@ -16,7 +16,6 @@ rustc_hir = { path = "../rustc_hir" }
 rustc_index = { path = "../rustc_index" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_middle = { path = "../rustc_middle" }
-rustc_serialize = { path = "../rustc_serialize" }
 rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
diff --git a/compiler/rustc_mir_dataflow/src/drop_flag_effects.rs b/compiler/rustc_mir_dataflow/src/drop_flag_effects.rs
index 163d74cc9..de1ca8d82 100644
--- a/compiler/rustc_mir_dataflow/src/drop_flag_effects.rs
+++ b/compiler/rustc_mir_dataflow/src/drop_flag_effects.rs
@@ -1,10 +1,8 @@
 use crate::elaborate_drops::DropFlagState;
 use rustc_middle::mir::{self, Body, Location, Terminator, TerminatorKind};
-use rustc_middle::ty::TyCtxt;
 use rustc_target::abi::VariantIdx;
 
-use super::indexes::MovePathIndex;
-use super::move_paths::{InitKind, LookupResult, MoveData};
+use super::move_paths::{InitKind, LookupResult, MoveData, MovePathIndex};
 use super::MoveDataParamEnv;
 
 pub fn move_path_children_matching<'tcx, F>(
@@ -30,8 +28,6 @@ where
 }
 
 pub fn on_lookup_result_bits<'tcx, F>(
-    tcx: TyCtxt<'tcx>,
-    body: &Body<'tcx>,
     move_data: &MoveData<'tcx>,
     lookup_result: LookupResult,
     each_child: F,
@@ -42,13 +38,11 @@ pub fn on_lookup_result_bits<'tcx, F>(
         LookupResult::Parent(..) => {
             // access to untracked value - do not touch children
         }
-        LookupResult::Exact(e) => on_all_children_bits(tcx, body, move_data, e, each_child),
+        LookupResult::Exact(e) => on_all_children_bits(move_data, e, each_child),
     }
 }
 
 pub fn on_all_children_bits<'tcx, F>(
-    tcx: TyCtxt<'tcx>,
-    body: &Body<'tcx>,
     move_data: &MoveData<'tcx>,
     move_path_index: MovePathIndex,
     mut each_child: F,
@@ -56,8 +50,6 @@ pub fn on_all_children_bits<'tcx, F>(
     F: FnMut(MovePathIndex),
 {
     fn on_all_children_bits<'tcx, F>(
-        tcx: TyCtxt<'tcx>,
-        body: &Body<'tcx>,
         move_data: &MoveData<'tcx>,
         move_path_index: MovePathIndex,
         each_child: &mut F,
@@ -68,15 +60,14 @@ pub fn on_all_children_bits<'tcx, F>(
 
         let mut next_child_index = move_data.move_paths[move_path_index].first_child;
         while let Some(child_index) = next_child_index {
-            on_all_children_bits(tcx, body, move_data, child_index, each_child);
+            on_all_children_bits(move_data, child_index, each_child);
             next_child_index = move_data.move_paths[child_index].next_sibling;
         }
     }
-    on_all_children_bits(tcx, body, move_data, move_path_index, &mut each_child);
+    on_all_children_bits(move_data, move_path_index, &mut each_child);
 }
 
 pub fn drop_flag_effects_for_function_entry<'tcx, F>(
-    tcx: TyCtxt<'tcx>,
     body: &Body<'tcx>,
     ctxt: &MoveDataParamEnv<'tcx>,
     mut callback: F,
@@ -87,14 +78,13 @@ pub fn drop_flag_effects_for_function_entry<'tcx, F>(
     for arg in body.args_iter() {
         let place = mir::Place::from(arg);
         let lookup_result = move_data.rev_lookup.find(place.as_ref());
-        on_lookup_result_bits(tcx, body, move_data, lookup_result, |mpi| {
+        on_lookup_result_bits(move_data, lookup_result, |mpi| {
             callback(mpi, DropFlagState::Present)
         });
     }
 }
 
 pub fn drop_flag_effects_for_location<'tcx, F>(
-    tcx: TyCtxt<'tcx>,
     body: &Body<'tcx>,
     ctxt: &MoveDataParamEnv<'tcx>,
     loc: Location,
@@ -110,7 +100,7 @@ pub fn drop_flag_effects_for_location<'tcx, F>(
         let path = mi.move_path_index(move_data);
         debug!("moving out of path {:?}", move_data.move_paths[path]);
 
-        on_all_children_bits(tcx, body, move_data, path, |mpi| callback(mpi, DropFlagState::Absent))
+        on_all_children_bits(move_data, path, |mpi| callback(mpi, DropFlagState::Absent))
     }
 
     // Drop does not count as a move but we should still consider the variable uninitialized.
@@ -118,24 +108,17 @@ pub fn drop_flag_effects_for_location<'tcx, F>(
         body.stmt_at(loc).right()
     {
         if let LookupResult::Exact(mpi) = move_data.rev_lookup.find(place.as_ref()) {
-            on_all_children_bits(tcx, body, move_data, mpi, |mpi| {
-                callback(mpi, DropFlagState::Absent)
-            })
+            on_all_children_bits(move_data, mpi, |mpi| callback(mpi, DropFlagState::Absent))
         }
     }
 
     debug!("drop_flag_effects: assignment for location({:?})", loc);
 
-    for_location_inits(tcx, body, move_data, loc, |mpi| callback(mpi, DropFlagState::Present));
+    for_location_inits(move_data, loc, |mpi| callback(mpi, DropFlagState::Present));
 }
 
-pub fn for_location_inits<'tcx, F>(
-    tcx: TyCtxt<'tcx>,
-    body: &Body<'tcx>,
-    move_data: &MoveData<'tcx>,
-    loc: Location,
-    mut callback: F,
-) where
+fn for_location_inits<'tcx, F>(move_data: &MoveData<'tcx>, loc: Location, mut callback: F)
+where
     F: FnMut(MovePathIndex),
 {
     for ii in &move_data.init_loc_map[loc] {
@@ -144,7 +127,7 @@ pub fn for_location_inits<'tcx, F>(
             InitKind::Deep => {
                 let path = init.path;
 
-                on_all_children_bits(tcx, body, move_data, path, &mut callback)
+                on_all_children_bits(move_data, path, &mut callback)
             }
             InitKind::Shallow => {
                 let mpi = init.path;
@@ -161,8 +144,6 @@ pub fn for_location_inits<'tcx, F>(
 /// NOTE: If there are no move paths corresponding to an inactive variant,
 /// `handle_inactive_variant` will not be called for that variant.
 pub(crate) fn on_all_inactive_variants<'tcx>(
-    tcx: TyCtxt<'tcx>,
-    body: &mir::Body<'tcx>,
     move_data: &MoveData<'tcx>,
     enum_place: mir::Place<'tcx>,
     active_variant: VariantIdx,
@@ -185,9 +166,7 @@ pub(crate) fn on_all_inactive_variants<'tcx>(
         };
 
         if variant_idx != active_variant {
-            on_all_children_bits(tcx, body, move_data, variant_mpi, |mpi| {
-                handle_inactive_variant(mpi)
-            });
+            on_all_children_bits(move_data, variant_mpi, |mpi| handle_inactive_variant(mpi));
         }
     }
 }
diff --git a/compiler/rustc_mir_dataflow/src/elaborate_drops.rs b/compiler/rustc_mir_dataflow/src/elaborate_drops.rs
index 25ba67a63..958fa0d17 100644
--- a/compiler/rustc_mir_dataflow/src/elaborate_drops.rs
+++ b/compiler/rustc_mir_dataflow/src/elaborate_drops.rs
@@ -481,12 +481,8 @@ where
     ) -> (BasicBlock, Unwind) {
         let (succ, unwind) = self.drop_ladder_bottom();
         if !adt.is_enum() {
-            let fields = self.move_paths_for_fields(
-                self.place,
-                self.path,
-                &adt.variant(FIRST_VARIANT),
-                args,
-            );
+            let fields =
+                self.move_paths_for_fields(self.place, self.path, adt.variant(FIRST_VARIANT), args);
             self.drop_ladder(fields, succ, unwind)
         } else {
             self.open_drop_for_multivariant(adt, args, succ, unwind)
@@ -518,7 +514,7 @@ where
                     self.place,
                     ProjectionElem::Downcast(Some(variant.name), variant_index),
                 );
-                let fields = self.move_paths_for_fields(base_place, variant_path, &variant, args);
+                let fields = self.move_paths_for_fields(base_place, variant_path, variant, args);
                 values.push(discr.val);
                 if let Unwind::To(unwind) = unwind {
                     // We can't use the half-ladder from the original
@@ -859,14 +855,14 @@ where
     fn open_drop(&mut self) -> BasicBlock {
         let ty = self.place_ty(self.place);
         match ty.kind() {
-            ty::Closure(_, args) => self.open_drop_for_tuple(&args.as_closure().upvar_tys()),
+            ty::Closure(_, args) => self.open_drop_for_tuple(args.as_closure().upvar_tys()),
             // Note that `elaborate_drops` only drops the upvars of a coroutine,
             // and this is ok because `open_drop` here can only be reached
             // within that own coroutine's resume function.
             // This should only happen for the self argument on the resume function.
             // It effectively only contains upvars until the coroutine transformation runs.
             // See librustc_body/transform/coroutine.rs for more details.
-            ty::Coroutine(_, args, _) => self.open_drop_for_tuple(&args.as_coroutine().upvar_tys()),
+            ty::Coroutine(_, args, _) => self.open_drop_for_tuple(args.as_coroutine().upvar_tys()),
             ty::Tuple(fields) => self.open_drop_for_tuple(fields),
             ty::Adt(def, args) => self.open_drop_for_adt(*def, args),
             ty::Dynamic(..) => self.complete_drop(self.succ, self.unwind),
diff --git a/compiler/rustc_mir_dataflow/src/framework/cursor.rs b/compiler/rustc_mir_dataflow/src/framework/cursor.rs
index c978bddfe..815f20459 100644
--- a/compiler/rustc_mir_dataflow/src/framework/cursor.rs
+++ b/compiler/rustc_mir_dataflow/src/framework/cursor.rs
@@ -1,60 +1,14 @@
 //! Random access inspection of the results of a dataflow analysis.
 
-use crate::{framework::BitSetExt, CloneAnalysis};
+use crate::framework::BitSetExt;
 
-use std::borrow::{Borrow, BorrowMut};
 use std::cmp::Ordering;
 
 #[cfg(debug_assertions)]
 use rustc_index::bit_set::BitSet;
 use rustc_middle::mir::{self, BasicBlock, Location};
 
-use super::{Analysis, Direction, Effect, EffectIndex, EntrySets, Results, ResultsCloned};
-
-// `AnalysisResults` is needed as an impl such as the following has an unconstrained type
-// parameter:
-// ```
-// impl<'tcx, A, E, R> ResultsCursor<'_, 'tcx, A, R>
-// where
-//     A: Analysis<'tcx>,
-//     E: Borrow<EntrySets<'tcx, A>>,
-//     R: Results<'tcx, A, E>,
-// {}
-// ```
-
-/// A type representing the analysis results consumed by a `ResultsCursor`.
-pub trait AnalysisResults<'tcx, A>: BorrowMut<Results<'tcx, A, Self::EntrySets>>
-where
-    A: Analysis<'tcx>,
-{
-    /// The type containing the entry sets for this `Results` type.
-    ///
-    /// Should be either `EntrySets<'tcx, A>` or `&EntrySets<'tcx, A>`.
-    type EntrySets: Borrow<EntrySets<'tcx, A>>;
-}
-impl<'tcx, A, E> AnalysisResults<'tcx, A> for Results<'tcx, A, E>
-where
-    A: Analysis<'tcx>,
-    E: Borrow<EntrySets<'tcx, A>>,
-{
-    type EntrySets = E;
-}
-impl<'a, 'tcx, A, E> AnalysisResults<'tcx, A> for &'a mut Results<'tcx, A, E>
-where
-    A: Analysis<'tcx>,
-    E: Borrow<EntrySets<'tcx, A>>,
-{
-    type EntrySets = E;
-}
-
-/// A `ResultsCursor` that borrows the underlying `Results`.
-pub type ResultsRefCursor<'res, 'mir, 'tcx, A> =
-    ResultsCursor<'mir, 'tcx, A, &'res mut Results<'tcx, A>>;
-
-/// A `ResultsCursor` which uses a cloned `Analysis` while borrowing the underlying `Results`. This
-/// allows multiple cursors over the same `Results`.
-pub type ResultsClonedCursor<'res, 'mir, 'tcx, A> =
-    ResultsCursor<'mir, 'tcx, A, ResultsCloned<'res, 'tcx, A>>;
+use super::{Analysis, Direction, Effect, EffectIndex, Results};
 
 /// Allows random access inspection of the results of a dataflow analysis.
 ///
@@ -62,15 +16,12 @@ pub type ResultsClonedCursor<'res, 'mir, 'tcx, A> =
 /// the same order as the `DIRECTION` of the analysis. In the worst case—when statements are
 /// visited in *reverse* order—performance will be quadratic in the number of statements in the
 /// block. The order in which basic blocks are inspected has no impact on performance.
-///
-/// A `ResultsCursor` can either own (the default) or borrow the dataflow results it inspects. The
-/// type of ownership is determined by `R` (see `ResultsRefCursor` above).
-pub struct ResultsCursor<'mir, 'tcx, A, R = Results<'tcx, A>>
+pub struct ResultsCursor<'mir, 'tcx, A>
 where
     A: Analysis<'tcx>,
 {
     body: &'mir mir::Body<'tcx>,
-    results: R,
+    results: Results<'tcx, A>,
     state: A::Domain,
 
     pos: CursorPosition,
@@ -84,7 +35,7 @@ where
     reachable_blocks: BitSet<BasicBlock>,
 }
 
-impl<'mir, 'tcx, A, R> ResultsCursor<'mir, 'tcx, A, R>
+impl<'mir, 'tcx, A> ResultsCursor<'mir, 'tcx, A>
 where
     A: Analysis<'tcx>,
 {
@@ -99,30 +50,13 @@ where
     }
 
     /// Unwraps this cursor, returning the underlying `Results`.
-    pub fn into_results(self) -> R {
+    pub fn into_results(self) -> Results<'tcx, A> {
         self.results
     }
-}
 
-impl<'res, 'mir, 'tcx, A> ResultsCursor<'mir, 'tcx, A, ResultsCloned<'res, 'tcx, A>>
-where
-    A: Analysis<'tcx> + CloneAnalysis,
-{
-    /// Creates a new cursor over the same `Results`. Note that the cursor's position is *not*
-    /// copied.
-    pub fn new_cursor(&self) -> Self {
-        Self::new(self.body, self.results.reclone_analysis())
-    }
-}
-
-impl<'mir, 'tcx, A, R> ResultsCursor<'mir, 'tcx, A, R>
-where
-    A: Analysis<'tcx>,
-    R: AnalysisResults<'tcx, A>,
-{
     /// Returns a new cursor that can inspect `results`.
-    pub fn new(body: &'mir mir::Body<'tcx>, results: R) -> Self {
-        let bottom_value = results.borrow().analysis.bottom_value(body);
+    pub fn new(body: &'mir mir::Body<'tcx>, results: Results<'tcx, A>) -> Self {
+        let bottom_value = results.analysis.bottom_value(body);
         ResultsCursor {
             body,
             results,
@@ -147,28 +81,23 @@ where
     }
 
     /// Returns the underlying `Results`.
-    pub fn results(&mut self) -> &Results<'tcx, A, R::EntrySets> {
-        self.results.borrow()
+    pub fn results(&self) -> &Results<'tcx, A> {
+        &self.results
     }
 
     /// Returns the underlying `Results`.
-    pub fn mut_results(&mut self) -> &mut Results<'tcx, A, R::EntrySets> {
-        self.results.borrow_mut()
+    pub fn mut_results(&mut self) -> &mut Results<'tcx, A> {
+        &mut self.results
     }
 
     /// Returns the `Analysis` used to generate the underlying `Results`.
     pub fn analysis(&self) -> &A {
-        &self.results.borrow().analysis
+        &self.results.analysis
     }
 
     /// Returns the `Analysis` used to generate the underlying `Results`.
     pub fn mut_analysis(&mut self) -> &mut A {
-        &mut self.results.borrow_mut().analysis
-    }
-
-    /// Returns both the dataflow state at the current location and the `Analysis`.
-    pub fn get_with_analysis(&mut self) -> (&A::Domain, &mut A) {
-        (&self.state, &mut self.results.borrow_mut().analysis)
+        &mut self.results.analysis
     }
 
     /// Resets the cursor to hold the entry set for the given basic block.
@@ -180,7 +109,7 @@ where
         #[cfg(debug_assertions)]
         assert!(self.reachable_blocks.contains(block));
 
-        self.state.clone_from(self.results.borrow().entry_set_for_block(block));
+        self.state.clone_from(self.results.entry_set_for_block(block));
         self.pos = CursorPosition::block_entry(block);
         self.state_needs_reset = false;
     }
@@ -269,11 +198,10 @@ where
             )
         };
 
-        let analysis = &mut self.results.borrow_mut().analysis;
         let target_effect_index = effect.at_index(target.statement_index);
 
         A::Direction::apply_effects_in_range(
-            analysis,
+            &mut self.results.analysis,
             &mut self.state,
             target.block,
             block_data,
@@ -289,12 +217,12 @@ where
     /// This can be used, e.g., to apply the call return effect directly to the cursor without
     /// creating an extra copy of the dataflow state.
     pub fn apply_custom_effect(&mut self, f: impl FnOnce(&mut A, &mut A::Domain)) {
-        f(&mut self.results.borrow_mut().analysis, &mut self.state);
+        f(&mut self.results.analysis, &mut self.state);
         self.state_needs_reset = true;
     }
 }
 
-impl<'mir, 'tcx, A, R> ResultsCursor<'mir, 'tcx, A, R>
+impl<'mir, 'tcx, A> ResultsCursor<'mir, 'tcx, A>
 where
     A: crate::GenKillAnalysis<'tcx>,
     A::Domain: BitSetExt<A::Idx>,
diff --git a/compiler/rustc_mir_dataflow/src/framework/direction.rs b/compiler/rustc_mir_dataflow/src/framework/direction.rs
index 70451edd5..4c3fadf48 100644
--- a/compiler/rustc_mir_dataflow/src/framework/direction.rs
+++ b/compiler/rustc_mir_dataflow/src/framework/direction.rs
@@ -196,7 +196,7 @@ impl Direction for Backward {
     {
         results.reset_to_block_entry(state, block);
 
-        vis.visit_block_end(results, &state, block_data, block);
+        vis.visit_block_end(state);
 
         // Terminator
         let loc = Location { block, statement_index: block_data.statements.len() };
@@ -214,7 +214,7 @@ impl Direction for Backward {
             vis.visit_statement_after_primary_effect(results, state, stmt, loc);
         }
 
-        vis.visit_block_start(results, state, block_data, block);
+        vis.visit_block_start(state);
     }
 
     fn join_state_into_successors_of<'tcx, A>(
@@ -287,12 +287,12 @@ impl Direction for Backward {
     }
 }
 
-struct BackwardSwitchIntEdgeEffectsApplier<'a, 'tcx, D, F> {
-    body: &'a mir::Body<'tcx>,
+struct BackwardSwitchIntEdgeEffectsApplier<'mir, 'tcx, D, F> {
+    body: &'mir mir::Body<'tcx>,
     pred: BasicBlock,
-    exit_state: &'a mut D,
+    exit_state: &'mir mut D,
     bb: BasicBlock,
-    propagate: &'a mut F,
+    propagate: &'mir mut F,
     effects_applied: bool,
 }
 
@@ -449,7 +449,7 @@ impl Direction for Forward {
     {
         results.reset_to_block_entry(state, block);
 
-        vis.visit_block_start(results, state, block_data, block);
+        vis.visit_block_start(state);
 
         for (statement_index, stmt) in block_data.statements.iter().enumerate() {
             let loc = Location { block, statement_index };
@@ -466,7 +466,7 @@ impl Direction for Forward {
         results.reconstruct_terminator_effect(state, term, loc);
         vis.visit_terminator_after_primary_effect(results, state, term, loc);
 
-        vis.visit_block_end(results, state, block_data, block);
+        vis.visit_block_end(state);
     }
 
     fn join_state_into_successors_of<'tcx, A>(
@@ -523,9 +523,9 @@ impl Direction for Forward {
     }
 }
 
-struct ForwardSwitchIntEdgeEffectsApplier<'a, D, F> {
-    exit_state: &'a mut D,
-    targets: &'a SwitchTargets,
+struct ForwardSwitchIntEdgeEffectsApplier<'mir, D, F> {
+    exit_state: &'mir mut D,
+    targets: &'mir SwitchTargets,
     propagate: F,
 
     effects_applied: bool,
diff --git a/compiler/rustc_mir_dataflow/src/framework/engine.rs b/compiler/rustc_mir_dataflow/src/framework/engine.rs
index a29962d77..784872c7e 100644
--- a/compiler/rustc_mir_dataflow/src/framework/engine.rs
+++ b/compiler/rustc_mir_dataflow/src/framework/engine.rs
@@ -5,9 +5,7 @@ use crate::errors::{
 };
 use crate::framework::BitSetExt;
 
-use std::borrow::Borrow;
 use std::ffi::OsString;
-use std::marker::PhantomData;
 use std::path::PathBuf;
 
 use rustc_ast as ast;
@@ -24,42 +22,37 @@ use rustc_span::symbol::{sym, Symbol};
 use super::fmt::DebugWithContext;
 use super::graphviz;
 use super::{
-    visit_results, Analysis, AnalysisDomain, CloneAnalysis, Direction, GenKill, GenKillAnalysis,
-    GenKillSet, JoinSemiLattice, ResultsClonedCursor, ResultsCursor, ResultsRefCursor,
-    ResultsVisitor,
+    visit_results, Analysis, AnalysisDomain, Direction, GenKill, GenKillAnalysis, GenKillSet,
+    JoinSemiLattice, ResultsCursor, ResultsVisitor,
 };
 
 pub type EntrySets<'tcx, A> = IndexVec<BasicBlock, <A as AnalysisDomain<'tcx>>::Domain>;
 
 /// A dataflow analysis that has converged to fixpoint.
-pub struct Results<'tcx, A, E = EntrySets<'tcx, A>>
+#[derive(Clone)]
+pub struct Results<'tcx, A>
 where
     A: Analysis<'tcx>,
 {
     pub analysis: A,
-    pub(super) entry_sets: E,
-    pub(super) _marker: PhantomData<&'tcx ()>,
+    pub(super) entry_sets: EntrySets<'tcx, A>,
 }
 
-/// `Results` type with a cloned `Analysis` and borrowed entry sets.
-pub type ResultsCloned<'res, 'tcx, A> = Results<'tcx, A, &'res EntrySets<'tcx, A>>;
-
-impl<'tcx, A, E> Results<'tcx, A, E>
+impl<'tcx, A> Results<'tcx, A>
 where
     A: Analysis<'tcx>,
-    E: Borrow<EntrySets<'tcx, A>>,
 {
     /// Creates a `ResultsCursor` that can inspect these `Results`.
     pub fn into_results_cursor<'mir>(
         self,
         body: &'mir mir::Body<'tcx>,
-    ) -> ResultsCursor<'mir, 'tcx, A, Self> {
+    ) -> ResultsCursor<'mir, 'tcx, A> {
         ResultsCursor::new(body, self)
     }
 
     /// Gets the dataflow state for the given block.
     pub fn entry_set_for_block(&self, block: BasicBlock) -> &A::Domain {
-        &self.entry_sets.borrow()[block]
+        &self.entry_sets[block]
     }
 
     pub fn visit_with<'mir>(
@@ -80,60 +73,14 @@ where
         visit_results(body, blocks.map(|(bb, _)| bb), self, vis)
     }
 }
-impl<'tcx, A> Results<'tcx, A>
-where
-    A: Analysis<'tcx>,
-{
-    /// Creates a `ResultsCursor` that can inspect these `Results`.
-    pub fn as_results_cursor<'a, 'mir>(
-        &'a mut self,
-        body: &'mir mir::Body<'tcx>,
-    ) -> ResultsRefCursor<'a, 'mir, 'tcx, A> {
-        ResultsCursor::new(body, self)
-    }
-}
-impl<'tcx, A> Results<'tcx, A>
-where
-    A: Analysis<'tcx> + CloneAnalysis,
-{
-    /// Creates a new `Results` type with a cloned `Analysis` and borrowed entry sets.
-    pub fn clone_analysis(&self) -> ResultsCloned<'_, 'tcx, A> {
-        Results {
-            analysis: self.analysis.clone_analysis(),
-            entry_sets: &self.entry_sets,
-            _marker: PhantomData,
-        }
-    }
-
-    /// Creates a `ResultsCursor` that can inspect these `Results`.
-    pub fn cloned_results_cursor<'mir>(
-        &self,
-        body: &'mir mir::Body<'tcx>,
-    ) -> ResultsClonedCursor<'_, 'mir, 'tcx, A> {
-        self.clone_analysis().into_results_cursor(body)
-    }
-}
-impl<'res, 'tcx, A> Results<'tcx, A, &'res EntrySets<'tcx, A>>
-where
-    A: Analysis<'tcx> + CloneAnalysis,
-{
-    /// Creates a new `Results` type with a cloned `Analysis` and borrowed entry sets.
-    pub fn reclone_analysis(&self) -> Self {
-        Results {
-            analysis: self.analysis.clone_analysis(),
-            entry_sets: self.entry_sets,
-            _marker: PhantomData,
-        }
-    }
-}
 
 /// A solver for dataflow problems.
-pub struct Engine<'a, 'tcx, A>
+pub struct Engine<'mir, 'tcx, A>
 where
     A: Analysis<'tcx>,
 {
     tcx: TyCtxt<'tcx>,
-    body: &'a mir::Body<'tcx>,
+    body: &'mir mir::Body<'tcx>,
     entry_sets: IndexVec<BasicBlock, A::Domain>,
     pass_name: Option<&'static str>,
     analysis: A,
@@ -147,14 +94,14 @@ where
     apply_statement_trans_for_block: Option<Box<dyn Fn(BasicBlock, &mut A::Domain)>>,
 }
 
-impl<'a, 'tcx, A, D, T> Engine<'a, 'tcx, A>
+impl<'mir, 'tcx, A, D, T> Engine<'mir, 'tcx, A>
 where
     A: GenKillAnalysis<'tcx, Idx = T, Domain = D>,
     D: Clone + JoinSemiLattice + GenKill<T> + BitSetExt<T>,
     T: Idx,
 {
     /// Creates a new `Engine` to solve a gen-kill dataflow problem.
-    pub fn new_gen_kill(tcx: TyCtxt<'tcx>, body: &'a mir::Body<'tcx>, mut analysis: A) -> Self {
+    pub fn new_gen_kill(tcx: TyCtxt<'tcx>, body: &'mir mir::Body<'tcx>, mut analysis: A) -> Self {
         // If there are no back-edges in the control-flow graph, we only ever need to apply the
         // transfer function for each block exactly once (assuming that we process blocks in RPO).
         //
@@ -186,7 +133,7 @@ where
     }
 }
 
-impl<'a, 'tcx, A, D> Engine<'a, 'tcx, A>
+impl<'mir, 'tcx, A, D> Engine<'mir, 'tcx, A>
 where
     A: Analysis<'tcx, Domain = D>,
     D: Clone + JoinSemiLattice,
@@ -196,13 +143,13 @@ where
     ///
     /// Gen-kill problems should use `new_gen_kill`, which will coalesce transfer functions for
     /// better performance.
-    pub fn new_generic(tcx: TyCtxt<'tcx>, body: &'a mir::Body<'tcx>, analysis: A) -> Self {
+    pub fn new_generic(tcx: TyCtxt<'tcx>, body: &'mir mir::Body<'tcx>, analysis: A) -> Self {
         Self::new(tcx, body, analysis, None)
     }
 
     fn new(
         tcx: TyCtxt<'tcx>,
-        body: &'a mir::Body<'tcx>,
+        body: &'mir mir::Body<'tcx>,
         analysis: A,
         apply_statement_trans_for_block: Option<Box<dyn Fn(BasicBlock, &mut A::Domain)>>,
     ) -> Self {
@@ -239,7 +186,6 @@ where
             tcx,
             apply_statement_trans_for_block,
             pass_name,
-            ..
         } = self;
 
         let mut dirty_queue: WorkQueue<BasicBlock> = WorkQueue::with_none(body.basic_blocks.len());
@@ -292,29 +238,31 @@ where
             );
         }
 
-        let mut results = Results { analysis, entry_sets, _marker: PhantomData };
+        let results = Results { analysis, entry_sets };
 
         if tcx.sess.opts.unstable_opts.dump_mir_dataflow {
-            let res = write_graphviz_results(tcx, &body, &mut results, pass_name);
+            let (res, results) = write_graphviz_results(tcx, body, results, pass_name);
             if let Err(e) = res {
                 error!("Failed to write graphviz dataflow results: {}", e);
             }
+            results
+        } else {
+            results
         }
-
-        results
     }
 }
 
 // Graphviz
 
 /// Writes a DOT file containing the results of a dataflow analysis if the user requested it via
-/// `rustc_mir` attributes and `-Z dump-mir-dataflow`.
+/// `rustc_mir` attributes and `-Z dump-mir-dataflow`. The `Result` in and the `Results` out are
+/// the same.
 fn write_graphviz_results<'tcx, A>(
     tcx: TyCtxt<'tcx>,
     body: &mir::Body<'tcx>,
-    results: &mut Results<'tcx, A>,
+    results: Results<'tcx, A>,
     pass_name: Option<&'static str>,
-) -> std::io::Result<()>
+) -> (std::io::Result<()>, Results<'tcx, A>)
 where
     A: Analysis<'tcx>,
     A::Domain: DebugWithContext<A>,
@@ -325,23 +273,30 @@ where
     let def_id = body.source.def_id();
     let Ok(attrs) = RustcMirAttrs::parse(tcx, def_id) else {
         // Invalid `rustc_mir` attrs are reported in `RustcMirAttrs::parse`
-        return Ok(());
+        return (Ok(()), results);
     };
 
-    let mut file = match attrs.output_path(A::NAME) {
-        Some(path) => {
-            debug!("printing dataflow results for {:?} to {}", def_id, path.display());
-            if let Some(parent) = path.parent() {
-                fs::create_dir_all(parent)?;
+    let file = try {
+        match attrs.output_path(A::NAME) {
+            Some(path) => {
+                debug!("printing dataflow results for {:?} to {}", def_id, path.display());
+                if let Some(parent) = path.parent() {
+                    fs::create_dir_all(parent)?;
+                }
+                let f = fs::File::create(&path)?;
+                io::BufWriter::new(f)
             }
-            io::BufWriter::new(fs::File::create(&path)?)
-        }
 
-        None if dump_enabled(tcx, A::NAME, def_id) => {
-            create_dump_file(tcx, ".dot", false, A::NAME, &pass_name.unwrap_or("-----"), body)?
-        }
+            None if dump_enabled(tcx, A::NAME, def_id) => {
+                create_dump_file(tcx, ".dot", false, A::NAME, &pass_name.unwrap_or("-----"), body)?
+            }
 
-        _ => return Ok(()),
+            _ => return (Ok(()), results),
+        }
+    };
+    let mut file = match file {
+        Ok(f) => f,
+        Err(e) => return (Err(e), results),
     };
 
     let style = match attrs.formatter {
@@ -357,11 +312,14 @@ where
     if tcx.sess.opts.unstable_opts.graphviz_dark_mode {
         render_opts.push(dot::RenderOption::DarkTheme);
     }
-    with_no_trimmed_paths!(dot::render_opts(&graphviz, &mut buf, &render_opts)?);
+    let r = with_no_trimmed_paths!(dot::render_opts(&graphviz, &mut buf, &render_opts));
 
-    file.write_all(&buf)?;
+    let lhs = try {
+        r?;
+        file.write_all(&buf)?;
+    };
 
-    Ok(())
+    (lhs, graphviz.into_results())
 }
 
 #[derive(Default)]
diff --git a/compiler/rustc_mir_dataflow/src/framework/graphviz.rs b/compiler/rustc_mir_dataflow/src/framework/graphviz.rs
index c12ccba1e..0270e059a 100644
--- a/compiler/rustc_mir_dataflow/src/framework/graphviz.rs
+++ b/compiler/rustc_mir_dataflow/src/framework/graphviz.rs
@@ -12,10 +12,10 @@ use rustc_middle::mir::graphviz_safe_def_name;
 use rustc_middle::mir::{self, BasicBlock, Body, Location};
 
 use super::fmt::{DebugDiffWithAdapter, DebugWithAdapter, DebugWithContext};
-use super::{Analysis, CallReturnPlaces, Direction, Results, ResultsRefCursor, ResultsVisitor};
+use super::{Analysis, CallReturnPlaces, Direction, Results, ResultsCursor, ResultsVisitor};
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub enum OutputStyle {
+pub(crate) enum OutputStyle {
     AfterOnly,
     BeforeAndAfter,
 }
@@ -29,33 +29,37 @@ impl OutputStyle {
     }
 }
 
-pub struct Formatter<'res, 'mir, 'tcx, A>
+pub(crate) struct Formatter<'mir, 'tcx, A>
 where
     A: Analysis<'tcx>,
 {
     body: &'mir Body<'tcx>,
-    results: RefCell<&'res mut Results<'tcx, A>>,
+    results: RefCell<Option<Results<'tcx, A>>>,
     style: OutputStyle,
     reachable: BitSet<BasicBlock>,
 }
 
-impl<'res, 'mir, 'tcx, A> Formatter<'res, 'mir, 'tcx, A>
+impl<'mir, 'tcx, A> Formatter<'mir, 'tcx, A>
 where
     A: Analysis<'tcx>,
 {
-    pub fn new(
+    pub(crate) fn new(
         body: &'mir Body<'tcx>,
-        results: &'res mut Results<'tcx, A>,
+        results: Results<'tcx, A>,
         style: OutputStyle,
     ) -> Self {
         let reachable = mir::traversal::reachable_as_bitset(body);
-        Formatter { body, results: results.into(), style, reachable }
+        Formatter { body, results: Some(results).into(), style, reachable }
+    }
+
+    pub(crate) fn into_results(self) -> Results<'tcx, A> {
+        self.results.into_inner().unwrap()
     }
 }
 
 /// A pair of a basic block and an index into that basic blocks `successors`.
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
-pub struct CfgEdge {
+pub(crate) struct CfgEdge {
     source: BasicBlock,
     index: usize,
 }
@@ -69,7 +73,7 @@ fn dataflow_successors(body: &Body<'_>, bb: BasicBlock) -> Vec<CfgEdge> {
         .collect()
 }
 
-impl<'tcx, A> dot::Labeller<'_> for Formatter<'_, '_, 'tcx, A>
+impl<'tcx, A> dot::Labeller<'_> for Formatter<'_, 'tcx, A>
 where
     A: Analysis<'tcx>,
     A::Domain: DebugWithContext<A>,
@@ -88,14 +92,19 @@ where
 
     fn node_label(&self, block: &Self::Node) -> dot::LabelText<'_> {
         let mut label = Vec::new();
-        let mut results = self.results.borrow_mut();
-        let mut fmt = BlockFormatter {
-            results: results.as_results_cursor(self.body),
-            style: self.style,
-            bg: Background::Light,
-        };
+        self.results.replace_with(|results| {
+            // `Formatter::result` is a `RefCell<Option<_>>` so we can replace
+            // the value with `None`, move it into the results cursor, move it
+            // back out, and return it to the refcell wrapped in `Some`.
+            let mut fmt = BlockFormatter {
+                results: results.take().unwrap().into_results_cursor(self.body),
+                style: self.style,
+                bg: Background::Light,
+            };
 
-        fmt.write_node_label(&mut label, *block).unwrap();
+            fmt.write_node_label(&mut label, *block).unwrap();
+            Some(fmt.results.into_results())
+        });
         dot::LabelText::html(String::from_utf8(label).unwrap())
     }
 
@@ -109,7 +118,7 @@ where
     }
 }
 
-impl<'mir, 'tcx, A> dot::GraphWalk<'mir> for Formatter<'_, 'mir, 'tcx, A>
+impl<'mir, 'tcx, A> dot::GraphWalk<'mir> for Formatter<'mir, 'tcx, A>
 where
     A: Analysis<'tcx>,
 {
@@ -143,16 +152,16 @@ where
     }
 }
 
-struct BlockFormatter<'res, 'mir, 'tcx, A>
+struct BlockFormatter<'mir, 'tcx, A>
 where
     A: Analysis<'tcx>,
 {
-    results: ResultsRefCursor<'res, 'mir, 'tcx, A>,
+    results: ResultsCursor<'mir, 'tcx, A>,
     bg: Background,
     style: OutputStyle,
 }
 
-impl<'res, 'mir, 'tcx, A> BlockFormatter<'res, 'mir, 'tcx, A>
+impl<'mir, 'tcx, A> BlockFormatter<'mir, 'tcx, A>
 where
     A: Analysis<'tcx>,
     A::Domain: DebugWithContext<A>,
@@ -536,25 +545,13 @@ where
 {
     type FlowState = A::Domain;
 
-    fn visit_block_start(
-        &mut self,
-        _results: &mut Results<'tcx, A>,
-        state: &Self::FlowState,
-        _block_data: &mir::BasicBlockData<'tcx>,
-        _block: BasicBlock,
-    ) {
+    fn visit_block_start(&mut self, state: &Self::FlowState) {
         if A::Direction::IS_FORWARD {
             self.prev_state.clone_from(state);
         }
     }
 
-    fn visit_block_end(
-        &mut self,
-        _results: &mut Results<'tcx, A>,
-        state: &Self::FlowState,
-        _block_data: &mir::BasicBlockData<'tcx>,
-        _block: BasicBlock,
-    ) {
+    fn visit_block_end(&mut self, state: &Self::FlowState) {
         if A::Direction::IS_BACKWARD {
             self.prev_state.clone_from(state);
         }
diff --git a/compiler/rustc_mir_dataflow/src/framework/lattice.rs b/compiler/rustc_mir_dataflow/src/framework/lattice.rs
index 3b89598d2..1c2b475a4 100644
--- a/compiler/rustc_mir_dataflow/src/framework/lattice.rs
+++ b/compiler/rustc_mir_dataflow/src/framework/lattice.rs
@@ -342,7 +342,7 @@ impl<V: Clone> Clone for MaybeReachable<V> {
     fn clone_from(&mut self, source: &Self) {
         match (&mut *self, source) {
             (MaybeReachable::Reachable(x), MaybeReachable::Reachable(y)) => {
-                x.clone_from(&y);
+                x.clone_from(y);
             }
             _ => *self = source.clone(),
         }
diff --git a/compiler/rustc_mir_dataflow/src/framework/mod.rs b/compiler/rustc_mir_dataflow/src/framework/mod.rs
index 5020a1cf0..09cdb055a 100644
--- a/compiler/rustc_mir_dataflow/src/framework/mod.rs
+++ b/compiler/rustc_mir_dataflow/src/framework/mod.rs
@@ -45,9 +45,9 @@ pub mod graphviz;
 pub mod lattice;
 mod visitor;
 
-pub use self::cursor::{AnalysisResults, ResultsClonedCursor, ResultsCursor, ResultsRefCursor};
+pub use self::cursor::ResultsCursor;
 pub use self::direction::{Backward, Direction, Forward};
-pub use self::engine::{Engine, EntrySets, Results, ResultsCloned};
+pub use self::engine::{Engine, Results};
 pub use self::lattice::{JoinSemiLattice, MaybeReachable};
 pub use self::visitor::{visit_results, ResultsVisitable, ResultsVisitor};
 
@@ -246,35 +246,21 @@ pub trait Analysis<'tcx>: AnalysisDomain<'tcx> {
     }
 }
 
-/// Defines an `Analysis` which can be cloned for use in multiple `ResultsCursor`s or
-/// `ResultsVisitor`s. Note this need not be a full clone, only enough of one to be used with a new
-/// `ResultsCursor` or `ResultsVisitor`
-pub trait CloneAnalysis {
-    fn clone_analysis(&self) -> Self;
-}
-impl<'tcx, A> CloneAnalysis for A
-where
-    A: Analysis<'tcx> + Copy,
-{
-    fn clone_analysis(&self) -> Self {
-        *self
-    }
-}
-
 /// A gen/kill dataflow problem.
 ///
-/// Each method in this trait has a corresponding one in `Analysis`. However, these methods only
-/// allow modification of the dataflow state via "gen" and "kill" operations. By defining transfer
-/// functions for each statement in this way, the transfer function for an entire basic block can
-/// be computed efficiently.
+/// Each method in this trait has a corresponding one in `Analysis`. However, the first two methods
+/// here only allow modification of the dataflow state via "gen" and "kill" operations. By defining
+/// transfer functions for each statement in this way, the transfer function for an entire basic
+/// block can be computed efficiently. The remaining methods match up with `Analysis` exactly.
 ///
-/// `Analysis` is automatically implemented for all implementers of `GenKillAnalysis`.
+/// `Analysis` is automatically implemented for all implementers of `GenKillAnalysis` via a blanket
+/// impl below.
 pub trait GenKillAnalysis<'tcx>: Analysis<'tcx> {
     type Idx: Idx;
 
     fn domain_size(&self, body: &mir::Body<'tcx>) -> usize;
 
-    /// See `Analysis::apply_statement_effect`.
+    /// See `Analysis::apply_statement_effect`. Note how the second arg differs.
     fn statement_effect(
         &mut self,
         trans: &mut impl GenKill<Self::Idx>,
@@ -282,7 +268,8 @@ pub trait GenKillAnalysis<'tcx>: Analysis<'tcx> {
         location: Location,
     );
 
-    /// See `Analysis::apply_before_statement_effect`.
+    /// See `Analysis::apply_before_statement_effect`. Note how the second arg
+    /// differs.
     fn before_statement_effect(
         &mut self,
         _trans: &mut impl GenKill<Self::Idx>,
@@ -302,7 +289,7 @@ pub trait GenKillAnalysis<'tcx>: Analysis<'tcx> {
     /// See `Analysis::apply_before_terminator_effect`.
     fn before_terminator_effect(
         &mut self,
-        _trans: &mut impl GenKill<Self::Idx>,
+        _trans: &mut Self::Domain,
         _terminator: &mir::Terminator<'tcx>,
         _location: Location,
     ) {
@@ -313,7 +300,7 @@ pub trait GenKillAnalysis<'tcx>: Analysis<'tcx> {
     /// See `Analysis::apply_call_return_effect`.
     fn call_return_effect(
         &mut self,
-        trans: &mut impl GenKill<Self::Idx>,
+        trans: &mut Self::Domain,
         block: BasicBlock,
         return_places: CallReturnPlaces<'_, 'tcx>,
     );
@@ -328,6 +315,7 @@ pub trait GenKillAnalysis<'tcx>: Analysis<'tcx> {
     }
 }
 
+// Blanket impl: any impl of `GenKillAnalysis` automatically impls `Analysis`.
 impl<'tcx, A> Analysis<'tcx> for A
 where
     A: GenKillAnalysis<'tcx>,
diff --git a/compiler/rustc_mir_dataflow/src/framework/tests.rs b/compiler/rustc_mir_dataflow/src/framework/tests.rs
index 9cce5b26c..6b338efd5 100644
--- a/compiler/rustc_mir_dataflow/src/framework/tests.rs
+++ b/compiler/rustc_mir_dataflow/src/framework/tests.rs
@@ -2,9 +2,7 @@
 
 use std::marker::PhantomData;
 
-use rustc_index::bit_set::BitSet;
 use rustc_index::IndexVec;
-use rustc_middle::mir::{self, BasicBlock, Location};
 use rustc_middle::ty;
 use rustc_span::DUMMY_SP;
 
@@ -267,8 +265,7 @@ fn test_cursor<D: Direction>(analysis: MockAnalysis<'_, D>) {
     let body = analysis.body;
 
     let mut cursor =
-        Results { entry_sets: analysis.mock_entry_sets(), analysis, _marker: PhantomData }
-            .into_results_cursor(body);
+        Results { entry_sets: analysis.mock_entry_sets(), analysis }.into_results_cursor(body);
 
     cursor.allow_unreachable();
 
diff --git a/compiler/rustc_mir_dataflow/src/framework/visitor.rs b/compiler/rustc_mir_dataflow/src/framework/visitor.rs
index 3cfa7cc1c..8b8a16bda 100644
--- a/compiler/rustc_mir_dataflow/src/framework/visitor.rs
+++ b/compiler/rustc_mir_dataflow/src/framework/visitor.rs
@@ -1,8 +1,6 @@
-use std::borrow::Borrow;
-
 use rustc_middle::mir::{self, BasicBlock, Location};
 
-use super::{Analysis, Direction, EntrySets, Results};
+use super::{Analysis, Direction, Results};
 
 /// Calls the corresponding method in `ResultsVisitor` for every location in a `mir::Body` with the
 /// dataflow state at that location.
@@ -33,14 +31,7 @@ pub fn visit_results<'mir, 'tcx, F, R>(
 pub trait ResultsVisitor<'mir, 'tcx, R> {
     type FlowState;
 
-    fn visit_block_start(
-        &mut self,
-        _results: &mut R,
-        _state: &Self::FlowState,
-        _block_data: &'mir mir::BasicBlockData<'tcx>,
-        _block: BasicBlock,
-    ) {
-    }
+    fn visit_block_start(&mut self, _state: &Self::FlowState) {}
 
     /// Called with the `before_statement_effect` of the given statement applied to `state` but not
     /// its `statement_effect`.
@@ -88,20 +79,13 @@ pub trait ResultsVisitor<'mir, 'tcx, R> {
     ) {
     }
 
-    fn visit_block_end(
-        &mut self,
-        _results: &mut R,
-        _state: &Self::FlowState,
-        _block_data: &'mir mir::BasicBlockData<'tcx>,
-        _block: BasicBlock,
-    ) {
-    }
+    fn visit_block_end(&mut self, _state: &Self::FlowState) {}
 }
 
 /// Things that can be visited by a `ResultsVisitor`.
 ///
-/// This trait exists so that we can visit the results of multiple dataflow analyses simultaneously.
-/// DO NOT IMPLEMENT MANUALLY. Instead, use the `impl_visitable` macro below.
+/// This trait exists so that we can visit the results of one or more dataflow analyses
+/// simultaneously.
 pub trait ResultsVisitable<'tcx> {
     type Direction: Direction;
     type FlowState;
@@ -143,10 +127,9 @@ pub trait ResultsVisitable<'tcx> {
     );
 }
 
-impl<'tcx, A, E> ResultsVisitable<'tcx> for Results<'tcx, A, E>
+impl<'tcx, A> ResultsVisitable<'tcx> for Results<'tcx, A>
 where
     A: Analysis<'tcx>,
-    E: Borrow<EntrySets<'tcx, A>>,
 {
     type FlowState = A::Domain;
 
diff --git a/compiler/rustc_mir_dataflow/src/impls/borrowed_locals.rs b/compiler/rustc_mir_dataflow/src/impls/borrowed_locals.rs
index 01acc380f..693994b5d 100644
--- a/compiler/rustc_mir_dataflow/src/impls/borrowed_locals.rs
+++ b/compiler/rustc_mir_dataflow/src/impls/borrowed_locals.rs
@@ -62,7 +62,7 @@ impl<'tcx> GenKillAnalysis<'tcx> for MaybeBorrowedLocals {
 
     fn call_return_effect(
         &mut self,
-        _trans: &mut impl GenKill<Self::Idx>,
+        _trans: &mut Self::Domain,
         _block: BasicBlock,
         _return_places: CallReturnPlaces<'_, 'tcx>,
     ) {
diff --git a/compiler/rustc_mir_dataflow/src/impls/initialized.rs b/compiler/rustc_mir_dataflow/src/impls/initialized.rs
index c968e7aea..6653b99b3 100644
--- a/compiler/rustc_mir_dataflow/src/impls/initialized.rs
+++ b/compiler/rustc_mir_dataflow/src/impls/initialized.rs
@@ -72,7 +72,7 @@ impl<'a, 'tcx> MaybeInitializedPlaces<'a, 'tcx> {
     ) -> bool {
         if let LookupResult::Exact(path) = self.move_data().rev_lookup.find(place.as_ref()) {
             let mut maybe_live = false;
-            on_all_children_bits(self.tcx, self.body, self.move_data(), path, |child| {
+            on_all_children_bits(self.move_data(), path, |child| {
                 maybe_live |= state.contains(child);
             });
             !maybe_live
@@ -203,14 +203,13 @@ impl<'a, 'tcx> HasMoveData<'tcx> for MaybeUninitializedPlaces<'a, 'tcx> {
 /// this data and `MaybeInitializedPlaces` yields the set of places
 /// that would require a dynamic drop-flag at that statement.
 pub struct DefinitelyInitializedPlaces<'a, 'tcx> {
-    tcx: TyCtxt<'tcx>,
     body: &'a Body<'tcx>,
     mdpe: &'a MoveDataParamEnv<'tcx>,
 }
 
 impl<'a, 'tcx> DefinitelyInitializedPlaces<'a, 'tcx> {
-    pub fn new(tcx: TyCtxt<'tcx>, body: &'a Body<'tcx>, mdpe: &'a MoveDataParamEnv<'tcx>) -> Self {
-        DefinitelyInitializedPlaces { tcx, body, mdpe }
+    pub fn new(body: &'a Body<'tcx>, mdpe: &'a MoveDataParamEnv<'tcx>) -> Self {
+        DefinitelyInitializedPlaces { body, mdpe }
     }
 }
 
@@ -250,15 +249,13 @@ impl<'a, 'tcx> HasMoveData<'tcx> for DefinitelyInitializedPlaces<'a, 'tcx> {
 /// }
 /// ```
 pub struct EverInitializedPlaces<'a, 'tcx> {
-    #[allow(dead_code)]
-    tcx: TyCtxt<'tcx>,
     body: &'a Body<'tcx>,
     mdpe: &'a MoveDataParamEnv<'tcx>,
 }
 
 impl<'a, 'tcx> EverInitializedPlaces<'a, 'tcx> {
-    pub fn new(tcx: TyCtxt<'tcx>, body: &'a Body<'tcx>, mdpe: &'a MoveDataParamEnv<'tcx>) -> Self {
-        EverInitializedPlaces { tcx, body, mdpe }
+    pub fn new(body: &'a Body<'tcx>, mdpe: &'a MoveDataParamEnv<'tcx>) -> Self {
+        EverInitializedPlaces { body, mdpe }
     }
 }
 
@@ -319,7 +316,7 @@ impl<'tcx> AnalysisDomain<'tcx> for MaybeInitializedPlaces<'_, 'tcx> {
     fn initialize_start_block(&self, _: &mir::Body<'tcx>, state: &mut Self::Domain) {
         *state =
             MaybeReachable::Reachable(ChunkedBitSet::new_empty(self.move_data().move_paths.len()));
-        drop_flag_effects_for_function_entry(self.tcx, self.body, self.mdpe, |path, s| {
+        drop_flag_effects_for_function_entry(self.body, self.mdpe, |path, s| {
             assert!(s == DropFlagState::Present);
             state.gen(path);
         });
@@ -339,7 +336,7 @@ impl<'tcx> GenKillAnalysis<'tcx> for MaybeInitializedPlaces<'_, 'tcx> {
         statement: &mir::Statement<'tcx>,
         location: Location,
     ) {
-        drop_flag_effects_for_location(self.tcx, self.body, self.mdpe, location, |path, s| {
+        drop_flag_effects_for_location(self.body, self.mdpe, location, |path, s| {
             Self::update_bits(trans, path, s)
         });
 
@@ -351,7 +348,7 @@ impl<'tcx> GenKillAnalysis<'tcx> for MaybeInitializedPlaces<'_, 'tcx> {
                 | mir::Rvalue::AddressOf(_, place) = rvalue
             && let LookupResult::Exact(mpi) = self.move_data().rev_lookup.find(place.as_ref())
         {
-            on_all_children_bits(self.tcx, self.body, self.move_data(), mpi, |child| {
+            on_all_children_bits(self.move_data(), mpi, |child| {
                 trans.gen(child);
             })
         }
@@ -371,7 +368,7 @@ impl<'tcx> GenKillAnalysis<'tcx> for MaybeInitializedPlaces<'_, 'tcx> {
         {
             edges = TerminatorEdges::Single(target);
         }
-        drop_flag_effects_for_location(self.tcx, self.body, self.mdpe, location, |path, s| {
+        drop_flag_effects_for_location(self.body, self.mdpe, location, |path, s| {
             Self::update_bits(state, path, s)
         });
         edges
@@ -379,7 +376,7 @@ impl<'tcx> GenKillAnalysis<'tcx> for MaybeInitializedPlaces<'_, 'tcx> {
 
     fn call_return_effect(
         &mut self,
-        trans: &mut impl GenKill<Self::Idx>,
+        trans: &mut Self::Domain,
         _block: mir::BasicBlock,
         return_places: CallReturnPlaces<'_, 'tcx>,
     ) {
@@ -387,8 +384,6 @@ impl<'tcx> GenKillAnalysis<'tcx> for MaybeInitializedPlaces<'_, 'tcx> {
             // when a call returns successfully, that means we need to set
             // the bits for that dest_place to 1 (initialized).
             on_lookup_result_bits(
-                self.tcx,
-                self.body,
                 self.move_data(),
                 self.move_data().rev_lookup.find(place.as_ref()),
                 |mpi| {
@@ -409,7 +404,7 @@ impl<'tcx> GenKillAnalysis<'tcx> for MaybeInitializedPlaces<'_, 'tcx> {
         }
 
         let enum_ = discr.place().and_then(|discr| {
-            switch_on_enum_discriminant(self.tcx, &self.body, &self.body[block], discr)
+            switch_on_enum_discriminant(self.tcx, self.body, &self.body[block], discr)
         });
 
         let Some((enum_place, enum_def)) = enum_ else {
@@ -432,8 +427,6 @@ impl<'tcx> GenKillAnalysis<'tcx> for MaybeInitializedPlaces<'_, 'tcx> {
             // Kill all move paths that correspond to variants we know to be inactive along this
             // particular outgoing edge of a `SwitchInt`.
             drop_flag_effects::on_all_inactive_variants(
-                self.tcx,
-                self.body,
                 self.move_data(),
                 enum_place,
                 variant,
@@ -458,7 +451,7 @@ impl<'tcx> AnalysisDomain<'tcx> for MaybeUninitializedPlaces<'_, 'tcx> {
         // set all bits to 1 (uninit) before gathering counter-evidence
         state.insert_all();
 
-        drop_flag_effects_for_function_entry(self.tcx, self.body, self.mdpe, |path, s| {
+        drop_flag_effects_for_function_entry(self.body, self.mdpe, |path, s| {
             assert!(s == DropFlagState::Present);
             state.remove(path);
         });
@@ -478,7 +471,7 @@ impl<'tcx> GenKillAnalysis<'tcx> for MaybeUninitializedPlaces<'_, 'tcx> {
         _statement: &mir::Statement<'tcx>,
         location: Location,
     ) {
-        drop_flag_effects_for_location(self.tcx, self.body, self.mdpe, location, |path, s| {
+        drop_flag_effects_for_location(self.body, self.mdpe, location, |path, s| {
             Self::update_bits(trans, path, s)
         });
 
@@ -492,7 +485,7 @@ impl<'tcx> GenKillAnalysis<'tcx> for MaybeUninitializedPlaces<'_, 'tcx> {
         terminator: &'mir mir::Terminator<'tcx>,
         location: Location,
     ) -> TerminatorEdges<'mir, 'tcx> {
-        drop_flag_effects_for_location(self.tcx, self.body, self.mdpe, location, |path, s| {
+        drop_flag_effects_for_location(self.body, self.mdpe, location, |path, s| {
             Self::update_bits(trans, path, s)
         });
         if self.skip_unreachable_unwind.contains(location.block) {
@@ -506,7 +499,7 @@ impl<'tcx> GenKillAnalysis<'tcx> for MaybeUninitializedPlaces<'_, 'tcx> {
 
     fn call_return_effect(
         &mut self,
-        trans: &mut impl GenKill<Self::Idx>,
+        trans: &mut Self::Domain,
         _block: mir::BasicBlock,
         return_places: CallReturnPlaces<'_, 'tcx>,
     ) {
@@ -514,8 +507,6 @@ impl<'tcx> GenKillAnalysis<'tcx> for MaybeUninitializedPlaces<'_, 'tcx> {
             // when a call returns successfully, that means we need to set
             // the bits for that dest_place to 0 (initialized).
             on_lookup_result_bits(
-                self.tcx,
-                self.body,
                 self.move_data(),
                 self.move_data().rev_lookup.find(place.as_ref()),
                 |mpi| {
@@ -540,7 +531,7 @@ impl<'tcx> GenKillAnalysis<'tcx> for MaybeUninitializedPlaces<'_, 'tcx> {
         }
 
         let enum_ = discr.place().and_then(|discr| {
-            switch_on_enum_discriminant(self.tcx, &self.body, &self.body[block], discr)
+            switch_on_enum_discriminant(self.tcx, self.body, &self.body[block], discr)
         });
 
         let Some((enum_place, enum_def)) = enum_ else {
@@ -563,8 +554,6 @@ impl<'tcx> GenKillAnalysis<'tcx> for MaybeUninitializedPlaces<'_, 'tcx> {
             // Mark all move paths that correspond to variants other than this one as maybe
             // uninitialized (in reality, they are *definitely* uninitialized).
             drop_flag_effects::on_all_inactive_variants(
-                self.tcx,
-                self.body,
                 self.move_data(),
                 enum_place,
                 variant,
@@ -589,7 +578,7 @@ impl<'a, 'tcx> AnalysisDomain<'tcx> for DefinitelyInitializedPlaces<'a, 'tcx> {
     fn initialize_start_block(&self, _: &mir::Body<'tcx>, state: &mut Self::Domain) {
         state.0.clear();
 
-        drop_flag_effects_for_function_entry(self.tcx, self.body, self.mdpe, |path, s| {
+        drop_flag_effects_for_function_entry(self.body, self.mdpe, |path, s| {
             assert!(s == DropFlagState::Present);
             state.0.insert(path);
         });
@@ -609,7 +598,7 @@ impl<'tcx> GenKillAnalysis<'tcx> for DefinitelyInitializedPlaces<'_, 'tcx> {
         _statement: &mir::Statement<'tcx>,
         location: Location,
     ) {
-        drop_flag_effects_for_location(self.tcx, self.body, self.mdpe, location, |path, s| {
+        drop_flag_effects_for_location(self.body, self.mdpe, location, |path, s| {
             Self::update_bits(trans, path, s)
         })
     }
@@ -620,7 +609,7 @@ impl<'tcx> GenKillAnalysis<'tcx> for DefinitelyInitializedPlaces<'_, 'tcx> {
         terminator: &'mir mir::Terminator<'tcx>,
         location: Location,
     ) -> TerminatorEdges<'mir, 'tcx> {
-        drop_flag_effects_for_location(self.tcx, self.body, self.mdpe, location, |path, s| {
+        drop_flag_effects_for_location(self.body, self.mdpe, location, |path, s| {
             Self::update_bits(trans, path, s)
         });
         terminator.edges()
@@ -628,7 +617,7 @@ impl<'tcx> GenKillAnalysis<'tcx> for DefinitelyInitializedPlaces<'_, 'tcx> {
 
     fn call_return_effect(
         &mut self,
-        trans: &mut impl GenKill<Self::Idx>,
+        trans: &mut Self::Domain,
         _block: mir::BasicBlock,
         return_places: CallReturnPlaces<'_, 'tcx>,
     ) {
@@ -636,8 +625,6 @@ impl<'tcx> GenKillAnalysis<'tcx> for DefinitelyInitializedPlaces<'_, 'tcx> {
             // when a call returns successfully, that means we need to set
             // the bits for that dest_place to 1 (initialized).
             on_lookup_result_bits(
-                self.tcx,
-                self.body,
                 self.move_data(),
                 self.move_data().rev_lookup.find(place.as_ref()),
                 |mpi| {
@@ -725,7 +712,7 @@ impl<'tcx> GenKillAnalysis<'tcx> for EverInitializedPlaces<'_, 'tcx> {
 
     fn call_return_effect(
         &mut self,
-        trans: &mut impl GenKill<Self::Idx>,
+        trans: &mut Self::Domain,
         block: mir::BasicBlock,
         _return_places: CallReturnPlaces<'_, 'tcx>,
     ) {
diff --git a/compiler/rustc_mir_dataflow/src/impls/liveness.rs b/compiler/rustc_mir_dataflow/src/impls/liveness.rs
index bdfb6a6ff..04bae6ae2 100644
--- a/compiler/rustc_mir_dataflow/src/impls/liveness.rs
+++ b/compiler/rustc_mir_dataflow/src/impls/liveness.rs
@@ -23,7 +23,6 @@ use crate::{Analysis, AnalysisDomain, Backward, GenKill, GenKillAnalysis};
 /// [`MaybeBorrowedLocals`]: super::MaybeBorrowedLocals
 /// [flow-test]: https://github.com/rust-lang/rust/blob/a08c47310c7d49cbdc5d7afb38408ba519967ecd/src/test/ui/mir-dataflow/liveness-ptr.rs
 /// [liveness]: https://en.wikipedia.org/wiki/Live_variable_analysis
-#[derive(Clone, Copy)]
 pub struct MaybeLiveLocals;
 
 impl<'tcx> AnalysisDomain<'tcx> for MaybeLiveLocals {
@@ -70,7 +69,7 @@ impl<'tcx> GenKillAnalysis<'tcx> for MaybeLiveLocals {
 
     fn call_return_effect(
         &mut self,
-        trans: &mut impl GenKill<Self::Idx>,
+        trans: &mut Self::Domain,
         _block: mir::BasicBlock,
         return_places: CallReturnPlaces<'_, 'tcx>,
     ) {
diff --git a/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs b/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs
index 5a58e3af8..646c70eb8 100644
--- a/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs
+++ b/compiler/rustc_mir_dataflow/src/impls/storage_liveness.rs
@@ -5,7 +5,7 @@ use rustc_middle::mir::*;
 use std::borrow::Cow;
 
 use super::MaybeBorrowedLocals;
-use crate::{GenKill, ResultsClonedCursor};
+use crate::{GenKill, ResultsCursor};
 
 #[derive(Clone)]
 pub struct MaybeStorageLive<'a> {
@@ -18,12 +18,6 @@ impl<'a> MaybeStorageLive<'a> {
     }
 }
 
-impl crate::CloneAnalysis for MaybeStorageLive<'_> {
-    fn clone_analysis(&self) -> Self {
-        self.clone()
-    }
-}
-
 impl<'tcx, 'a> crate::AnalysisDomain<'tcx> for MaybeStorageLive<'a> {
     type Domain = BitSet<Local>;
 
@@ -78,7 +72,7 @@ impl<'tcx, 'a> crate::GenKillAnalysis<'tcx> for MaybeStorageLive<'a> {
 
     fn call_return_effect(
         &mut self,
-        _trans: &mut impl GenKill<Self::Idx>,
+        _trans: &mut Self::Domain,
         _block: BasicBlock,
         _return_places: CallReturnPlaces<'_, 'tcx>,
     ) {
@@ -150,7 +144,7 @@ impl<'tcx> crate::GenKillAnalysis<'tcx> for MaybeStorageDead {
 
     fn call_return_effect(
         &mut self,
-        _trans: &mut impl GenKill<Self::Idx>,
+        _trans: &mut Self::Domain,
         _block: BasicBlock,
         _return_places: CallReturnPlaces<'_, 'tcx>,
     ) {
@@ -158,28 +152,21 @@ impl<'tcx> crate::GenKillAnalysis<'tcx> for MaybeStorageDead {
     }
 }
 
-type BorrowedLocalsResults<'res, 'mir, 'tcx> =
-    ResultsClonedCursor<'res, 'mir, 'tcx, MaybeBorrowedLocals>;
+type BorrowedLocalsResults<'mir, 'tcx> = ResultsCursor<'mir, 'tcx, MaybeBorrowedLocals>;
 
 /// Dataflow analysis that determines whether each local requires storage at a
 /// given location; i.e. whether its storage can go away without being observed.
-pub struct MaybeRequiresStorage<'res, 'mir, 'tcx> {
-    borrowed_locals: BorrowedLocalsResults<'res, 'mir, 'tcx>,
+pub struct MaybeRequiresStorage<'mir, 'tcx> {
+    borrowed_locals: BorrowedLocalsResults<'mir, 'tcx>,
 }
 
-impl<'res, 'mir, 'tcx> MaybeRequiresStorage<'res, 'mir, 'tcx> {
-    pub fn new(borrowed_locals: BorrowedLocalsResults<'res, 'mir, 'tcx>) -> Self {
+impl<'mir, 'tcx> MaybeRequiresStorage<'mir, 'tcx> {
+    pub fn new(borrowed_locals: BorrowedLocalsResults<'mir, 'tcx>) -> Self {
         MaybeRequiresStorage { borrowed_locals }
     }
 }
 
-impl crate::CloneAnalysis for MaybeRequiresStorage<'_, '_, '_> {
-    fn clone_analysis(&self) -> Self {
-        Self { borrowed_locals: self.borrowed_locals.new_cursor() }
-    }
-}
-
-impl<'tcx> crate::AnalysisDomain<'tcx> for MaybeRequiresStorage<'_, '_, 'tcx> {
+impl<'tcx> crate::AnalysisDomain<'tcx> for MaybeRequiresStorage<'_, 'tcx> {
     type Domain = BitSet<Local>;
 
     const NAME: &'static str = "requires_storage";
@@ -198,7 +185,7 @@ impl<'tcx> crate::AnalysisDomain<'tcx> for MaybeRequiresStorage<'_, '_, 'tcx> {
     }
 }
 
-impl<'tcx> crate::GenKillAnalysis<'tcx> for MaybeRequiresStorage<'_, '_, 'tcx> {
+impl<'tcx> crate::GenKillAnalysis<'tcx> for MaybeRequiresStorage<'_, 'tcx> {
     type Idx = Local;
 
     fn domain_size(&self, body: &Body<'tcx>) -> usize {
@@ -251,7 +238,7 @@ impl<'tcx> crate::GenKillAnalysis<'tcx> for MaybeRequiresStorage<'_, '_, 'tcx> {
 
     fn before_terminator_effect(
         &mut self,
-        trans: &mut impl GenKill<Self::Idx>,
+        trans: &mut Self::Domain,
         terminator: &Terminator<'tcx>,
         loc: Location,
     ) {
@@ -347,7 +334,7 @@ impl<'tcx> crate::GenKillAnalysis<'tcx> for MaybeRequiresStorage<'_, '_, 'tcx> {
 
     fn call_return_effect(
         &mut self,
-        trans: &mut impl GenKill<Self::Idx>,
+        trans: &mut Self::Domain,
         _block: BasicBlock,
         return_places: CallReturnPlaces<'_, 'tcx>,
     ) {
@@ -355,7 +342,7 @@ impl<'tcx> crate::GenKillAnalysis<'tcx> for MaybeRequiresStorage<'_, '_, 'tcx> {
     }
 }
 
-impl<'tcx> MaybeRequiresStorage<'_, '_, 'tcx> {
+impl<'tcx> MaybeRequiresStorage<'_, 'tcx> {
     /// Kill locals that are fully moved and have not been borrowed.
     fn check_for_move(&mut self, trans: &mut impl GenKill<Local>, loc: Location) {
         let body = self.borrowed_locals.body();
@@ -364,12 +351,12 @@ impl<'tcx> MaybeRequiresStorage<'_, '_, 'tcx> {
     }
 }
 
-struct MoveVisitor<'a, 'res, 'mir, 'tcx, T> {
-    borrowed_locals: &'a mut BorrowedLocalsResults<'res, 'mir, 'tcx>,
+struct MoveVisitor<'a, 'mir, 'tcx, T> {
+    borrowed_locals: &'a mut BorrowedLocalsResults<'mir, 'tcx>,
     trans: &'a mut T,
 }
 
-impl<'tcx, T> Visitor<'tcx> for MoveVisitor<'_, '_, '_, 'tcx, T>
+impl<'tcx, T> Visitor<'tcx> for MoveVisitor<'_, '_, 'tcx, T>
 where
     T: GenKill<Local>,
 {
diff --git a/compiler/rustc_mir_dataflow/src/lib.rs b/compiler/rustc_mir_dataflow/src/lib.rs
index eea0e030e..f0b21fd41 100644
--- a/compiler/rustc_mir_dataflow/src/lib.rs
+++ b/compiler/rustc_mir_dataflow/src/lib.rs
@@ -4,7 +4,7 @@
 #![feature(let_chains)]
 #![feature(min_specialization)]
 #![feature(stmt_expr_attributes)]
-#![feature(trusted_step)]
+#![feature(try_blocks)]
 #![recursion_limit = "256"]
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
@@ -14,24 +14,17 @@ extern crate tracing;
 #[macro_use]
 extern crate rustc_middle;
 
-use rustc_ast::MetaItem;
-use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
-use rustc_fluent_macro::fluent_messages;
-use rustc_hir::def_id::DefId;
-use rustc_middle::ty::{self, TyCtxt};
-use rustc_span::symbol::{sym, Symbol};
+use rustc_middle::ty;
 
 pub use self::drop_flag_effects::{
     drop_flag_effects_for_function_entry, drop_flag_effects_for_location,
     move_path_children_matching, on_all_children_bits, on_lookup_result_bits,
 };
 pub use self::framework::{
-    fmt, graphviz, lattice, visit_results, Analysis, AnalysisDomain, AnalysisResults, Backward,
-    CloneAnalysis, Direction, Engine, Forward, GenKill, GenKillAnalysis, JoinSemiLattice,
-    MaybeReachable, Results, ResultsCloned, ResultsClonedCursor, ResultsCursor, ResultsRefCursor,
-    ResultsVisitable, ResultsVisitor, SwitchIntEdgeEffects,
+    fmt, lattice, visit_results, Analysis, AnalysisDomain, Direction, GenKill, GenKillAnalysis,
+    JoinSemiLattice, MaybeReachable, Results, ResultsCursor, ResultsVisitable, ResultsVisitor,
 };
-
+use self::framework::{Backward, SwitchIntEdgeEffects};
 use self::move_paths::MoveData;
 
 pub mod debuginfo;
@@ -46,26 +39,9 @@ pub mod storage;
 pub mod un_derefer;
 pub mod value_analysis;
 
-fluent_messages! { "../messages.ftl" }
-
-pub(crate) mod indexes {
-    pub(crate) use super::move_paths::MovePathIndex;
-}
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
 
 pub struct MoveDataParamEnv<'tcx> {
     pub move_data: MoveData<'tcx>,
     pub param_env: ty::ParamEnv<'tcx>,
 }
-
-pub fn has_rustc_mir_with(tcx: TyCtxt<'_>, def_id: DefId, name: Symbol) -> Option<MetaItem> {
-    for attr in tcx.get_attrs(def_id, sym::rustc_mir) {
-        let items = attr.meta_item_list();
-        for item in items.iter().flat_map(|l| l.iter()) {
-            match item.meta_item() {
-                Some(mi) if mi.has_name(name) => return Some(mi.clone()),
-                _ => continue,
-            }
-        }
-    }
-    None
-}
diff --git a/compiler/rustc_mir_dataflow/src/move_paths/mod.rs b/compiler/rustc_mir_dataflow/src/move_paths/mod.rs
index 7ab1a9ed0..22cf39992 100644
--- a/compiler/rustc_mir_dataflow/src/move_paths/mod.rs
+++ b/compiler/rustc_mir_dataflow/src/move_paths/mod.rs
@@ -14,6 +14,7 @@ use self::abs_domain::{AbstractElem, Lift};
 mod abs_domain;
 
 rustc_index::newtype_index! {
+    #[orderable]
     #[debug_format = "mp{}"]
     pub struct MovePathIndex {}
 }
@@ -25,6 +26,7 @@ impl polonius_engine::Atom for MovePathIndex {
 }
 
 rustc_index::newtype_index! {
+    #[orderable]
     #[debug_format = "mo{}"]
     pub struct MoveOutIndex {}
 }
diff --git a/compiler/rustc_mir_dataflow/src/rustc_peek.rs b/compiler/rustc_mir_dataflow/src/rustc_peek.rs
index d3dce641b..448128b69 100644
--- a/compiler/rustc_mir_dataflow/src/rustc_peek.rs
+++ b/compiler/rustc_mir_dataflow/src/rustc_peek.rs
@@ -1,11 +1,3 @@
-use rustc_span::symbol::sym;
-use rustc_span::Span;
-
-use rustc_index::bit_set::ChunkedBitSet;
-use rustc_middle::mir::MirPass;
-use rustc_middle::mir::{self, Body, Local, Location};
-use rustc_middle::ty::{self, Ty, TyCtxt};
-
 use crate::errors::{
     PeekArgumentNotALocal, PeekArgumentUntracked, PeekBitNotSet, PeekMustBeNotTemporary,
     PeekMustBePlaceOrRefPlace, StopAfterDataFlowEndedCompilation,
@@ -18,13 +10,33 @@ use crate::move_paths::{HasMoveData, MoveData};
 use crate::move_paths::{LookupResult, MovePathIndex};
 use crate::MoveDataParamEnv;
 use crate::{Analysis, JoinSemiLattice, ResultsCursor};
+use rustc_ast::MetaItem;
+use rustc_hir::def_id::DefId;
+use rustc_index::bit_set::ChunkedBitSet;
+use rustc_middle::mir::MirPass;
+use rustc_middle::mir::{self, Body, Local, Location};
+use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_span::symbol::{sym, Symbol};
+use rustc_span::Span;
 
 pub struct SanityCheck;
 
+fn has_rustc_mir_with(tcx: TyCtxt<'_>, def_id: DefId, name: Symbol) -> Option<MetaItem> {
+    for attr in tcx.get_attrs(def_id, sym::rustc_mir) {
+        let items = attr.meta_item_list();
+        for item in items.iter().flat_map(|l| l.iter()) {
+            match item.meta_item() {
+                Some(mi) if mi.has_name(name) => return Some(mi.clone()),
+                _ => continue,
+            }
+        }
+    }
+    None
+}
+
 // FIXME: This should be a `MirLint`, but it needs to be moved back to `rustc_mir_transform` first.
 impl<'tcx> MirPass<'tcx> for SanityCheck {
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
-        use crate::has_rustc_mir_with;
         let def_id = body.source.def_id();
         if !tcx.has_attr(def_id, sym::rustc_mir) {
             debug!("skipping rustc_peek::SanityCheck on {}", tcx.def_path_str(def_id));
@@ -34,7 +46,7 @@ impl<'tcx> MirPass<'tcx> for SanityCheck {
         }
 
         let param_env = tcx.param_env(def_id);
-        let move_data = MoveData::gather_moves(&body, tcx, param_env, |_| true);
+        let move_data = MoveData::gather_moves(body, tcx, param_env, |_| true);
         let mdpe = MoveDataParamEnv { move_data, param_env };
 
         if has_rustc_mir_with(tcx, def_id, sym::rustc_peek_maybe_init).is_some() {
@@ -54,7 +66,7 @@ impl<'tcx> MirPass<'tcx> for SanityCheck {
         }
 
         if has_rustc_mir_with(tcx, def_id, sym::rustc_peek_definite_init).is_some() {
-            let flow_def_inits = DefinitelyInitializedPlaces::new(tcx, body, &mdpe)
+            let flow_def_inits = DefinitelyInitializedPlaces::new(body, &mdpe)
                 .into_engine(tcx, body)
                 .iterate_to_fixpoint();
 
@@ -89,10 +101,8 @@ impl<'tcx> MirPass<'tcx> for SanityCheck {
 /// (If there are any calls to `rustc_peek` that do not match the
 /// expression form above, then that emits an error as well, but those
 /// errors are not intended to be used for unit tests.)
-pub fn sanity_check_via_rustc_peek<'tcx, A>(
-    tcx: TyCtxt<'tcx>,
-    mut cursor: ResultsCursor<'_, 'tcx, A>,
-) where
+fn sanity_check_via_rustc_peek<'tcx, A>(tcx: TyCtxt<'tcx>, mut cursor: ResultsCursor<'_, 'tcx, A>)
+where
     A: RustcPeekAt<'tcx>,
 {
     let def_id = cursor.body().source.def_id();
@@ -129,7 +139,8 @@ pub fn sanity_check_via_rustc_peek<'tcx, A>(
             ) => {
                 let loc = Location { block: bb, statement_index };
                 cursor.seek_before_primary_effect(loc);
-                let (state, analysis) = cursor.get_with_analysis();
+                let state = cursor.get();
+                let analysis = cursor.analysis();
                 analysis.peek_at(tcx, *place, state, call);
             }
 
@@ -173,7 +184,7 @@ impl PeekCallKind {
 }
 
 #[derive(Clone, Copy, Debug)]
-pub struct PeekCall {
+struct PeekCall {
     arg: Local,
     kind: PeekCallKind,
     span: Span,
@@ -221,7 +232,7 @@ impl PeekCall {
     }
 }
 
-pub trait RustcPeekAt<'tcx>: Analysis<'tcx> {
+trait RustcPeekAt<'tcx>: Analysis<'tcx> {
     fn peek_at(
         &self,
         tcx: TyCtxt<'tcx>,
diff --git a/compiler/rustc_mir_dataflow/src/un_derefer.rs b/compiler/rustc_mir_dataflow/src/un_derefer.rs
index 874d50ffd..b803ecc57 100644
--- a/compiler/rustc_mir_dataflow/src/un_derefer.rs
+++ b/compiler/rustc_mir_dataflow/src/un_derefer.rs
@@ -3,13 +3,13 @@ use rustc_middle::mir::*;
 
 /// Used for reverting changes made by `DerefSeparator`
 #[derive(Default, Debug)]
-pub struct UnDerefer<'tcx> {
+pub(crate) struct UnDerefer<'tcx> {
     deref_chains: FxHashMap<Local, Vec<PlaceRef<'tcx>>>,
 }
 
 impl<'tcx> UnDerefer<'tcx> {
     #[inline]
-    pub fn insert(&mut self, local: Local, reffed: PlaceRef<'tcx>) {
+    pub(crate) fn insert(&mut self, local: Local, reffed: PlaceRef<'tcx>) {
         let mut chain = self.deref_chains.remove(&reffed.local).unwrap_or_default();
         chain.push(reffed);
         self.deref_chains.insert(local, chain);
@@ -17,7 +17,7 @@ impl<'tcx> UnDerefer<'tcx> {
 
     /// Returns the chain of places behind `DerefTemp` locals
     #[inline]
-    pub fn deref_chain(&self, local: Local) -> &[PlaceRef<'tcx>] {
+    pub(crate) fn deref_chain(&self, local: Local) -> &[PlaceRef<'tcx>] {
         self.deref_chains.get(&local).map(Vec::as_slice).unwrap_or_default()
     }
 
@@ -25,7 +25,7 @@ impl<'tcx> UnDerefer<'tcx> {
     ///
     /// See [`PlaceRef::iter_projections`]
     #[inline]
-    pub fn iter_projections(
+    pub(crate) fn iter_projections(
         &self,
         place: PlaceRef<'tcx>,
     ) -> impl Iterator<Item = (PlaceRef<'tcx>, PlaceElem<'tcx>)> + '_ {
diff --git a/compiler/rustc_mir_dataflow/src/value_analysis.rs b/compiler/rustc_mir_dataflow/src/value_analysis.rs
index 025d2ddfd..2802f5491 100644
--- a/compiler/rustc_mir_dataflow/src/value_analysis.rs
+++ b/compiler/rustc_mir_dataflow/src/value_analysis.rs
@@ -332,8 +332,6 @@ pub struct ValueAnalysisWrapper<T>(pub T);
 impl<'tcx, T: ValueAnalysis<'tcx>> AnalysisDomain<'tcx> for ValueAnalysisWrapper<T> {
     type Domain = State<T::Value>;
 
-    type Direction = crate::Forward;
-
     const NAME: &'static str = T::NAME;
 
     fn bottom_value(&self, _body: &Body<'tcx>) -> Self::Domain {
@@ -476,26 +474,10 @@ impl<V: Clone> State<V> {
         }
     }
 
-    pub fn is_reachable(&self) -> bool {
+    fn is_reachable(&self) -> bool {
         matches!(&self.0, StateData::Reachable(_))
     }
 
-    pub fn mark_unreachable(&mut self) {
-        self.0 = StateData::Unreachable;
-    }
-
-    pub fn flood_all(&mut self)
-    where
-        V: HasTop,
-    {
-        self.flood_all_with(V::TOP)
-    }
-
-    pub fn flood_all_with(&mut self, value: V) {
-        let StateData::Reachable(values) = &mut self.0 else { return };
-        values.raw.fill(value);
-    }
-
     /// Assign `value` to all places that are contained in `place` or may alias one.
     pub fn flood_with(&mut self, place: PlaceRef<'_>, map: &Map, value: V) {
         self.flood_with_tail_elem(place, None, map, value)
@@ -510,7 +492,7 @@ impl<V: Clone> State<V> {
     }
 
     /// Assign `value` to the discriminant of `place` and all places that may alias it.
-    pub fn flood_discr_with(&mut self, place: PlaceRef<'_>, map: &Map, value: V) {
+    fn flood_discr_with(&mut self, place: PlaceRef<'_>, map: &Map, value: V) {
         self.flood_with_tail_elem(place, Some(TrackElem::Discriminant), map, value)
     }
 
@@ -546,7 +528,7 @@ impl<V: Clone> State<V> {
     /// This does nothing if the place is not tracked.
     ///
     /// The target place must have been flooded before calling this method.
-    pub fn insert_idx(&mut self, target: PlaceIndex, result: ValueOrPlace<V>, map: &Map) {
+    fn insert_idx(&mut self, target: PlaceIndex, result: ValueOrPlace<V>, map: &Map) {
         match result {
             ValueOrPlace::Value(value) => self.insert_value_idx(target, value, map),
             ValueOrPlace::Place(source) => self.insert_place_idx(target, source, map),
@@ -841,7 +823,7 @@ impl Map {
     ) {
         // Allocate a value slot if it doesn't have one, and the user requested one.
         assert!(self.places[place].value_index.is_none());
-        if tcx.layout_of(param_env.and(ty)).map_or(false, |layout| layout.abi.is_scalar()) {
+        if tcx.layout_of(param_env.and(ty)).is_ok_and(|layout| layout.abi.is_scalar()) {
             self.places[place].value_index = Some(self.value_count.into());
             self.value_count += 1;
         }
@@ -910,18 +892,13 @@ impl Map {
         self.inner_values[root] = start..end;
     }
 
-    /// Returns the number of tracked places, i.e., those for which a value can be stored.
-    pub fn tracked_places(&self) -> usize {
-        self.value_count
-    }
-
     /// Applies a single projection element, yielding the corresponding child.
     pub fn apply(&self, place: PlaceIndex, elem: TrackElem) -> Option<PlaceIndex> {
         self.projections.get(&(place, elem)).copied()
     }
 
     /// Locates the given place, if it exists in the tree.
-    pub fn find_extra(
+    fn find_extra(
         &self,
         place: PlaceRef<'_>,
         extra: impl IntoIterator<Item = TrackElem>,
@@ -954,7 +931,7 @@ impl Map {
     }
 
     /// Iterate over all direct children.
-    pub fn children(&self, parent: PlaceIndex) -> impl Iterator<Item = PlaceIndex> + '_ {
+    fn children(&self, parent: PlaceIndex) -> impl Iterator<Item = PlaceIndex> + '_ {
         Children::new(self, parent)
     }
 
@@ -979,11 +956,7 @@ impl Map {
             // The local is not tracked at all, so it does not alias anything.
             return;
         };
-        let elems = place
-            .projection
-            .iter()
-            .map(|&elem| elem.try_into())
-            .chain(tail_elem.map(Ok).into_iter());
+        let elems = place.projection.iter().map(|&elem| elem.try_into()).chain(tail_elem.map(Ok));
         for elem in elems {
             // A field aliases the parent place.
             if let Some(vi) = self.places[index].value_index {
diff --git a/compiler/rustc_mir_transform/Cargo.toml b/compiler/rustc_mir_transform/Cargo.toml
index 0448e9d27..9cc083edb 100644
--- a/compiler/rustc_mir_transform/Cargo.toml
+++ b/compiler/rustc_mir_transform/Cargo.toml
@@ -6,7 +6,7 @@ edition = "2021"
 [dependencies]
 # tidy-alphabetical-start
 either = "1"
-itertools = "0.10.1"
+itertools = "0.11"
 rustc_arena = { path = "../rustc_arena" }
 rustc_ast = { path = "../rustc_ast" }
 rustc_attr = { path = "../rustc_attr" }
@@ -20,7 +20,6 @@ rustc_macros = { path = "../rustc_macros" }
 rustc_middle = { path = "../rustc_middle" }
 rustc_mir_build = { path = "../rustc_mir_build" }
 rustc_mir_dataflow = { path = "../rustc_mir_dataflow" }
-rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
@@ -28,8 +27,3 @@ rustc_trait_selection = { path = "../rustc_trait_selection" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
 tracing = "0.1"
 # tidy-alphabetical-end
-
-[dev-dependencies]
-# tidy-alphabetical-start
-coverage_test_macros = { path = "src/coverage/test_macros" }
-# tidy-alphabetical-end
diff --git a/compiler/rustc_mir_transform/messages.ftl b/compiler/rustc_mir_transform/messages.ftl
index 5a99afc45..b8dbdf18d 100644
--- a/compiler/rustc_mir_transform/messages.ftl
+++ b/compiler/rustc_mir_transform/messages.ftl
@@ -42,8 +42,19 @@ mir_transform_requires_unsafe = {$details} is unsafe and requires unsafe {$op_in
     }
     .not_inherited = items do not inherit unsafety from separate enclosing items
 
+mir_transform_target_feature_call_help = in order for the call to be safe, the context requires the following additional target {$missing_target_features_count ->
+    [1] feature
+    *[count] features
+    }: {$missing_target_features}
+
 mir_transform_target_feature_call_label = call to function with `#[target_feature]`
-mir_transform_target_feature_call_note = can only be called if the required target features are available
+mir_transform_target_feature_call_note = the {$build_target_features} target {$build_target_features_count ->
+    [1] feature
+    *[count] features
+    } being enabled in the build configuration does not remove the requirement to list {$build_target_features_count ->
+    [1] it
+    *[count] them
+    } in `#[target_feature]`
 
 mir_transform_unaligned_packed_ref = reference to packed field is unaligned
     .note = packed structs are only aligned by one byte, and many modern architectures penalize unaligned field accesses
diff --git a/compiler/rustc_mir_transform/src/abort_unwinding_calls.rs b/compiler/rustc_mir_transform/src/abort_unwinding_calls.rs
index 2b3d423ea..dfc7a9891 100644
--- a/compiler/rustc_mir_transform/src/abort_unwinding_calls.rs
+++ b/compiler/rustc_mir_transform/src/abort_unwinding_calls.rs
@@ -1,4 +1,3 @@
-use crate::MirPass;
 use rustc_ast::InlineAsmOptions;
 use rustc_middle::mir::*;
 use rustc_middle::ty::layout;
diff --git a/compiler/rustc_mir_transform/src/add_call_guards.rs b/compiler/rustc_mir_transform/src/add_call_guards.rs
index b814fbf32..a47c8d94b 100644
--- a/compiler/rustc_mir_transform/src/add_call_guards.rs
+++ b/compiler/rustc_mir_transform/src/add_call_guards.rs
@@ -1,4 +1,3 @@
-use crate::MirPass;
 use rustc_index::{Idx, IndexVec};
 use rustc_middle::mir::*;
 use rustc_middle::ty::TyCtxt;
diff --git a/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs b/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs
index ef2a0c790..de6d20ae3 100644
--- a/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs
+++ b/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs
@@ -2,7 +2,6 @@ use rustc_middle::mir::*;
 use rustc_middle::ty::TyCtxt;
 
 use crate::util;
-use crate::MirPass;
 use rustc_middle::mir::patch::MirPatch;
 
 /// This pass moves values being dropped that are within a packed
diff --git a/compiler/rustc_mir_transform/src/add_retag.rs b/compiler/rustc_mir_transform/src/add_retag.rs
index 75473ca53..94077c630 100644
--- a/compiler/rustc_mir_transform/src/add_retag.rs
+++ b/compiler/rustc_mir_transform/src/add_retag.rs
@@ -4,7 +4,6 @@
 //! of MIR building, and only after this pass we think of the program has having the
 //! normal MIR semantics.
 
-use crate::MirPass;
 use rustc_middle::mir::*;
 use rustc_middle::ty::{self, Ty, TyCtxt};
 
diff --git a/compiler/rustc_mir_transform/src/add_subtyping_projections.rs b/compiler/rustc_mir_transform/src/add_subtyping_projections.rs
index e5be7c0ca..04204c68f 100644
--- a/compiler/rustc_mir_transform/src/add_subtyping_projections.rs
+++ b/compiler/rustc_mir_transform/src/add_subtyping_projections.rs
@@ -1,4 +1,3 @@
-use crate::MirPass;
 use rustc_index::IndexVec;
 use rustc_middle::mir::patch::MirPatch;
 use rustc_middle::mir::visit::MutVisitor;
diff --git a/compiler/rustc_mir_transform/src/check_alignment.rs b/compiler/rustc_mir_transform/src/check_alignment.rs
index 42b2f1886..9eec724ef 100644
--- a/compiler/rustc_mir_transform/src/check_alignment.rs
+++ b/compiler/rustc_mir_transform/src/check_alignment.rs
@@ -1,4 +1,3 @@
-use crate::MirPass;
 use rustc_hir::lang_items::LangItem;
 use rustc_index::IndexVec;
 use rustc_middle::mir::*;
diff --git a/compiler/rustc_mir_transform/src/check_const_item_mutation.rs b/compiler/rustc_mir_transform/src/check_const_item_mutation.rs
index 61bf530f1..3195cd362 100644
--- a/compiler/rustc_mir_transform/src/check_const_item_mutation.rs
+++ b/compiler/rustc_mir_transform/src/check_const_item_mutation.rs
@@ -13,7 +13,7 @@ pub struct CheckConstItemMutation;
 impl<'tcx> MirLint<'tcx> for CheckConstItemMutation {
     fn run_lint(&self, tcx: TyCtxt<'tcx>, body: &Body<'tcx>) {
         let mut checker = ConstMutationChecker { body, tcx, target_local: None };
-        checker.visit_body(&body);
+        checker.visit_body(body);
     }
 }
 
@@ -98,7 +98,7 @@ impl<'tcx> Visitor<'tcx> for ConstMutationChecker<'_, 'tcx> {
             if !lhs.projection.is_empty() {
                 if let Some(def_id) = self.is_const_item_without_destructor(lhs.local)
                     && let Some((lint_root, span, item)) =
-                        self.should_lint_const_item_usage(&lhs, def_id, loc)
+                        self.should_lint_const_item_usage(lhs, def_id, loc)
                 {
                     self.tcx.emit_spanned_lint(
                         CONST_ITEM_MUTATION,
@@ -132,12 +132,7 @@ impl<'tcx> Visitor<'tcx> for ConstMutationChecker<'_, 'tcx> {
                 // the `self` parameter of a method call (as the terminator of our current
                 // BasicBlock). If so, we emit a more specific lint.
                 let method_did = self.target_local.and_then(|target_local| {
-                    rustc_middle::util::find_self_call(
-                        self.tcx,
-                        &self.body,
-                        target_local,
-                        loc.block,
-                    )
+                    rustc_middle::util::find_self_call(self.tcx, self.body, target_local, loc.block)
                 });
                 let lint_loc =
                     if method_did.is_some() { self.body.terminator_loc(loc.block) } else { loc };
diff --git a/compiler/rustc_mir_transform/src/check_packed_ref.rs b/compiler/rustc_mir_transform/src/check_packed_ref.rs
index 9ee0a7040..77bcba50a 100644
--- a/compiler/rustc_mir_transform/src/check_packed_ref.rs
+++ b/compiler/rustc_mir_transform/src/check_packed_ref.rs
@@ -12,7 +12,7 @@ impl<'tcx> MirLint<'tcx> for CheckPackedRef {
         let param_env = tcx.param_env(body.source.def_id());
         let source_info = SourceInfo::outermost(body.span);
         let mut checker = PackedRefChecker { body, tcx, param_env, source_info };
-        checker.visit_body(&body);
+        checker.visit_body(body);
     }
 }
 
diff --git a/compiler/rustc_mir_transform/src/check_unsafety.rs b/compiler/rustc_mir_transform/src/check_unsafety.rs
index 8872f9a97..f246de55c 100644
--- a/compiler/rustc_mir_transform/src/check_unsafety.rs
+++ b/compiler/rustc_mir_transform/src/check_unsafety.rs
@@ -242,7 +242,7 @@ impl<'tcx> Visitor<'tcx> for UnsafetyChecker<'_, 'tcx> {
                     let assigned_ty = place.ty(&self.body.local_decls, self.tcx).ty;
                     if assigned_ty.needs_drop(self.tcx, self.param_env) {
                         // This would be unsafe, but should be outright impossible since we reject such unions.
-                        self.tcx.sess.delay_span_bug(
+                        self.tcx.sess.span_delayed_bug(
                             self.source_info.span,
                             format!("union fields that need dropping should be impossible: {assigned_ty}")
                         );
@@ -287,19 +287,20 @@ impl<'tcx> UnsafetyChecker<'_, 'tcx> {
             .safety;
         match safety {
             // `unsafe` blocks are required in safe code
-            Safety::Safe => violations.into_iter().for_each(|&violation| {
+            Safety::Safe => violations.into_iter().for_each(|violation| {
                 match violation.kind {
                     UnsafetyViolationKind::General => {}
                     UnsafetyViolationKind::UnsafeFn => {
                         bug!("`UnsafetyViolationKind::UnsafeFn` in an `Safe` context")
                     }
                 }
-                if !self.violations.contains(&violation) {
-                    self.violations.push(violation)
+                if !self.violations.contains(violation) {
+                    self.violations.push(violation.clone())
                 }
             }),
             // With the RFC 2585, no longer allow `unsafe` operations in `unsafe fn`s
-            Safety::FnUnsafe => violations.into_iter().for_each(|&(mut violation)| {
+            Safety::FnUnsafe => violations.into_iter().for_each(|violation| {
+                let mut violation = violation.clone();
                 violation.kind = UnsafetyViolationKind::UnsafeFn;
                 if !self.violations.contains(&violation) {
                     self.violations.push(violation)
@@ -367,9 +368,22 @@ impl<'tcx> UnsafetyChecker<'_, 'tcx> {
 
         // Is `callee_features` a subset of `calling_features`?
         if !callee_features.iter().all(|feature| self_features.contains(feature)) {
+            let missing: Vec<_> = callee_features
+                .iter()
+                .copied()
+                .filter(|feature| !self_features.contains(feature))
+                .collect();
+            let build_enabled = self
+                .tcx
+                .sess
+                .target_features
+                .iter()
+                .copied()
+                .filter(|feature| missing.contains(feature))
+                .collect();
             self.require_unsafe(
                 UnsafetyViolationKind::General,
-                UnsafetyViolationDetails::CallToFunctionWith,
+                UnsafetyViolationDetails::CallToFunctionWith { missing, build_enabled },
             )
         }
     }
@@ -385,7 +399,7 @@ pub(crate) fn provide(providers: &mut Providers) {
 enum Context {
     Safe,
     /// in an `unsafe fn`
-    UnsafeFn(HirId),
+    UnsafeFn,
     /// in a *used* `unsafe` block
     /// (i.e. a block without unused-unsafe warning)
     UnsafeBlock(HirId),
@@ -407,7 +421,7 @@ impl<'tcx> intravisit::Visitor<'tcx> for UnusedUnsafeVisitor<'_, 'tcx> {
             };
             let unused_unsafe = match (self.context, used) {
                 (_, false) => UnusedUnsafe::Unused,
-                (Context::Safe, true) | (Context::UnsafeFn(_), true) => {
+                (Context::Safe, true) | (Context::UnsafeFn, true) => {
                     let previous_context = self.context;
                     self.context = Context::UnsafeBlock(block.hir_id);
                     intravisit::walk_block(self, block);
@@ -452,9 +466,9 @@ fn check_unused_unsafe(
     };
 
     let body = tcx.hir().body(body_id);
-    let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
+    let hir_id = tcx.local_def_id_to_hir_id(def_id);
     let context = match tcx.hir().fn_sig_by_hir_id(hir_id) {
-        Some(sig) if sig.header.unsafety == hir::Unsafety::Unsafe => Context::UnsafeFn(hir_id),
+        Some(sig) if sig.header.unsafety == hir::Unsafety::Unsafe => Context::UnsafeFn,
         _ => Context::Safe,
     };
 
@@ -494,7 +508,7 @@ fn unsafety_check_result(tcx: TyCtxt<'_>, def: LocalDefId) -> &UnsafetyCheckResu
     let param_env = tcx.param_env(def);
 
     let mut checker = UnsafetyChecker::new(body, def, tcx, param_env);
-    checker.visit_body(&body);
+    checker.visit_body(body);
 
     let unused_unsafes = (!tcx.is_typeck_child(def.to_def_id()))
         .then(|| check_unused_unsafe(tcx, def, &checker.used_unsafe_blocks));
@@ -528,8 +542,9 @@ pub fn check_unsafety(tcx: TyCtxt<'_>, def_id: LocalDefId) {
     // Only suggest wrapping the entire function body in an unsafe block once
     let mut suggest_unsafe_block = true;
 
-    for &UnsafetyViolation { source_info, lint_root, kind, details } in violations.iter() {
-        let details = errors::RequiresUnsafeDetail { violation: details, span: source_info.span };
+    for &UnsafetyViolation { source_info, lint_root, kind, ref details } in violations.iter() {
+        let details =
+            errors::RequiresUnsafeDetail { violation: details.clone(), span: source_info.span };
 
         match kind {
             UnsafetyViolationKind::General => {
@@ -568,7 +583,7 @@ pub fn check_unsafety(tcx: TyCtxt<'_>, def_id: LocalDefId) {
                     errors::UnsafeOpInUnsafeFn {
                         details,
                         suggest_unsafe_block: suggest_unsafe_block.then(|| {
-                            let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
+                            let hir_id = tcx.local_def_id_to_hir_id(def_id);
                             let fn_sig = tcx
                                 .hir()
                                 .fn_sig_by_hir_id(hir_id)
diff --git a/compiler/rustc_mir_transform/src/const_goto.rs b/compiler/rustc_mir_transform/src/const_goto.rs
index fd2d37dbe..388434607 100644
--- a/compiler/rustc_mir_transform/src/const_goto.rs
+++ b/compiler/rustc_mir_transform/src/const_goto.rs
@@ -17,7 +17,6 @@
 //! }
 //! ```
 
-use crate::MirPass;
 use rustc_middle::mir::*;
 use rustc_middle::ty::TyCtxt;
 use rustc_middle::{mir::visit::Visitor, ty::ParamEnv};
diff --git a/compiler/rustc_mir_transform/src/const_prop.rs b/compiler/rustc_mir_transform/src/const_prop.rs
index f7f882310..e66d5e0a9 100644
--- a/compiler/rustc_mir_transform/src/const_prop.rs
+++ b/compiler/rustc_mir_transform/src/const_prop.rs
@@ -11,6 +11,7 @@ use rustc_middle::mir::visit::{
     MutVisitor, MutatingUseContext, NonMutatingUseContext, PlaceContext, Visitor,
 };
 use rustc_middle::mir::*;
+use rustc_middle::query::TyCtxtAt;
 use rustc_middle::ty::layout::{LayoutError, LayoutOf, LayoutOfHelpers, TyAndLayout};
 use rustc_middle::ty::{self, GenericArgs, Instance, ParamEnv, Ty, TyCtxt, TypeVisitableExt};
 use rustc_span::{def_id::DefId, Span};
@@ -18,7 +19,6 @@ use rustc_target::abi::{self, HasDataLayout, Size, TargetDataLayout};
 use rustc_target::spec::abi::Abi as CallAbi;
 
 use crate::dataflow_const_prop::Patch;
-use crate::MirPass;
 use rustc_const_eval::interpret::{
     self, compile_time_machine, AllocId, ConstAllocation, FnArg, Frame, ImmTy, Immediate, InterpCx,
     InterpResult, MemoryKind, OpTy, PlaceTy, Pointer, Scalar, StackPopCleanup,
@@ -84,8 +84,7 @@ impl<'tcx> MirPass<'tcx> for ConstProp {
 
         // FIXME(welseywiser) const prop doesn't work on coroutines because of query cycles
         // computing their layout.
-        let is_coroutine = def_kind == DefKind::Coroutine;
-        if is_coroutine {
+        if tcx.is_coroutine(def_id.to_def_id()) {
             trace!("ConstProp skipped for coroutine {:?}", def_id);
             return;
         }
@@ -221,7 +220,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx>
     }
 
     fn before_access_global(
-        _tcx: TyCtxt<'tcx>,
+        _tcx: TyCtxtAt<'tcx>,
         _machine: &Self,
         _alloc_id: AllocId,
         alloc: ConstAllocation<'tcx>,
@@ -241,10 +240,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx>
     }
 
     #[inline(always)]
-    fn expose_ptr(
-        _ecx: &mut InterpCx<'mir, 'tcx, Self>,
-        _ptr: Pointer<AllocId>,
-    ) -> InterpResult<'tcx> {
+    fn expose_ptr(_ecx: &mut InterpCx<'mir, 'tcx, Self>, _ptr: Pointer) -> InterpResult<'tcx> {
         throw_machine_stop_str!("exposing pointers isn't supported in ConstProp")
     }
 
@@ -607,7 +603,7 @@ impl CanConstProp {
         for arg in body.args_iter() {
             cpv.found_assignment.insert(arg);
         }
-        cpv.visit_body(&body);
+        cpv.visit_body(body);
         cpv.can_const_prop
     }
 }
diff --git a/compiler/rustc_mir_transform/src/const_prop_lint.rs b/compiler/rustc_mir_transform/src/const_prop_lint.rs
index a23ba9c4a..99eecb567 100644
--- a/compiler/rustc_mir_transform/src/const_prop_lint.rs
+++ b/compiler/rustc_mir_transform/src/const_prop_lint.rs
@@ -61,7 +61,7 @@ impl<'tcx> MirLint<'tcx> for ConstPropLint {
 
         // FIXME(welseywiser) const prop doesn't work on coroutines because of query cycles
         // computing their layout.
-        if let DefKind::Coroutine = def_kind {
+        if tcx.is_coroutine(def_id.to_def_id()) {
             trace!("ConstPropLint skipped for coroutine {:?}", def_id);
             return;
         }
@@ -453,7 +453,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
         cond: &Operand<'tcx>,
         location: Location,
     ) -> Option<!> {
-        let value = &self.eval_operand(&cond, location)?;
+        let value = &self.eval_operand(cond, location)?;
         trace!("assertion on {:?} should be {:?}", value, expected);
 
         let expected = Scalar::from_bool(expected);
@@ -626,7 +626,7 @@ impl<'tcx> Visitor<'tcx> for ConstPropagator<'_, 'tcx> {
                 self.check_assertion(*expected, msg, cond, location);
             }
             TerminatorKind::SwitchInt { ref discr, ref targets } => {
-                if let Some(ref value) = self.eval_operand(&discr, location)
+                if let Some(ref value) = self.eval_operand(discr, location)
                     && let Some(value_const) =
                         self.use_ecx(location, |this| this.ecx.read_scalar(value))
                     && let Ok(constant) = value_const.try_to_int()
diff --git a/compiler/rustc_mir_transform/src/copy_prop.rs b/compiler/rustc_mir_transform/src/copy_prop.rs
index f5db7ce97..0119b95cc 100644
--- a/compiler/rustc_mir_transform/src/copy_prop.rs
+++ b/compiler/rustc_mir_transform/src/copy_prop.rs
@@ -6,7 +6,6 @@ use rustc_middle::ty::TyCtxt;
 use rustc_mir_dataflow::impls::borrowed_locals;
 
 use crate::ssa::SsaLocals;
-use crate::MirPass;
 
 /// Unify locals that copy each other.
 ///
@@ -50,7 +49,7 @@ fn propagate_ssa<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
 
     Replacer {
         tcx,
-        copy_classes: &ssa.copy_classes(),
+        copy_classes: ssa.copy_classes(),
         fully_moved,
         borrowed_locals,
         storage_to_remove,
@@ -124,7 +123,7 @@ impl<'tcx> MutVisitor<'tcx> for Replacer<'_, 'tcx> {
     }
 
     fn visit_place(&mut self, place: &mut Place<'tcx>, ctxt: PlaceContext, loc: Location) {
-        if let Some(new_projection) = self.process_projection(&place.projection, loc) {
+        if let Some(new_projection) = self.process_projection(place.projection, loc) {
             place.projection = self.tcx().mk_place_elems(&new_projection);
         }
 
diff --git a/compiler/rustc_mir_transform/src/coroutine.rs b/compiler/rustc_mir_transform/src/coroutine.rs
index abaed103f..d7dd44af7 100644
--- a/compiler/rustc_mir_transform/src/coroutine.rs
+++ b/compiler/rustc_mir_transform/src/coroutine.rs
@@ -55,7 +55,6 @@ use crate::deref_separator::deref_finder;
 use crate::errors;
 use crate::pass_manager as pm;
 use crate::simplify;
-use crate::MirPass;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_errors::pluralize;
 use rustc_hir as hir;
@@ -63,17 +62,16 @@ use rustc_hir::lang_items::LangItem;
 use rustc_hir::CoroutineKind;
 use rustc_index::bit_set::{BitMatrix, BitSet, GrowableBitSet};
 use rustc_index::{Idx, IndexVec};
-use rustc_middle::mir::dump_mir;
 use rustc_middle::mir::visit::{MutVisitor, PlaceContext, Visitor};
 use rustc_middle::mir::*;
+use rustc_middle::ty::CoroutineArgs;
 use rustc_middle::ty::InstanceDef;
-use rustc_middle::ty::{self, AdtDef, Ty, TyCtxt};
-use rustc_middle::ty::{CoroutineArgs, GenericArgsRef};
+use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_mir_dataflow::impls::{
     MaybeBorrowedLocals, MaybeLiveLocals, MaybeRequiresStorage, MaybeStorageLive,
 };
 use rustc_mir_dataflow::storage::always_storage_live_locals;
-use rustc_mir_dataflow::{self, Analysis};
+use rustc_mir_dataflow::Analysis;
 use rustc_span::def_id::{DefId, LocalDefId};
 use rustc_span::symbol::sym;
 use rustc_span::Span;
@@ -225,8 +223,6 @@ struct SuspensionPoint<'tcx> {
 struct TransformVisitor<'tcx> {
     tcx: TyCtxt<'tcx>,
     coroutine_kind: hir::CoroutineKind,
-    state_adt_ref: AdtDef<'tcx>,
-    state_args: GenericArgsRef<'tcx>,
 
     // The type of the discriminant in the coroutine struct
     discr_ty: Ty<'tcx>,
@@ -245,22 +241,56 @@ struct TransformVisitor<'tcx> {
     always_live_locals: BitSet<Local>,
 
     // The original RETURN_PLACE local
-    new_ret_local: Local,
+    old_ret_local: Local,
+
+    old_yield_ty: Ty<'tcx>,
+
+    old_ret_ty: Ty<'tcx>,
 }
 
 impl<'tcx> TransformVisitor<'tcx> {
     fn insert_none_ret_block(&self, body: &mut Body<'tcx>) -> BasicBlock {
         let block = BasicBlock::new(body.basic_blocks.len());
-
         let source_info = SourceInfo::outermost(body.span);
 
-        let (kind, idx) = self.coroutine_state_adt_and_variant_idx(true);
-        assert_eq!(self.state_adt_ref.variant(idx).fields.len(), 0);
+        let none_value = match self.coroutine_kind {
+            CoroutineKind::Async(_) => span_bug!(body.span, "`Future`s are not fused inherently"),
+            CoroutineKind::Coroutine => span_bug!(body.span, "`Coroutine`s cannot be fused"),
+            // `gen` continues return `None`
+            CoroutineKind::Gen(_) => {
+                let option_def_id = self.tcx.require_lang_item(LangItem::Option, None);
+                Rvalue::Aggregate(
+                    Box::new(AggregateKind::Adt(
+                        option_def_id,
+                        VariantIdx::from_usize(0),
+                        self.tcx.mk_args(&[self.old_yield_ty.into()]),
+                        None,
+                        None,
+                    )),
+                    IndexVec::new(),
+                )
+            }
+            // `async gen` continues to return `Poll::Ready(None)`
+            CoroutineKind::AsyncGen(_) => {
+                let ty::Adt(_poll_adt, args) = *self.old_yield_ty.kind() else { bug!() };
+                let ty::Adt(_option_adt, args) = *args.type_at(0).kind() else { bug!() };
+                let yield_ty = args.type_at(0);
+                Rvalue::Use(Operand::Constant(Box::new(ConstOperand {
+                    span: source_info.span,
+                    const_: Const::Unevaluated(
+                        UnevaluatedConst::new(
+                            self.tcx.require_lang_item(LangItem::AsyncGenFinished, None),
+                            self.tcx.mk_args(&[yield_ty.into()]),
+                        ),
+                        self.old_yield_ty,
+                    ),
+                    user_ty: None,
+                })))
+            }
+        };
+
         let statements = vec![Statement {
-            kind: StatementKind::Assign(Box::new((
-                Place::return_place(),
-                Rvalue::Aggregate(Box::new(kind), IndexVec::new()),
-            ))),
+            kind: StatementKind::Assign(Box::new((Place::return_place(), none_value))),
             source_info,
         }];
 
@@ -273,23 +303,6 @@ impl<'tcx> TransformVisitor<'tcx> {
         block
     }
 
-    fn coroutine_state_adt_and_variant_idx(
-        &self,
-        is_return: bool,
-    ) -> (AggregateKind<'tcx>, VariantIdx) {
-        let idx = VariantIdx::new(match (is_return, self.coroutine_kind) {
-            (true, hir::CoroutineKind::Coroutine) => 1, // CoroutineState::Complete
-            (false, hir::CoroutineKind::Coroutine) => 0, // CoroutineState::Yielded
-            (true, hir::CoroutineKind::Async(_)) => 0,  // Poll::Ready
-            (false, hir::CoroutineKind::Async(_)) => 1, // Poll::Pending
-            (true, hir::CoroutineKind::Gen(_)) => 0,    // Option::None
-            (false, hir::CoroutineKind::Gen(_)) => 1,   // Option::Some
-        });
-
-        let kind = AggregateKind::Adt(self.state_adt_ref.did(), idx, self.state_args, None, None);
-        (kind, idx)
-    }
-
     // Make a `CoroutineState` or `Poll` variant assignment.
     //
     // `core::ops::CoroutineState` only has single element tuple variants,
@@ -302,51 +315,119 @@ impl<'tcx> TransformVisitor<'tcx> {
         is_return: bool,
         statements: &mut Vec<Statement<'tcx>>,
     ) {
-        let (kind, idx) = self.coroutine_state_adt_and_variant_idx(is_return);
-
-        match self.coroutine_kind {
-            // `Poll::Pending`
+        let rvalue = match self.coroutine_kind {
             CoroutineKind::Async(_) => {
-                if !is_return {
-                    assert_eq!(self.state_adt_ref.variant(idx).fields.len(), 0);
-
-                    // FIXME(swatinem): assert that `val` is indeed unit?
-                    statements.push(Statement {
-                        kind: StatementKind::Assign(Box::new((
-                            Place::return_place(),
-                            Rvalue::Aggregate(Box::new(kind), IndexVec::new()),
-                        ))),
-                        source_info,
-                    });
-                    return;
+                let poll_def_id = self.tcx.require_lang_item(LangItem::Poll, None);
+                let args = self.tcx.mk_args(&[self.old_ret_ty.into()]);
+                if is_return {
+                    // Poll::Ready(val)
+                    Rvalue::Aggregate(
+                        Box::new(AggregateKind::Adt(
+                            poll_def_id,
+                            VariantIdx::from_usize(0),
+                            args,
+                            None,
+                            None,
+                        )),
+                        IndexVec::from_raw(vec![val]),
+                    )
+                } else {
+                    // Poll::Pending
+                    Rvalue::Aggregate(
+                        Box::new(AggregateKind::Adt(
+                            poll_def_id,
+                            VariantIdx::from_usize(1),
+                            args,
+                            None,
+                            None,
+                        )),
+                        IndexVec::new(),
+                    )
                 }
             }
-            // `Option::None`
             CoroutineKind::Gen(_) => {
+                let option_def_id = self.tcx.require_lang_item(LangItem::Option, None);
+                let args = self.tcx.mk_args(&[self.old_yield_ty.into()]);
                 if is_return {
-                    assert_eq!(self.state_adt_ref.variant(idx).fields.len(), 0);
-
-                    statements.push(Statement {
-                        kind: StatementKind::Assign(Box::new((
-                            Place::return_place(),
-                            Rvalue::Aggregate(Box::new(kind), IndexVec::new()),
-                        ))),
-                        source_info,
-                    });
-                    return;
+                    // None
+                    Rvalue::Aggregate(
+                        Box::new(AggregateKind::Adt(
+                            option_def_id,
+                            VariantIdx::from_usize(0),
+                            args,
+                            None,
+                            None,
+                        )),
+                        IndexVec::new(),
+                    )
+                } else {
+                    // Some(val)
+                    Rvalue::Aggregate(
+                        Box::new(AggregateKind::Adt(
+                            option_def_id,
+                            VariantIdx::from_usize(1),
+                            args,
+                            None,
+                            None,
+                        )),
+                        IndexVec::from_raw(vec![val]),
+                    )
                 }
             }
-            CoroutineKind::Coroutine => {}
-        }
-
-        // else: `Poll::Ready(x)`, `CoroutineState::Yielded(x)`, `CoroutineState::Complete(x)`, or `Option::Some(x)`
-        assert_eq!(self.state_adt_ref.variant(idx).fields.len(), 1);
+            CoroutineKind::AsyncGen(_) => {
+                if is_return {
+                    let ty::Adt(_poll_adt, args) = *self.old_yield_ty.kind() else { bug!() };
+                    let ty::Adt(_option_adt, args) = *args.type_at(0).kind() else { bug!() };
+                    let yield_ty = args.type_at(0);
+                    Rvalue::Use(Operand::Constant(Box::new(ConstOperand {
+                        span: source_info.span,
+                        const_: Const::Unevaluated(
+                            UnevaluatedConst::new(
+                                self.tcx.require_lang_item(LangItem::AsyncGenFinished, None),
+                                self.tcx.mk_args(&[yield_ty.into()]),
+                            ),
+                            self.old_yield_ty,
+                        ),
+                        user_ty: None,
+                    })))
+                } else {
+                    Rvalue::Use(val)
+                }
+            }
+            CoroutineKind::Coroutine => {
+                let coroutine_state_def_id =
+                    self.tcx.require_lang_item(LangItem::CoroutineState, None);
+                let args = self.tcx.mk_args(&[self.old_yield_ty.into(), self.old_ret_ty.into()]);
+                if is_return {
+                    // CoroutineState::Complete(val)
+                    Rvalue::Aggregate(
+                        Box::new(AggregateKind::Adt(
+                            coroutine_state_def_id,
+                            VariantIdx::from_usize(1),
+                            args,
+                            None,
+                            None,
+                        )),
+                        IndexVec::from_raw(vec![val]),
+                    )
+                } else {
+                    // CoroutineState::Yielded(val)
+                    Rvalue::Aggregate(
+                        Box::new(AggregateKind::Adt(
+                            coroutine_state_def_id,
+                            VariantIdx::from_usize(0),
+                            args,
+                            None,
+                            None,
+                        )),
+                        IndexVec::from_raw(vec![val]),
+                    )
+                }
+            }
+        };
 
         statements.push(Statement {
-            kind: StatementKind::Assign(Box::new((
-                Place::return_place(),
-                Rvalue::Aggregate(Box::new(kind), [val].into()),
-            ))),
+            kind: StatementKind::Assign(Box::new((Place::return_place(), rvalue))),
             source_info,
         });
     }
@@ -420,7 +501,7 @@ impl<'tcx> MutVisitor<'tcx> for TransformVisitor<'tcx> {
 
         let ret_val = match data.terminator().kind {
             TerminatorKind::Return => {
-                Some((true, None, Operand::Move(Place::from(self.new_ret_local)), None))
+                Some((true, None, Operand::Move(Place::from(self.old_ret_local)), None))
             }
             TerminatorKind::Yield { ref value, resume, resume_arg, drop } => {
                 Some((false, Some((resume, resume_arg)), value.clone(), drop))
@@ -446,12 +527,26 @@ impl<'tcx> MutVisitor<'tcx> for TransformVisitor<'tcx> {
                         resume_arg
                     };
 
+                let storage_liveness: GrowableBitSet<Local> =
+                    self.storage_liveness[block].clone().unwrap().into();
+
+                for i in 0..self.always_live_locals.domain_size() {
+                    let l = Local::new(i);
+                    let needs_storage_dead = storage_liveness.contains(l)
+                        && !self.remap.contains_key(&l)
+                        && !self.always_live_locals.contains(l);
+                    if needs_storage_dead {
+                        data.statements
+                            .push(Statement { source_info, kind: StatementKind::StorageDead(l) });
+                    }
+                }
+
                 self.suspension_points.push(SuspensionPoint {
                     state,
                     resume,
                     resume_arg,
                     drop,
-                    storage_liveness: self.storage_liveness[block].clone().unwrap().into(),
+                    storage_liveness,
                 });
 
                 VariantIdx::new(state)
@@ -617,6 +712,22 @@ fn replace_resume_ty_local<'tcx>(
     }
 }
 
+/// Transforms the `body` of the coroutine applying the following transform:
+///
+/// - Remove the `resume` argument.
+///
+/// Ideally the async lowering would not add the `resume` argument.
+///
+/// The async lowering step and the type / lifetime inference / checking are
+/// still using the `resume` argument for the time being. After this transform,
+/// the coroutine body doesn't have the `resume` argument.
+fn transform_gen_context<'tcx>(_tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+    // This leaves the local representing the `resume` argument in place,
+    // but turns it into a regular local variable. This is cheaper than
+    // adjusting all local references in the body after removing it.
+    body.arg_count = 1;
+}
+
 struct LivenessInfo {
     /// Which locals are live across any suspension point.
     saved_locals: CoroutineSavedLocals,
@@ -651,36 +762,34 @@ fn locals_live_across_suspend_points<'tcx>(
     always_live_locals: &BitSet<Local>,
     movable: bool,
 ) -> LivenessInfo {
-    let body_ref: &Body<'_> = &body;
-
     // Calculate when MIR locals have live storage. This gives us an upper bound of their
     // lifetimes.
     let mut storage_live = MaybeStorageLive::new(std::borrow::Cow::Borrowed(always_live_locals))
-        .into_engine(tcx, body_ref)
+        .into_engine(tcx, body)
         .iterate_to_fixpoint()
-        .into_results_cursor(body_ref);
+        .into_results_cursor(body);
 
     // Calculate the MIR locals which have been previously
     // borrowed (even if they are still active).
     let borrowed_locals_results =
-        MaybeBorrowedLocals.into_engine(tcx, body_ref).pass_name("coroutine").iterate_to_fixpoint();
+        MaybeBorrowedLocals.into_engine(tcx, body).pass_name("coroutine").iterate_to_fixpoint();
 
-    let mut borrowed_locals_cursor = borrowed_locals_results.cloned_results_cursor(body_ref);
+    let mut borrowed_locals_cursor = borrowed_locals_results.clone().into_results_cursor(body);
 
     // Calculate the MIR locals that we actually need to keep storage around
     // for.
-    let mut requires_storage_results =
-        MaybeRequiresStorage::new(borrowed_locals_results.cloned_results_cursor(body))
-            .into_engine(tcx, body_ref)
-            .iterate_to_fixpoint();
-    let mut requires_storage_cursor = requires_storage_results.as_results_cursor(body_ref);
+    let mut requires_storage_cursor =
+        MaybeRequiresStorage::new(borrowed_locals_results.into_results_cursor(body))
+            .into_engine(tcx, body)
+            .iterate_to_fixpoint()
+            .into_results_cursor(body);
 
     // Calculate the liveness of MIR locals ignoring borrows.
     let mut liveness = MaybeLiveLocals
-        .into_engine(tcx, body_ref)
+        .into_engine(tcx, body)
         .pass_name("coroutine")
         .iterate_to_fixpoint()
-        .into_results_cursor(body_ref);
+        .into_results_cursor(body);
 
     let mut storage_liveness_map = IndexVec::from_elem(None, &body.basic_blocks);
     let mut live_locals_at_suspension_points = Vec::new();
@@ -742,14 +851,14 @@ fn locals_live_across_suspend_points<'tcx>(
     // saving.
     let live_locals_at_suspension_points = live_locals_at_suspension_points
         .iter()
-        .map(|live_here| saved_locals.renumber_bitset(&live_here))
+        .map(|live_here| saved_locals.renumber_bitset(live_here))
         .collect();
 
     let storage_conflicts = compute_storage_conflicts(
-        body_ref,
+        body,
         &saved_locals,
         always_live_locals.clone(),
-        requires_storage_results,
+        requires_storage_cursor.into_results(),
     );
 
     LivenessInfo {
@@ -778,7 +887,7 @@ impl CoroutineSavedLocals {
     /// Transforms a `BitSet<Local>` that contains only locals saved across yield points to the
     /// equivalent `BitSet<CoroutineSavedLocal>`.
     fn renumber_bitset(&self, input: &BitSet<Local>) -> BitSet<CoroutineSavedLocal> {
-        assert!(self.superset(&input), "{:?} not a superset of {:?}", self.0, input);
+        assert!(self.superset(input), "{:?} not a superset of {:?}", self.0, input);
         let mut out = BitSet::new_empty(self.count());
         for (saved_local, local) in self.iter_enumerated() {
             if input.contains(local) {
@@ -814,7 +923,7 @@ fn compute_storage_conflicts<'mir, 'tcx>(
     body: &'mir Body<'tcx>,
     saved_locals: &CoroutineSavedLocals,
     always_live_locals: BitSet<Local>,
-    mut requires_storage: rustc_mir_dataflow::Results<'tcx, MaybeRequiresStorage<'_, 'mir, 'tcx>>,
+    mut requires_storage: rustc_mir_dataflow::Results<'tcx, MaybeRequiresStorage<'mir, 'tcx>>,
 ) -> BitMatrix<CoroutineSavedLocal, CoroutineSavedLocal> {
     assert_eq!(body.local_decls.len(), saved_locals.domain_size());
 
@@ -829,7 +938,7 @@ fn compute_storage_conflicts<'mir, 'tcx>(
     // Compute the storage conflicts for all eligible locals.
     let mut visitor = StorageConflictVisitor {
         body,
-        saved_locals: &saved_locals,
+        saved_locals: saved_locals,
         local_conflicts: BitMatrix::from_row_n(&ineligible_locals, body.local_decls.len()),
     };
 
@@ -1128,7 +1237,7 @@ fn create_coroutine_drop_shim<'tcx>(
     // The returned state and the poisoned state fall through to the default
     // case which is just to return
 
-    insert_switch(&mut body, cases, &transform, TerminatorKind::Return);
+    insert_switch(&mut body, cases, transform, TerminatorKind::Return);
 
     for block in body.basic_blocks_mut() {
         let kind = &mut block.terminator_mut().kind;
@@ -1320,7 +1429,9 @@ fn create_coroutine_resume_function<'tcx>(
             CoroutineKind::Async(_) | CoroutineKind::Coroutine => {
                 insert_panic_block(tcx, body, ResumedAfterReturn(coroutine_kind))
             }
-            CoroutineKind::Gen(_) => transform.insert_none_ret_block(body),
+            CoroutineKind::AsyncGen(_) | CoroutineKind::Gen(_) => {
+                transform.insert_none_ret_block(body)
+            }
         };
         cases.insert(1, (RETURNED, block));
     }
@@ -1328,7 +1439,15 @@ fn create_coroutine_resume_function<'tcx>(
     insert_switch(body, cases, &transform, TerminatorKind::Unreachable);
 
     make_coroutine_state_argument_indirect(tcx, body);
-    make_coroutine_state_argument_pinned(tcx, body);
+
+    match coroutine_kind {
+        // Iterator::next doesn't accept a pinned argument,
+        // unlike for all other coroutine kinds.
+        CoroutineKind::Gen(_) => {}
+        _ => {
+            make_coroutine_state_argument_pinned(tcx, body);
+        }
+    }
 
     // Make sure we remove dead blocks to remove
     // unrelated code from the drop part of the function
@@ -1391,13 +1510,6 @@ fn create_cases<'tcx>(
 
                 // Create StorageLive instructions for locals with live storage
                 for i in 0..(body.local_decls.len()) {
-                    if i == 2 {
-                        // The resume argument is live on function entry. Don't insert a
-                        // `StorageLive`, or the following `Assign` will read from uninitialized
-                        // memory.
-                        continue;
-                    }
-
                     let l = Local::new(i);
                     let needs_storage_live = point.storage_liveness.contains(l)
                         && !transform.remap.contains_key(&l)
@@ -1456,7 +1568,7 @@ pub(crate) fn mir_coroutine_witnesses<'tcx>(
 
     // The witness simply contains all locals live across suspend points.
 
-    let always_live_locals = always_storage_live_locals(&body);
+    let always_live_locals = always_storage_live_locals(body);
     let liveness_info = locals_live_across_suspend_points(tcx, body, &always_live_locals, movable);
 
     // Extract locals which are live across suspension point into `layout`
@@ -1464,17 +1576,18 @@ pub(crate) fn mir_coroutine_witnesses<'tcx>(
     // `storage_liveness` tells us which locals have live storage at suspension points
     let (_, coroutine_layout, _) = compute_layout(liveness_info, body);
 
-    check_suspend_tys(tcx, &coroutine_layout, &body);
+    check_suspend_tys(tcx, &coroutine_layout, body);
 
     Some(coroutine_layout)
 }
 
 impl<'tcx> MirPass<'tcx> for StateTransform {
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
-        let Some(yield_ty) = body.yield_ty() else {
+        let Some(old_yield_ty) = body.yield_ty() else {
             // This only applies to coroutines
             return;
         };
+        let old_ret_ty = body.return_ty();
 
         assert!(body.coroutine_drop().is_none());
 
@@ -1488,44 +1601,51 @@ impl<'tcx> MirPass<'tcx> for StateTransform {
                 (args.discr_ty(tcx), movability == hir::Movability::Movable)
             }
             _ => {
-                tcx.sess
-                    .delay_span_bug(body.span, format!("unexpected coroutine type {coroutine_ty}"));
+                tcx.sess.span_delayed_bug(
+                    body.span,
+                    format!("unexpected coroutine type {coroutine_ty}"),
+                );
                 return;
             }
         };
 
         let is_async_kind = matches!(body.coroutine_kind(), Some(CoroutineKind::Async(_)));
-        let (state_adt_ref, state_args) = match body.coroutine_kind().unwrap() {
+        let is_async_gen_kind = matches!(body.coroutine_kind(), Some(CoroutineKind::AsyncGen(_)));
+        let is_gen_kind = matches!(body.coroutine_kind(), Some(CoroutineKind::Gen(_)));
+        let new_ret_ty = match body.coroutine_kind().unwrap() {
             CoroutineKind::Async(_) => {
                 // Compute Poll<return_ty>
                 let poll_did = tcx.require_lang_item(LangItem::Poll, None);
                 let poll_adt_ref = tcx.adt_def(poll_did);
-                let poll_args = tcx.mk_args(&[body.return_ty().into()]);
-                (poll_adt_ref, poll_args)
+                let poll_args = tcx.mk_args(&[old_ret_ty.into()]);
+                Ty::new_adt(tcx, poll_adt_ref, poll_args)
             }
             CoroutineKind::Gen(_) => {
                 // Compute Option<yield_ty>
                 let option_did = tcx.require_lang_item(LangItem::Option, None);
                 let option_adt_ref = tcx.adt_def(option_did);
-                let option_args = tcx.mk_args(&[body.yield_ty().unwrap().into()]);
-                (option_adt_ref, option_args)
+                let option_args = tcx.mk_args(&[old_yield_ty.into()]);
+                Ty::new_adt(tcx, option_adt_ref, option_args)
+            }
+            CoroutineKind::AsyncGen(_) => {
+                // The yield ty is already `Poll<Option<yield_ty>>`
+                old_yield_ty
             }
             CoroutineKind::Coroutine => {
                 // Compute CoroutineState<yield_ty, return_ty>
                 let state_did = tcx.require_lang_item(LangItem::CoroutineState, None);
                 let state_adt_ref = tcx.adt_def(state_did);
-                let state_args = tcx.mk_args(&[yield_ty.into(), body.return_ty().into()]);
-                (state_adt_ref, state_args)
+                let state_args = tcx.mk_args(&[old_yield_ty.into(), old_ret_ty.into()]);
+                Ty::new_adt(tcx, state_adt_ref, state_args)
             }
         };
-        let ret_ty = Ty::new_adt(tcx, state_adt_ref, state_args);
 
-        // We rename RETURN_PLACE which has type mir.return_ty to new_ret_local
+        // We rename RETURN_PLACE which has type mir.return_ty to old_ret_local
         // RETURN_PLACE then is a fresh unused local with type ret_ty.
-        let new_ret_local = replace_local(RETURN_PLACE, ret_ty, body, tcx);
+        let old_ret_local = replace_local(RETURN_PLACE, new_ret_ty, body, tcx);
 
         // Replace all occurrences of `ResumeTy` with `&mut Context<'_>` within async bodies.
-        if is_async_kind {
+        if is_async_kind || is_async_gen_kind {
             transform_async_context(tcx, body);
         }
 
@@ -1539,9 +1659,10 @@ impl<'tcx> MirPass<'tcx> for StateTransform {
         } else {
             body.local_decls[resume_local].ty
         };
-        let new_resume_local = replace_local(resume_local, resume_ty, body, tcx);
+        let old_resume_local = replace_local(resume_local, resume_ty, body, tcx);
 
-        // When first entering the coroutine, move the resume argument into its new local.
+        // When first entering the coroutine, move the resume argument into its old local
+        // (which is now a generator interior).
         let source_info = SourceInfo::outermost(body.span);
         let stmts = &mut body.basic_blocks_mut()[START_BLOCK].statements;
         stmts.insert(
@@ -1549,13 +1670,13 @@ impl<'tcx> MirPass<'tcx> for StateTransform {
             Statement {
                 source_info,
                 kind: StatementKind::Assign(Box::new((
-                    new_resume_local.into(),
+                    old_resume_local.into(),
                     Rvalue::Use(Operand::Move(resume_local.into())),
                 ))),
             },
         );
 
-        let always_live_locals = always_storage_live_locals(&body);
+        let always_live_locals = always_storage_live_locals(body);
 
         let liveness_info =
             locals_live_across_suspend_points(tcx, body, &always_live_locals, movable);
@@ -1585,14 +1706,14 @@ impl<'tcx> MirPass<'tcx> for StateTransform {
         let mut transform = TransformVisitor {
             tcx,
             coroutine_kind: body.coroutine_kind().unwrap(),
-            state_adt_ref,
-            state_args,
             remap,
             storage_liveness,
             always_live_locals,
             suspension_points: Vec::new(),
-            new_ret_local,
+            old_ret_local,
             discr_ty,
+            old_ret_ty,
+            old_yield_ty,
         };
         transform.visit_body(body);
 
@@ -1600,6 +1721,11 @@ impl<'tcx> MirPass<'tcx> for StateTransform {
         body.arg_count = 2; // self, resume arg
         body.spread_arg = None;
 
+        // Remove the context argument within generator bodies.
+        if is_gen_kind {
+            transform_gen_context(tcx, body);
+        }
+
         // The original arguments to the function are no longer arguments, mark them as such.
         // Otherwise they'll conflict with our new arguments, which although they don't have
         // argument_index set, will get emitted as unnamed arguments.
diff --git a/compiler/rustc_mir_transform/src/cost_checker.rs b/compiler/rustc_mir_transform/src/cost_checker.rs
index 9bb26693c..79bed960b 100644
--- a/compiler/rustc_mir_transform/src/cost_checker.rs
+++ b/compiler/rustc_mir_transform/src/cost_checker.rs
@@ -69,7 +69,9 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> {
             }
             TerminatorKind::Call { func: Operand::Constant(ref f), unwind, .. } => {
                 let fn_ty = self.instantiate_ty(f.const_.ty());
-                self.cost += if let ty::FnDef(def_id, _) = *fn_ty.kind() && tcx.is_intrinsic(def_id) {
+                self.cost += if let ty::FnDef(def_id, _) = *fn_ty.kind()
+                    && tcx.is_intrinsic(def_id)
+                {
                     // Don't give intrinsics the extra penalty for calls
                     INSTR_COST
                 } else {
diff --git a/compiler/rustc_mir_transform/src/coverage/counters.rs b/compiler/rustc_mir_transform/src/coverage/counters.rs
index b34ec95b4..604589e5b 100644
--- a/compiler/rustc_mir_transform/src/coverage/counters.rs
+++ b/compiler/rustc_mir_transform/src/coverage/counters.rs
@@ -1,18 +1,16 @@
-use super::graph;
-
-use graph::{BasicCoverageBlock, BcbBranch, CoverageGraph, TraverseCoverageGraphWithLoops};
-
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::graph::WithNumNodes;
 use rustc_index::bit_set::BitSet;
 use rustc_index::IndexVec;
 use rustc_middle::mir::coverage::*;
 
+use super::graph::{BasicCoverageBlock, CoverageGraph, TraverseCoverageGraphWithLoops};
+
 use std::fmt::{self, Debug};
 
 /// The coverage counter or counter expression associated with a particular
 /// BCB node or BCB edge.
-#[derive(Clone)]
+#[derive(Clone, Copy)]
 pub(super) enum BcbCounter {
     Counter { id: CounterId },
     Expression { id: ExpressionId },
@@ -88,11 +86,20 @@ impl CoverageCounters {
         BcbCounter::Counter { id }
     }
 
-    fn make_expression(&mut self, lhs: CovTerm, op: Op, rhs: CovTerm) -> BcbCounter {
-        let id = self.expressions.push(Expression { lhs, op, rhs });
+    fn make_expression(&mut self, lhs: BcbCounter, op: Op, rhs: BcbCounter) -> BcbCounter {
+        let expression = Expression { lhs: lhs.as_term(), op, rhs: rhs.as_term() };
+        let id = self.expressions.push(expression);
         BcbCounter::Expression { id }
     }
 
+    /// Variant of `make_expression` that makes `lhs` optional and assumes [`Op::Add`].
+    ///
+    /// This is useful when using [`Iterator::fold`] to build an arbitrary-length sum.
+    fn make_sum_expression(&mut self, lhs: Option<BcbCounter>, rhs: BcbCounter) -> BcbCounter {
+        let Some(lhs) = lhs else { return rhs };
+        self.make_expression(lhs, Op::Add, rhs)
+    }
+
     /// Counter IDs start from one and go up.
     fn next_counter(&mut self) -> CounterId {
         let next = self.next_counter_id;
@@ -109,7 +116,7 @@ impl CoverageCounters {
         self.expressions.len()
     }
 
-    fn set_bcb_counter(&mut self, bcb: BasicCoverageBlock, counter_kind: BcbCounter) -> CovTerm {
+    fn set_bcb_counter(&mut self, bcb: BasicCoverageBlock, counter_kind: BcbCounter) -> BcbCounter {
         assert!(
             // If the BCB has an edge counter (to be injected into a new `BasicBlock`), it can also
             // have an expression (to be injected into an existing `BasicBlock` represented by this
@@ -118,14 +125,13 @@ impl CoverageCounters {
             "attempt to add a `Counter` to a BCB target with existing incoming edge counters"
         );
 
-        let term = counter_kind.as_term();
         if let Some(replaced) = self.bcb_counters[bcb].replace(counter_kind) {
             bug!(
                 "attempt to set a BasicCoverageBlock coverage counter more than once; \
                 {bcb:?} already had counter {replaced:?}",
             );
         } else {
-            term
+            counter_kind
         }
     }
 
@@ -134,11 +140,13 @@ impl CoverageCounters {
         from_bcb: BasicCoverageBlock,
         to_bcb: BasicCoverageBlock,
         counter_kind: BcbCounter,
-    ) -> CovTerm {
+    ) -> BcbCounter {
         // If the BCB has an edge counter (to be injected into a new `BasicBlock`), it can also
         // have an expression (to be injected into an existing `BasicBlock` represented by this
         // `BasicCoverageBlock`).
-        if let Some(node_counter) = self.bcb_counter(to_bcb) && !node_counter.is_expression() {
+        if let Some(node_counter) = self.bcb_counter(to_bcb)
+            && !node_counter.is_expression()
+        {
             bug!(
                 "attempt to add an incoming edge counter from {from_bcb:?} \
                 when the target BCB already has {node_counter:?}"
@@ -146,19 +154,18 @@ impl CoverageCounters {
         }
 
         self.bcb_has_incoming_edge_counters.insert(to_bcb);
-        let term = counter_kind.as_term();
         if let Some(replaced) = self.bcb_edge_counters.insert((from_bcb, to_bcb), counter_kind) {
             bug!(
                 "attempt to set an edge counter more than once; from_bcb: \
                 {from_bcb:?} already had counter {replaced:?}",
             );
         } else {
-            term
+            counter_kind
         }
     }
 
-    pub(super) fn bcb_counter(&self, bcb: BasicCoverageBlock) -> Option<&BcbCounter> {
-        self.bcb_counters[bcb].as_ref()
+    pub(super) fn bcb_counter(&self, bcb: BasicCoverageBlock) -> Option<BcbCounter> {
+        self.bcb_counters[bcb]
     }
 
     pub(super) fn bcb_node_counters(
@@ -220,15 +227,11 @@ impl<'a> MakeBcbCounters<'a> {
         // all `BasicCoverageBlock` nodes in the loop are visited before visiting any node outside
         // the loop. The `traversal` state includes a `context_stack`, providing a way to know if
         // the current BCB is in one or more nested loops or not.
-        let mut traversal = TraverseCoverageGraphWithLoops::new(&self.basic_coverage_blocks);
+        let mut traversal = TraverseCoverageGraphWithLoops::new(self.basic_coverage_blocks);
         while let Some(bcb) = traversal.next() {
             if bcb_has_coverage_spans(bcb) {
                 debug!("{:?} has at least one coverage span. Get or make its counter", bcb);
-                let branching_counter_operand = self.get_or_make_counter_operand(bcb);
-
-                if self.bcb_needs_branch_counters(bcb) {
-                    self.make_branch_counters(&traversal, bcb, branching_counter_operand);
-                }
+                self.make_node_and_branch_counters(&traversal, bcb);
             } else {
                 debug!(
                     "{:?} does not have any coverage spans. A counter will only be added if \
@@ -245,100 +248,93 @@ impl<'a> MakeBcbCounters<'a> {
         );
     }
 
-    fn make_branch_counters(
+    fn make_node_and_branch_counters(
         &mut self,
         traversal: &TraverseCoverageGraphWithLoops<'_>,
-        branching_bcb: BasicCoverageBlock,
-        branching_counter_operand: CovTerm,
+        from_bcb: BasicCoverageBlock,
     ) {
-        let branches = self.bcb_branches(branching_bcb);
+        // First, ensure that this node has a counter of some kind.
+        // We might also use its term later to compute one of the branch counters.
+        let from_bcb_operand = self.get_or_make_counter_operand(from_bcb);
+
+        let branch_target_bcbs = self.basic_coverage_blocks.successors[from_bcb].as_slice();
+
+        // If this node doesn't have multiple out-edges, or all of its out-edges
+        // already have counters, then we don't need to create edge counters.
+        let needs_branch_counters = branch_target_bcbs.len() > 1
+            && branch_target_bcbs
+                .iter()
+                .any(|&to_bcb| self.branch_has_no_counter(from_bcb, to_bcb));
+        if !needs_branch_counters {
+            return;
+        }
+
         debug!(
-            "{:?} has some branch(es) without counters:\n  {}",
-            branching_bcb,
-            branches
+            "{from_bcb:?} has some branch(es) without counters:\n  {}",
+            branch_target_bcbs
                 .iter()
-                .map(|branch| { format!("{:?}: {:?}", branch, self.branch_counter(branch)) })
+                .map(|&to_bcb| {
+                    format!("{from_bcb:?}->{to_bcb:?}: {:?}", self.branch_counter(from_bcb, to_bcb))
+                })
                 .collect::<Vec<_>>()
                 .join("\n  "),
         );
 
-        // Use the `traversal` state to decide if a subset of the branches exit a loop, making it
-        // likely that branch is executed less than branches that do not exit the same loop. In this
-        // case, any branch that does not exit the loop (and has not already been assigned a
-        // counter) should be counted by expression, if possible. (If a preferred expression branch
-        // is not selected based on the loop context, select any branch without an existing
-        // counter.)
-        let expression_branch = self.choose_preferred_expression_branch(traversal, &branches);
-
-        // Assign a Counter or Expression to each branch, plus additional `Expression`s, as needed,
-        // to sum up intermediate results.
-        let mut some_sumup_counter_operand = None;
-        for branch in branches {
-            // Skip the selected `expression_branch`, if any. It's expression will be assigned after
-            // all others.
-            if branch != expression_branch {
-                let branch_counter_operand = if branch.is_only_path_to_target() {
-                    debug!(
-                        "  {:?} has only one incoming edge (from {:?}), so adding a \
-                        counter",
-                        branch, branching_bcb
-                    );
-                    self.get_or_make_counter_operand(branch.target_bcb)
-                } else {
-                    debug!("  {:?} has multiple incoming edges, so adding an edge counter", branch);
-                    self.get_or_make_edge_counter_operand(branching_bcb, branch.target_bcb)
-                };
-                if let Some(sumup_counter_operand) =
-                    some_sumup_counter_operand.replace(branch_counter_operand)
-                {
-                    let intermediate_expression = self.coverage_counters.make_expression(
-                        branch_counter_operand,
-                        Op::Add,
-                        sumup_counter_operand,
-                    );
-                    debug!("  [new intermediate expression: {:?}]", intermediate_expression);
-                    let intermediate_expression_operand = intermediate_expression.as_term();
-                    some_sumup_counter_operand.replace(intermediate_expression_operand);
-                }
-            }
-        }
+        // Of the branch edges that don't have counters yet, one can be given an expression
+        // (computed from the other edges) instead of a dedicated counter.
+        let expression_to_bcb = self.choose_preferred_expression_branch(traversal, from_bcb);
 
-        // Assign the final expression to the `expression_branch` by subtracting the total of all
-        // other branches from the counter of the branching BCB.
-        let sumup_counter_operand =
-            some_sumup_counter_operand.expect("sumup_counter_operand should have a value");
+        // For each branch arm other than the one that was chosen to get an expression,
+        // ensure that it has a counter (existing counter/expression or a new counter),
+        // and accumulate the corresponding terms into a single sum term.
+        let sum_of_all_other_branches: BcbCounter = {
+            let _span = debug_span!("sum_of_all_other_branches", ?expression_to_bcb).entered();
+            branch_target_bcbs
+                .iter()
+                .copied()
+                // Skip the chosen branch, since we'll calculate it from the other branches.
+                .filter(|&to_bcb| to_bcb != expression_to_bcb)
+                .fold(None, |accum, to_bcb| {
+                    let _span = debug_span!("to_bcb", ?accum, ?to_bcb).entered();
+                    let branch_counter = self.get_or_make_edge_counter_operand(from_bcb, to_bcb);
+                    Some(self.coverage_counters.make_sum_expression(accum, branch_counter))
+                })
+                .expect("there must be at least one other branch")
+        };
+
+        // For the branch that was chosen to get an expression, create that expression
+        // by taking the count of the node we're branching from, and subtracting the
+        // sum of all the other branches.
         debug!(
-            "Making an expression for the selected expression_branch: {:?} \
-            (expression_branch predecessors: {:?})",
-            expression_branch,
-            self.bcb_predecessors(expression_branch.target_bcb),
+            "Making an expression for the selected expression_branch: \
+            {expression_to_bcb:?} (expression_branch predecessors: {:?})",
+            self.bcb_predecessors(expression_to_bcb),
         );
         let expression = self.coverage_counters.make_expression(
-            branching_counter_operand,
+            from_bcb_operand,
             Op::Subtract,
-            sumup_counter_operand,
+            sum_of_all_other_branches,
         );
-        debug!("{:?} gets an expression: {:?}", expression_branch, expression);
-        let bcb = expression_branch.target_bcb;
-        if expression_branch.is_only_path_to_target() {
-            self.coverage_counters.set_bcb_counter(bcb, expression);
+        debug!("{expression_to_bcb:?} gets an expression: {expression:?}");
+        if self.basic_coverage_blocks.bcb_has_multiple_in_edges(expression_to_bcb) {
+            self.coverage_counters.set_bcb_edge_counter(from_bcb, expression_to_bcb, expression);
         } else {
-            self.coverage_counters.set_bcb_edge_counter(branching_bcb, bcb, expression);
+            self.coverage_counters.set_bcb_counter(expression_to_bcb, expression);
         }
     }
 
     #[instrument(level = "debug", skip(self))]
-    fn get_or_make_counter_operand(&mut self, bcb: BasicCoverageBlock) -> CovTerm {
+    fn get_or_make_counter_operand(&mut self, bcb: BasicCoverageBlock) -> BcbCounter {
         // If the BCB already has a counter, return it.
-        if let Some(counter_kind) = &self.coverage_counters.bcb_counters[bcb] {
+        if let Some(counter_kind) = self.coverage_counters.bcb_counters[bcb] {
             debug!("{bcb:?} already has a counter: {counter_kind:?}");
-            return counter_kind.as_term();
+            return counter_kind;
         }
 
         // A BCB with only one incoming edge gets a simple `Counter` (via `make_counter()`).
         // Also, a BCB that loops back to itself gets a simple `Counter`. This may indicate the
         // program results in a tight infinite loop, but it should still compile.
-        let one_path_to_target = self.bcb_has_one_path_to_target(bcb);
+        let one_path_to_target = !self.basic_coverage_blocks.bcb_has_multiple_in_edges(bcb);
         if one_path_to_target || self.bcb_predecessors(bcb).contains(&bcb) {
             let counter_kind = self.coverage_counters.make_counter();
             if one_path_to_target {
@@ -353,40 +349,25 @@ impl<'a> MakeBcbCounters<'a> {
             return self.coverage_counters.set_bcb_counter(bcb, counter_kind);
         }
 
-        // A BCB with multiple incoming edges can compute its count by `Expression`, summing up the
-        // counters and/or expressions of its incoming edges. This will recursively get or create
-        // counters for those incoming edges first, then call `make_expression()` to sum them up,
-        // with additional intermediate expressions as needed.
-        let _sumup_debug_span = debug_span!("(preparing sum-up expression)").entered();
-
-        let mut predecessors = self.bcb_predecessors(bcb).to_owned().into_iter();
-        let first_edge_counter_operand =
-            self.get_or_make_edge_counter_operand(predecessors.next().unwrap(), bcb);
-        let mut some_sumup_edge_counter_operand = None;
-        for predecessor in predecessors {
-            let edge_counter_operand = self.get_or_make_edge_counter_operand(predecessor, bcb);
-            if let Some(sumup_edge_counter_operand) =
-                some_sumup_edge_counter_operand.replace(edge_counter_operand)
-            {
-                let intermediate_expression = self.coverage_counters.make_expression(
-                    sumup_edge_counter_operand,
-                    Op::Add,
-                    edge_counter_operand,
-                );
-                debug!("new intermediate expression: {intermediate_expression:?}");
-                let intermediate_expression_operand = intermediate_expression.as_term();
-                some_sumup_edge_counter_operand.replace(intermediate_expression_operand);
-            }
-        }
-        let counter_kind = self.coverage_counters.make_expression(
-            first_edge_counter_operand,
-            Op::Add,
-            some_sumup_edge_counter_operand.unwrap(),
-        );
-        drop(_sumup_debug_span);
-
-        debug!("{bcb:?} gets a new counter (sum of predecessor counters): {counter_kind:?}");
-        self.coverage_counters.set_bcb_counter(bcb, counter_kind)
+        // A BCB with multiple incoming edges can compute its count by ensuring that counters
+        // exist for each of those edges, and then adding them up to get a total count.
+        let sum_of_in_edges: BcbCounter = {
+            let _span = debug_span!("sum_of_in_edges", ?bcb).entered();
+            // We avoid calling `self.bcb_predecessors` here so that we can
+            // call methods on `&mut self` inside the fold.
+            self.basic_coverage_blocks.predecessors[bcb]
+                .iter()
+                .copied()
+                .fold(None, |accum, from_bcb| {
+                    let _span = debug_span!("from_bcb", ?accum, ?from_bcb).entered();
+                    let edge_counter = self.get_or_make_edge_counter_operand(from_bcb, bcb);
+                    Some(self.coverage_counters.make_sum_expression(accum, edge_counter))
+                })
+                .expect("there must be at least one in-edge")
+        };
+
+        debug!("{bcb:?} gets a new counter (sum of predecessor counters): {sum_of_in_edges:?}");
+        self.coverage_counters.set_bcb_counter(bcb, sum_of_in_edges)
     }
 
     #[instrument(level = "debug", skip(self))]
@@ -394,20 +375,26 @@ impl<'a> MakeBcbCounters<'a> {
         &mut self,
         from_bcb: BasicCoverageBlock,
         to_bcb: BasicCoverageBlock,
-    ) -> CovTerm {
+    ) -> BcbCounter {
+        // If the target BCB has only one in-edge (i.e. this one), then create
+        // a node counter instead, since it will have the same value.
+        if !self.basic_coverage_blocks.bcb_has_multiple_in_edges(to_bcb) {
+            assert_eq!([from_bcb].as_slice(), self.basic_coverage_blocks.predecessors[to_bcb]);
+            return self.get_or_make_counter_operand(to_bcb);
+        }
+
         // If the source BCB has only one successor (assumed to be the given target), an edge
         // counter is unnecessary. Just get or make a counter for the source BCB.
-        let successors = self.bcb_successors(from_bcb).iter();
-        if successors.len() == 1 {
+        if self.bcb_successors(from_bcb).len() == 1 {
             return self.get_or_make_counter_operand(from_bcb);
         }
 
         // If the edge already has a counter, return it.
-        if let Some(counter_kind) =
+        if let Some(&counter_kind) =
             self.coverage_counters.bcb_edge_counters.get(&(from_bcb, to_bcb))
         {
             debug!("Edge {from_bcb:?}->{to_bcb:?} already has a counter: {counter_kind:?}");
-            return counter_kind.as_term();
+            return counter_kind;
         }
 
         // Make a new counter to count this edge.
@@ -421,16 +408,19 @@ impl<'a> MakeBcbCounters<'a> {
     fn choose_preferred_expression_branch(
         &self,
         traversal: &TraverseCoverageGraphWithLoops<'_>,
-        branches: &[BcbBranch],
-    ) -> BcbBranch {
-        let good_reloop_branch = self.find_good_reloop_branch(traversal, &branches);
-        if let Some(reloop_branch) = good_reloop_branch {
-            assert!(self.branch_has_no_counter(&reloop_branch));
-            debug!("Selecting reloop branch {reloop_branch:?} to get an expression");
-            reloop_branch
+        from_bcb: BasicCoverageBlock,
+    ) -> BasicCoverageBlock {
+        let good_reloop_branch = self.find_good_reloop_branch(traversal, from_bcb);
+        if let Some(reloop_target) = good_reloop_branch {
+            assert!(self.branch_has_no_counter(from_bcb, reloop_target));
+            debug!("Selecting reloop target {reloop_target:?} to get an expression");
+            reloop_target
         } else {
-            let &branch_without_counter =
-                branches.iter().find(|&branch| self.branch_has_no_counter(branch)).expect(
+            let &branch_without_counter = self
+                .bcb_successors(from_bcb)
+                .iter()
+                .find(|&&to_bcb| self.branch_has_no_counter(from_bcb, to_bcb))
+                .expect(
                     "needs_branch_counters was `true` so there should be at least one \
                     branch",
                 );
@@ -451,26 +441,28 @@ impl<'a> MakeBcbCounters<'a> {
     fn find_good_reloop_branch(
         &self,
         traversal: &TraverseCoverageGraphWithLoops<'_>,
-        branches: &[BcbBranch],
-    ) -> Option<BcbBranch> {
+        from_bcb: BasicCoverageBlock,
+    ) -> Option<BasicCoverageBlock> {
+        let branch_target_bcbs = self.bcb_successors(from_bcb);
+
         // Consider each loop on the current traversal context stack, top-down.
         for reloop_bcbs in traversal.reloop_bcbs_per_loop() {
             let mut all_branches_exit_this_loop = true;
 
             // Try to find a branch that doesn't exit this loop and doesn't
             // already have a counter.
-            for &branch in branches {
+            for &branch_target_bcb in branch_target_bcbs {
                 // A branch is a reloop branch if it dominates any BCB that has
                 // an edge back to the loop header. (Other branches are exits.)
                 let is_reloop_branch = reloop_bcbs.iter().any(|&reloop_bcb| {
-                    self.basic_coverage_blocks.dominates(branch.target_bcb, reloop_bcb)
+                    self.basic_coverage_blocks.dominates(branch_target_bcb, reloop_bcb)
                 });
 
                 if is_reloop_branch {
                     all_branches_exit_this_loop = false;
-                    if self.branch_has_no_counter(&branch) {
+                    if self.branch_has_no_counter(from_bcb, branch_target_bcb) {
                         // We found a good branch to be given an expression.
-                        return Some(branch);
+                        return Some(branch_target_bcb);
                     }
                     // Keep looking for another reloop branch without a counter.
                 } else {
@@ -503,36 +495,23 @@ impl<'a> MakeBcbCounters<'a> {
     }
 
     #[inline]
-    fn bcb_branches(&self, from_bcb: BasicCoverageBlock) -> Vec<BcbBranch> {
-        self.bcb_successors(from_bcb)
-            .iter()
-            .map(|&to_bcb| BcbBranch::from_to(from_bcb, to_bcb, &self.basic_coverage_blocks))
-            .collect::<Vec<_>>()
-    }
-
-    fn bcb_needs_branch_counters(&self, bcb: BasicCoverageBlock) -> bool {
-        let branch_needs_a_counter = |branch: &BcbBranch| self.branch_has_no_counter(branch);
-        let branches = self.bcb_branches(bcb);
-        branches.len() > 1 && branches.iter().any(branch_needs_a_counter)
-    }
-
-    fn branch_has_no_counter(&self, branch: &BcbBranch) -> bool {
-        self.branch_counter(branch).is_none()
+    fn branch_has_no_counter(
+        &self,
+        from_bcb: BasicCoverageBlock,
+        to_bcb: BasicCoverageBlock,
+    ) -> bool {
+        self.branch_counter(from_bcb, to_bcb).is_none()
     }
 
-    fn branch_counter(&self, branch: &BcbBranch) -> Option<&BcbCounter> {
-        let to_bcb = branch.target_bcb;
-        if let Some(from_bcb) = branch.edge_from_bcb {
+    fn branch_counter(
+        &self,
+        from_bcb: BasicCoverageBlock,
+        to_bcb: BasicCoverageBlock,
+    ) -> Option<&BcbCounter> {
+        if self.basic_coverage_blocks.bcb_has_multiple_in_edges(to_bcb) {
             self.coverage_counters.bcb_edge_counters.get(&(from_bcb, to_bcb))
         } else {
             self.coverage_counters.bcb_counters[to_bcb].as_ref()
         }
     }
-
-    /// Returns true if the BasicCoverageBlock has zero or one incoming edge. (If zero, it should be
-    /// the entry point for the function.)
-    #[inline]
-    fn bcb_has_one_path_to_target(&self, bcb: BasicCoverageBlock) -> bool {
-        self.bcb_predecessors(bcb).len() <= 1
-    }
 }
diff --git a/compiler/rustc_mir_transform/src/coverage/graph.rs b/compiler/rustc_mir_transform/src/coverage/graph.rs
index 6bab62aa8..263bfdaaa 100644
--- a/compiler/rustc_mir_transform/src/coverage/graph.rs
+++ b/compiler/rustc_mir_transform/src/coverage/graph.rs
@@ -38,7 +38,7 @@ impl CoverageGraph {
                 }
                 let bcb_data = &bcbs[bcb];
                 let mut bcb_successors = Vec::new();
-                for successor in bcb_filtered_successors(&mir_body, bcb_data.last_bb())
+                for successor in bcb_filtered_successors(mir_body, bcb_data.last_bb())
                     .filter_map(|successor_bb| bb_to_bcb[successor_bb])
                 {
                     if !seen[successor] {
@@ -62,6 +62,14 @@ impl CoverageGraph {
             Self { bcbs, bb_to_bcb, successors, predecessors, dominators: None };
         let dominators = dominators::dominators(&basic_coverage_blocks);
         basic_coverage_blocks.dominators = Some(dominators);
+
+        // The coverage graph's entry-point node (bcb0) always starts with bb0,
+        // which never has predecessors. Any other blocks merged into bcb0 can't
+        // have multiple (coverage-relevant) predecessors, so bcb0 always has
+        // zero in-edges.
+        assert!(basic_coverage_blocks[START_BCB].leader_bb() == mir::START_BLOCK);
+        assert!(basic_coverage_blocks.predecessors[START_BCB].is_empty());
+
         basic_coverage_blocks
     }
 
@@ -199,6 +207,25 @@ impl CoverageGraph {
     pub fn cmp_in_dominator_order(&self, a: BasicCoverageBlock, b: BasicCoverageBlock) -> Ordering {
         self.dominators.as_ref().unwrap().cmp_in_dominator_order(a, b)
     }
+
+    /// Returns true if the given node has 2 or more in-edges, i.e. 2 or more
+    /// predecessors.
+    ///
+    /// This property is interesting to code that assigns counters to nodes and
+    /// edges, because if a node _doesn't_ have multiple in-edges, then there's
+    /// no benefit in having a separate counter for its in-edge, because it
+    /// would have the same value as the node's own counter.
+    ///
+    /// FIXME: That assumption might not be true for [`TerminatorKind::Yield`]?
+    #[inline(always)]
+    pub(super) fn bcb_has_multiple_in_edges(&self, bcb: BasicCoverageBlock) -> bool {
+        // Even though bcb0 conceptually has an extra virtual in-edge due to
+        // being the entry point, we've already asserted that it has no _other_
+        // in-edges, so there's no possibility of it having _multiple_ in-edges.
+        // (And since its virtual in-edge doesn't exist in the graph, that edge
+        // can't have a separate counter anyway.)
+        self.predecessors[bcb].len() > 1
+    }
 }
 
 impl Index<BasicCoverageBlock> for CoverageGraph {
@@ -264,6 +291,7 @@ impl graph::WithPredecessors for CoverageGraph {
 
 rustc_index::newtype_index! {
     /// A node in the control-flow graph of CoverageGraph.
+    #[orderable]
     #[debug_format = "bcb{}"]
     pub(super) struct BasicCoverageBlock {
         const START_BCB = 0;
@@ -318,45 +346,6 @@ impl BasicCoverageBlockData {
     }
 }
 
-/// Represents a successor from a branching BasicCoverageBlock (such as the arms of a `SwitchInt`)
-/// as either the successor BCB itself, if it has only one incoming edge, or the successor _plus_
-/// the specific branching BCB, representing the edge between the two. The latter case
-/// distinguishes this incoming edge from other incoming edges to the same `target_bcb`.
-#[derive(Clone, Copy, PartialEq, Eq)]
-pub(super) struct BcbBranch {
-    pub edge_from_bcb: Option<BasicCoverageBlock>,
-    pub target_bcb: BasicCoverageBlock,
-}
-
-impl BcbBranch {
-    pub fn from_to(
-        from_bcb: BasicCoverageBlock,
-        to_bcb: BasicCoverageBlock,
-        basic_coverage_blocks: &CoverageGraph,
-    ) -> Self {
-        let edge_from_bcb = if basic_coverage_blocks.predecessors[to_bcb].len() > 1 {
-            Some(from_bcb)
-        } else {
-            None
-        };
-        Self { edge_from_bcb, target_bcb: to_bcb }
-    }
-
-    pub fn is_only_path_to_target(&self) -> bool {
-        self.edge_from_bcb.is_none()
-    }
-}
-
-impl std::fmt::Debug for BcbBranch {
-    fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        if let Some(from_bcb) = self.edge_from_bcb {
-            write!(fmt, "{:?}->{:?}", from_bcb, self.target_bcb)
-        } else {
-            write!(fmt, "{:?}", self.target_bcb)
-        }
-    }
-}
-
 // Returns the subset of a block's successors that are relevant to the coverage
 // graph, i.e. those that do not represent unwinds or unreachable branches.
 // FIXME(#78544): MIR InstrumentCoverage: Improve coverage of `#[should_panic]` tests and
diff --git a/compiler/rustc_mir_transform/src/coverage/mod.rs b/compiler/rustc_mir_transform/src/coverage/mod.rs
index 97e4468a0..c5a339128 100644
--- a/compiler/rustc_mir_transform/src/coverage/mod.rs
+++ b/compiler/rustc_mir_transform/src/coverage/mod.rs
@@ -13,7 +13,6 @@ use self::spans::CoverageSpans;
 
 use crate::MirPass;
 
-use rustc_data_structures::sync::Lrc;
 use rustc_middle::hir;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::mir::coverage::*;
@@ -22,9 +21,9 @@ use rustc_middle::mir::{
     TerminatorKind,
 };
 use rustc_middle::ty::TyCtxt;
-use rustc_span::def_id::DefId;
+use rustc_span::def_id::LocalDefId;
 use rustc_span::source_map::SourceMap;
-use rustc_span::{ExpnKind, SourceFile, Span, Symbol};
+use rustc_span::{ExpnKind, Span, Symbol};
 
 /// Inserts `StatementKind::Coverage` statements that either instrument the binary with injected
 /// counters, via intrinsic `llvm.instrprof.increment`, and/or inject metadata used during codegen
@@ -39,31 +38,19 @@ impl<'tcx> MirPass<'tcx> for InstrumentCoverage {
     fn run_pass(&self, tcx: TyCtxt<'tcx>, mir_body: &mut mir::Body<'tcx>) {
         let mir_source = mir_body.source;
 
-        // If the InstrumentCoverage pass is called on promoted MIRs, skip them.
-        // See: https://github.com/rust-lang/rust/pull/73011#discussion_r438317601
-        if mir_source.promoted.is_some() {
-            trace!(
-                "InstrumentCoverage skipped for {:?} (already promoted for Miri evaluation)",
-                mir_source.def_id()
-            );
-            return;
-        }
+        // This pass runs after MIR promotion, but before promoted MIR starts to
+        // be transformed, so it should never see promoted MIR.
+        assert!(mir_source.promoted.is_none());
+
+        let def_id = mir_source.def_id().expect_local();
 
-        let is_fn_like =
-            tcx.hir().get_by_def_id(mir_source.def_id().expect_local()).fn_kind().is_some();
-
-        // Only instrument functions, methods, and closures (not constants since they are evaluated
-        // at compile time by Miri).
-        // FIXME(#73156): Handle source code coverage in const eval, but note, if and when const
-        // expressions get coverage spans, we will probably have to "carve out" space for const
-        // expressions from coverage spans in enclosing MIR's, like we do for closures. (That might
-        // be tricky if const expressions have no corresponding statements in the enclosing MIR.
-        // Closures are carved out by their initial `Assign` statement.)
-        if !is_fn_like {
-            trace!("InstrumentCoverage skipped for {:?} (not an fn-like)", mir_source.def_id());
+        if !is_eligible_for_coverage(tcx, def_id) {
+            trace!("InstrumentCoverage skipped for {def_id:?} (not eligible)");
             return;
         }
 
+        // An otherwise-eligible function is still skipped if its start block
+        // is known to be unreachable.
         match mir_body.basic_blocks[mir::START_BLOCK].terminator().kind {
             TerminatorKind::Unreachable => {
                 trace!("InstrumentCoverage skipped for unreachable `START_BLOCK`");
@@ -72,81 +59,43 @@ impl<'tcx> MirPass<'tcx> for InstrumentCoverage {
             _ => {}
         }
 
-        let codegen_fn_attrs = tcx.codegen_fn_attrs(mir_source.def_id());
-        if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NO_COVERAGE) {
-            return;
-        }
-
-        trace!("InstrumentCoverage starting for {:?}", mir_source.def_id());
+        trace!("InstrumentCoverage starting for {def_id:?}");
         Instrumentor::new(tcx, mir_body).inject_counters();
-        trace!("InstrumentCoverage done for {:?}", mir_source.def_id());
+        trace!("InstrumentCoverage done for {def_id:?}");
     }
 }
 
 struct Instrumentor<'a, 'tcx> {
     tcx: TyCtxt<'tcx>,
     mir_body: &'a mut mir::Body<'tcx>,
-    source_file: Lrc<SourceFile>,
-    fn_sig_span: Span,
-    body_span: Span,
-    function_source_hash: u64,
+    hir_info: ExtractedHirInfo,
     basic_coverage_blocks: CoverageGraph,
     coverage_counters: CoverageCounters,
 }
 
 impl<'a, 'tcx> Instrumentor<'a, 'tcx> {
     fn new(tcx: TyCtxt<'tcx>, mir_body: &'a mut mir::Body<'tcx>) -> Self {
-        let source_map = tcx.sess.source_map();
-        let def_id = mir_body.source.def_id();
-        let (some_fn_sig, hir_body) = fn_sig_and_body(tcx, def_id);
-
-        let body_span = get_body_span(tcx, hir_body, mir_body);
-
-        let source_file = source_map.lookup_source_file(body_span.lo());
-        let fn_sig_span = match some_fn_sig.filter(|fn_sig| {
-            fn_sig.span.eq_ctxt(body_span)
-                && Lrc::ptr_eq(&source_file, &source_map.lookup_source_file(fn_sig.span.lo()))
-        }) {
-            Some(fn_sig) => fn_sig.span.with_hi(body_span.lo()),
-            None => body_span.shrink_to_lo(),
-        };
+        let hir_info = extract_hir_info(tcx, mir_body.source.def_id().expect_local());
 
-        debug!(
-            "instrumenting {}: {:?}, fn sig span: {:?}, body span: {:?}",
-            if tcx.is_closure(def_id) { "closure" } else { "function" },
-            def_id,
-            fn_sig_span,
-            body_span
-        );
+        debug!(?hir_info, "instrumenting {:?}", mir_body.source.def_id());
 
-        let function_source_hash = hash_mir_source(tcx, hir_body);
         let basic_coverage_blocks = CoverageGraph::from_mir(mir_body);
         let coverage_counters = CoverageCounters::new(&basic_coverage_blocks);
 
-        Self {
-            tcx,
-            mir_body,
-            source_file,
-            fn_sig_span,
-            body_span,
-            function_source_hash,
-            basic_coverage_blocks,
-            coverage_counters,
-        }
+        Self { tcx, mir_body, hir_info, basic_coverage_blocks, coverage_counters }
     }
 
     fn inject_counters(&'a mut self) {
-        let fn_sig_span = self.fn_sig_span;
-        let body_span = self.body_span;
-
         ////////////////////////////////////////////////////
         // Compute coverage spans from the `CoverageGraph`.
-        let coverage_spans = CoverageSpans::generate_coverage_spans(
-            &self.mir_body,
-            fn_sig_span,
-            body_span,
+        let Some(coverage_spans) = CoverageSpans::generate_coverage_spans(
+            self.mir_body,
+            &self.hir_info,
             &self.basic_coverage_blocks,
-        );
+        ) else {
+            // No relevant spans were found in MIR, so skip instrumenting this function.
+            return;
+        };
 
         ////////////////////////////////////////////////////
         // Create an optimized mix of `Counter`s and `Expression`s for the `CoverageGraph`. Ensure
@@ -160,7 +109,7 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> {
         let mappings = self.create_mappings_and_inject_coverage_statements(&coverage_spans);
 
         self.mir_body.function_coverage_info = Some(Box::new(FunctionCoverageInfo {
-            function_source_hash: self.function_source_hash,
+            function_source_hash: self.hir_info.function_source_hash,
             num_counters: self.coverage_counters.num_counters(),
             expressions: self.coverage_counters.take_expressions(),
             mappings,
@@ -175,11 +124,12 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> {
         coverage_spans: &CoverageSpans,
     ) -> Vec<Mapping> {
         let source_map = self.tcx.sess.source_map();
-        let body_span = self.body_span;
+        let body_span = self.hir_info.body_span;
 
+        let source_file = source_map.lookup_source_file(body_span.lo());
         use rustc_session::RemapFileNameExt;
         let file_name =
-            Symbol::intern(&self.source_file.name.for_codegen(self.tcx.sess).to_string_lossy());
+            Symbol::intern(&source_file.name.for_codegen(self.tcx.sess).to_string_lossy());
 
         let mut mappings = Vec::new();
 
@@ -240,7 +190,7 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> {
             );
 
             // Inject a counter into the newly-created BB.
-            inject_statement(self.mir_body, self.make_mir_coverage_kind(&counter_kind), new_bb);
+            inject_statement(self.mir_body, self.make_mir_coverage_kind(counter_kind), new_bb);
         }
 
         mappings
@@ -325,27 +275,77 @@ fn make_code_region(
     }
 }
 
-fn fn_sig_and_body(
-    tcx: TyCtxt<'_>,
-    def_id: DefId,
-) -> (Option<&rustc_hir::FnSig<'_>>, &rustc_hir::Body<'_>) {
+fn is_eligible_for_coverage(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
+    // Only instrument functions, methods, and closures (not constants since they are evaluated
+    // at compile time by Miri).
+    // FIXME(#73156): Handle source code coverage in const eval, but note, if and when const
+    // expressions get coverage spans, we will probably have to "carve out" space for const
+    // expressions from coverage spans in enclosing MIR's, like we do for closures. (That might
+    // be tricky if const expressions have no corresponding statements in the enclosing MIR.
+    // Closures are carved out by their initial `Assign` statement.)
+    if !tcx.def_kind(def_id).is_fn_like() {
+        trace!("InstrumentCoverage skipped for {def_id:?} (not an fn-like)");
+        return false;
+    }
+
+    if tcx.codegen_fn_attrs(def_id).flags.contains(CodegenFnAttrFlags::NO_COVERAGE) {
+        return false;
+    }
+
+    true
+}
+
+/// Function information extracted from HIR by the coverage instrumentor.
+#[derive(Debug)]
+struct ExtractedHirInfo {
+    function_source_hash: u64,
+    is_async_fn: bool,
+    fn_sig_span: Span,
+    body_span: Span,
+}
+
+fn extract_hir_info<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalDefId) -> ExtractedHirInfo {
     // FIXME(#79625): Consider improving MIR to provide the information needed, to avoid going back
     // to HIR for it.
-    let hir_node = tcx.hir().get_if_local(def_id).expect("expected DefId is local");
+
+    let hir_node = tcx.hir_node_by_def_id(def_id);
     let (_, fn_body_id) =
         hir::map::associated_body(hir_node).expect("HIR node is a function with body");
-    (hir_node.fn_sig(), tcx.hir().body(fn_body_id))
+    let hir_body = tcx.hir().body(fn_body_id);
+
+    let is_async_fn = hir_node.fn_sig().is_some_and(|fn_sig| fn_sig.header.is_async());
+    let body_span = get_body_span(tcx, hir_body, def_id);
+
+    // The actual signature span is only used if it has the same context and
+    // filename as the body, and precedes the body.
+    let maybe_fn_sig_span = hir_node.fn_sig().map(|fn_sig| fn_sig.span);
+    let fn_sig_span = maybe_fn_sig_span
+        .filter(|&fn_sig_span| {
+            let source_map = tcx.sess.source_map();
+            let file_idx = |span: Span| source_map.lookup_source_file_idx(span.lo());
+
+            fn_sig_span.eq_ctxt(body_span)
+                && fn_sig_span.hi() <= body_span.lo()
+                && file_idx(fn_sig_span) == file_idx(body_span)
+        })
+        // If so, extend it to the start of the body span.
+        .map(|fn_sig_span| fn_sig_span.with_hi(body_span.lo()))
+        // Otherwise, create a dummy signature span at the start of the body.
+        .unwrap_or_else(|| body_span.shrink_to_lo());
+
+    let function_source_hash = hash_mir_source(tcx, hir_body);
+
+    ExtractedHirInfo { function_source_hash, is_async_fn, fn_sig_span, body_span }
 }
 
 fn get_body_span<'tcx>(
     tcx: TyCtxt<'tcx>,
     hir_body: &rustc_hir::Body<'tcx>,
-    mir_body: &mut mir::Body<'tcx>,
+    def_id: LocalDefId,
 ) -> Span {
     let mut body_span = hir_body.value.span;
-    let def_id = mir_body.source.def_id();
 
-    if tcx.is_closure(def_id) {
+    if tcx.is_closure(def_id.to_def_id()) {
         // If the MIR function is a closure, and if the closure body span
         // starts from a macro, but it's content is not in that macro, try
         // to find a non-macro callsite, and instrument the spans there
diff --git a/compiler/rustc_mir_transform/src/coverage/query.rs b/compiler/rustc_mir_transform/src/coverage/query.rs
index 809407f89..dfc7c3a71 100644
--- a/compiler/rustc_mir_transform/src/coverage/query.rs
+++ b/compiler/rustc_mir_transform/src/coverage/query.rs
@@ -2,9 +2,9 @@ use super::*;
 
 use rustc_data_structures::captures::Captures;
 use rustc_middle::mir::coverage::*;
-use rustc_middle::mir::{Body, Coverage, CoverageIdsInfo};
+use rustc_middle::mir::{Body, CoverageIdsInfo};
 use rustc_middle::query::Providers;
-use rustc_middle::ty::{self, TyCtxt};
+use rustc_middle::ty::{self};
 
 /// A `query` provider for retrieving coverage information injected into MIR.
 pub(crate) fn provide(providers: &mut Providers) {
diff --git a/compiler/rustc_mir_transform/src/coverage/spans.rs b/compiler/rustc_mir_transform/src/coverage/spans.rs
index b318134ae..ae43a18ad 100644
--- a/compiler/rustc_mir_transform/src/coverage/spans.rs
+++ b/compiler/rustc_mir_transform/src/coverage/spans.rs
@@ -6,6 +6,7 @@ use rustc_middle::mir;
 use rustc_span::{BytePos, ExpnKind, MacroKind, Span, Symbol, DUMMY_SP};
 
 use super::graph::{BasicCoverageBlock, CoverageGraph, START_BCB};
+use crate::coverage::ExtractedHirInfo;
 
 mod from_mir;
 
@@ -15,26 +16,32 @@ pub(super) struct CoverageSpans {
 }
 
 impl CoverageSpans {
+    /// Extracts coverage-relevant spans from MIR, and associates them with
+    /// their corresponding BCBs.
+    ///
+    /// Returns `None` if no coverage-relevant spans could be extracted.
     pub(super) fn generate_coverage_spans(
         mir_body: &mir::Body<'_>,
-        fn_sig_span: Span,
-        body_span: Span,
+        hir_info: &ExtractedHirInfo,
         basic_coverage_blocks: &CoverageGraph,
-    ) -> Self {
+    ) -> Option<Self> {
         let coverage_spans = CoverageSpansGenerator::generate_coverage_spans(
             mir_body,
-            fn_sig_span,
-            body_span,
+            hir_info,
             basic_coverage_blocks,
         );
 
+        if coverage_spans.is_empty() {
+            return None;
+        }
+
         // Group the coverage spans by BCB, with the BCBs in sorted order.
         let mut bcb_to_spans = IndexVec::from_elem_n(Vec::new(), basic_coverage_blocks.num_nodes());
         for CoverageSpan { bcb, span, .. } in coverage_spans {
             bcb_to_spans[bcb].push(span);
         }
 
-        Self { bcb_to_spans }
+        Some(Self { bcb_to_spans })
     }
 
     pub(super) fn bcb_has_coverage_spans(&self, bcb: BasicCoverageBlock) -> bool {
@@ -89,10 +96,10 @@ impl CoverageSpan {
         }
     }
 
-    pub fn merge_from(&mut self, mut other: CoverageSpan) {
-        debug_assert!(self.is_mergeable(&other));
+    pub fn merge_from(&mut self, other: &Self) {
+        debug_assert!(self.is_mergeable(other));
         self.span = self.span.to(other.span);
-        self.merged_spans.append(&mut other.merged_spans);
+        self.merged_spans.extend_from_slice(&other.merged_spans);
     }
 
     pub fn cutoff_statements_at(&mut self, cutoff_pos: BytePos) {
@@ -129,16 +136,14 @@ impl CoverageSpan {
     /// If the span is part of a macro, and the macro is visible (expands directly to the given
     /// body_span), returns the macro name symbol.
     pub fn visible_macro(&self, body_span: Span) -> Option<Symbol> {
-        if let Some(current_macro) = self.current_macro()
-            && self
-                .expn_span
-                .parent_callsite()
-                .unwrap_or_else(|| bug!("macro must have a parent"))
-                .eq_ctxt(body_span)
-        {
-            return Some(current_macro);
-        }
-        None
+        let current_macro = self.current_macro()?;
+        let parent_callsite = self.expn_span.parent_callsite()?;
+
+        // In addition to matching the context of the body span, the parent callsite
+        // must also be the source callsite, i.e. the parent must have no parent.
+        let is_visible_macro =
+            parent_callsite.parent_callsite().is_none() && parent_callsite.eq_ctxt(body_span);
+        is_visible_macro.then_some(current_macro)
     }
 
     pub fn is_macro_expansion(&self) -> bool {
@@ -224,19 +229,17 @@ impl<'a> CoverageSpansGenerator<'a> {
     /// to be).
     pub(super) fn generate_coverage_spans(
         mir_body: &mir::Body<'_>,
-        fn_sig_span: Span, // Ensured to be same SourceFile and SyntaxContext as `body_span`
-        body_span: Span,
+        hir_info: &ExtractedHirInfo,
         basic_coverage_blocks: &'a CoverageGraph,
     ) -> Vec<CoverageSpan> {
         let sorted_spans = from_mir::mir_to_initial_sorted_coverage_spans(
             mir_body,
-            fn_sig_span,
-            body_span,
+            hir_info,
             basic_coverage_blocks,
         );
 
         let coverage_spans = Self {
-            body_span,
+            body_span: hir_info.body_span,
             basic_coverage_blocks,
             sorted_spans_iter: sorted_spans.into_iter(),
             some_curr: None,
@@ -269,7 +272,7 @@ impl<'a> CoverageSpansGenerator<'a> {
             if curr.is_mergeable(prev) {
                 debug!("  same bcb (and neither is a closure), merge with prev={prev:?}");
                 let prev = self.take_prev();
-                self.curr_mut().merge_from(prev);
+                self.curr_mut().merge_from(&prev);
                 self.maybe_push_macro_name_span();
             // Note that curr.span may now differ from curr_original_span
             } else if prev.span.hi() <= curr.span.lo() {
@@ -277,7 +280,7 @@ impl<'a> CoverageSpansGenerator<'a> {
                     "  different bcbs and disjoint spans, so keep curr for next iter, and add prev={prev:?}",
                 );
                 let prev = self.take_prev();
-                self.push_refined_span(prev);
+                self.refined_spans.push(prev);
                 self.maybe_push_macro_name_span();
             } else if prev.is_closure {
                 // drop any equal or overlapping span (`curr`) and keep `prev` to test again in the
@@ -321,33 +324,30 @@ impl<'a> CoverageSpansGenerator<'a> {
             }
         }
 
-        let prev = self.take_prev();
-        debug!("    AT END, adding last prev={prev:?}");
-
-        // Take `pending_dups` so that we can drain it while calling self methods.
-        // It is never used as a field after this point.
-        for dup in std::mem::take(&mut self.pending_dups) {
+        // Drain any remaining dups into the output.
+        for dup in self.pending_dups.drain(..) {
             debug!("    ...adding at least one pending dup={:?}", dup);
-            self.push_refined_span(dup);
+            self.refined_spans.push(dup);
         }
 
-        // Async functions wrap a closure that implements the body to be executed. The enclosing
-        // function is called and returns an `impl Future` without initially executing any of the
-        // body. To avoid showing the return from the enclosing function as a "covered" return from
-        // the closure, the enclosing function's `TerminatorKind::Return`s `CoverageSpan` is
-        // excluded. The closure's `Return` is the only one that will be counted. This provides
-        // adequate coverage, and more intuitive counts. (Avoids double-counting the closing brace
-        // of the function body.)
-        let body_ends_with_closure = if let Some(last_covspan) = self.refined_spans.last() {
-            last_covspan.is_closure && last_covspan.span.hi() == self.body_span.hi()
-        } else {
-            false
-        };
-
-        if !body_ends_with_closure {
-            self.push_refined_span(prev);
+        // There is usually a final span remaining in `prev` after the loop ends,
+        // so add it to the output as well.
+        if let Some(prev) = self.some_prev.take() {
+            debug!("    AT END, adding last prev={prev:?}");
+            self.refined_spans.push(prev);
         }
 
+        // Do one last merge pass, to simplify the output.
+        self.refined_spans.dedup_by(|b, a| {
+            if a.is_mergeable(b) {
+                debug!(?a, ?b, "merging list-adjacent refined spans");
+                a.merge_from(b);
+                true
+            } else {
+                false
+            }
+        });
+
         // Remove `CoverageSpan`s derived from closures, originally added to ensure the coverage
         // regions for the current function leave room for the closure's own coverage regions
         // (injected separately, from the closure's own MIR).
@@ -355,18 +355,6 @@ impl<'a> CoverageSpansGenerator<'a> {
         self.refined_spans
     }
 
-    fn push_refined_span(&mut self, covspan: CoverageSpan) {
-        if let Some(last) = self.refined_spans.last_mut()
-            && last.is_mergeable(&covspan)
-        {
-            // Instead of pushing the new span, merge it with the last refined span.
-            debug!(?last, ?covspan, "merging new refined span with last refined span");
-            last.merge_from(covspan);
-        } else {
-            self.refined_spans.push(covspan);
-        }
-    }
-
     /// If `curr` is part of a new macro expansion, carve out and push a separate
     /// span that ends just after the macro name and its subsequent `!`.
     fn maybe_push_macro_name_span(&mut self) {
@@ -379,57 +367,59 @@ impl<'a> CoverageSpansGenerator<'a> {
             return;
         }
 
-        let merged_prefix_len = self.curr_original_span.lo() - curr.span.lo();
-        let after_macro_bang = merged_prefix_len + BytePos(visible_macro.as_str().len() as u32 + 1);
-        if self.curr().span.lo() + after_macro_bang > self.curr().span.hi() {
+        // The split point is relative to `curr_original_span`,
+        // because `curr.span` may have been merged with preceding spans.
+        let split_point_after_macro_bang = self.curr_original_span.lo()
+            + BytePos(visible_macro.as_str().len() as u32)
+            + BytePos(1); // add 1 for the `!`
+        debug_assert!(split_point_after_macro_bang <= curr.span.hi());
+        if split_point_after_macro_bang > curr.span.hi() {
             // Something is wrong with the macro name span;
-            // return now to avoid emitting malformed mappings.
-            // FIXME(#117788): Track down why this happens.
+            // return now to avoid emitting malformed mappings (e.g. #117788).
             return;
         }
+
         let mut macro_name_cov = curr.clone();
-        self.curr_mut().span = curr.span.with_lo(curr.span.lo() + after_macro_bang);
-        macro_name_cov.span =
-            macro_name_cov.span.with_hi(macro_name_cov.span.lo() + after_macro_bang);
+        macro_name_cov.span = macro_name_cov.span.with_hi(split_point_after_macro_bang);
+        self.curr_mut().span = curr.span.with_lo(split_point_after_macro_bang);
+
         debug!(
             "  and curr starts a new macro expansion, so add a new span just for \
             the macro `{visible_macro}!`, new span={macro_name_cov:?}",
         );
-        self.push_refined_span(macro_name_cov);
+        self.refined_spans.push(macro_name_cov);
     }
 
+    #[track_caller]
     fn curr(&self) -> &CoverageSpan {
-        self.some_curr
-            .as_ref()
-            .unwrap_or_else(|| bug!("invalid attempt to unwrap a None some_curr"))
+        self.some_curr.as_ref().unwrap_or_else(|| bug!("some_curr is None (curr)"))
     }
 
+    #[track_caller]
     fn curr_mut(&mut self) -> &mut CoverageSpan {
-        self.some_curr
-            .as_mut()
-            .unwrap_or_else(|| bug!("invalid attempt to unwrap a None some_curr"))
+        self.some_curr.as_mut().unwrap_or_else(|| bug!("some_curr is None (curr_mut)"))
     }
 
     /// If called, then the next call to `next_coverage_span()` will *not* update `prev` with the
     /// `curr` coverage span.
+    #[track_caller]
     fn take_curr(&mut self) -> CoverageSpan {
-        self.some_curr.take().unwrap_or_else(|| bug!("invalid attempt to unwrap a None some_curr"))
+        self.some_curr.take().unwrap_or_else(|| bug!("some_curr is None (take_curr)"))
     }
 
+    #[track_caller]
     fn prev(&self) -> &CoverageSpan {
-        self.some_prev
-            .as_ref()
-            .unwrap_or_else(|| bug!("invalid attempt to unwrap a None some_prev"))
+        self.some_prev.as_ref().unwrap_or_else(|| bug!("some_prev is None (prev)"))
     }
 
+    #[track_caller]
     fn prev_mut(&mut self) -> &mut CoverageSpan {
-        self.some_prev
-            .as_mut()
-            .unwrap_or_else(|| bug!("invalid attempt to unwrap a None some_prev"))
+        self.some_prev.as_mut().unwrap_or_else(|| bug!("some_prev is None (prev_mut)"))
     }
 
+    #[track_caller]
     fn take_prev(&mut self) -> CoverageSpan {
-        self.some_prev.take().unwrap_or_else(|| bug!("invalid attempt to unwrap a None some_prev"))
+        self.some_prev.take().unwrap_or_else(|| bug!("some_prev is None (take_prev)"))
     }
 
     /// If there are `pending_dups` but `prev` is not a matching dup (`prev.span` doesn't match the
@@ -452,19 +442,14 @@ impl<'a> CoverageSpansGenerator<'a> {
             previous iteration, or prev started a new disjoint span"
         );
         if last_dup.span.hi() <= self.curr().span.lo() {
-            // Temporarily steal `pending_dups` into a local, so that we can
-            // drain it while calling other self methods.
-            let mut pending_dups = std::mem::take(&mut self.pending_dups);
-            for dup in pending_dups.drain(..) {
+            for dup in self.pending_dups.drain(..) {
                 debug!("    ...adding at least one pending={:?}", dup);
-                self.push_refined_span(dup);
+                self.refined_spans.push(dup);
             }
-            // The list of dups is now empty, but we can recycle its capacity.
-            assert!(pending_dups.is_empty() && self.pending_dups.is_empty());
-            self.pending_dups = pending_dups;
         } else {
             self.pending_dups.clear();
         }
+        assert!(self.pending_dups.is_empty());
     }
 
     /// Advance `prev` to `curr` (if any), and `curr` to the next `CoverageSpan` in sorted order.
@@ -475,7 +460,9 @@ impl<'a> CoverageSpansGenerator<'a> {
         }
         while let Some(curr) = self.sorted_spans_iter.next() {
             debug!("FOR curr={:?}", curr);
-            if let Some(prev) = &self.some_prev && prev.span.lo() > curr.span.lo() {
+            if let Some(prev) = &self.some_prev
+                && prev.span.lo() > curr.span.lo()
+            {
                 // Skip curr because prev has already advanced beyond the end of curr.
                 // This can only happen if a prior iteration updated `prev` to skip past
                 // a region of code, such as skipping past a closure.
@@ -509,22 +496,18 @@ impl<'a> CoverageSpansGenerator<'a> {
         let has_pre_closure_span = prev.span.lo() < right_cutoff;
         let has_post_closure_span = prev.span.hi() > right_cutoff;
 
-        // Temporarily steal `pending_dups` into a local, so that we can
-        // mutate and/or drain it while calling other self methods.
-        let mut pending_dups = std::mem::take(&mut self.pending_dups);
-
         if has_pre_closure_span {
             let mut pre_closure = self.prev().clone();
             pre_closure.span = pre_closure.span.with_hi(left_cutoff);
             debug!("  prev overlaps a closure. Adding span for pre_closure={:?}", pre_closure);
-            if !pending_dups.is_empty() {
-                for mut dup in pending_dups.iter().cloned() {
-                    dup.span = dup.span.with_hi(left_cutoff);
-                    debug!("    ...and at least one pre_closure dup={:?}", dup);
-                    self.push_refined_span(dup);
-                }
+
+            for mut dup in self.pending_dups.iter().cloned() {
+                dup.span = dup.span.with_hi(left_cutoff);
+                debug!("    ...and at least one pre_closure dup={:?}", dup);
+                self.refined_spans.push(dup);
             }
-            self.push_refined_span(pre_closure);
+
+            self.refined_spans.push(pre_closure);
         }
 
         if has_post_closure_span {
@@ -533,19 +516,17 @@ impl<'a> CoverageSpansGenerator<'a> {
             // about how the `CoverageSpan`s are ordered.)
             self.prev_mut().span = self.prev().span.with_lo(right_cutoff);
             debug!("  Mutated prev.span to start after the closure. prev={:?}", self.prev());
-            for dup in pending_dups.iter_mut() {
+
+            for dup in &mut self.pending_dups {
                 debug!("    ...and at least one overlapping dup={:?}", dup);
                 dup.span = dup.span.with_lo(right_cutoff);
             }
+
             let closure_covspan = self.take_curr(); // Prevent this curr from becoming prev.
-            self.push_refined_span(closure_covspan); // since self.prev() was already updated
+            self.refined_spans.push(closure_covspan); // since self.prev() was already updated
         } else {
-            pending_dups.clear();
+            self.pending_dups.clear();
         }
-
-        // Restore the modified post-closure spans, or the empty vector's capacity.
-        assert!(self.pending_dups.is_empty());
-        self.pending_dups = pending_dups;
     }
 
     /// Called if `curr.span` equals `prev_original_span` (and potentially equal to all
@@ -641,7 +622,7 @@ impl<'a> CoverageSpansGenerator<'a> {
             } else {
                 debug!("  ... adding modified prev={:?}", self.prev());
                 let prev = self.take_prev();
-                self.push_refined_span(prev);
+                self.refined_spans.push(prev);
             }
         } else {
             // with `pending_dups`, `prev` cannot have any statements that don't overlap
diff --git a/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs b/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs
index 6189e5379..a9c4ea33d 100644
--- a/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs
+++ b/compiler/rustc_mir_transform/src/coverage/spans/from_mir.rs
@@ -7,13 +7,22 @@ use rustc_span::Span;
 
 use crate::coverage::graph::{BasicCoverageBlock, BasicCoverageBlockData, CoverageGraph};
 use crate::coverage::spans::CoverageSpan;
+use crate::coverage::ExtractedHirInfo;
 
 pub(super) fn mir_to_initial_sorted_coverage_spans(
     mir_body: &mir::Body<'_>,
-    fn_sig_span: Span,
-    body_span: Span,
+    hir_info: &ExtractedHirInfo,
     basic_coverage_blocks: &CoverageGraph,
 ) -> Vec<CoverageSpan> {
+    let &ExtractedHirInfo { is_async_fn, fn_sig_span, body_span, .. } = hir_info;
+    if is_async_fn {
+        // An async function desugars into a function that returns a future,
+        // with the user code wrapped in a closure. Any spans in the desugared
+        // outer function will be unhelpful, so just produce a single span
+        // associating the function signature with its entry BCB.
+        return vec![CoverageSpan::for_fn_sig(fn_sig_span)];
+    }
+
     let mut initial_spans = Vec::with_capacity(mir_body.basic_blocks.len() * 2);
     for (bcb, bcb_data) in basic_coverage_blocks.iter_enumerated() {
         initial_spans.extend(bcb_to_initial_coverage_spans(mir_body, body_span, bcb, bcb_data));
@@ -63,14 +72,14 @@ fn bcb_to_initial_coverage_spans<'a, 'tcx>(
 
         let statement_spans = data.statements.iter().filter_map(move |statement| {
             let expn_span = filtered_statement_span(statement)?;
-            let span = function_source_span(expn_span, body_span);
+            let span = unexpand_into_body_span(expn_span, body_span)?;
 
             Some(CoverageSpan::new(span, expn_span, bcb, is_closure(statement)))
         });
 
         let terminator_span = Some(data.terminator()).into_iter().filter_map(move |terminator| {
             let expn_span = filtered_terminator_span(terminator)?;
-            let span = function_source_span(expn_span, body_span);
+            let span = unexpand_into_body_span(expn_span, body_span)?;
 
             Some(CoverageSpan::new(span, expn_span, bcb, false))
         });
@@ -92,13 +101,13 @@ fn is_closure(statement: &Statement<'_>) -> bool {
 /// If the MIR `Statement` has a span contributive to computing coverage spans,
 /// return it; otherwise return `None`.
 fn filtered_statement_span(statement: &Statement<'_>) -> Option<Span> {
+    use mir::coverage::CoverageKind;
+
     match statement.kind {
         // These statements have spans that are often outside the scope of the executed source code
         // for their parent `BasicBlock`.
         StatementKind::StorageLive(_)
         | StatementKind::StorageDead(_)
-        // Coverage should not be encountered, but don't inject coverage coverage
-        | StatementKind::Coverage(_)
         // Ignore `ConstEvalCounter`s
         | StatementKind::ConstEvalCounter
         // Ignore `Nop`s
@@ -122,9 +131,13 @@ fn filtered_statement_span(statement: &Statement<'_>) -> Option<Span> {
         // If and when the Issue is resolved, remove this special case match pattern:
         StatementKind::FakeRead(box (FakeReadCause::ForGuardBinding, _)) => None,
 
-        // Retain spans from all other statements
+        // Retain spans from most other statements.
         StatementKind::FakeRead(box (_, _)) // Not including `ForGuardBinding`
         | StatementKind::Intrinsic(..)
+        | StatementKind::Coverage(box mir::Coverage {
+            // The purpose of `SpanMarker` is to be matched and accepted here.
+            kind: CoverageKind::SpanMarker
+        })
         | StatementKind::Assign(_)
         | StatementKind::SetDiscriminant { .. }
         | StatementKind::Deinit(..)
@@ -133,6 +146,11 @@ fn filtered_statement_span(statement: &Statement<'_>) -> Option<Span> {
         | StatementKind::AscribeUserType(_, _) => {
             Some(statement.source_info.span)
         }
+
+        StatementKind::Coverage(box mir::Coverage {
+            // These coverage statements should not exist prior to coverage instrumentation.
+            kind: CoverageKind::CounterIncrement { .. } | CoverageKind::ExpressionUsed { .. }
+        }) => bug!("Unexpected coverage statement found during coverage instrumentation: {statement:?}"),
     }
 }
 
@@ -180,14 +198,16 @@ fn filtered_terminator_span(terminator: &Terminator<'_>) -> Option<Span> {
 /// Returns an extrapolated span (pre-expansion[^1]) corresponding to a range
 /// within the function's body source. This span is guaranteed to be contained
 /// within, or equal to, the `body_span`. If the extrapolated span is not
-/// contained within the `body_span`, the `body_span` is returned.
+/// contained within the `body_span`, `None` is returned.
 ///
 /// [^1]Expansions result from Rust syntax including macros, syntactic sugar,
 /// etc.).
 #[inline]
-fn function_source_span(span: Span, body_span: Span) -> Span {
+fn unexpand_into_body_span(span: Span, body_span: Span) -> Option<Span> {
     use rustc_span::source_map::original_sp;
 
+    // FIXME(#118525): Consider switching from `original_sp` to `Span::find_ancestor_inside`,
+    // which is similar but gives slightly different results in some edge cases.
     let original_span = original_sp(span, body_span).with_ctxt(body_span.ctxt());
-    if body_span.contains(original_span) { original_span } else { body_span }
+    body_span.contains(original_span).then_some(original_span)
 }
diff --git a/compiler/rustc_mir_transform/src/coverage/test_macros/Cargo.toml b/compiler/rustc_mir_transform/src/coverage/test_macros/Cargo.toml
deleted file mode 100644
index f753caa91..000000000
--- a/compiler/rustc_mir_transform/src/coverage/test_macros/Cargo.toml
+++ /dev/null
@@ -1,7 +0,0 @@
-[package]
-name = "coverage_test_macros"
-version = "0.0.0"
-edition = "2021"
-
-[lib]
-proc-macro = true
diff --git a/compiler/rustc_mir_transform/src/coverage/test_macros/src/lib.rs b/compiler/rustc_mir_transform/src/coverage/test_macros/src/lib.rs
deleted file mode 100644
index f41adf667..000000000
--- a/compiler/rustc_mir_transform/src/coverage/test_macros/src/lib.rs
+++ /dev/null
@@ -1,6 +0,0 @@
-use proc_macro::TokenStream;
-
-#[proc_macro]
-pub fn let_bcb(item: TokenStream) -> TokenStream {
-    format!("let bcb{item} = graph::BasicCoverageBlock::from_usize({item});").parse().unwrap()
-}
diff --git a/compiler/rustc_mir_transform/src/coverage/tests.rs b/compiler/rustc_mir_transform/src/coverage/tests.rs
index 702fe5f56..931bc8e58 100644
--- a/compiler/rustc_mir_transform/src/coverage/tests.rs
+++ b/compiler/rustc_mir_transform/src/coverage/tests.rs
@@ -27,15 +27,17 @@
 use super::counters;
 use super::graph::{self, BasicCoverageBlock};
 
-use coverage_test_macros::let_bcb;
-
 use itertools::Itertools;
 use rustc_data_structures::graph::WithNumNodes;
 use rustc_data_structures::graph::WithSuccessors;
 use rustc_index::{Idx, IndexVec};
 use rustc_middle::mir::*;
 use rustc_middle::ty;
-use rustc_span::{self, BytePos, Pos, Span, DUMMY_SP};
+use rustc_span::{BytePos, Pos, Span, DUMMY_SP};
+
+fn bcb(index: u32) -> BasicCoverageBlock {
+    BasicCoverageBlock::from_u32(index)
+}
 
 // All `TEMP_BLOCK` targets should be replaced before calling `to_body() -> mir::Body`.
 const TEMP_BLOCK: BasicBlock = BasicBlock::MAX;
@@ -300,12 +302,15 @@ fn goto_switchint<'a>() -> Body<'a> {
     mir_body
 }
 
-macro_rules! assert_successors {
-    ($basic_coverage_blocks:ident, $i:ident, [$($successor:ident),*]) => {
-        let mut successors = $basic_coverage_blocks.successors[$i].clone();
-        successors.sort_unstable();
-        assert_eq!(successors, vec![$($successor),*]);
-    }
+#[track_caller]
+fn assert_successors(
+    basic_coverage_blocks: &graph::CoverageGraph,
+    bcb: BasicCoverageBlock,
+    expected_successors: &[BasicCoverageBlock],
+) {
+    let mut successors = basic_coverage_blocks.successors[bcb].clone();
+    successors.sort_unstable();
+    assert_eq!(successors, expected_successors);
 }
 
 #[test]
@@ -334,13 +339,9 @@ fn test_covgraph_goto_switchint() {
         basic_coverage_blocks.iter_enumerated().collect::<Vec<_>>()
     );
 
-    let_bcb!(0);
-    let_bcb!(1);
-    let_bcb!(2);
-
-    assert_successors!(basic_coverage_blocks, bcb0, [bcb1, bcb2]);
-    assert_successors!(basic_coverage_blocks, bcb1, []);
-    assert_successors!(basic_coverage_blocks, bcb2, []);
+    assert_successors(&basic_coverage_blocks, bcb(0), &[bcb(1), bcb(2)]);
+    assert_successors(&basic_coverage_blocks, bcb(1), &[]);
+    assert_successors(&basic_coverage_blocks, bcb(2), &[]);
 }
 
 /// Create a mock `Body` with a loop.
@@ -418,15 +419,10 @@ fn test_covgraph_switchint_then_loop_else_return() {
         basic_coverage_blocks.iter_enumerated().collect::<Vec<_>>()
     );
 
-    let_bcb!(0);
-    let_bcb!(1);
-    let_bcb!(2);
-    let_bcb!(3);
-
-    assert_successors!(basic_coverage_blocks, bcb0, [bcb1]);
-    assert_successors!(basic_coverage_blocks, bcb1, [bcb2, bcb3]);
-    assert_successors!(basic_coverage_blocks, bcb2, []);
-    assert_successors!(basic_coverage_blocks, bcb3, [bcb1]);
+    assert_successors(&basic_coverage_blocks, bcb(0), &[bcb(1)]);
+    assert_successors(&basic_coverage_blocks, bcb(1), &[bcb(2), bcb(3)]);
+    assert_successors(&basic_coverage_blocks, bcb(2), &[]);
+    assert_successors(&basic_coverage_blocks, bcb(3), &[bcb(1)]);
 }
 
 /// Create a mock `Body` with nested loops.
@@ -546,21 +542,13 @@ fn test_covgraph_switchint_loop_then_inner_loop_else_break() {
         basic_coverage_blocks.iter_enumerated().collect::<Vec<_>>()
     );
 
-    let_bcb!(0);
-    let_bcb!(1);
-    let_bcb!(2);
-    let_bcb!(3);
-    let_bcb!(4);
-    let_bcb!(5);
-    let_bcb!(6);
-
-    assert_successors!(basic_coverage_blocks, bcb0, [bcb1]);
-    assert_successors!(basic_coverage_blocks, bcb1, [bcb2, bcb3]);
-    assert_successors!(basic_coverage_blocks, bcb2, []);
-    assert_successors!(basic_coverage_blocks, bcb3, [bcb4]);
-    assert_successors!(basic_coverage_blocks, bcb4, [bcb5, bcb6]);
-    assert_successors!(basic_coverage_blocks, bcb5, [bcb1]);
-    assert_successors!(basic_coverage_blocks, bcb6, [bcb4]);
+    assert_successors(&basic_coverage_blocks, bcb(0), &[bcb(1)]);
+    assert_successors(&basic_coverage_blocks, bcb(1), &[bcb(2), bcb(3)]);
+    assert_successors(&basic_coverage_blocks, bcb(2), &[]);
+    assert_successors(&basic_coverage_blocks, bcb(3), &[bcb(4)]);
+    assert_successors(&basic_coverage_blocks, bcb(4), &[bcb(5), bcb(6)]);
+    assert_successors(&basic_coverage_blocks, bcb(5), &[bcb(1)]);
+    assert_successors(&basic_coverage_blocks, bcb(6), &[bcb(4)]);
 }
 
 #[test]
@@ -595,10 +583,7 @@ fn test_find_loop_backedges_one() {
         backedges
     );
 
-    let_bcb!(1);
-    let_bcb!(3);
-
-    assert_eq!(backedges[bcb1], vec![bcb3]);
+    assert_eq!(backedges[bcb(1)], &[bcb(3)]);
 }
 
 #[test]
@@ -613,13 +598,8 @@ fn test_find_loop_backedges_two() {
         backedges
     );
 
-    let_bcb!(1);
-    let_bcb!(4);
-    let_bcb!(5);
-    let_bcb!(6);
-
-    assert_eq!(backedges[bcb1], vec![bcb5]);
-    assert_eq!(backedges[bcb4], vec![bcb6]);
+    assert_eq!(backedges[bcb(1)], &[bcb(5)]);
+    assert_eq!(backedges[bcb(4)], &[bcb(6)]);
 }
 
 #[test]
@@ -632,13 +612,11 @@ fn test_traverse_coverage_with_loops() {
         traversed_in_order.push(bcb);
     }
 
-    let_bcb!(6);
-
     // bcb0 is visited first. Then bcb1 starts the first loop, and all remaining nodes, *except*
     // bcb6 are inside the first loop.
     assert_eq!(
         *traversed_in_order.last().expect("should have elements"),
-        bcb6,
+        bcb(6),
         "bcb6 should not be visited until all nodes inside the first loop have been visited"
     );
 }
@@ -656,20 +634,18 @@ fn test_make_bcb_counters() {
         coverage_counters.make_bcb_counters(&basic_coverage_blocks, bcb_has_coverage_spans);
         assert_eq!(coverage_counters.num_expressions(), 0);
 
-        let_bcb!(1);
         assert_eq!(
             0, // bcb1 has a `Counter` with id = 0
-            match coverage_counters.bcb_counter(bcb1).expect("should have a counter") {
+            match coverage_counters.bcb_counter(bcb(1)).expect("should have a counter") {
                 counters::BcbCounter::Counter { id, .. } => id,
                 _ => panic!("expected a Counter"),
             }
             .as_u32()
         );
 
-        let_bcb!(2);
         assert_eq!(
             1, // bcb2 has a `Counter` with id = 1
-            match coverage_counters.bcb_counter(bcb2).expect("should have a counter") {
+            match coverage_counters.bcb_counter(bcb(2)).expect("should have a counter") {
                 counters::BcbCounter::Counter { id, .. } => id,
                 _ => panic!("expected a Counter"),
             }
diff --git a/compiler/rustc_mir_transform/src/cross_crate_inline.rs b/compiler/rustc_mir_transform/src/cross_crate_inline.rs
index 261d9dd44..5f01b8418 100644
--- a/compiler/rustc_mir_transform/src/cross_crate_inline.rs
+++ b/compiler/rustc_mir_transform/src/cross_crate_inline.rs
@@ -22,6 +22,18 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
         return false;
     }
 
+    // This just reproduces the logic from Instance::requires_inline.
+    match tcx.def_kind(def_id) {
+        DefKind::Ctor(..) | DefKind::Closure => return true,
+        DefKind::Fn | DefKind::AssocFn => {}
+        _ => return false,
+    }
+
+    // From this point on, it is valid to return true or false.
+    if tcx.sess.opts.unstable_opts.cross_crate_inline_threshold == InliningThreshold::Always {
+        return true;
+    }
+
     // Obey source annotations first; this is important because it means we can use
     // #[inline(never)] to force code generation.
     match codegen_fn_attrs.inline {
@@ -30,13 +42,6 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
         _ => {}
     }
 
-    // This just reproduces the logic from Instance::requires_inline.
-    match tcx.def_kind(def_id) {
-        DefKind::Ctor(..) | DefKind::Closure => return true,
-        DefKind::Fn | DefKind::AssocFn => {}
-        _ => return false,
-    }
-
     // Don't do any inference when incremental compilation is enabled; the additional inlining that
     // inference permits also creates more work for small edits.
     if tcx.sess.opts.incremental.is_some() {
diff --git a/compiler/rustc_mir_transform/src/ctfe_limit.rs b/compiler/rustc_mir_transform/src/ctfe_limit.rs
index bf5722b3d..dcc960e1e 100644
--- a/compiler/rustc_mir_transform/src/ctfe_limit.rs
+++ b/compiler/rustc_mir_transform/src/ctfe_limit.rs
@@ -20,7 +20,7 @@ impl<'tcx> MirPass<'tcx> for CtfeLimit {
             .filter_map(|(node, node_data)| {
                 if matches!(node_data.terminator().kind, TerminatorKind::Call { .. })
                     // Back edges in a CFG indicate loops
-                    || has_back_edge(&doms, node, &node_data)
+                    || has_back_edge(doms, node, node_data)
                 {
                     Some(node)
                 } else {
diff --git a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs
index 81d2bba98..ad12bce9b 100644
--- a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs
+++ b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs
@@ -8,6 +8,7 @@ use rustc_hir::def::DefKind;
 use rustc_middle::mir::interpret::{AllocId, ConstAllocation, InterpResult, Scalar};
 use rustc_middle::mir::visit::{MutVisitor, PlaceContext, Visitor};
 use rustc_middle::mir::*;
+use rustc_middle::query::TyCtxtAt;
 use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_mir_dataflow::value_analysis::{
@@ -19,7 +20,6 @@ use rustc_span::DUMMY_SP;
 use rustc_target::abi::{Abi, FieldIdx, Size, VariantIdx, FIRST_VARIANT};
 
 use crate::const_prop::throw_machine_stop_str;
-use crate::MirPass;
 
 // These constants are somewhat random guesses and have not been optimized.
 // If `tcx.sess.mir_opt_level() >= 4`, we ignore the limits (this can become very expensive).
@@ -362,7 +362,7 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> {
                     && let Ok(rhs_layout) = self.tcx.layout_of(self.param_env.and(rhs_ty))
                 {
                     let op = ImmTy::from_scalar(pointer, rhs_layout).into();
-                    self.assign_constant(state, place, op, &rhs.projection);
+                    self.assign_constant(state, place, op, rhs.projection);
                 }
             }
             Operand::Constant(box constant) => {
@@ -496,7 +496,7 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> {
             FlatSet::Elem(scalar) => {
                 let ty = op.ty(self.local_decls, self.tcx);
                 self.tcx.layout_of(self.param_env.and(ty)).map_or(FlatSet::Top, |layout| {
-                    FlatSet::Elem(ImmTy::from_scalar(scalar.into(), layout))
+                    FlatSet::Elem(ImmTy::from_scalar(scalar, layout))
                 })
             }
             FlatSet::Bottom => FlatSet::Bottom,
@@ -597,7 +597,9 @@ fn propagatable_scalar(
     state: &State<FlatSet<Scalar>>,
     map: &Map,
 ) -> Option<Scalar> {
-    if let FlatSet::Elem(value) = state.get_idx(place, map) && value.try_to_int().is_ok() {
+    if let FlatSet::Elem(value) = state.get_idx(place, map)
+        && value.try_to_int().is_ok()
+    {
         // Do not attempt to propagate pointers, as we may fail to preserve their identity.
         Some(value)
     } else {
@@ -836,7 +838,8 @@ impl<'tcx> Visitor<'tcx> for OperandCollector<'tcx, '_, '_, '_> {
         location: Location,
     ) {
         if let PlaceElem::Index(local) = elem
-            && let Some(value) = self.visitor.try_make_constant(self.ecx, local.into(), self.state, self.map)
+            && let Some(value) =
+                self.visitor.try_make_constant(self.ecx, local.into(), self.state, self.map)
         {
             self.visitor.patch.before_effect.insert((location, local.into()), value);
         }
@@ -873,7 +876,7 @@ impl<'mir, 'tcx: 'mir> rustc_const_eval::interpret::Machine<'mir, 'tcx> for Dumm
     }
 
     fn before_access_global(
-        _tcx: TyCtxt<'tcx>,
+        _tcx: TyCtxtAt<'tcx>,
         _machine: &Self,
         _alloc_id: AllocId,
         alloc: ConstAllocation<'tcx>,
diff --git a/compiler/rustc_mir_transform/src/deduplicate_blocks.rs b/compiler/rustc_mir_transform/src/deduplicate_blocks.rs
index 666293cbc..b40b2ec8b 100644
--- a/compiler/rustc_mir_transform/src/deduplicate_blocks.rs
+++ b/compiler/rustc_mir_transform/src/deduplicate_blocks.rs
@@ -3,8 +3,6 @@
 
 use std::{collections::hash_map::Entry, hash::Hash, hash::Hasher, iter};
 
-use crate::MirPass;
-
 use rustc_data_structures::fx::FxHashMap;
 use rustc_middle::mir::visit::MutVisitor;
 use rustc_middle::mir::*;
diff --git a/compiler/rustc_mir_transform/src/deref_separator.rs b/compiler/rustc_mir_transform/src/deref_separator.rs
index 42be74570..0e2fccc85 100644
--- a/compiler/rustc_mir_transform/src/deref_separator.rs
+++ b/compiler/rustc_mir_transform/src/deref_separator.rs
@@ -1,4 +1,3 @@
-use crate::MirPass;
 use rustc_index::IndexVec;
 use rustc_middle::mir::patch::MirPatch;
 use rustc_middle::mir::visit::NonUseContext::VarDebugInfo;
diff --git a/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs b/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs
index 1c917a85c..96943435b 100644
--- a/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs
+++ b/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs
@@ -2,7 +2,6 @@
 //!
 //! Box is not actually a pointer so it is incorrect to dereference it directly.
 
-use crate::MirPass;
 use rustc_hir::def_id::DefId;
 use rustc_index::Idx;
 use rustc_middle::mir::patch::MirPatch;
diff --git a/compiler/rustc_mir_transform/src/elaborate_drops.rs b/compiler/rustc_mir_transform/src/elaborate_drops.rs
index 59156b242..c45badbc5 100644
--- a/compiler/rustc_mir_transform/src/elaborate_drops.rs
+++ b/compiler/rustc_mir_transform/src/elaborate_drops.rs
@@ -1,5 +1,4 @@
 use crate::deref_separator::deref_finder;
-use crate::MirPass;
 use rustc_index::bit_set::BitSet;
 use rustc_index::IndexVec;
 use rustc_middle::mir::patch::MirPatch;
@@ -57,7 +56,7 @@ impl<'tcx> MirPass<'tcx> for ElaborateDrops {
         // For types that do not need dropping, the behaviour is trivial. So we only need to track
         // init/uninit for types that do need dropping.
         let move_data =
-            MoveData::gather_moves(&body, tcx, param_env, |ty| ty.needs_drop(tcx, param_env));
+            MoveData::gather_moves(body, tcx, param_env, |ty| ty.needs_drop(tcx, param_env));
         let elaborate_patch = {
             let env = MoveDataParamEnv { move_data, param_env };
 
@@ -67,7 +66,7 @@ impl<'tcx> MirPass<'tcx> for ElaborateDrops {
                 .pass_name("elaborate_drops")
                 .iterate_to_fixpoint()
                 .into_results_cursor(body);
-            let dead_unwinds = compute_dead_unwinds(&body, &mut inits);
+            let dead_unwinds = compute_dead_unwinds(body, &mut inits);
 
             let uninits = MaybeUninitializedPlaces::new(tcx, body, &env)
                 .mark_inactive_variants_as_uninit()
@@ -172,19 +171,13 @@ impl<'a, 'tcx> DropElaborator<'a, 'tcx> for Elaborator<'a, '_, 'tcx> {
                 let mut some_live = false;
                 let mut some_dead = false;
                 let mut children_count = 0;
-                on_all_children_bits(
-                    self.tcx(),
-                    self.body(),
-                    self.ctxt.move_data(),
-                    path,
-                    |child| {
-                        let (live, dead) = self.ctxt.init_data.maybe_live_dead(child);
-                        debug!("elaborate_drop: state({:?}) = {:?}", child, (live, dead));
-                        some_live |= live;
-                        some_dead |= dead;
-                        children_count += 1;
-                    },
-                );
+                on_all_children_bits(self.ctxt.move_data(), path, |child| {
+                    let (live, dead) = self.ctxt.init_data.maybe_live_dead(child);
+                    debug!("elaborate_drop: state({:?}) = {:?}", child, (live, dead));
+                    some_live |= live;
+                    some_dead |= dead;
+                    children_count += 1;
+                });
                 ((some_live, some_dead), children_count != 1)
             }
         };
@@ -202,13 +195,9 @@ impl<'a, 'tcx> DropElaborator<'a, 'tcx> for Elaborator<'a, '_, 'tcx> {
                 self.ctxt.set_drop_flag(loc, path, DropFlagState::Absent);
             }
             DropFlagMode::Deep => {
-                on_all_children_bits(
-                    self.tcx(),
-                    self.body(),
-                    self.ctxt.move_data(),
-                    path,
-                    |child| self.ctxt.set_drop_flag(loc, child, DropFlagState::Absent),
-                );
+                on_all_children_bits(self.ctxt.move_data(), path, |child| {
+                    self.ctxt.set_drop_flag(loc, child, DropFlagState::Absent)
+                });
             }
         }
     }
@@ -268,10 +257,9 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> {
     }
 
     fn create_drop_flag(&mut self, index: MovePathIndex, span: Span) {
-        let tcx = self.tcx;
         let patch = &mut self.patch;
         debug!("create_drop_flag({:?})", self.body.span);
-        self.drop_flags[index].get_or_insert_with(|| patch.new_temp(tcx.types.bool, span));
+        self.drop_flags[index].get_or_insert_with(|| patch.new_temp(self.tcx.types.bool, span));
     }
 
     fn drop_flag(&mut self, index: MovePathIndex) -> Option<Place<'tcx>> {
@@ -304,7 +292,7 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> {
             match path {
                 LookupResult::Exact(path) => {
                     self.init_data.seek_before(self.body.terminator_loc(bb));
-                    on_all_children_bits(self.tcx, self.body, self.move_data(), path, |child| {
+                    on_all_children_bits(self.move_data(), path, |child| {
                         let (maybe_live, maybe_dead) = self.init_data.maybe_live_dead(child);
                         debug!(
                             "collect_drop_flags: collecting {:?} from {:?}@{:?} - {:?}",
@@ -327,7 +315,7 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> {
                     self.init_data.seek_before(self.body.terminator_loc(bb));
                     let (_maybe_live, maybe_dead) = self.init_data.maybe_live_dead(parent);
                     if maybe_dead {
-                        self.tcx.sess.delay_span_bug(
+                        self.tcx.sess.span_delayed_bug(
                             terminator.source_info.span,
                             format!(
                                 "drop of untracked, uninitialized value {bb:?}, place {place:?} ({path:?})"
@@ -392,7 +380,7 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> {
                 LookupResult::Parent(None) => {}
                 LookupResult::Parent(Some(_)) => {
                     if !replace {
-                        self.tcx.sess.delay_span_bug(
+                        self.tcx.sess.span_delayed_bug(
                             terminator.source_info.span,
                             format!("drop of untracked value {bb:?}"),
                         );
@@ -444,7 +432,7 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> {
 
                 let loc = Location { block: tgt, statement_index: 0 };
                 let path = self.move_data().rev_lookup.find(destination.as_ref());
-                on_lookup_result_bits(self.tcx, self.body, self.move_data(), path, |child| {
+                on_lookup_result_bits(self.move_data(), path, |child| {
                     self.set_drop_flag(loc, child, DropFlagState::Present)
                 });
             }
@@ -453,14 +441,9 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> {
 
     fn drop_flags_for_args(&mut self) {
         let loc = Location::START;
-        rustc_mir_dataflow::drop_flag_effects_for_function_entry(
-            self.tcx,
-            self.body,
-            self.env,
-            |path, ds| {
-                self.set_drop_flag(loc, path, ds);
-            },
-        )
+        rustc_mir_dataflow::drop_flag_effects_for_function_entry(self.body, self.env, |path, ds| {
+            self.set_drop_flag(loc, path, ds);
+        })
     }
 
     fn drop_flags_for_locs(&mut self) {
@@ -492,7 +475,6 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> {
                 }
                 let loc = Location { block: bb, statement_index: i };
                 rustc_mir_dataflow::drop_flag_effects_for_location(
-                    self.tcx,
                     self.body,
                     self.env,
                     loc,
@@ -515,7 +497,7 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> {
 
                 let loc = Location { block: bb, statement_index: data.statements.len() };
                 let path = self.move_data().rev_lookup.find(destination.as_ref());
-                on_lookup_result_bits(self.tcx, self.body, self.move_data(), path, |child| {
+                on_lookup_result_bits(self.move_data(), path, |child| {
                     self.set_drop_flag(loc, child, DropFlagState::Present)
                 });
             }
diff --git a/compiler/rustc_mir_transform/src/errors.rs b/compiler/rustc_mir_transform/src/errors.rs
index 5879a8039..fd4af3150 100644
--- a/compiler/rustc_mir_transform/src/errors.rs
+++ b/compiler/rustc_mir_transform/src/errors.rs
@@ -1,6 +1,8 @@
+use std::borrow::Cow;
+
 use rustc_errors::{
-    Applicability, DecorateLint, DiagnosticBuilder, DiagnosticMessage, EmissionGuarantee, Handler,
-    IntoDiagnostic,
+    Applicability, DecorateLint, DiagCtxt, DiagnosticArgValue, DiagnosticBuilder,
+    DiagnosticMessage, EmissionGuarantee, ErrorGuaranteed, IntoDiagnostic,
 };
 use rustc_macros::{Diagnostic, LintDiagnostic, Subdiagnostic};
 use rustc_middle::mir::{AssertKind, UnsafetyViolationDetails};
@@ -9,6 +11,8 @@ use rustc_session::lint::{self, Lint};
 use rustc_span::def_id::DefId;
 use rustc_span::Span;
 
+use crate::fluent_generated as fluent;
+
 #[derive(LintDiagnostic)]
 pub(crate) enum ConstMutate {
     #[diag(mir_transform_const_modify)]
@@ -58,75 +62,108 @@ pub(crate) struct RequiresUnsafe {
 // so we need to eagerly translate the label here, which isn't supported by the derive API
 // We could also exhaustively list out the primary messages for all unsafe violations,
 // but this would result in a lot of duplication.
-impl<'sess, G: EmissionGuarantee> IntoDiagnostic<'sess, G> for RequiresUnsafe {
+impl<'sess> IntoDiagnostic<'sess> for RequiresUnsafe {
     #[track_caller]
-    fn into_diagnostic(self, handler: &'sess Handler) -> DiagnosticBuilder<'sess, G> {
-        let mut diag =
-            handler.struct_diagnostic(crate::fluent_generated::mir_transform_requires_unsafe);
+    fn into_diagnostic(self, dcx: &'sess DiagCtxt) -> DiagnosticBuilder<'sess, ErrorGuaranteed> {
+        let mut diag = dcx.struct_err(fluent::mir_transform_requires_unsafe);
         diag.code(rustc_errors::DiagnosticId::Error("E0133".to_string()));
         diag.set_span(self.span);
         diag.span_label(self.span, self.details.label());
-        diag.note(self.details.note());
-        let desc = handler.eagerly_translate_to_string(self.details.label(), [].into_iter());
+        let desc = dcx.eagerly_translate_to_string(self.details.label(), [].into_iter());
         diag.set_arg("details", desc);
         diag.set_arg("op_in_unsafe_fn_allowed", self.op_in_unsafe_fn_allowed);
+        self.details.add_subdiagnostics(&mut diag);
         if let Some(sp) = self.enclosing {
-            diag.span_label(sp, crate::fluent_generated::mir_transform_not_inherited);
+            diag.span_label(sp, fluent::mir_transform_not_inherited);
         }
         diag
     }
 }
 
-#[derive(Copy, Clone)]
+#[derive(Clone)]
 pub(crate) struct RequiresUnsafeDetail {
     pub span: Span,
     pub violation: UnsafetyViolationDetails,
 }
 
 impl RequiresUnsafeDetail {
-    fn note(self) -> DiagnosticMessage {
+    fn add_subdiagnostics<G: EmissionGuarantee>(&self, diag: &mut DiagnosticBuilder<'_, G>) {
         use UnsafetyViolationDetails::*;
         match self.violation {
-            CallToUnsafeFunction => crate::fluent_generated::mir_transform_call_to_unsafe_note,
-            UseOfInlineAssembly => crate::fluent_generated::mir_transform_use_of_asm_note,
+            CallToUnsafeFunction => {
+                diag.note(fluent::mir_transform_call_to_unsafe_note);
+            }
+            UseOfInlineAssembly => {
+                diag.note(fluent::mir_transform_use_of_asm_note);
+            }
             InitializingTypeWith => {
-                crate::fluent_generated::mir_transform_initializing_valid_range_note
+                diag.note(fluent::mir_transform_initializing_valid_range_note);
+            }
+            CastOfPointerToInt => {
+                diag.note(fluent::mir_transform_const_ptr2int_note);
+            }
+            UseOfMutableStatic => {
+                diag.note(fluent::mir_transform_use_of_static_mut_note);
+            }
+            UseOfExternStatic => {
+                diag.note(fluent::mir_transform_use_of_extern_static_note);
+            }
+            DerefOfRawPointer => {
+                diag.note(fluent::mir_transform_deref_ptr_note);
+            }
+            AccessToUnionField => {
+                diag.note(fluent::mir_transform_union_access_note);
             }
-            CastOfPointerToInt => crate::fluent_generated::mir_transform_const_ptr2int_note,
-            UseOfMutableStatic => crate::fluent_generated::mir_transform_use_of_static_mut_note,
-            UseOfExternStatic => crate::fluent_generated::mir_transform_use_of_extern_static_note,
-            DerefOfRawPointer => crate::fluent_generated::mir_transform_deref_ptr_note,
-            AccessToUnionField => crate::fluent_generated::mir_transform_union_access_note,
             MutationOfLayoutConstrainedField => {
-                crate::fluent_generated::mir_transform_mutation_layout_constrained_note
+                diag.note(fluent::mir_transform_mutation_layout_constrained_note);
             }
             BorrowOfLayoutConstrainedField => {
-                crate::fluent_generated::mir_transform_mutation_layout_constrained_borrow_note
+                diag.note(fluent::mir_transform_mutation_layout_constrained_borrow_note);
+            }
+            CallToFunctionWith { ref missing, ref build_enabled } => {
+                diag.help(fluent::mir_transform_target_feature_call_help);
+                diag.set_arg(
+                    "missing_target_features",
+                    DiagnosticArgValue::StrListSepByAnd(
+                        missing.iter().map(|feature| Cow::from(feature.as_str())).collect(),
+                    ),
+                );
+                diag.set_arg("missing_target_features_count", missing.len());
+                if !build_enabled.is_empty() {
+                    diag.note(fluent::mir_transform_target_feature_call_note);
+                    diag.set_arg(
+                        "build_target_features",
+                        DiagnosticArgValue::StrListSepByAnd(
+                            build_enabled
+                                .iter()
+                                .map(|feature| Cow::from(feature.as_str()))
+                                .collect(),
+                        ),
+                    );
+                    diag.set_arg("build_target_features_count", build_enabled.len());
+                }
             }
-            CallToFunctionWith => crate::fluent_generated::mir_transform_target_feature_call_note,
         }
     }
 
-    fn label(self) -> DiagnosticMessage {
+    fn label(&self) -> DiagnosticMessage {
         use UnsafetyViolationDetails::*;
         match self.violation {
-            CallToUnsafeFunction => crate::fluent_generated::mir_transform_call_to_unsafe_label,
-            UseOfInlineAssembly => crate::fluent_generated::mir_transform_use_of_asm_label,
-            InitializingTypeWith => {
-                crate::fluent_generated::mir_transform_initializing_valid_range_label
-            }
-            CastOfPointerToInt => crate::fluent_generated::mir_transform_const_ptr2int_label,
-            UseOfMutableStatic => crate::fluent_generated::mir_transform_use_of_static_mut_label,
-            UseOfExternStatic => crate::fluent_generated::mir_transform_use_of_extern_static_label,
-            DerefOfRawPointer => crate::fluent_generated::mir_transform_deref_ptr_label,
-            AccessToUnionField => crate::fluent_generated::mir_transform_union_access_label,
+            CallToUnsafeFunction => fluent::mir_transform_call_to_unsafe_label,
+            UseOfInlineAssembly => fluent::mir_transform_use_of_asm_label,
+            InitializingTypeWith => fluent::mir_transform_initializing_valid_range_label,
+            CastOfPointerToInt => fluent::mir_transform_const_ptr2int_label,
+            UseOfMutableStatic => fluent::mir_transform_use_of_static_mut_label,
+            UseOfExternStatic => fluent::mir_transform_use_of_extern_static_label,
+            DerefOfRawPointer => fluent::mir_transform_deref_ptr_label,
+            AccessToUnionField => fluent::mir_transform_union_access_label,
             MutationOfLayoutConstrainedField => {
-                crate::fluent_generated::mir_transform_mutation_layout_constrained_label
+                fluent::mir_transform_mutation_layout_constrained_label
             }
             BorrowOfLayoutConstrainedField => {
-                crate::fluent_generated::mir_transform_mutation_layout_constrained_borrow_label
+                fluent::mir_transform_mutation_layout_constrained_borrow_label
             }
-            CallToFunctionWith => crate::fluent_generated::mir_transform_target_feature_call_label,
+            CallToFunctionWith { .. } => fluent::mir_transform_target_feature_call_label,
         }
     }
 }
@@ -143,30 +180,25 @@ pub(crate) struct UnsafeOpInUnsafeFn {
 
 impl<'a> DecorateLint<'a, ()> for UnsafeOpInUnsafeFn {
     #[track_caller]
-    fn decorate_lint<'b>(
-        self,
-        diag: &'b mut DiagnosticBuilder<'a, ()>,
-    ) -> &'b mut DiagnosticBuilder<'a, ()> {
-        let handler = diag.handler().expect("lint should not yet be emitted");
-        let desc = handler.eagerly_translate_to_string(self.details.label(), [].into_iter());
+    fn decorate_lint<'b>(self, diag: &'b mut DiagnosticBuilder<'a, ()>) {
+        let dcx = diag.dcx().expect("lint should not yet be emitted");
+        let desc = dcx.eagerly_translate_to_string(self.details.label(), [].into_iter());
         diag.set_arg("details", desc);
         diag.span_label(self.details.span, self.details.label());
-        diag.note(self.details.note());
+        self.details.add_subdiagnostics(diag);
 
         if let Some((start, end, fn_sig)) = self.suggest_unsafe_block {
-            diag.span_note(fn_sig, crate::fluent_generated::mir_transform_note);
+            diag.span_note(fn_sig, fluent::mir_transform_note);
             diag.tool_only_multipart_suggestion(
-                crate::fluent_generated::mir_transform_suggestion,
+                fluent::mir_transform_suggestion,
                 vec![(start, " unsafe {".into()), (end, "}".into())],
                 Applicability::MaybeIncorrect,
             );
         }
-
-        diag
     }
 
     fn msg(&self) -> DiagnosticMessage {
-        crate::fluent_generated::mir_transform_unsafe_op_in_unsafe_fn
+        fluent::mir_transform_unsafe_op_in_unsafe_fn
     }
 }
 
@@ -176,10 +208,7 @@ pub(crate) enum AssertLint<P> {
 }
 
 impl<'a, P: std::fmt::Debug> DecorateLint<'a, ()> for AssertLint<P> {
-    fn decorate_lint<'b>(
-        self,
-        diag: &'b mut DiagnosticBuilder<'a, ()>,
-    ) -> &'b mut DiagnosticBuilder<'a, ()> {
+    fn decorate_lint<'b>(self, diag: &'b mut DiagnosticBuilder<'a, ()>) {
         let span = self.span();
         let assert_kind = self.panic();
         let message = assert_kind.diagnostic_message();
@@ -187,18 +216,12 @@ impl<'a, P: std::fmt::Debug> DecorateLint<'a, ()> for AssertLint<P> {
             diag.set_arg(name, value);
         });
         diag.span_label(span, message);
-
-        diag
     }
 
     fn msg(&self) -> DiagnosticMessage {
         match self {
-            AssertLint::ArithmeticOverflow(..) => {
-                crate::fluent_generated::mir_transform_arithmetic_overflow
-            }
-            AssertLint::UnconditionalPanic(..) => {
-                crate::fluent_generated::mir_transform_operation_will_panic
-            }
+            AssertLint::ArithmeticOverflow(..) => fluent::mir_transform_arithmetic_overflow,
+            AssertLint::UnconditionalPanic(..) => fluent::mir_transform_operation_will_panic,
         }
     }
 }
@@ -251,23 +274,19 @@ pub(crate) struct MustNotSupend<'tcx, 'a> {
 
 // Needed for def_path_str
 impl<'a> DecorateLint<'a, ()> for MustNotSupend<'_, '_> {
-    fn decorate_lint<'b>(
-        self,
-        diag: &'b mut rustc_errors::DiagnosticBuilder<'a, ()>,
-    ) -> &'b mut rustc_errors::DiagnosticBuilder<'a, ()> {
-        diag.span_label(self.yield_sp, crate::fluent_generated::_subdiag::label);
+    fn decorate_lint<'b>(self, diag: &'b mut rustc_errors::DiagnosticBuilder<'a, ()>) {
+        diag.span_label(self.yield_sp, fluent::_subdiag::label);
         if let Some(reason) = self.reason {
             diag.subdiagnostic(reason);
         }
-        diag.span_help(self.src_sp, crate::fluent_generated::_subdiag::help);
+        diag.span_help(self.src_sp, fluent::_subdiag::help);
         diag.set_arg("pre", self.pre);
         diag.set_arg("def_path", self.tcx.def_path_str(self.def_id));
         diag.set_arg("post", self.post);
-        diag
     }
 
     fn msg(&self) -> rustc_errors::DiagnosticMessage {
-        crate::fluent_generated::mir_transform_must_not_suspend
+        fluent::mir_transform_must_not_suspend
     }
 }
 
diff --git a/compiler/rustc_mir_transform/src/function_item_references.rs b/compiler/rustc_mir_transform/src/function_item_references.rs
index a42eacbf2..340bb1948 100644
--- a/compiler/rustc_mir_transform/src/function_item_references.rs
+++ b/compiler/rustc_mir_transform/src/function_item_references.rs
@@ -14,7 +14,7 @@ pub struct FunctionItemReferences;
 impl<'tcx> MirLint<'tcx> for FunctionItemReferences {
     fn run_lint(&self, tcx: TyCtxt<'tcx>, body: &Body<'tcx>) {
         let mut checker = FunctionItemRefChecker { tcx, body };
-        checker.visit_body(&body);
+        checker.visit_body(body);
     }
 }
 
@@ -47,12 +47,12 @@ impl<'tcx> Visitor<'tcx> for FunctionItemRefChecker<'_, 'tcx> {
                     for inner_ty in arg_ty.walk().filter_map(|arg| arg.as_type()) {
                         if let Some((fn_id, fn_args)) = FunctionItemRefChecker::is_fn_ref(inner_ty)
                         {
-                            let span = self.nth_arg_span(&args, 0);
+                            let span = self.nth_arg_span(args, 0);
                             self.emit_lint(fn_id, fn_args, source_info, span);
                         }
                     }
                 } else {
-                    self.check_bound_args(def_id, args_ref, &args, source_info);
+                    self.check_bound_args(def_id, args_ref, args, source_info);
                 }
             }
         }
diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs
index dce298e92..3b8adf7e8 100644
--- a/compiler/rustc_mir_transform/src/gvn.rs
+++ b/compiler/rustc_mir_transform/src/gvn.rs
@@ -88,8 +88,8 @@ use rustc_data_structures::fx::{FxHashMap, FxIndexSet};
 use rustc_data_structures::graph::dominators::Dominators;
 use rustc_hir::def::DefKind;
 use rustc_index::bit_set::BitSet;
+use rustc_index::newtype_index;
 use rustc_index::IndexVec;
-use rustc_macros::newtype_index;
 use rustc_middle::mir::interpret::GlobalAlloc;
 use rustc_middle::mir::visit::*;
 use rustc_middle::mir::*;
@@ -103,7 +103,6 @@ use std::borrow::Cow;
 
 use crate::dataflow_const_prop::DummyMachine;
 use crate::ssa::{AssignedValue, SsaLocals};
-use crate::MirPass;
 use either::Either;
 
 pub struct GVN;
@@ -388,7 +387,9 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
                         self.ecx.copy_op(op, &field_dest, /*allow_transmute*/ false).ok()?;
                     }
                     self.ecx.write_discriminant(variant.unwrap_or(FIRST_VARIANT), &dest).ok()?;
-                    self.ecx.alloc_mark_immutable(dest.ptr().provenance.unwrap()).ok()?;
+                    self.ecx
+                        .alloc_mark_immutable(dest.ptr().provenance.unwrap().alloc_id())
+                        .ok()?;
                     dest.into()
                 } else {
                     return None;
@@ -461,7 +462,9 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
             }
             NullaryOp(null_op, ty) => {
                 let layout = self.ecx.layout_of(ty).ok()?;
-                if let NullOp::SizeOf | NullOp::AlignOf = null_op && layout.is_unsized() {
+                if let NullOp::SizeOf | NullOp::AlignOf = null_op
+                    && layout.is_unsized()
+                {
                     return None;
                 }
                 let val = match null_op {
@@ -641,12 +644,10 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
             {
                 if let Some(offset) = self.evaluated[idx].as_ref()
                     && let Ok(offset) = self.ecx.read_target_usize(offset)
+                    && let Some(min_length) = offset.checked_add(1)
                 {
-                    projection.to_mut()[i] = ProjectionElem::ConstantIndex {
-                        offset,
-                        min_length: offset + 1,
-                        from_end: false,
-                    };
+                    projection.to_mut()[i] =
+                        ProjectionElem::ConstantIndex { offset, min_length, from_end: false };
                 } else if let Some(new_idx) = self.try_as_local(idx, location) {
                     projection.to_mut()[i] = ProjectionElem::Index(new_idx);
                     self.reused_locals.insert(new_idx);
@@ -865,7 +866,9 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
             .collect();
         let fields = fields?;
 
-        if let AggregateTy::Array = ty && fields.len() > 4 {
+        if let AggregateTy::Array = ty
+            && fields.len() > 4
+        {
             let first = fields[0];
             if fields.iter().all(|&v| v == first) {
                 let len = ty::Const::from_target_usize(self.tcx, fields.len().try_into().unwrap());
@@ -924,7 +927,8 @@ fn op_to_prop_const<'tcx>(
         }
 
         let pointer = mplace.ptr().into_pointer_or_addr().ok()?;
-        let (alloc_id, offset) = pointer.into_parts();
+        let (prov, offset) = pointer.into_parts();
+        let alloc_id = prov.alloc_id();
         intern_const_alloc_for_constprop(ecx, alloc_id).ok()?;
         if matches!(ecx.tcx.global_alloc(alloc_id), GlobalAlloc::Memory(_)) {
             // `alloc_id` may point to a static. Codegen will choke on an `Indirect` with anything
@@ -1008,8 +1012,7 @@ impl<'tcx> MutVisitor<'tcx> for VnState<'_, 'tcx> {
             // Do not try to simplify a constant, it's already in canonical shape.
             && !matches!(rvalue, Rvalue::Use(Operand::Constant(_)))
         {
-            if let Some(value) = self.simplify_rvalue(rvalue, location)
-            {
+            if let Some(value) = self.simplify_rvalue(rvalue, location) {
                 if let Some(const_) = self.try_as_constant(value) {
                     *rvalue = Rvalue::Use(Operand::Constant(Box::new(const_)));
                 } else if let Some(local) = self.try_as_local(value, location)
diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs
index 793dcf0d9..8ad804bf3 100644
--- a/compiler/rustc_mir_transform/src/inline.rs
+++ b/compiler/rustc_mir_transform/src/inline.rs
@@ -17,7 +17,6 @@ use rustc_target::spec::abi::Abi;
 use crate::cost_checker::CostChecker;
 use crate::simplify::{remove_dead_blocks, CfgSimplifier};
 use crate::util;
-use crate::MirPass;
 use std::iter;
 use std::ops::{Range, RangeFrom};
 
@@ -32,7 +31,6 @@ struct CallSite<'tcx> {
     callee: Instance<'tcx>,
     fn_sig: ty::PolyFnSig<'tcx>,
     block: BasicBlock,
-    target: Option<BasicBlock>,
     source_info: SourceInfo,
 }
 
@@ -281,7 +279,7 @@ impl<'tcx> Inliner<'tcx> {
         }
 
         let old_blocks = caller_body.basic_blocks.next_index();
-        self.inline_call(caller_body, &callsite, callee_body);
+        self.inline_call(caller_body, callsite, callee_body);
         let new_blocks = old_blocks..caller_body.basic_blocks.next_index();
 
         Ok(new_blocks)
@@ -367,7 +365,7 @@ impl<'tcx> Inliner<'tcx> {
     ) -> Option<CallSite<'tcx>> {
         // Only consider direct calls to functions
         let terminator = bb_data.terminator();
-        if let TerminatorKind::Call { ref func, target, fn_span, .. } = terminator.kind {
+        if let TerminatorKind::Call { ref func, fn_span, .. } = terminator.kind {
             let func_ty = func.ty(caller_body, self.tcx);
             if let ty::FnDef(def_id, args) = *func_ty.kind() {
                 // To resolve an instance its args have to be fully normalized.
@@ -386,7 +384,7 @@ impl<'tcx> Inliner<'tcx> {
                 let fn_sig = self.tcx.fn_sig(def_id).instantiate(self.tcx, args);
                 let source_info = SourceInfo { span: fn_span, ..terminator.source_info };
 
-                return Some(CallSite { callee, fn_sig, block: bb, target, source_info });
+                return Some(CallSite { callee, fn_sig, block: bb, source_info });
             }
         }
 
@@ -541,142 +539,158 @@ impl<'tcx> Inliner<'tcx> {
         mut callee_body: Body<'tcx>,
     ) {
         let terminator = caller_body[callsite.block].terminator.take().unwrap();
-        match terminator.kind {
-            TerminatorKind::Call { args, destination, unwind, .. } => {
-                // If the call is something like `a[*i] = f(i)`, where
-                // `i : &mut usize`, then just duplicating the `a[*i]`
-                // Place could result in two different locations if `f`
-                // writes to `i`. To prevent this we need to create a temporary
-                // borrow of the place and pass the destination as `*temp` instead.
-                fn dest_needs_borrow(place: Place<'_>) -> bool {
-                    for elem in place.projection.iter() {
-                        match elem {
-                            ProjectionElem::Deref | ProjectionElem::Index(_) => return true,
-                            _ => {}
-                        }
-                    }
+        let TerminatorKind::Call { args, destination, unwind, target, .. } = terminator.kind else {
+            bug!("unexpected terminator kind {:?}", terminator.kind);
+        };
+
+        let return_block = if let Some(block) = target {
+            // Prepare a new block for code that should execute when call returns. We don't use
+            // target block directly since it might have other predecessors.
+            let mut data = BasicBlockData::new(Some(Terminator {
+                source_info: terminator.source_info,
+                kind: TerminatorKind::Goto { target: block },
+            }));
+            data.is_cleanup = caller_body[block].is_cleanup;
+            Some(caller_body.basic_blocks_mut().push(data))
+        } else {
+            None
+        };
 
-                    false
+        // If the call is something like `a[*i] = f(i)`, where
+        // `i : &mut usize`, then just duplicating the `a[*i]`
+        // Place could result in two different locations if `f`
+        // writes to `i`. To prevent this we need to create a temporary
+        // borrow of the place and pass the destination as `*temp` instead.
+        fn dest_needs_borrow(place: Place<'_>) -> bool {
+            for elem in place.projection.iter() {
+                match elem {
+                    ProjectionElem::Deref | ProjectionElem::Index(_) => return true,
+                    _ => {}
                 }
+            }
 
-                let dest = if dest_needs_borrow(destination) {
-                    trace!("creating temp for return destination");
-                    let dest = Rvalue::Ref(
-                        self.tcx.lifetimes.re_erased,
-                        BorrowKind::Mut { kind: MutBorrowKind::Default },
-                        destination,
-                    );
-                    let dest_ty = dest.ty(caller_body, self.tcx);
-                    let temp = Place::from(self.new_call_temp(caller_body, &callsite, dest_ty));
-                    caller_body[callsite.block].statements.push(Statement {
-                        source_info: callsite.source_info,
-                        kind: StatementKind::Assign(Box::new((temp, dest))),
-                    });
-                    self.tcx.mk_place_deref(temp)
-                } else {
-                    destination
-                };
+            false
+        }
 
-                // Always create a local to hold the destination, as `RETURN_PLACE` may appear
-                // where a full `Place` is not allowed.
-                let (remap_destination, destination_local) = if let Some(d) = dest.as_local() {
-                    (false, d)
-                } else {
-                    (
-                        true,
-                        self.new_call_temp(
-                            caller_body,
-                            &callsite,
-                            destination.ty(caller_body, self.tcx).ty,
-                        ),
-                    )
-                };
+        let dest = if dest_needs_borrow(destination) {
+            trace!("creating temp for return destination");
+            let dest = Rvalue::Ref(
+                self.tcx.lifetimes.re_erased,
+                BorrowKind::Mut { kind: MutBorrowKind::Default },
+                destination,
+            );
+            let dest_ty = dest.ty(caller_body, self.tcx);
+            let temp =
+                Place::from(self.new_call_temp(caller_body, &callsite, dest_ty, return_block));
+            caller_body[callsite.block].statements.push(Statement {
+                source_info: callsite.source_info,
+                kind: StatementKind::Assign(Box::new((temp, dest))),
+            });
+            self.tcx.mk_place_deref(temp)
+        } else {
+            destination
+        };
 
-                // Copy the arguments if needed.
-                let args: Vec<_> = self.make_call_args(args, &callsite, caller_body, &callee_body);
-
-                let mut integrator = Integrator {
-                    args: &args,
-                    new_locals: Local::new(caller_body.local_decls.len())..,
-                    new_scopes: SourceScope::new(caller_body.source_scopes.len())..,
-                    new_blocks: BasicBlock::new(caller_body.basic_blocks.len())..,
-                    destination: destination_local,
-                    callsite_scope: caller_body.source_scopes[callsite.source_info.scope].clone(),
-                    callsite,
-                    cleanup_block: unwind,
-                    in_cleanup_block: false,
-                    tcx: self.tcx,
-                    always_live_locals: BitSet::new_filled(callee_body.local_decls.len()),
-                };
+        // Always create a local to hold the destination, as `RETURN_PLACE` may appear
+        // where a full `Place` is not allowed.
+        let (remap_destination, destination_local) = if let Some(d) = dest.as_local() {
+            (false, d)
+        } else {
+            (
+                true,
+                self.new_call_temp(
+                    caller_body,
+                    &callsite,
+                    destination.ty(caller_body, self.tcx).ty,
+                    return_block,
+                ),
+            )
+        };
 
-                // Map all `Local`s, `SourceScope`s and `BasicBlock`s to new ones
-                // (or existing ones, in a few special cases) in the caller.
-                integrator.visit_body(&mut callee_body);
-
-                // If there are any locals without storage markers, give them storage only for the
-                // duration of the call.
-                for local in callee_body.vars_and_temps_iter() {
-                    if integrator.always_live_locals.contains(local) {
-                        let new_local = integrator.map_local(local);
-                        caller_body[callsite.block].statements.push(Statement {
-                            source_info: callsite.source_info,
-                            kind: StatementKind::StorageLive(new_local),
-                        });
-                    }
-                }
-                if let Some(block) = callsite.target {
-                    // To avoid repeated O(n) insert, push any new statements to the end and rotate
-                    // the slice once.
-                    let mut n = 0;
-                    if remap_destination {
-                        caller_body[block].statements.push(Statement {
-                            source_info: callsite.source_info,
-                            kind: StatementKind::Assign(Box::new((
-                                dest,
-                                Rvalue::Use(Operand::Move(destination_local.into())),
-                            ))),
-                        });
-                        n += 1;
-                    }
-                    for local in callee_body.vars_and_temps_iter().rev() {
-                        if integrator.always_live_locals.contains(local) {
-                            let new_local = integrator.map_local(local);
-                            caller_body[block].statements.push(Statement {
-                                source_info: callsite.source_info,
-                                kind: StatementKind::StorageDead(new_local),
-                            });
-                            n += 1;
-                        }
-                    }
-                    caller_body[block].statements.rotate_right(n);
-                }
+        // Copy the arguments if needed.
+        let args: Vec<_> =
+            self.make_call_args(args, &callsite, caller_body, &callee_body, return_block);
+
+        let mut integrator = Integrator {
+            args: &args,
+            new_locals: Local::new(caller_body.local_decls.len())..,
+            new_scopes: SourceScope::new(caller_body.source_scopes.len())..,
+            new_blocks: BasicBlock::new(caller_body.basic_blocks.len())..,
+            destination: destination_local,
+            callsite_scope: caller_body.source_scopes[callsite.source_info.scope].clone(),
+            callsite,
+            cleanup_block: unwind,
+            in_cleanup_block: false,
+            return_block,
+            tcx: self.tcx,
+            always_live_locals: BitSet::new_filled(callee_body.local_decls.len()),
+        };
 
-                // Insert all of the (mapped) parts of the callee body into the caller.
-                caller_body.local_decls.extend(callee_body.drain_vars_and_temps());
-                caller_body.source_scopes.extend(&mut callee_body.source_scopes.drain(..));
-                caller_body.var_debug_info.append(&mut callee_body.var_debug_info);
-                caller_body.basic_blocks_mut().extend(callee_body.basic_blocks_mut().drain(..));
+        // Map all `Local`s, `SourceScope`s and `BasicBlock`s to new ones
+        // (or existing ones, in a few special cases) in the caller.
+        integrator.visit_body(&mut callee_body);
 
-                caller_body[callsite.block].terminator = Some(Terminator {
+        // If there are any locals without storage markers, give them storage only for the
+        // duration of the call.
+        for local in callee_body.vars_and_temps_iter() {
+            if integrator.always_live_locals.contains(local) {
+                let new_local = integrator.map_local(local);
+                caller_body[callsite.block].statements.push(Statement {
                     source_info: callsite.source_info,
-                    kind: TerminatorKind::Goto { target: integrator.map_block(START_BLOCK) },
+                    kind: StatementKind::StorageLive(new_local),
                 });
-
-                // Copy only unevaluated constants from the callee_body into the caller_body.
-                // Although we are only pushing `ConstKind::Unevaluated` consts to
-                // `required_consts`, here we may not only have `ConstKind::Unevaluated`
-                // because we are calling `instantiate_and_normalize_erasing_regions`.
-                caller_body.required_consts.extend(
-                    callee_body.required_consts.iter().copied().filter(|&ct| match ct.const_ {
-                        Const::Ty(_) => {
-                            bug!("should never encounter ty::UnevaluatedConst in `required_consts`")
-                        }
-                        Const::Val(..) | Const::Unevaluated(..) => true,
-                    }),
-                );
             }
-            kind => bug!("unexpected terminator kind {:?}", kind),
         }
+        if let Some(block) = return_block {
+            // To avoid repeated O(n) insert, push any new statements to the end and rotate
+            // the slice once.
+            let mut n = 0;
+            if remap_destination {
+                caller_body[block].statements.push(Statement {
+                    source_info: callsite.source_info,
+                    kind: StatementKind::Assign(Box::new((
+                        dest,
+                        Rvalue::Use(Operand::Move(destination_local.into())),
+                    ))),
+                });
+                n += 1;
+            }
+            for local in callee_body.vars_and_temps_iter().rev() {
+                if integrator.always_live_locals.contains(local) {
+                    let new_local = integrator.map_local(local);
+                    caller_body[block].statements.push(Statement {
+                        source_info: callsite.source_info,
+                        kind: StatementKind::StorageDead(new_local),
+                    });
+                    n += 1;
+                }
+            }
+            caller_body[block].statements.rotate_right(n);
+        }
+
+        // Insert all of the (mapped) parts of the callee body into the caller.
+        caller_body.local_decls.extend(callee_body.drain_vars_and_temps());
+        caller_body.source_scopes.extend(&mut callee_body.source_scopes.drain(..));
+        caller_body.var_debug_info.append(&mut callee_body.var_debug_info);
+        caller_body.basic_blocks_mut().extend(callee_body.basic_blocks_mut().drain(..));
+
+        caller_body[callsite.block].terminator = Some(Terminator {
+            source_info: callsite.source_info,
+            kind: TerminatorKind::Goto { target: integrator.map_block(START_BLOCK) },
+        });
+
+        // Copy only unevaluated constants from the callee_body into the caller_body.
+        // Although we are only pushing `ConstKind::Unevaluated` consts to
+        // `required_consts`, here we may not only have `ConstKind::Unevaluated`
+        // because we are calling `instantiate_and_normalize_erasing_regions`.
+        caller_body.required_consts.extend(callee_body.required_consts.iter().copied().filter(
+            |&ct| match ct.const_ {
+                Const::Ty(_) => {
+                    bug!("should never encounter ty::UnevaluatedConst in `required_consts`")
+                }
+                Const::Val(..) | Const::Unevaluated(..) => true,
+            },
+        ));
     }
 
     fn make_call_args(
@@ -685,6 +699,7 @@ impl<'tcx> Inliner<'tcx> {
         callsite: &CallSite<'tcx>,
         caller_body: &mut Body<'tcx>,
         callee_body: &Body<'tcx>,
+        return_block: Option<BasicBlock>,
     ) -> Vec<Local> {
         let tcx = self.tcx;
 
@@ -713,8 +728,18 @@ impl<'tcx> Inliner<'tcx> {
         // and the vector is `[closure_ref, tmp0, tmp1, tmp2]`.
         if callsite.fn_sig.abi() == Abi::RustCall && callee_body.spread_arg.is_none() {
             let mut args = args.into_iter();
-            let self_ = self.create_temp_if_necessary(args.next().unwrap(), callsite, caller_body);
-            let tuple = self.create_temp_if_necessary(args.next().unwrap(), callsite, caller_body);
+            let self_ = self.create_temp_if_necessary(
+                args.next().unwrap(),
+                callsite,
+                caller_body,
+                return_block,
+            );
+            let tuple = self.create_temp_if_necessary(
+                args.next().unwrap(),
+                callsite,
+                caller_body,
+                return_block,
+            );
             assert!(args.next().is_none());
 
             let tuple = Place::from(tuple);
@@ -731,13 +756,13 @@ impl<'tcx> Inliner<'tcx> {
                 let tuple_field = Operand::Move(tcx.mk_place_field(tuple, FieldIdx::new(i), ty));
 
                 // Spill to a local to make e.g., `tmp0`.
-                self.create_temp_if_necessary(tuple_field, callsite, caller_body)
+                self.create_temp_if_necessary(tuple_field, callsite, caller_body, return_block)
             });
 
             closure_ref_arg.chain(tuple_tmp_args).collect()
         } else {
             args.into_iter()
-                .map(|a| self.create_temp_if_necessary(a, callsite, caller_body))
+                .map(|a| self.create_temp_if_necessary(a, callsite, caller_body, return_block))
                 .collect()
         }
     }
@@ -749,6 +774,7 @@ impl<'tcx> Inliner<'tcx> {
         arg: Operand<'tcx>,
         callsite: &CallSite<'tcx>,
         caller_body: &mut Body<'tcx>,
+        return_block: Option<BasicBlock>,
     ) -> Local {
         // Reuse the operand if it is a moved temporary.
         if let Operand::Move(place) = &arg
@@ -761,7 +787,7 @@ impl<'tcx> Inliner<'tcx> {
         // Otherwise, create a temporary for the argument.
         trace!("creating temp for argument {:?}", arg);
         let arg_ty = arg.ty(caller_body, self.tcx);
-        let local = self.new_call_temp(caller_body, callsite, arg_ty);
+        let local = self.new_call_temp(caller_body, callsite, arg_ty, return_block);
         caller_body[callsite.block].statements.push(Statement {
             source_info: callsite.source_info,
             kind: StatementKind::Assign(Box::new((Place::from(local), Rvalue::Use(arg)))),
@@ -775,6 +801,7 @@ impl<'tcx> Inliner<'tcx> {
         caller_body: &mut Body<'tcx>,
         callsite: &CallSite<'tcx>,
         ty: Ty<'tcx>,
+        return_block: Option<BasicBlock>,
     ) -> Local {
         let local = caller_body.local_decls.push(LocalDecl::new(ty, callsite.source_info.span));
 
@@ -783,7 +810,7 @@ impl<'tcx> Inliner<'tcx> {
             kind: StatementKind::StorageLive(local),
         });
 
-        if let Some(block) = callsite.target {
+        if let Some(block) = return_block {
             caller_body[block].statements.insert(
                 0,
                 Statement {
@@ -814,6 +841,7 @@ struct Integrator<'a, 'tcx> {
     callsite: &'a CallSite<'tcx>,
     cleanup_block: UnwindAction,
     in_cleanup_block: bool,
+    return_block: Option<BasicBlock>,
     tcx: TyCtxt<'tcx>,
     always_live_locals: BitSet<Local>,
 }
@@ -957,7 +985,7 @@ impl<'tcx> MutVisitor<'tcx> for Integrator<'_, 'tcx> {
                 *unwind = self.map_unwind(*unwind);
             }
             TerminatorKind::Return => {
-                terminator.kind = if let Some(tgt) = self.callsite.target {
+                terminator.kind = if let Some(tgt) = self.return_block {
                     TerminatorKind::Goto { target: tgt }
                 } else {
                     TerminatorKind::Unreachable
diff --git a/compiler/rustc_mir_transform/src/instsimplify.rs b/compiler/rustc_mir_transform/src/instsimplify.rs
index fbcd6e75a..4f0f63d22 100644
--- a/compiler/rustc_mir_transform/src/instsimplify.rs
+++ b/compiler/rustc_mir_transform/src/instsimplify.rs
@@ -1,8 +1,6 @@
 //! Performs various peephole optimizations.
 
 use crate::simplify::simplify_duplicate_switch_targets;
-use crate::MirPass;
-use rustc_hir::Mutability;
 use rustc_middle::mir::*;
 use rustc_middle::ty::layout::ValidityRequirement;
 use rustc_middle::ty::{self, GenericArgsRef, ParamEnv, Ty, TyCtxt};
@@ -35,12 +33,9 @@ impl<'tcx> MirPass<'tcx> for InstSimplify {
                 }
             }
 
-            ctx.simplify_primitive_clone(
-                &mut block.terminator.as_mut().unwrap(),
-                &mut block.statements,
-            );
+            ctx.simplify_primitive_clone(block.terminator.as_mut().unwrap(), &mut block.statements);
             ctx.simplify_intrinsic_assert(
-                &mut block.terminator.as_mut().unwrap(),
+                block.terminator.as_mut().unwrap(),
                 &mut block.statements,
             );
             simplify_duplicate_switch_targets(block.terminator.as_mut().unwrap());
diff --git a/compiler/rustc_mir_transform/src/jump_threading.rs b/compiler/rustc_mir_transform/src/jump_threading.rs
index 7b918be44..a41d8e212 100644
--- a/compiler/rustc_mir_transform/src/jump_threading.rs
+++ b/compiler/rustc_mir_transform/src/jump_threading.rs
@@ -45,7 +45,6 @@ use rustc_middle::ty::{self, ScalarInt, Ty, TyCtxt};
 use rustc_mir_dataflow::value_analysis::{Map, PlaceIndex, State, TrackElem};
 
 use crate::cost_checker::CostChecker;
-use crate::MirPass;
 
 pub struct JumpThreading;
 
@@ -95,7 +94,7 @@ impl<'tcx> MirPass<'tcx> for JumpThreading {
 
             let cost = CostChecker::new(tcx, param_env, None, body);
 
-            let mut state = State::new(ConditionSet::default(), &finder.map);
+            let mut state = State::new(ConditionSet::default(), finder.map);
 
             let conds = if let Some((value, then, else_)) = targets.as_static_if() {
                 let Some(value) = ScalarInt::try_from_uint(value, discr_layout.size) else {
@@ -112,7 +111,7 @@ impl<'tcx> MirPass<'tcx> for JumpThreading {
                 }))
             };
             let conds = ConditionSet(conds);
-            state.insert_value_idx(discr, conds, &finder.map);
+            state.insert_value_idx(discr, conds, finder.map);
 
             finder.find_opportunity(bb, state, cost, 0);
         }
@@ -247,7 +246,9 @@ impl<'tcx, 'a> TOFinder<'tcx, 'a> {
         let last_non_rec = self.opportunities.len();
 
         let predecessors = &self.body.basic_blocks.predecessors()[bb];
-        if let &[pred] = &predecessors[..] && bb != START_BLOCK {
+        if let &[pred] = &predecessors[..]
+            && bb != START_BLOCK
+        {
             let term = self.body.basic_blocks[pred].terminator();
             match term.kind {
                 TerminatorKind::SwitchInt { ref discr, ref targets } => {
@@ -419,8 +420,10 @@ impl<'tcx, 'a> TOFinder<'tcx, 'a> {
                                 // Do not support unions.
                                 AggregateKind::Adt(.., Some(_)) => return None,
                                 AggregateKind::Adt(_, variant_index, ..) if agg_ty.is_enum() => {
-                                    if let Some(discr_target) = self.map.apply(lhs, TrackElem::Discriminant)
-                                        && let Some(discr_value) = discriminant_for_variant(agg_ty, *variant_index)
+                                    if let Some(discr_target) =
+                                        self.map.apply(lhs, TrackElem::Discriminant)
+                                        && let Some(discr_value) =
+                                            discriminant_for_variant(agg_ty, *variant_index)
                                     {
                                         self.process_operand(bb, discr_target, &discr_value, state);
                                     }
@@ -646,7 +649,7 @@ impl OpportunitySet {
 
             // `succ` must be a successor of `current`. If it is not, this means this TO is not
             // satisfiable and a previous TO erased this edge, so we bail out.
-            if basic_blocks[current].terminator().successors().find(|s| *s == succ).is_none() {
+            if !basic_blocks[current].terminator().successors().any(|s| s == succ) {
                 debug!("impossible");
                 return;
             }
diff --git a/compiler/rustc_mir_transform/src/large_enums.rs b/compiler/rustc_mir_transform/src/large_enums.rs
index 0a8b13d66..1d788a55f 100644
--- a/compiler/rustc_mir_transform/src/large_enums.rs
+++ b/compiler/rustc_mir_transform/src/large_enums.rs
@@ -1,5 +1,4 @@
 use crate::rustc_middle::ty::util::IntTypeExt;
-use crate::MirPass;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_middle::mir::interpret::AllocId;
 use rustc_middle::mir::*;
diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs
index bf5f0ca7c..89e897191 100644
--- a/compiler/rustc_mir_transform/src/lib.rs
+++ b/compiler/rustc_mir_transform/src/lib.rs
@@ -118,10 +118,7 @@ use rustc_const_eval::transform::promote_consts;
 use rustc_const_eval::transform::validate;
 use rustc_mir_dataflow::rustc_peek;
 
-use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
-use rustc_fluent_macro::fluent_messages;
-
-fluent_messages! { "../messages.ftl" }
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
 
 pub fn provide(providers: &mut Providers) {
     check_unsafety::provide(providers);
@@ -269,7 +266,7 @@ fn mir_const_qualif(tcx: TyCtxt<'_>, def: LocalDefId) -> ConstQualifs {
     let body = &tcx.mir_const(def).borrow();
 
     if body.return_ty().references_error() {
-        tcx.sess.delay_span_bug(body.span, "mir_const_qualif: MIR had errors");
+        tcx.sess.span_delayed_bug(body.span, "mir_const_qualif: MIR had errors");
         return Default::default();
     }
 
@@ -398,7 +395,7 @@ fn inner_mir_for_ctfe(tcx: TyCtxt<'_>, def: LocalDefId) -> Body<'_> {
 /// mir borrowck *before* doing so in order to ensure that borrowck can be run and doesn't
 /// end up missing the source MIR due to stealing happening.
 fn mir_drops_elaborated_and_const_checked(tcx: TyCtxt<'_>, def: LocalDefId) -> &Steal<Body<'_>> {
-    if let DefKind::Coroutine = tcx.def_kind(def) {
+    if tcx.is_coroutine(def.to_def_id()) {
         tcx.ensure_with_value().mir_coroutine_witnesses(def);
     }
     let mir_borrowck = tcx.mir_borrowck(def);
@@ -481,14 +478,14 @@ pub fn run_analysis_to_runtime_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'
     assert!(body.phase == MirPhase::Analysis(AnalysisPhase::PostCleanup));
 
     // Do a little drop elaboration before const-checking if `const_precise_live_drops` is enabled.
-    if check_consts::post_drop_elaboration::checking_enabled(&ConstCx::new(tcx, &body)) {
+    if check_consts::post_drop_elaboration::checking_enabled(&ConstCx::new(tcx, body)) {
         pm::run_passes(
             tcx,
             body,
             &[&remove_uninit_drops::RemoveUninitDrops, &simplify::SimplifyCfg::RemoveFalseEdges],
             None,
         );
-        check_consts::post_drop_elaboration::check_live_drops(tcx, &body); // FIXME: make this a MIR lint
+        check_consts::post_drop_elaboration::check_live_drops(tcx, body); // FIXME: make this a MIR lint
     }
 
     debug!("runtime_mir_lowering({:?})", did);
diff --git a/compiler/rustc_mir_transform/src/lower_intrinsics.rs b/compiler/rustc_mir_transform/src/lower_intrinsics.rs
index 5f3d8dfc6..18f588dcc 100644
--- a/compiler/rustc_mir_transform/src/lower_intrinsics.rs
+++ b/compiler/rustc_mir_transform/src/lower_intrinsics.rs
@@ -1,10 +1,8 @@
 //! Lowers intrinsic calls
 
-use crate::MirPass;
 use rustc_middle::mir::*;
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_span::symbol::sym;
-use rustc_target::abi::{FieldIdx, VariantIdx};
 
 pub struct LowerIntrinsics;
 
@@ -251,37 +249,6 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics {
                         });
                         terminator.kind = TerminatorKind::Goto { target };
                     }
-                    sym::option_payload_ptr => {
-                        if let (Some(target), Some(arg)) = (*target, args[0].place()) {
-                            let ty::RawPtr(ty::TypeAndMut { ty: dest_ty, .. }) =
-                                destination.ty(local_decls, tcx).ty.kind()
-                            else {
-                                bug!();
-                            };
-
-                            block.statements.push(Statement {
-                                source_info: terminator.source_info,
-                                kind: StatementKind::Assign(Box::new((
-                                    *destination,
-                                    Rvalue::AddressOf(
-                                        Mutability::Not,
-                                        arg.project_deeper(
-                                            &[
-                                                PlaceElem::Deref,
-                                                PlaceElem::Downcast(
-                                                    Some(sym::Some),
-                                                    VariantIdx::from_u32(1),
-                                                ),
-                                                PlaceElem::Field(FieldIdx::from_u32(0), *dest_ty),
-                                            ],
-                                            tcx,
-                                        ),
-                                    ),
-                                ))),
-                            });
-                            terminator.kind = TerminatorKind::Goto { target };
-                        }
-                    }
                     sym::transmute | sym::transmute_unchecked => {
                         let dst_ty = destination.ty(local_decls, tcx).ty;
                         let Ok([arg]) = <[_; 1]>::try_from(std::mem::take(args)) else {
diff --git a/compiler/rustc_mir_transform/src/lower_slice_len.rs b/compiler/rustc_mir_transform/src/lower_slice_len.rs
index ae4878411..daeb56666 100644
--- a/compiler/rustc_mir_transform/src/lower_slice_len.rs
+++ b/compiler/rustc_mir_transform/src/lower_slice_len.rs
@@ -1,7 +1,6 @@
 //! This pass lowers calls to core::slice::len to just Len op.
 //! It should run before inlining!
 
-use crate::MirPass;
 use rustc_hir::def_id::DefId;
 use rustc_index::IndexSlice;
 use rustc_middle::mir::*;
diff --git a/compiler/rustc_mir_transform/src/match_branches.rs b/compiler/rustc_mir_transform/src/match_branches.rs
index 3dc627b61..1c4aa37d5 100644
--- a/compiler/rustc_mir_transform/src/match_branches.rs
+++ b/compiler/rustc_mir_transform/src/match_branches.rs
@@ -1,4 +1,3 @@
-use crate::MirPass;
 use rustc_middle::mir::*;
 use rustc_middle::ty::TyCtxt;
 use std::iter;
diff --git a/compiler/rustc_mir_transform/src/multiple_return_terminators.rs b/compiler/rustc_mir_transform/src/multiple_return_terminators.rs
index c9b42e75c..64749a4b5 100644
--- a/compiler/rustc_mir_transform/src/multiple_return_terminators.rs
+++ b/compiler/rustc_mir_transform/src/multiple_return_terminators.rs
@@ -1,7 +1,7 @@
 //! This pass removes jumps to basic blocks containing only a return, and replaces them with a
 //! return instead.
 
-use crate::{simplify, MirPass};
+use crate::simplify;
 use rustc_index::bit_set::BitSet;
 use rustc_middle::mir::*;
 use rustc_middle::ty::TyCtxt;
diff --git a/compiler/rustc_mir_transform/src/normalize_array_len.rs b/compiler/rustc_mir_transform/src/normalize_array_len.rs
index 206cdf9fe..128634bd7 100644
--- a/compiler/rustc_mir_transform/src/normalize_array_len.rs
+++ b/compiler/rustc_mir_transform/src/normalize_array_len.rs
@@ -2,7 +2,6 @@
 //! is taken using `.len()` method. Handy to preserve information in MIR for const prop
 
 use crate::ssa::SsaLocals;
-use crate::MirPass;
 use rustc_index::IndexVec;
 use rustc_middle::mir::visit::*;
 use rustc_middle::mir::*;
diff --git a/compiler/rustc_mir_transform/src/pass_manager.rs b/compiler/rustc_mir_transform/src/pass_manager.rs
index a8aba29ad..c4eca18ff 100644
--- a/compiler/rustc_mir_transform/src/pass_manager.rs
+++ b/compiler/rustc_mir_transform/src/pass_manager.rs
@@ -99,7 +99,7 @@ where
             );
             *polarity
         });
-    overridden.unwrap_or_else(|| pass.is_enabled(&tcx.sess))
+    overridden.unwrap_or_else(|| pass.is_enabled(tcx.sess))
 }
 
 fn run_passes_inner<'tcx>(
@@ -126,7 +126,7 @@ fn run_passes_inner<'tcx>(
             let dump_enabled = pass.is_mir_dump_enabled();
 
             if dump_enabled {
-                dump_mir_for_pass(tcx, body, &name, false);
+                dump_mir_for_pass(tcx, body, name, false);
             }
             if validate {
                 validate_body(tcx, body, format!("before pass {name}"));
@@ -142,7 +142,7 @@ fn run_passes_inner<'tcx>(
             }
 
             if dump_enabled {
-                dump_mir_for_pass(tcx, body, &name, true);
+                dump_mir_for_pass(tcx, body, name, true);
             }
             if validate {
                 validate_body(tcx, body, format!("after pass {name}"));
diff --git a/compiler/rustc_mir_transform/src/prettify.rs b/compiler/rustc_mir_transform/src/prettify.rs
index 745fa3084..7b77d0323 100644
--- a/compiler/rustc_mir_transform/src/prettify.rs
+++ b/compiler/rustc_mir_transform/src/prettify.rs
@@ -4,7 +4,6 @@
 //! (`-Zmir-enable-passes=+ReorderBasicBlocks,+ReorderLocals`)
 //! to make the MIR easier to read for humans.
 
-use crate::MirPass;
 use rustc_index::{bit_set::BitSet, IndexSlice, IndexVec};
 use rustc_middle::mir::visit::{MutVisitor, PlaceContext, Visitor};
 use rustc_middle::mir::*;
diff --git a/compiler/rustc_mir_transform/src/ref_prop.rs b/compiler/rustc_mir_transform/src/ref_prop.rs
index df39c819b..f13ab5b0f 100644
--- a/compiler/rustc_mir_transform/src/ref_prop.rs
+++ b/compiler/rustc_mir_transform/src/ref_prop.rs
@@ -9,7 +9,6 @@ use rustc_mir_dataflow::storage::always_storage_live_locals;
 use rustc_mir_dataflow::Analysis;
 
 use crate::ssa::{SsaLocals, StorageLiveLocals};
-use crate::MirPass;
 
 /// Propagate references using SSA analysis.
 ///
diff --git a/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs b/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs
index 54892442c..095119e2e 100644
--- a/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs
+++ b/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs
@@ -1,4 +1,3 @@
-use crate::MirPass;
 use rustc_index::bit_set::BitSet;
 use rustc_middle::mir::patch::MirPatch;
 use rustc_middle::mir::*;
diff --git a/compiler/rustc_mir_transform/src/remove_place_mention.rs b/compiler/rustc_mir_transform/src/remove_place_mention.rs
index 8be1c3757..78335b3b5 100644
--- a/compiler/rustc_mir_transform/src/remove_place_mention.rs
+++ b/compiler/rustc_mir_transform/src/remove_place_mention.rs
@@ -1,6 +1,5 @@
 //! This pass removes `PlaceMention` statement, which has no effect at codegen.
 
-use crate::MirPass;
 use rustc_middle::mir::*;
 use rustc_middle::ty::TyCtxt;
 
diff --git a/compiler/rustc_mir_transform/src/remove_storage_markers.rs b/compiler/rustc_mir_transform/src/remove_storage_markers.rs
index dbe082e90..795f5232e 100644
--- a/compiler/rustc_mir_transform/src/remove_storage_markers.rs
+++ b/compiler/rustc_mir_transform/src/remove_storage_markers.rs
@@ -1,6 +1,5 @@
 //! This pass removes storage markers if they won't be emitted during codegen.
 
-use crate::MirPass;
 use rustc_middle::mir::*;
 use rustc_middle::ty::TyCtxt;
 
diff --git a/compiler/rustc_mir_transform/src/remove_uninit_drops.rs b/compiler/rustc_mir_transform/src/remove_uninit_drops.rs
index 87fee2410..7d12bcf2f 100644
--- a/compiler/rustc_mir_transform/src/remove_uninit_drops.rs
+++ b/compiler/rustc_mir_transform/src/remove_uninit_drops.rs
@@ -4,9 +4,7 @@ use rustc_middle::ty::GenericArgsRef;
 use rustc_middle::ty::{self, ParamEnv, Ty, TyCtxt, VariantDef};
 use rustc_mir_dataflow::impls::MaybeInitializedPlaces;
 use rustc_mir_dataflow::move_paths::{LookupResult, MoveData, MovePathIndex};
-use rustc_mir_dataflow::{
-    self, move_path_children_matching, Analysis, MaybeReachable, MoveDataParamEnv,
-};
+use rustc_mir_dataflow::{move_path_children_matching, Analysis, MaybeReachable, MoveDataParamEnv};
 use rustc_target::abi::FieldIdx;
 
 use crate::MirPass;
@@ -25,7 +23,7 @@ impl<'tcx> MirPass<'tcx> for RemoveUninitDrops {
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
         let param_env = tcx.param_env(body.source.def_id());
         let move_data =
-            MoveData::gather_moves(&body, tcx, param_env, |ty| ty.needs_drop(tcx, param_env));
+            MoveData::gather_moves(body, tcx, param_env, |ty| ty.needs_drop(tcx, param_env));
 
         let mdpe = MoveDataParamEnv { move_data, param_env };
         let mut maybe_inits = MaybeInitializedPlaces::new(tcx, body, &mdpe)
diff --git a/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs b/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs
index 08b2a6537..5d528bed3 100644
--- a/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs
+++ b/compiler/rustc_mir_transform/src/remove_unneeded_drops.rs
@@ -4,7 +4,6 @@
 //! useful because (unlike MIR building) it runs after type checking, so it can make use of
 //! `Reveal::All` to provide more precise type information.
 
-use crate::MirPass;
 use rustc_middle::mir::*;
 use rustc_middle::ty::TyCtxt;
 
diff --git a/compiler/rustc_mir_transform/src/remove_zsts.rs b/compiler/rustc_mir_transform/src/remove_zsts.rs
index 5aa3c3cfe..34d57a453 100644
--- a/compiler/rustc_mir_transform/src/remove_zsts.rs
+++ b/compiler/rustc_mir_transform/src/remove_zsts.rs
@@ -1,6 +1,5 @@
 //! Removes operations on ZST places, and convert ZST operands to constants.
 
-use crate::MirPass;
 use rustc_middle::mir::visit::*;
 use rustc_middle::mir::*;
 use rustc_middle::ty::{self, Ty, TyCtxt};
@@ -17,6 +16,11 @@ impl<'tcx> MirPass<'tcx> for RemoveZsts {
         if tcx.type_of(body.source.def_id()).instantiate_identity().is_coroutine() {
             return;
         }
+
+        if !tcx.consider_optimizing(|| format!("RemoveZsts - {:?}", body.source.def_id())) {
+            return;
+        }
+
         let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id());
         let local_decls = &body.local_decls;
         let mut replacer = Replacer { tcx, param_env, local_decls };
@@ -125,12 +129,6 @@ impl<'tcx> MutVisitor<'tcx> for Replacer<'_, 'tcx> {
         if let Some(place_for_ty) = place_for_ty
             && let ty = place_for_ty.ty(self.local_decls, self.tcx).ty
             && self.known_to_be_zst(ty)
-            && self.tcx.consider_optimizing(|| {
-                format!(
-                    "RemoveZsts - Place: {:?} SourceInfo: {:?}",
-                    place_for_ty, statement.source_info
-                )
-            })
         {
             statement.make_nop();
         } else {
diff --git a/compiler/rustc_mir_transform/src/reveal_all.rs b/compiler/rustc_mir_transform/src/reveal_all.rs
index 1626cf3c0..4d2eca578 100644
--- a/compiler/rustc_mir_transform/src/reveal_all.rs
+++ b/compiler/rustc_mir_transform/src/reveal_all.rs
@@ -1,6 +1,5 @@
 //! Normalizes MIR in RevealAll mode.
 
-use crate::MirPass;
 use rustc_middle::mir::visit::*;
 use rustc_middle::mir::*;
 use rustc_middle::ty::{self, Ty, TyCtxt};
diff --git a/compiler/rustc_mir_transform/src/separate_const_switch.rs b/compiler/rustc_mir_transform/src/separate_const_switch.rs
index 907cfe758..6e22690d8 100644
--- a/compiler/rustc_mir_transform/src/separate_const_switch.rs
+++ b/compiler/rustc_mir_transform/src/separate_const_switch.rs
@@ -37,7 +37,6 @@
 //! simplicity rather than completeness (it notably
 //! sometimes duplicates abusively).
 
-use crate::MirPass;
 use rustc_middle::mir::*;
 use rustc_middle::ty::TyCtxt;
 use smallvec::SmallVec;
diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs
index ab7961321..fba73d519 100644
--- a/compiler/rustc_mir_transform/src/shim.rs
+++ b/compiler/rustc_mir_transform/src/shim.rs
@@ -113,8 +113,8 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> Body<'
             &deref_separator::Derefer,
             &remove_noop_landing_pads::RemoveNoopLandingPads,
             &simplify::SimplifyCfg::MakeShim,
-            &add_call_guards::CriticalCallEdges,
             &abort_unwinding_calls::AbortUnwindingCalls,
+            &add_call_guards::CriticalCallEdges,
         ],
         Some(MirPhase::Runtime(RuntimePhase::Optimized)),
     );
@@ -181,7 +181,7 @@ fn build_drop_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, ty: Option<Ty<'tcx>>)
         GenericArgs::identity_for_item(tcx, def_id)
     };
     let sig = tcx.fn_sig(def_id).instantiate(tcx, args);
-    let sig = tcx.erase_late_bound_regions(sig);
+    let sig = tcx.instantiate_bound_regions_with_erased(sig);
     let span = tcx.def_span(def_id);
 
     let source_info = SourceInfo::outermost(span);
@@ -418,7 +418,7 @@ impl<'tcx> CloneShimBuilder<'tcx> {
         // otherwise going to be TySelf and we can't index
         // or access fields of a Place of type TySelf.
         let sig = tcx.fn_sig(def_id).instantiate(tcx, &[self_ty.into()]);
-        let sig = tcx.erase_late_bound_regions(sig);
+        let sig = tcx.instantiate_bound_regions_with_erased(sig);
         let span = tcx.def_span(def_id);
 
         CloneShimBuilder {
@@ -656,7 +656,7 @@ fn build_call_shim<'tcx>(
     // to substitute into the signature of the shim. It is not necessary for users of this
     // MIR body to perform further substitutions (see `InstanceDef::has_polymorphic_mir_body`).
     let (sig_args, untuple_args) = if let ty::InstanceDef::FnPtrShim(_, ty) = instance {
-        let sig = tcx.erase_late_bound_regions(ty.fn_sig(tcx));
+        let sig = tcx.instantiate_bound_regions_with_erased(ty.fn_sig(tcx));
 
         let untuple_args = sig.inputs();
 
@@ -670,7 +670,7 @@ fn build_call_shim<'tcx>(
 
     let def_id = instance.def_id();
     let sig = tcx.fn_sig(def_id);
-    let sig = sig.map_bound(|sig| tcx.erase_late_bound_regions(sig));
+    let sig = sig.map_bound(|sig| tcx.instantiate_bound_regions_with_erased(sig));
 
     assert_eq!(sig_args.is_some(), !instance.has_polymorphic_mir_body());
     let mut sig = if let Some(sig_args) = sig_args {
diff --git a/compiler/rustc_mir_transform/src/simplify.rs b/compiler/rustc_mir_transform/src/simplify.rs
index 0a1c01114..856a0f227 100644
--- a/compiler/rustc_mir_transform/src/simplify.rs
+++ b/compiler/rustc_mir_transform/src/simplify.rs
@@ -27,7 +27,6 @@
 //! naively generate still contains the `_a = ()` write in the unreachable block "after" the
 //! return.
 
-use crate::MirPass;
 use rustc_data_structures::fx::FxIndexSet;
 use rustc_index::{Idx, IndexSlice, IndexVec};
 use rustc_middle::mir::visit::{MutVisitor, MutatingUseContext, PlaceContext, Visitor};
@@ -74,7 +73,7 @@ pub fn simplify_cfg<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
 
 impl<'tcx> MirPass<'tcx> for SimplifyCfg {
     fn name(&self) -> &'static str {
-        &self.name()
+        self.name()
     }
 
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
diff --git a/compiler/rustc_mir_transform/src/simplify_branches.rs b/compiler/rustc_mir_transform/src/simplify_branches.rs
index 1f0e605c3..35a052166 100644
--- a/compiler/rustc_mir_transform/src/simplify_branches.rs
+++ b/compiler/rustc_mir_transform/src/simplify_branches.rs
@@ -1,4 +1,3 @@
-use crate::MirPass;
 use rustc_middle::mir::*;
 use rustc_middle::ty::TyCtxt;
 
diff --git a/compiler/rustc_mir_transform/src/sroa.rs b/compiler/rustc_mir_transform/src/sroa.rs
index 7de4ca667..06d5e17fd 100644
--- a/compiler/rustc_mir_transform/src/sroa.rs
+++ b/compiler/rustc_mir_transform/src/sroa.rs
@@ -1,4 +1,3 @@
-use crate::MirPass;
 use rustc_data_structures::flat_map_in_place::FlatMapInPlace;
 use rustc_index::bit_set::{BitSet, GrowableBitSet};
 use rustc_index::IndexVec;
@@ -167,7 +166,7 @@ impl<'tcx> ReplacementMap<'tcx> {
         };
         let fields = self.fragments[place.local].as_ref()?;
         let (_, new_local) = fields[f]?;
-        Some(Place { local: new_local, projection: tcx.mk_place_elems(&rest) })
+        Some(Place { local: new_local, projection: tcx.mk_place_elems(rest) })
     }
 
     fn place_fragments(
diff --git a/compiler/rustc_mir_transform/src/ssa.rs b/compiler/rustc_mir_transform/src/ssa.rs
index 1f59c790b..3a6e1ef34 100644
--- a/compiler/rustc_mir_transform/src/ssa.rs
+++ b/compiler/rustc_mir_transform/src/ssa.rs
@@ -40,7 +40,8 @@ impl SsaLocals {
         let dominators = body.basic_blocks.dominators();
 
         let direct_uses = IndexVec::from_elem(0, &body.local_decls);
-        let mut visitor = SsaVisitor { assignments, assignment_order, dominators, direct_uses };
+        let mut visitor =
+            SsaVisitor { body, assignments, assignment_order, dominators, direct_uses };
 
         for local in body.args_iter() {
             visitor.assignments[local] = Set1::One(DefLocation::Argument);
@@ -110,7 +111,7 @@ impl SsaLocals {
         body: &'a Body<'tcx>,
     ) -> impl Iterator<Item = (Local, &'a Rvalue<'tcx>, Location)> + 'a {
         self.assignment_order.iter().filter_map(|&local| {
-            if let Set1::One(DefLocation::Body(loc)) = self.assignments[local] {
+            if let Set1::One(DefLocation::Assignment(loc)) = self.assignments[local] {
                 let stmt = body.stmt_at(loc).left()?;
                 // `loc` must point to a direct assignment to `local`.
                 let Some((target, rvalue)) = stmt.kind.as_assign() else { bug!() };
@@ -134,21 +135,21 @@ impl SsaLocals {
                     AssignedValue::Arg,
                     Location { block: START_BLOCK, statement_index: 0 },
                 ),
-                Set1::One(DefLocation::Body(loc)) => {
+                Set1::One(DefLocation::Assignment(loc)) => {
                     let bb = &mut basic_blocks[loc.block];
-                    let value = if loc.statement_index < bb.statements.len() {
-                        // `loc` must point to a direct assignment to `local`.
-                        let stmt = &mut bb.statements[loc.statement_index];
-                        let StatementKind::Assign(box (target, ref mut rvalue)) = stmt.kind else {
-                            bug!()
-                        };
-                        assert_eq!(target.as_local(), Some(local));
-                        AssignedValue::Rvalue(rvalue)
-                    } else {
-                        let term = bb.terminator_mut();
-                        AssignedValue::Terminator(&mut term.kind)
+                    // `loc` must point to a direct assignment to `local`.
+                    let stmt = &mut bb.statements[loc.statement_index];
+                    let StatementKind::Assign(box (target, ref mut rvalue)) = stmt.kind else {
+                        bug!()
                     };
-                    f(local, value, loc)
+                    assert_eq!(target.as_local(), Some(local));
+                    f(local, AssignedValue::Rvalue(rvalue), loc)
+                }
+                Set1::One(DefLocation::CallReturn { call, .. }) => {
+                    let bb = &mut basic_blocks[call];
+                    let loc = Location { block: call, statement_index: bb.statements.len() };
+                    let term = bb.terminator_mut();
+                    f(local, AssignedValue::Terminator(&mut term.kind), loc)
                 }
                 _ => {}
             }
@@ -201,14 +202,15 @@ impl SsaLocals {
     }
 }
 
-struct SsaVisitor<'a> {
+struct SsaVisitor<'tcx, 'a> {
+    body: &'a Body<'tcx>,
     dominators: &'a Dominators<BasicBlock>,
     assignments: IndexVec<Local, Set1<DefLocation>>,
     assignment_order: Vec<Local>,
     direct_uses: IndexVec<Local, u32>,
 }
 
-impl SsaVisitor<'_> {
+impl SsaVisitor<'_, '_> {
     fn check_dominates(&mut self, local: Local, loc: Location) {
         let set = &mut self.assignments[local];
         let assign_dominates = match *set {
@@ -224,7 +226,7 @@ impl SsaVisitor<'_> {
     }
 }
 
-impl<'tcx> Visitor<'tcx> for SsaVisitor<'_> {
+impl<'tcx> Visitor<'tcx> for SsaVisitor<'tcx, '_> {
     fn visit_local(&mut self, local: Local, ctxt: PlaceContext, loc: Location) {
         match ctxt {
             PlaceContext::MutatingUse(MutatingUseContext::Projection)
@@ -250,9 +252,18 @@ impl<'tcx> Visitor<'tcx> for SsaVisitor<'_> {
 
     fn visit_place(&mut self, place: &Place<'tcx>, ctxt: PlaceContext, loc: Location) {
         let location = match ctxt {
-            PlaceContext::MutatingUse(
-                MutatingUseContext::Store | MutatingUseContext::Call | MutatingUseContext::Yield,
-            ) => Some(DefLocation::Body(loc)),
+            PlaceContext::MutatingUse(MutatingUseContext::Store) => {
+                Some(DefLocation::Assignment(loc))
+            }
+            PlaceContext::MutatingUse(MutatingUseContext::Call) => {
+                let call = loc.block;
+                let TerminatorKind::Call { target, .. } =
+                    self.body.basic_blocks[call].terminator().kind
+                else {
+                    bug!()
+                };
+                Some(DefLocation::CallReturn { call, target })
+            }
             _ => None,
         };
         if let Some(location) = location
@@ -359,7 +370,7 @@ impl StorageLiveLocals {
             for (statement_index, statement) in bbdata.statements.iter().enumerate() {
                 if let StatementKind::StorageLive(local) = statement.kind {
                     storage_live[local]
-                        .insert(DefLocation::Body(Location { block, statement_index }));
+                        .insert(DefLocation::Assignment(Location { block, statement_index }));
                 }
             }
         }
diff --git a/compiler/rustc_mir_transform/src/uninhabited_enum_branching.rs b/compiler/rustc_mir_transform/src/uninhabited_enum_branching.rs
index 98f67e18a..e68d37f4c 100644
--- a/compiler/rustc_mir_transform/src/uninhabited_enum_branching.rs
+++ b/compiler/rustc_mir_transform/src/uninhabited_enum_branching.rs
@@ -86,7 +86,7 @@ impl<'tcx> MirPass<'tcx> for UninhabitedEnumBranching {
                 continue;
             }
 
-            let Some(discriminant_ty) = get_switched_on_type(&bb_data, tcx, body) else { continue };
+            let Some(discriminant_ty) = get_switched_on_type(bb_data, tcx, body) else { continue };
 
             let layout = tcx.layout_of(
                 tcx.param_env_reveal_all_normalized(body.source.def_id()).and(discriminant_ty),
diff --git a/compiler/rustc_mir_transform/src/unreachable_prop.rs b/compiler/rustc_mir_transform/src/unreachable_prop.rs
index 919e8d6a2..f12a6aa24 100644
--- a/compiler/rustc_mir_transform/src/unreachable_prop.rs
+++ b/compiler/rustc_mir_transform/src/unreachable_prop.rs
@@ -2,7 +2,6 @@
 //! when all of their successors are unreachable. This is achieved through a
 //! post-order traversal of the blocks.
 
-use crate::MirPass;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_middle::mir::interpret::Scalar;
 use rustc_middle::mir::patch::MirPatch;
diff --git a/compiler/rustc_monomorphize/src/collector.rs b/compiler/rustc_monomorphize/src/collector.rs
index 65bdcf107..a68bfcd06 100644
--- a/compiler/rustc_monomorphize/src/collector.rs
+++ b/compiler/rustc_monomorphize/src/collector.rs
@@ -176,6 +176,7 @@ use rustc_middle::mir::visit::Visitor as MirVisitor;
 use rustc_middle::mir::{self, Location};
 use rustc_middle::query::TyCtxtAt;
 use rustc_middle::ty::adjustment::{CustomCoerceUnsized, PointerCoercion};
+use rustc_middle::ty::layout::ValidityRequirement;
 use rustc_middle::ty::print::with_no_trimmed_paths;
 use rustc_middle::ty::{
     self, AssocKind, GenericParamDefKind, Instance, InstanceDef, Ty, TyCtxt, TypeFoldable,
@@ -370,7 +371,7 @@ fn collect_items_rec<'tcx>(
     // current step of mono items collection.
     //
     // FIXME: don't rely on global state, instead bubble up errors. Note: this is very hard to do.
-    let error_count = tcx.sess.diagnostic().err_count();
+    let error_count = tcx.sess.dcx().err_count();
 
     match starting_item.node {
         MonoItem::Static(def_id) => {
@@ -385,8 +386,8 @@ fn collect_items_rec<'tcx>(
             recursion_depth_reset = None;
 
             if let Ok(alloc) = tcx.eval_static_initializer(def_id) {
-                for &id in alloc.inner().provenance().ptrs().values() {
-                    collect_alloc(tcx, id, &mut used_items);
+                for &prov in alloc.inner().provenance().ptrs().values() {
+                    collect_alloc(tcx, prov.alloc_id(), &mut used_items);
                 }
             }
 
@@ -458,7 +459,7 @@ fn collect_items_rec<'tcx>(
 
     // Check for PMEs and emit a diagnostic if one happened. To try to show relevant edges of the
     // mono item graph.
-    if tcx.sess.diagnostic().err_count() > error_count
+    if tcx.sess.dcx().err_count() > error_count
         && starting_item.node.is_generic_fn(tcx)
         && starting_item.node.is_user_defined()
     {
@@ -740,7 +741,7 @@ impl<'a, 'tcx> MirVisitor<'tcx> for MirUsedCollector<'a, 'tcx> {
             ) => {
                 let fn_ty = operand.ty(self.body, self.tcx);
                 let fn_ty = self.monomorphize(fn_ty);
-                visit_fn_use(self.tcx, fn_ty, false, span, &mut self.output);
+                visit_fn_use(self.tcx, fn_ty, false, span, self.output);
             }
             mir::Rvalue::Cast(
                 mir::CastKind::PointerCoercion(PointerCoercion::ClosureFnPointer(_)),
@@ -816,7 +817,7 @@ impl<'a, 'tcx> MirVisitor<'tcx> for MirUsedCollector<'a, 'tcx> {
                 let callee_ty = func.ty(self.body, tcx);
                 let callee_ty = self.monomorphize(callee_ty);
                 self.check_fn_args_move_size(callee_ty, args, location);
-                visit_fn_use(self.tcx, callee_ty, true, source, &mut self.output)
+                visit_fn_use(self.tcx, callee_ty, true, source, self.output)
             }
             mir::TerminatorKind::Drop { ref place, .. } => {
                 let ty = place.ty(self.body, self.tcx).ty;
@@ -828,7 +829,7 @@ impl<'a, 'tcx> MirVisitor<'tcx> for MirUsedCollector<'a, 'tcx> {
                     match *op {
                         mir::InlineAsmOperand::SymFn { ref value } => {
                             let fn_ty = self.monomorphize(value.const_.ty());
-                            visit_fn_use(self.tcx, fn_ty, false, source, &mut self.output);
+                            visit_fn_use(self.tcx, fn_ty, false, source, self.output);
                         }
                         mir::InlineAsmOperand::SymStatic { def_id } => {
                             let instance = Instance::mono(self.tcx, def_id);
@@ -844,6 +845,9 @@ impl<'a, 'tcx> MirVisitor<'tcx> for MirUsedCollector<'a, 'tcx> {
             mir::TerminatorKind::Assert { ref msg, .. } => {
                 let lang_item = match &**msg {
                     mir::AssertKind::BoundsCheck { .. } => LangItem::PanicBoundsCheck,
+                    mir::AssertKind::MisalignedPointerDereference { .. } => {
+                        LangItem::PanicMisalignedPointerDereference
+                    }
                     _ => LangItem::Panic,
                 };
                 push_mono_lang_item(self, lang_item);
@@ -920,6 +924,21 @@ fn visit_instance_use<'tcx>(
         return;
     }
 
+    // The intrinsics assert_inhabited, assert_zero_valid, and assert_mem_uninitialized_valid will
+    // be lowered in codegen to nothing or a call to panic_nounwind. So if we encounter any
+    // of those intrinsics, we need to include a mono item for panic_nounwind, else we may try to
+    // codegen a call to that function without generating code for the function itself.
+    if let ty::InstanceDef::Intrinsic(def_id) = instance.def {
+        let name = tcx.item_name(def_id);
+        if let Some(_requirement) = ValidityRequirement::from_intrinsic(name) {
+            let def_id = tcx.lang_items().get(LangItem::PanicNounwind).unwrap();
+            let panic_instance = Instance::mono(tcx, def_id);
+            if should_codegen_locally(tcx, &panic_instance) {
+                output.push(create_fn_mono_item(tcx, panic_instance, source));
+            }
+        }
+    }
+
     match instance.def {
         ty::InstanceDef::Virtual(..) | ty::InstanceDef::Intrinsic(_) => {
             if !is_direct_call {
@@ -1118,7 +1137,7 @@ fn create_mono_items_for_vtable_methods<'tcx>(
 ) {
     assert!(!trait_ty.has_escaping_bound_vars() && !impl_ty.has_escaping_bound_vars());
 
-    if let ty::Dynamic(ref trait_ty, ..) = trait_ty.kind() {
+    if let ty::Dynamic(trait_ty, ..) = trait_ty.kind() {
         if let Some(principal) = trait_ty.principal() {
             let poly_trait_ref = principal.with_self_ty(tcx, impl_ty);
             assert!(!poly_trait_ref.has_escaping_bound_vars());
@@ -1191,7 +1210,7 @@ impl<'v> RootCollector<'_, 'v> {
 
                 // but even just declaring them must collect the items they refer to
                 if let Ok(val) = self.tcx.const_eval_poly(id.owner_id.to_def_id()) {
-                    collect_const_value(self.tcx, val, &mut self.output);
+                    collect_const_value(self.tcx, val, self.output);
                 }
             }
             DefKind::Impl { .. } => {
@@ -1363,9 +1382,9 @@ fn collect_alloc<'tcx>(tcx: TyCtxt<'tcx>, alloc_id: AllocId, output: &mut MonoIt
         }
         GlobalAlloc::Memory(alloc) => {
             trace!("collecting {:?} with {:#?}", alloc_id, alloc);
-            for &inner in alloc.inner().provenance().ptrs().values() {
+            for &prov in alloc.inner().provenance().ptrs().values() {
                 rustc_data_structures::stack::ensure_sufficient_stack(|| {
-                    collect_alloc(tcx, inner, output);
+                    collect_alloc(tcx, prov.alloc_id(), output);
                 });
             }
         }
@@ -1422,14 +1441,14 @@ fn collect_used_items<'tcx>(
     // and abort compilation if any of them errors.
     MirUsedCollector {
         tcx,
-        body: &body,
+        body: body,
         output,
         instance,
         move_size_spans: vec![],
         visiting_call_terminator: false,
         skip_move_check_fns: None,
     }
-    .visit_body(&body);
+    .visit_body(body);
 }
 
 #[instrument(skip(tcx, output), level = "debug")]
@@ -1440,12 +1459,12 @@ fn collect_const_value<'tcx>(
 ) {
     match value {
         mir::ConstValue::Scalar(Scalar::Ptr(ptr, _size)) => {
-            collect_alloc(tcx, ptr.provenance, output)
+            collect_alloc(tcx, ptr.provenance.alloc_id(), output)
         }
         mir::ConstValue::Indirect { alloc_id, .. } => collect_alloc(tcx, alloc_id, output),
         mir::ConstValue::Slice { data, meta: _ } => {
-            for &id in data.inner().provenance().ptrs().values() {
-                collect_alloc(tcx, id, output);
+            for &prov in data.inner().provenance().ptrs().values() {
+                collect_alloc(tcx, prov.alloc_id(), output);
             }
         }
         _ => {}
diff --git a/compiler/rustc_monomorphize/src/errors.rs b/compiler/rustc_monomorphize/src/errors.rs
index d242a7bae..247b22455 100644
--- a/compiler/rustc_monomorphize/src/errors.rs
+++ b/compiler/rustc_monomorphize/src/errors.rs
@@ -51,9 +51,9 @@ impl IntoDiagnostic<'_> for UnusedGenericParamsHint {
     #[track_caller]
     fn into_diagnostic(
         self,
-        handler: &'_ rustc_errors::Handler,
+        dcx: &'_ rustc_errors::DiagCtxt,
     ) -> rustc_errors::DiagnosticBuilder<'_, ErrorGuaranteed> {
-        let mut diag = handler.struct_err(fluent::monomorphize_unused_generic_params);
+        let mut diag = dcx.struct_err(fluent::monomorphize_unused_generic_params);
         diag.set_span(self.span);
         for (span, name) in self.param_spans.into_iter().zip(self.param_names) {
             // FIXME: I can figure out how to do a label with a fluent string with a fixed message,
diff --git a/compiler/rustc_monomorphize/src/lib.rs b/compiler/rustc_monomorphize/src/lib.rs
index 5f05020ac..2f5f2d15c 100644
--- a/compiler/rustc_monomorphize/src/lib.rs
+++ b/compiler/rustc_monomorphize/src/lib.rs
@@ -10,8 +10,6 @@ extern crate tracing;
 #[macro_use]
 extern crate rustc_middle;
 
-use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
-use rustc_fluent_macro::fluent_messages;
 use rustc_hir::lang_items::LangItem;
 use rustc_middle::query::{Providers, TyCtxtAt};
 use rustc_middle::traits;
@@ -24,7 +22,7 @@ mod partitioning;
 mod polymorphize;
 mod util;
 
-fluent_messages! { "../messages.ftl" }
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
 
 fn custom_coerce_unsize_info<'tcx>(
     tcx: TyCtxtAt<'tcx>,
diff --git a/compiler/rustc_monomorphize/src/partitioning.rs b/compiler/rustc_monomorphize/src/partitioning.rs
index 4009e2892..d47d3e5e7 100644
--- a/compiler/rustc_monomorphize/src/partitioning.rs
+++ b/compiler/rustc_monomorphize/src/partitioning.rs
@@ -212,11 +212,17 @@ where
     let cgu_name_cache = &mut FxHashMap::default();
 
     for mono_item in mono_items {
-        // Handle only root items directly here. Inlined items are handled at
-        // the bottom of the loop based on reachability.
+        // Handle only root (GloballyShared) items directly here. Inlined (LocalCopy) items
+        // are handled at the bottom of the loop based on reachability, with one exception.
+        // The #[lang = "start"] item is the program entrypoint, so there are no calls to it in MIR.
+        // So even if its mode is LocalCopy, we need to treat it like a root.
         match mono_item.instantiation_mode(cx.tcx) {
             InstantiationMode::GloballyShared { .. } => {}
-            InstantiationMode::LocalCopy => continue,
+            InstantiationMode::LocalCopy => {
+                if Some(mono_item.def_id()) != cx.tcx.lang_items().start_fn() {
+                    continue;
+                }
+            }
         }
 
         let characteristic_def_id = characteristic_def_id_of_mono_item(cx.tcx, mono_item);
@@ -436,12 +442,12 @@ fn merge_codegen_units<'tcx>(
         for cgu in codegen_units.iter_mut() {
             if let Some(new_cgu_name) = new_cgu_names.get(&cgu.name()) {
                 if cx.tcx.sess.opts.unstable_opts.human_readable_cgu_names {
-                    cgu.set_name(Symbol::intern(&new_cgu_name));
+                    cgu.set_name(Symbol::intern(new_cgu_name));
                 } else {
                     // If we don't require CGU names to be human-readable,
                     // we use a fixed length hash of the composite CGU name
                     // instead.
-                    let new_cgu_name = CodegenUnit::mangle_name(&new_cgu_name);
+                    let new_cgu_name = CodegenUnit::mangle_name(new_cgu_name);
                     cgu.set_name(Symbol::intern(&new_cgu_name));
                 }
             }
@@ -883,7 +889,7 @@ fn mono_item_visibility<'tcx>(
 }
 
 fn default_visibility(tcx: TyCtxt<'_>, id: DefId, is_generic: bool) -> Visibility {
-    if !tcx.sess.target.default_hidden_visibility {
+    if !tcx.sess.default_hidden_visibility() {
         return Visibility::Default;
     }
 
@@ -1140,7 +1146,7 @@ fn collect_and_partition_mono_items(tcx: TyCtxt<'_>, (): ()) -> (&DefIdSet, &[Co
     // Output monomorphization stats per def_id
     if let SwitchWithOptPath::Enabled(ref path) = tcx.sess.opts.unstable_opts.dump_mono_stats {
         if let Err(err) =
-            dump_mono_items_stats(tcx, &codegen_units, path, tcx.crate_name(LOCAL_CRATE))
+            dump_mono_items_stats(tcx, codegen_units, path, tcx.crate_name(LOCAL_CRATE))
         {
             tcx.sess.emit_fatal(CouldntDumpMonoStats { error: err.to_string() });
         }
diff --git a/compiler/rustc_monomorphize/src/polymorphize.rs b/compiler/rustc_monomorphize/src/polymorphize.rs
index a3b35eab4..91abbb216 100644
--- a/compiler/rustc_monomorphize/src/polymorphize.rs
+++ b/compiler/rustc_monomorphize/src/polymorphize.rs
@@ -131,7 +131,7 @@ fn mark_used_by_default_parameters<'tcx>(
     unused_parameters: &mut UnusedGenericParams,
 ) {
     match tcx.def_kind(def_id) {
-        DefKind::Closure | DefKind::Coroutine => {
+        DefKind::Closure => {
             for param in &generics.params {
                 debug!(?param, "(closure/gen)");
                 unused_parameters.mark_used(param.index);
@@ -248,7 +248,7 @@ impl<'a, 'tcx> Visitor<'tcx> for MarkUsedGenericParams<'a, 'tcx> {
     fn visit_local_decl(&mut self, local: Local, local_decl: &LocalDecl<'tcx>) {
         if local == Local::from_usize(1) {
             let def_kind = self.tcx.def_kind(self.def_id);
-            if matches!(def_kind, DefKind::Closure | DefKind::Coroutine) {
+            if matches!(def_kind, DefKind::Closure) {
                 // Skip visiting the closure/coroutine that is currently being processed. This only
                 // happens because the first argument to the closure is a reference to itself and
                 // that will call `visit_args`, resulting in each generic parameter captured being
diff --git a/compiler/rustc_next_trait_solver/Cargo.toml b/compiler/rustc_next_trait_solver/Cargo.toml
new file mode 100644
index 000000000..9d496fd8e
--- /dev/null
+++ b/compiler/rustc_next_trait_solver/Cargo.toml
@@ -0,0 +1,13 @@
+[package]
+name = "rustc_next_trait_solver"
+version = "0.0.0"
+edition = "2021"
+
+[dependencies]
+rustc_type_ir = { path = "../rustc_type_ir", default-features = false }
+
+[features]
+default = ["nightly"]
+nightly = [
+    "rustc_type_ir/nightly",
+]
\ No newline at end of file
diff --git a/compiler/rustc_trait_selection/src/solve/canonicalize.rs b/compiler/rustc_next_trait_solver/src/canonicalizer.rs
index 377ae1b4e..ac2e8960b 100644
--- a/compiler/rustc_trait_selection/src/solve/canonicalize.rs
+++ b/compiler/rustc_next_trait_solver/src/canonicalizer.rs
@@ -1,17 +1,10 @@
 use std::cmp::Ordering;
 
-use crate::infer::InferCtxt;
-use rustc_middle::infer::canonical::Canonical;
-use rustc_middle::infer::canonical::CanonicalTyVarKind;
-use rustc_middle::infer::canonical::CanonicalVarInfo;
-use rustc_middle::infer::canonical::CanonicalVarInfos;
-use rustc_middle::infer::canonical::CanonicalVarKind;
-use rustc_middle::ty::BoundRegionKind::BrAnon;
-use rustc_middle::ty::BoundTyKind;
-use rustc_middle::ty::TyCtxt;
-use rustc_middle::ty::TypeVisitableExt;
-use rustc_middle::ty::{self, Ty};
-use rustc_middle::ty::{TypeFoldable, TypeFolder, TypeSuperFoldable};
+use rustc_type_ir::fold::{TypeFoldable, TypeFolder, TypeSuperFoldable};
+use rustc_type_ir::{
+    self as ty, Canonical, CanonicalTyVarKind, CanonicalVarInfo, CanonicalVarKind, ConstTy,
+    InferCtxtLike, Interner, IntoKind, PlaceholderLike,
+};
 
 /// Whether we're canonicalizing a query input or the query response.
 ///
@@ -42,23 +35,22 @@ pub enum CanonicalizeMode {
     },
 }
 
-pub struct Canonicalizer<'a, 'tcx> {
-    infcx: &'a InferCtxt<'tcx>,
+pub struct Canonicalizer<'a, Infcx: InferCtxtLike<Interner = I>, I: Interner> {
+    infcx: &'a Infcx,
     canonicalize_mode: CanonicalizeMode,
 
-    variables: &'a mut Vec<ty::GenericArg<'tcx>>,
-    primitive_var_infos: Vec<CanonicalVarInfo<'tcx>>,
+    variables: &'a mut Vec<I::GenericArg>,
+    primitive_var_infos: Vec<CanonicalVarInfo<I>>,
     binder_index: ty::DebruijnIndex,
 }
 
-impl<'a, 'tcx> Canonicalizer<'a, 'tcx> {
-    #[instrument(level = "debug", skip(infcx), ret)]
-    pub fn canonicalize<T: TypeFoldable<TyCtxt<'tcx>>>(
-        infcx: &'a InferCtxt<'tcx>,
+impl<'a, Infcx: InferCtxtLike<Interner = I>, I: Interner> Canonicalizer<'a, Infcx, I> {
+    pub fn canonicalize<T: TypeFoldable<I>>(
+        infcx: &'a Infcx,
         canonicalize_mode: CanonicalizeMode,
-        variables: &'a mut Vec<ty::GenericArg<'tcx>>,
+        variables: &'a mut Vec<I::GenericArg>,
         value: T,
-    ) -> Canonical<'tcx, T> {
+    ) -> ty::Canonical<I, T> {
         let mut canonicalizer = Canonicalizer {
             infcx,
             canonicalize_mode,
@@ -69,15 +61,16 @@ impl<'a, 'tcx> Canonicalizer<'a, 'tcx> {
         };
 
         let value = value.fold_with(&mut canonicalizer);
-        assert!(!value.has_infer());
-        assert!(!value.has_placeholders());
+        // FIXME: Restore these assertions. Should we uplift type flags?
+        // assert!(!value.has_infer(), "unexpected infer in {value:?}");
+        // assert!(!value.has_placeholders(), "unexpected placeholders in {value:?}");
 
         let (max_universe, variables) = canonicalizer.finalize();
 
         Canonical { max_universe, variables, value }
     }
 
-    fn finalize(self) -> (ty::UniverseIndex, CanonicalVarInfos<'tcx>) {
+    fn finalize(self) -> (ty::UniverseIndex, I::CanonicalVars) {
         let mut var_infos = self.primitive_var_infos;
         // See the rustc-dev-guide section about how we deal with universes
         // during canonicalization in the new solver.
@@ -105,7 +98,7 @@ impl<'a, 'tcx> Canonicalizer<'a, 'tcx> {
                     .max()
                     .unwrap_or(ty::UniverseIndex::ROOT);
 
-                let var_infos = self.infcx.tcx.mk_canonical_var_infos(&var_infos);
+                let var_infos = self.infcx.interner().mk_canonical_var_infos(&var_infos);
                 return (max_universe, var_infos);
             }
         }
@@ -131,7 +124,7 @@ impl<'a, 'tcx> Canonicalizer<'a, 'tcx> {
         let mut existential_in_new_uv = false;
         let mut next_orig_uv = Some(ty::UniverseIndex::ROOT);
         while let Some(orig_uv) = next_orig_uv.take() {
-            let mut update_uv = |var: &mut CanonicalVarInfo<'tcx>, orig_uv, is_existential| {
+            let mut update_uv = |var: &mut CanonicalVarInfo<I>, orig_uv, is_existential| {
                 let uv = var.universe();
                 match uv.cmp(&orig_uv) {
                     Ordering::Less => (), // Already updated
@@ -187,19 +180,22 @@ impl<'a, 'tcx> Canonicalizer<'a, 'tcx> {
             }
         }
 
-        let var_infos = self.infcx.tcx.mk_canonical_var_infos(&var_infos);
+        let var_infos = self.infcx.interner().mk_canonical_var_infos(&var_infos);
         (curr_compressed_uv, var_infos)
     }
 }
 
-impl<'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'_, 'tcx> {
-    fn interner(&self) -> TyCtxt<'tcx> {
-        self.infcx.tcx
+impl<Infcx: InferCtxtLike<Interner = I>, I: Interner> TypeFolder<I>
+    for Canonicalizer<'_, Infcx, I>
+{
+    fn interner(&self) -> I {
+        self.infcx.interner()
     }
 
-    fn fold_binder<T>(&mut self, t: ty::Binder<'tcx, T>) -> ty::Binder<'tcx, T>
+    fn fold_binder<T>(&mut self, t: I::Binder<T>) -> I::Binder<T>
     where
-        T: TypeFoldable<TyCtxt<'tcx>>,
+        T: TypeFoldable<I>,
+        I::Binder<T>: TypeSuperFoldable<I>,
     {
         self.binder_index.shift_in(1);
         let t = t.super_fold_with(self);
@@ -207,22 +203,9 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'_, 'tcx> {
         t
     }
 
-    fn fold_region(&mut self, r: ty::Region<'tcx>) -> ty::Region<'tcx> {
-        if let ty::ReVar(vid) = *r {
-            let resolved_region = self
-                .infcx
-                .inner
-                .borrow_mut()
-                .unwrap_region_constraints()
-                .opportunistic_resolve_var(self.infcx.tcx, vid);
-            assert_eq!(
-                r, resolved_region,
-                "region var should have been resolved, {r} -> {resolved_region}"
-            );
-        }
-
-        let kind = match *r {
-            ty::ReLateBound(..) => return r,
+    fn fold_region(&mut self, r: I::Region) -> I::Region {
+        let kind = match r.kind() {
+            ty::ReBound(..) => return r,
 
             // We may encounter `ReStatic` in item signatures or the hidden type
             // of an opaque. `ReErased` should only be encountered in the hidden
@@ -237,9 +220,11 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'_, 'tcx> {
                 CanonicalizeMode::Response { .. } => return r,
             },
 
-            ty::ReFree(_) | ty::ReEarlyBound(_) => match self.canonicalize_mode {
+            ty::ReEarlyParam(_) | ty::ReLateParam(_) => match self.canonicalize_mode {
                 CanonicalizeMode::Input => CanonicalVarKind::Region(ty::UniverseIndex::ROOT),
-                CanonicalizeMode::Response { .. } => bug!("unexpected region in response: {r:?}"),
+                CanonicalizeMode::Response { .. } => {
+                    panic!("unexpected region in response: {r:?}")
+                }
             },
 
             ty::RePlaceholder(placeholder) => match self.canonicalize_mode {
@@ -248,20 +233,26 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'_, 'tcx> {
                 CanonicalizeMode::Response { max_input_universe } => {
                     // If we have a placeholder region inside of a query, it must be from
                     // a new universe.
-                    if max_input_universe.can_name(placeholder.universe) {
-                        bug!("new placeholder in universe {max_input_universe:?}: {r:?}");
+                    if max_input_universe.can_name(placeholder.universe()) {
+                        panic!("new placeholder in universe {max_input_universe:?}: {r:?}");
                     }
                     CanonicalVarKind::PlaceholderRegion(placeholder)
                 }
             },
 
-            ty::ReVar(_) => match self.canonicalize_mode {
-                CanonicalizeMode::Input => CanonicalVarKind::Region(ty::UniverseIndex::ROOT),
-                CanonicalizeMode::Response { .. } => {
-                    CanonicalVarKind::Region(self.infcx.universe_of_region(r))
+            ty::ReVar(vid) => {
+                assert_eq!(
+                    self.infcx.opportunistic_resolve_lt_var(vid),
+                    None,
+                    "region vid should have been resolved fully before canonicalization"
+                );
+                match self.canonicalize_mode {
+                    CanonicalizeMode::Input => CanonicalVarKind::Region(ty::UniverseIndex::ROOT),
+                    CanonicalizeMode::Response { .. } => {
+                        CanonicalVarKind::Region(self.infcx.universe_of_lt(vid).unwrap())
+                    }
                 }
-            },
-
+            }
             ty::ReError(_) => return r,
         };
 
@@ -271,55 +262,60 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'_, 'tcx> {
                 self.variables.iter().position(|&v| v == r.into()).map(ty::BoundVar::from)
             }
         };
+
         let var = existing_bound_var.unwrap_or_else(|| {
             let var = ty::BoundVar::from(self.variables.len());
             self.variables.push(r.into());
             self.primitive_var_infos.push(CanonicalVarInfo { kind });
             var
         });
-        let br = ty::BoundRegion { var, kind: BrAnon };
-        ty::Region::new_late_bound(self.interner(), self.binder_index, br)
+
+        self.interner().mk_bound_region(self.binder_index, var)
     }
 
-    fn fold_ty(&mut self, t: Ty<'tcx>) -> Ty<'tcx> {
-        let kind = match *t.kind() {
-            ty::Infer(ty::TyVar(vid)) => {
-                assert_eq!(self.infcx.root_var(vid), vid, "ty vid should have been resolved");
-                let Err(ui) = self.infcx.probe_ty_var(vid) else {
-                    bug!("ty var should have been resolved: {t}");
-                };
-                CanonicalVarKind::Ty(CanonicalTyVarKind::General(ui))
-            }
-            ty::Infer(ty::IntVar(vid)) => {
-                assert_eq!(self.infcx.opportunistic_resolve_int_var(vid), t);
-                CanonicalVarKind::Ty(CanonicalTyVarKind::Int)
-            }
-            ty::Infer(ty::FloatVar(vid)) => {
-                assert_eq!(self.infcx.opportunistic_resolve_float_var(vid), t);
-                CanonicalVarKind::Ty(CanonicalTyVarKind::Float)
-            }
-            ty::Infer(ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_)) => {
-                bug!("fresh var during canonicalization: {t:?}")
-            }
+    fn fold_ty(&mut self, t: I::Ty) -> I::Ty
+    where
+        I::Ty: TypeSuperFoldable<I>,
+    {
+        let kind = match t.kind() {
+            ty::Infer(i) => match i {
+                ty::TyVar(vid) => {
+                    assert_eq!(
+                        self.infcx.root_ty_var(vid),
+                        vid,
+                        "ty vid should have been resolved fully before canonicalization"
+                    );
+                    assert_eq!(
+                        self.infcx.probe_ty_var(vid),
+                        None,
+                        "ty vid should have been resolved fully before canonicalization"
+                    );
+
+                    CanonicalVarKind::Ty(CanonicalTyVarKind::General(
+                        self.infcx
+                            .universe_of_ty(vid)
+                            .unwrap_or_else(|| panic!("ty var should have been resolved: {t:?}")),
+                    ))
+                }
+                ty::IntVar(_) => CanonicalVarKind::Ty(CanonicalTyVarKind::Int),
+                ty::FloatVar(_) => CanonicalVarKind::Ty(CanonicalTyVarKind::Float),
+                ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_) => {
+                    todo!()
+                }
+            },
             ty::Placeholder(placeholder) => match self.canonicalize_mode {
-                CanonicalizeMode::Input => CanonicalVarKind::PlaceholderTy(ty::Placeholder {
-                    universe: placeholder.universe,
-                    bound: ty::BoundTy {
-                        var: ty::BoundVar::from_usize(self.variables.len()),
-                        kind: ty::BoundTyKind::Anon,
-                    },
-                }),
+                CanonicalizeMode::Input => CanonicalVarKind::PlaceholderTy(PlaceholderLike::new(
+                    placeholder.universe(),
+                    self.variables.len().into(),
+                )),
                 CanonicalizeMode::Response { .. } => CanonicalVarKind::PlaceholderTy(placeholder),
             },
             ty::Param(_) => match self.canonicalize_mode {
-                CanonicalizeMode::Input => CanonicalVarKind::PlaceholderTy(ty::Placeholder {
-                    universe: ty::UniverseIndex::ROOT,
-                    bound: ty::BoundTy {
-                        var: ty::BoundVar::from_usize(self.variables.len()),
-                        kind: ty::BoundTyKind::Anon,
-                    },
-                }),
-                CanonicalizeMode::Response { .. } => bug!("param ty in response: {t:?}"),
+                CanonicalizeMode::Input => CanonicalVarKind::PlaceholderTy(PlaceholderLike::new(
+                    ty::UniverseIndex::ROOT,
+                    self.variables.len().into(),
+                )),
+                CanonicalizeMode::Response { .. } => panic!("param ty in response: {t:?}"),
             },
             ty::Bool
             | ty::Char
@@ -354,44 +350,38 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'_, 'tcx> {
                 var
             }),
         );
-        let bt = ty::BoundTy { var, kind: BoundTyKind::Anon };
-        Ty::new_bound(self.infcx.tcx, self.binder_index, bt)
+
+        self.interner().mk_bound_ty(self.binder_index, var)
     }
 
-    fn fold_const(&mut self, c: ty::Const<'tcx>) -> ty::Const<'tcx> {
+    fn fold_const(&mut self, c: I::Const) -> I::Const
+    where
+        I::Const: TypeSuperFoldable<I>,
+    {
         let kind = match c.kind() {
-            ty::ConstKind::Infer(ty::InferConst::Var(vid)) => {
-                assert_eq!(
-                    self.infcx.root_const_var(vid),
-                    vid,
-                    "const var should have been resolved"
-                );
-                let Err(ui) = self.infcx.probe_const_var(vid) else {
-                    bug!("const var should have been resolved");
-                };
-                // FIXME: we should fold this ty eventually
-                CanonicalVarKind::Const(ui, c.ty())
-            }
-            ty::ConstKind::Infer(ty::InferConst::EffectVar(vid)) => {
-                assert_eq!(
-                    self.infcx.root_effect_var(vid),
-                    vid,
-                    "effect var should have been resolved"
-                );
-                let None = self.infcx.probe_effect_var(vid) else {
-                    bug!("effect var should have been resolved");
-                };
-                CanonicalVarKind::Effect
-            }
-            ty::ConstKind::Infer(ty::InferConst::Fresh(_)) => {
-                bug!("fresh var during canonicalization: {c:?}")
+            ty::ConstKind::Infer(i) => {
+                // FIXME: we should fold the ty too eventually
+                match i {
+                    ty::InferConst::Var(vid) => {
+                        assert_eq!(
+                            self.infcx.root_ct_var(vid),
+                            vid,
+                            "region vid should have been resolved fully before canonicalization"
+                        );
+                        assert_eq!(
+                            self.infcx.probe_ct_var(vid),
+                            None,
+                            "region vid should have been resolved fully before canonicalization"
+                        );
+                        CanonicalVarKind::Const(self.infcx.universe_of_ct(vid).unwrap(), c.ty())
+                    }
+                    ty::InferConst::EffectVar(_) => CanonicalVarKind::Effect,
+                    ty::InferConst::Fresh(_) => todo!(),
+                }
             }
             ty::ConstKind::Placeholder(placeholder) => match self.canonicalize_mode {
                 CanonicalizeMode::Input => CanonicalVarKind::PlaceholderConst(
-                    ty::Placeholder {
-                        universe: placeholder.universe,
-                        bound: ty::BoundVar::from(self.variables.len()),
-                    },
+                    PlaceholderLike::new(placeholder.universe(), self.variables.len().into()),
                     c.ty(),
                 ),
                 CanonicalizeMode::Response { .. } => {
@@ -400,13 +390,10 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'_, 'tcx> {
             },
             ty::ConstKind::Param(_) => match self.canonicalize_mode {
                 CanonicalizeMode::Input => CanonicalVarKind::PlaceholderConst(
-                    ty::Placeholder {
-                        universe: ty::UniverseIndex::ROOT,
-                        bound: ty::BoundVar::from(self.variables.len()),
-                    },
+                    PlaceholderLike::new(ty::UniverseIndex::ROOT, self.variables.len().into()),
                     c.ty(),
                 ),
-                CanonicalizeMode::Response { .. } => bug!("param ty in response: {c:?}"),
+                CanonicalizeMode::Response { .. } => panic!("param ty in response: {c:?}"),
             },
             ty::ConstKind::Bound(_, _)
             | ty::ConstKind::Unevaluated(_)
@@ -423,6 +410,7 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'_, 'tcx> {
                 var
             }),
         );
-        ty::Const::new_bound(self.infcx.tcx, self.binder_index, var, c.ty())
+
+        self.interner().mk_bound_const(self.binder_index, var, c.ty())
     }
 }
diff --git a/compiler/rustc_next_trait_solver/src/lib.rs b/compiler/rustc_next_trait_solver/src/lib.rs
new file mode 100644
index 000000000..e5fc8f755
--- /dev/null
+++ b/compiler/rustc_next_trait_solver/src/lib.rs
@@ -0,0 +1 @@
+pub mod canonicalizer;
diff --git a/compiler/rustc_parse/messages.ftl b/compiler/rustc_parse/messages.ftl
index 266190da0..363b8f4bf 100644
--- a/compiler/rustc_parse/messages.ftl
+++ b/compiler/rustc_parse/messages.ftl
@@ -456,8 +456,6 @@ parse_macro_expands_to_adt_field = macros cannot expand to {$adt_ty} fields
 
 parse_macro_expands_to_enum_variant = macros cannot expand to enum variants
 
-parse_macro_expands_to_match_arm = macros cannot expand to match arms
-
 parse_macro_invocation_visibility = can't qualify macro invocation with `pub`
     .suggestion = remove the visibility
     .help = try adjusting the macro to put `{$vis}` inside the invocation
@@ -492,9 +490,13 @@ parse_match_arm_body_without_braces = `match` arm body without braces
         } with a body
     .suggestion_use_comma_not_semicolon = replace `;` with `,` to end a `match` arm expression
 
+parse_maybe_comparison = you might have meant to compare for equality
+
 parse_maybe_fn_typo_with_impl = you might have meant to write `impl` instead of `fn`
     .suggestion = replace `fn` with `impl` here
 
+parse_maybe_missing_let = you might have meant to continue the let-chain
+
 parse_maybe_recover_from_bad_qpath_stage_2 =
     missing angle brackets in associated item path
     .suggestion = types that don't start with an identifier need to be surrounded with angle brackets in qualified paths
@@ -721,6 +723,9 @@ parse_sugg_wrap_pattern_in_parens = wrap the pattern in parentheses
 parse_switch_mut_let_order =
     switch the order of `mut` and `let`
 
+parse_switch_ref_box_order = switch the order of `ref` and `box`
+    .suggestion = swap them
+
 parse_ternary_operator = Rust has no ternary operator
     .help = use an `if-else` expression instead
 
@@ -739,6 +744,9 @@ parse_trailing_vert_not_allowed = a trailing `|` is not allowed in an or-pattern
 parse_trait_alias_cannot_be_auto = trait aliases cannot be `auto`
 parse_trait_alias_cannot_be_unsafe = trait aliases cannot be `unsafe`
 
+parse_transpose_dyn_or_impl = `for<...>` expected after `{$kw}`, not before
+    .suggestion = move `{$kw}` before the `for<...>`
+
 parse_type_ascription_removed =
     if you meant to annotate an expression with a type, the type ascription syntax has been removed, see issue #101728 <https://github.com/rust-lang/rust/issues/101728>
 
@@ -767,6 +775,9 @@ parse_unexpected_lifetime_in_pattern = unexpected lifetime `{$symbol}` in patter
 parse_unexpected_parentheses_in_for_head = unexpected parentheses surrounding `for` loop head
     .suggestion = remove parentheses in `for` loop
 
+parse_unexpected_parentheses_in_match_arm_pattern = unexpected parentheses surrounding `match` arm pattern
+    .suggestion = remove parentheses surrounding the pattern
+
 parse_unexpected_self_in_generic_parameters = unexpected keyword `Self` in generic parameters
     .note = you cannot use `Self` as a generic parameter because it is reserved for associated items
 
diff --git a/compiler/rustc_parse/src/errors.rs b/compiler/rustc_parse/src/errors.rs
index 8ab1ec298..008adcc83 100644
--- a/compiler/rustc_parse/src/errors.rs
+++ b/compiler/rustc_parse/src/errors.rs
@@ -2,7 +2,7 @@ use std::borrow::Cow;
 
 use rustc_ast::token::Token;
 use rustc_ast::{Path, Visibility};
-use rustc_errors::{AddToDiagnostic, Applicability, EmissionGuarantee, IntoDiagnostic};
+use rustc_errors::{AddToDiagnostic, Applicability, ErrorGuaranteed, IntoDiagnostic};
 use rustc_macros::{Diagnostic, Subdiagnostic};
 use rustc_session::errors::ExprParenthesesNeeded;
 use rustc_span::edition::{Edition, LATEST_STABLE_EDITION};
@@ -138,6 +138,14 @@ pub(crate) enum InvalidVariableDeclarationSub {
 }
 
 #[derive(Diagnostic)]
+#[diag(parse_switch_ref_box_order)]
+pub(crate) struct SwitchRefBoxOrder {
+    #[primary_span]
+    #[suggestion(applicability = "machine-applicable", code = "box ref")]
+    pub span: Span,
+}
+
+#[derive(Diagnostic)]
 #[diag(parse_invalid_comparison_operator)]
 pub(crate) struct InvalidComparisonOperator {
     #[primary_span]
@@ -407,6 +415,32 @@ pub(crate) struct ExpectedExpressionFoundLet {
     pub span: Span,
     #[subdiagnostic]
     pub reason: ForbiddenLetReason,
+    #[subdiagnostic]
+    pub missing_let: Option<MaybeMissingLet>,
+    #[subdiagnostic]
+    pub comparison: Option<MaybeComparison>,
+}
+
+#[derive(Subdiagnostic, Clone, Copy)]
+#[multipart_suggestion(
+    parse_maybe_missing_let,
+    applicability = "maybe-incorrect",
+    style = "verbose"
+)]
+pub(crate) struct MaybeMissingLet {
+    #[suggestion_part(code = "let ")]
+    pub span: Span,
+}
+
+#[derive(Subdiagnostic, Clone, Copy)]
+#[multipart_suggestion(
+    parse_maybe_comparison,
+    applicability = "maybe-incorrect",
+    style = "verbose"
+)]
+pub(crate) struct MaybeComparison {
+    #[suggestion_part(code = "=")]
+    pub span: Span,
 }
 
 #[derive(Diagnostic)]
@@ -1004,15 +1038,15 @@ pub(crate) struct ExpectedIdentifier {
     pub help_cannot_start_number: Option<HelpIdentifierStartsWithNumber>,
 }
 
-impl<'a, G: EmissionGuarantee> IntoDiagnostic<'a, G> for ExpectedIdentifier {
+impl<'a> IntoDiagnostic<'a> for ExpectedIdentifier {
     #[track_caller]
     fn into_diagnostic(
         self,
-        handler: &'a rustc_errors::Handler,
-    ) -> rustc_errors::DiagnosticBuilder<'a, G> {
+        dcx: &'a rustc_errors::DiagCtxt,
+    ) -> rustc_errors::DiagnosticBuilder<'a, ErrorGuaranteed> {
         let token_descr = TokenDescription::from_token(&self.token);
 
-        let mut diag = handler.struct_diagnostic(match token_descr {
+        let mut diag = dcx.struct_err(match token_descr {
             Some(TokenDescription::ReservedIdentifier) => {
                 fluent::parse_expected_identifier_found_reserved_identifier_str
             }
@@ -1061,15 +1095,15 @@ pub(crate) struct ExpectedSemi {
     pub sugg: ExpectedSemiSugg,
 }
 
-impl<'a, G: EmissionGuarantee> IntoDiagnostic<'a, G> for ExpectedSemi {
+impl<'a> IntoDiagnostic<'a> for ExpectedSemi {
     #[track_caller]
     fn into_diagnostic(
         self,
-        handler: &'a rustc_errors::Handler,
-    ) -> rustc_errors::DiagnosticBuilder<'a, G> {
+        dcx: &'a rustc_errors::DiagCtxt,
+    ) -> rustc_errors::DiagnosticBuilder<'a, ErrorGuaranteed> {
         let token_descr = TokenDescription::from_token(&self.token);
 
-        let mut diag = handler.struct_diagnostic(match token_descr {
+        let mut diag = dcx.struct_err(match token_descr {
             Some(TokenDescription::ReservedIdentifier) => {
                 fluent::parse_expected_semi_found_reserved_identifier_str
             }
@@ -1241,12 +1275,28 @@ pub(crate) struct ParenthesesInForHead {
 #[derive(Subdiagnostic)]
 #[multipart_suggestion(parse_suggestion, applicability = "machine-applicable")]
 pub(crate) struct ParenthesesInForHeadSugg {
-    #[suggestion_part(code = "{left_snippet}")]
+    #[suggestion_part(code = " ")]
     pub left: Span,
-    pub left_snippet: String,
-    #[suggestion_part(code = "{right_snippet}")]
+    #[suggestion_part(code = " ")]
+    pub right: Span,
+}
+
+#[derive(Diagnostic)]
+#[diag(parse_unexpected_parentheses_in_match_arm_pattern)]
+pub(crate) struct ParenthesesInMatchPat {
+    #[primary_span]
+    pub span: Vec<Span>,
+    #[subdiagnostic]
+    pub sugg: ParenthesesInMatchPatSugg,
+}
+
+#[derive(Subdiagnostic)]
+#[multipart_suggestion(parse_suggestion, applicability = "machine-applicable")]
+pub(crate) struct ParenthesesInMatchPatSugg {
+    #[suggestion_part(code = "")]
+    pub left: Span,
+    #[suggestion_part(code = "")]
     pub right: Span,
-    pub right_snippet: String,
 }
 
 #[derive(Diagnostic)]
@@ -1676,7 +1726,7 @@ pub(crate) struct ExternItemCannotBeConst {
     #[primary_span]
     pub ident_span: Span,
     #[suggestion(code = "static ", applicability = "machine-applicable")]
-    pub const_span: Span,
+    pub const_span: Option<Span>,
 }
 
 #[derive(Diagnostic)]
@@ -2278,9 +2328,8 @@ pub(crate) enum InvalidMutInPattern {
     #[note(parse_note_mut_pattern_usage)]
     NonIdent {
         #[primary_span]
-        #[suggestion(code = "{pat}", applicability = "machine-applicable")]
+        #[suggestion(code = "", applicability = "machine-applicable")]
         span: Span,
-        pat: String,
     },
 }
 
@@ -2828,3 +2877,23 @@ pub(crate) struct GenericArgsInPatRequireTurbofishSyntax {
     )]
     pub suggest_turbofish: Span,
 }
+
+#[derive(Diagnostic)]
+#[diag(parse_transpose_dyn_or_impl)]
+pub(crate) struct TransposeDynOrImpl<'a> {
+    #[primary_span]
+    pub span: Span,
+    pub kw: &'a str,
+    #[subdiagnostic]
+    pub sugg: TransposeDynOrImplSugg<'a>,
+}
+
+#[derive(Subdiagnostic)]
+#[multipart_suggestion(parse_suggestion, applicability = "machine-applicable")]
+pub(crate) struct TransposeDynOrImplSugg<'a> {
+    #[suggestion_part(code = "")]
+    pub removal_span: Span,
+    #[suggestion_part(code = "{kw} ")]
+    pub insertion_span: Span,
+    pub kw: &'a str,
+}
diff --git a/compiler/rustc_parse/src/lexer/mod.rs b/compiler/rustc_parse/src/lexer/mod.rs
index f2eed5c9b..a91fbdff4 100644
--- a/compiler/rustc_parse/src/lexer/mod.rs
+++ b/compiler/rustc_parse/src/lexer/mod.rs
@@ -67,7 +67,7 @@ pub(crate) fn parse_token_trees<'a>(
     let (stream, res, unmatched_delims) =
         tokentrees::TokenTreesReader::parse_all_token_trees(string_reader);
     match res {
-        Ok(()) if unmatched_delims.is_empty() => Ok(stream),
+        Ok(_open_spacing) if unmatched_delims.is_empty() => Ok(stream),
         _ => {
             // Return error if there are unmatched delimiters or unclosed delimiters.
             // We emit delimiter mismatch errors first, then emit the unclosing delimiter mismatch
@@ -75,7 +75,7 @@ pub(crate) fn parse_token_trees<'a>(
 
             let mut buffer = Vec::with_capacity(1);
             for unmatched in unmatched_delims {
-                if let Some(err) = make_unclosed_delims_error(unmatched, &sess) {
+                if let Some(err) = make_unclosed_delims_error(unmatched, sess) {
                     err.buffer(&mut buffer);
                 }
             }
@@ -230,7 +230,7 @@ impl<'a> StringReader<'a> {
                         let string = self.str_from(suffix_start);
                         if string == "_" {
                             self.sess
-                                .span_diagnostic
+                                .dcx
                                 .emit_err(errors::UnderscoreLiteralSuffix { span: self.mk_sp(suffix_start, self.pos) });
                             None
                         } else {
@@ -349,7 +349,7 @@ impl<'a> StringReader<'a> {
         c: char,
     ) -> DiagnosticBuilder<'a, !> {
         self.sess
-            .span_diagnostic
+            .dcx
             .struct_span_fatal(self.mk_sp(from_pos, to_pos), format!("{}: {}", m, escaped_char(c)))
     }
 
@@ -362,7 +362,7 @@ impl<'a> StringReader<'a> {
         if contains_text_flow_control_chars(content) {
             let span = self.mk_sp(start, self.pos);
             self.sess.buffer_lint_with_diagnostic(
-                &TEXT_DIRECTION_CODEPOINT_IN_COMMENT,
+                TEXT_DIRECTION_CODEPOINT_IN_COMMENT,
                 span,
                 ast::CRATE_NODE_ID,
                 "unicode codepoint changing visible direction of text present in comment",
@@ -406,7 +406,7 @@ impl<'a> StringReader<'a> {
         match kind {
             rustc_lexer::LiteralKind::Char { terminated } => {
                 if !terminated {
-                    self.sess.span_diagnostic.span_fatal_with_code(
+                    self.sess.dcx.span_fatal_with_code(
                         self.mk_sp(start, end),
                         "unterminated character literal",
                         error_code!(E0762),
@@ -416,7 +416,7 @@ impl<'a> StringReader<'a> {
             }
             rustc_lexer::LiteralKind::Byte { terminated } => {
                 if !terminated {
-                    self.sess.span_diagnostic.span_fatal_with_code(
+                    self.sess.dcx.span_fatal_with_code(
                         self.mk_sp(start + BytePos(1), end),
                         "unterminated byte constant",
                         error_code!(E0763),
@@ -426,7 +426,7 @@ impl<'a> StringReader<'a> {
             }
             rustc_lexer::LiteralKind::Str { terminated } => {
                 if !terminated {
-                    self.sess.span_diagnostic.span_fatal_with_code(
+                    self.sess.dcx.span_fatal_with_code(
                         self.mk_sp(start, end),
                         "unterminated double quote string",
                         error_code!(E0765),
@@ -436,7 +436,7 @@ impl<'a> StringReader<'a> {
             }
             rustc_lexer::LiteralKind::ByteStr { terminated } => {
                 if !terminated {
-                    self.sess.span_diagnostic.span_fatal_with_code(
+                    self.sess.dcx.span_fatal_with_code(
                         self.mk_sp(start + BytePos(1), end),
                         "unterminated double quote byte string",
                         error_code!(E0766),
@@ -446,7 +446,7 @@ impl<'a> StringReader<'a> {
             }
             rustc_lexer::LiteralKind::CStr { terminated } => {
                 if !terminated {
-                    self.sess.span_diagnostic.span_fatal_with_code(
+                    self.sess.dcx.span_fatal_with_code(
                         self.mk_sp(start + BytePos(1), end),
                         "unterminated C string",
                         error_code!(E0767),
@@ -581,7 +581,7 @@ impl<'a> StringReader<'a> {
         possible_offset: Option<u32>,
         found_terminators: u32,
     ) -> ! {
-        let mut err = self.sess.span_diagnostic.struct_span_fatal_with_code(
+        let mut err = self.sess.dcx.struct_span_fatal_with_code(
             self.mk_sp(start, start),
             "unterminated raw string",
             error_code!(E0748),
@@ -617,7 +617,7 @@ impl<'a> StringReader<'a> {
             None => "unterminated block comment",
         };
         let last_bpos = self.pos;
-        let mut err = self.sess.span_diagnostic.struct_span_fatal_with_code(
+        let mut err = self.sess.dcx.struct_span_fatal_with_code(
             self.mk_sp(start, last_bpos),
             msg,
             error_code!(E0758),
@@ -683,7 +683,7 @@ impl<'a> StringReader<'a> {
         } else {
             // Before Rust 2021, only emit a lint for migration.
             self.sess.buffer_lint_with_diagnostic(
-                &RUST_2021_PREFIXES_INCOMPATIBLE_SYNTAX,
+                RUST_2021_PREFIXES_INCOMPATIBLE_SYNTAX,
                 prefix_span,
                 ast::CRATE_NODE_ID,
                 format!("prefix `{prefix}` is unknown"),
@@ -722,7 +722,7 @@ impl<'a> StringReader<'a> {
                     has_fatal_err = true;
                 }
                 emit_unescape_error(
-                    &self.sess.span_diagnostic,
+                    &self.sess.dcx,
                     lit_content,
                     span_with_quotes,
                     span,
diff --git a/compiler/rustc_parse/src/lexer/tokentrees.rs b/compiler/rustc_parse/src/lexer/tokentrees.rs
index 31d91fe80..2bc2789a4 100644
--- a/compiler/rustc_parse/src/lexer/tokentrees.rs
+++ b/compiler/rustc_parse/src/lexer/tokentrees.rs
@@ -3,9 +3,10 @@ use super::diagnostics::same_indentation_level;
 use super::diagnostics::TokenTreeDiagInfo;
 use super::{StringReader, UnmatchedDelim};
 use rustc_ast::token::{self, Delimiter, Token};
-use rustc_ast::tokenstream::{DelimSpan, Spacing, TokenStream, TokenTree};
+use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree};
 use rustc_ast_pretty::pprust::token_to_string;
-use rustc_errors::PErr;
+use rustc_errors::{Applicability, PErr};
+use rustc_span::symbol::kw;
 
 pub(super) struct TokenTreesReader<'a> {
     string_reader: StringReader<'a>,
@@ -24,54 +25,46 @@ impl<'a> TokenTreesReader<'a> {
             token: Token::dummy(),
             diag_info: TokenTreeDiagInfo::default(),
         };
-        let (stream, res) = tt_reader.parse_token_trees(/* is_delimited */ false);
+        let (_open_spacing, stream, res) =
+            tt_reader.parse_token_trees(/* is_delimited */ false);
         (stream, res, tt_reader.diag_info.unmatched_delims)
     }
 
-    // Parse a stream of tokens into a list of `TokenTree`s.
+    // Parse a stream of tokens into a list of `TokenTree`s. The `Spacing` in
+    // the result is that of the opening delimiter.
     fn parse_token_trees(
         &mut self,
         is_delimited: bool,
-    ) -> (TokenStream, Result<(), Vec<PErr<'a>>>) {
-        self.token = self.string_reader.next_token().0;
+    ) -> (Spacing, TokenStream, Result<(), Vec<PErr<'a>>>) {
+        // Move past the opening delimiter.
+        let (_, open_spacing) = self.bump(false);
+
         let mut buf = Vec::new();
         loop {
             match self.token.kind {
                 token::OpenDelim(delim) => {
                     buf.push(match self.parse_token_tree_open_delim(delim) {
                         Ok(val) => val,
-                        Err(errs) => return (TokenStream::new(buf), Err(errs)),
+                        Err(errs) => return (open_spacing, TokenStream::new(buf), Err(errs)),
                     })
                 }
                 token::CloseDelim(delim) => {
                     return (
+                        open_spacing,
                         TokenStream::new(buf),
                         if is_delimited { Ok(()) } else { Err(vec![self.close_delim_err(delim)]) },
                     );
                 }
                 token::Eof => {
                     return (
+                        open_spacing,
                         TokenStream::new(buf),
                         if is_delimited { Err(vec![self.eof_err()]) } else { Ok(()) },
                     );
                 }
                 _ => {
-                    // Get the next normal token. This might require getting multiple adjacent
-                    // single-char tokens and joining them together.
-                    let (this_spacing, next_tok) = loop {
-                        let (next_tok, is_next_tok_preceded_by_whitespace) =
-                            self.string_reader.next_token();
-                        if is_next_tok_preceded_by_whitespace {
-                            break (Spacing::Alone, next_tok);
-                        } else if let Some(glued) = self.token.glue(&next_tok) {
-                            self.token = glued;
-                        } else {
-                            let this_spacing =
-                                if next_tok.is_punct() { Spacing::Joint } else { Spacing::Alone };
-                            break (this_spacing, next_tok);
-                        }
-                    };
-                    let this_tok = std::mem::replace(&mut self.token, next_tok);
+                    // Get the next normal token.
+                    let (this_tok, this_spacing) = self.bump(true);
                     buf.push(TokenTree::Token(this_tok, this_spacing));
                 }
             }
@@ -80,7 +73,7 @@ impl<'a> TokenTreesReader<'a> {
 
     fn eof_err(&mut self) -> PErr<'a> {
         let msg = "this file contains an unclosed delimiter";
-        let mut err = self.string_reader.sess.span_diagnostic.struct_span_err(self.token.span, msg);
+        let mut err = self.string_reader.sess.dcx.struct_span_err(self.token.span, msg);
         for &(_, sp) in &self.diag_info.open_braces {
             err.span_label(sp, "unclosed delimiter");
             self.diag_info.unmatched_delims.push(UnmatchedDelim {
@@ -96,7 +89,7 @@ impl<'a> TokenTreesReader<'a> {
             report_suspicious_mismatch_block(
                 &mut err,
                 &self.diag_info,
-                &self.string_reader.sess.source_map(),
+                self.string_reader.sess.source_map(),
                 *delim,
             )
         }
@@ -115,32 +108,16 @@ impl<'a> TokenTreesReader<'a> {
         // Parse the token trees within the delimiters.
         // We stop at any delimiter so we can try to recover if the user
         // uses an incorrect delimiter.
-        let (tts, res) = self.parse_token_trees(/* is_delimited */ true);
-        if let Err(mut errs) = res {
-            // If there are unclosed delims, see if there are diff markers and if so, point them
-            // out instead of complaining about the unclosed delims.
-            let mut parser = crate::stream_to_parser(self.string_reader.sess, tts, None);
-            let mut diff_errs = vec![];
-            while parser.token != token::Eof {
-                if let Err(diff_err) = parser.err_diff_marker() {
-                    diff_errs.push(diff_err);
-                }
-                parser.bump();
-            }
-            if !diff_errs.is_empty() {
-                errs.iter_mut().for_each(|err| {
-                    err.delay_as_bug();
-                });
-                return Err(diff_errs);
-            }
-            return Err(errs);
+        let (open_spacing, tts, res) = self.parse_token_trees(/* is_delimited */ true);
+        if let Err(errs) = res {
+            return Err(self.unclosed_delim_err(tts, errs));
         }
 
         // Expand to cover the entire delimited token tree
         let delim_span = DelimSpan::from_pair(pre_span, self.token.span);
         let sm = self.string_reader.sess.source_map();
 
-        match self.token.kind {
+        let close_spacing = match self.token.kind {
             // Correct delimiter.
             token::CloseDelim(close_delim) if close_delim == open_delim => {
                 let (open_brace, open_brace_span) = self.diag_info.open_braces.pop().unwrap();
@@ -162,7 +139,7 @@ impl<'a> TokenTreesReader<'a> {
                 }
 
                 // Move past the closing delimiter.
-                self.token = self.string_reader.next_token().0;
+                self.bump(false).1
             }
             // Incorrect delimiter.
             token::CloseDelim(close_delim) => {
@@ -179,7 +156,7 @@ impl<'a> TokenTreesReader<'a> {
                         unclosed_delimiter = Some(sp);
                     };
                     for (brace, brace_span) in &self.diag_info.open_braces {
-                        if same_indentation_level(&sm, self.token.span, *brace_span)
+                        if same_indentation_level(sm, self.token.span, *brace_span)
                             && brace == &close_delim
                         {
                             // high likelihood of these two corresponding
@@ -206,18 +183,106 @@ impl<'a> TokenTreesReader<'a> {
                 //     bar(baz(
                 // }  // Incorrect delimiter but matches the earlier `{`
                 if !self.diag_info.open_braces.iter().any(|&(b, _)| b == close_delim) {
-                    self.token = self.string_reader.next_token().0;
+                    self.bump(false).1
+                } else {
+                    // The choice of value here doesn't matter.
+                    Spacing::Alone
                 }
             }
             token::Eof => {
                 // Silently recover, the EOF token will be seen again
                 // and an error emitted then. Thus we don't pop from
-                // self.open_braces here.
+                // self.open_braces here. The choice of spacing value here
+                // doesn't matter.
+                Spacing::Alone
             }
             _ => unreachable!(),
-        }
+        };
+
+        let spacing = DelimSpacing::new(open_spacing, close_spacing);
+
+        Ok(TokenTree::Delimited(delim_span, spacing, open_delim, tts))
+    }
+
+    // Move on to the next token, returning the current token and its spacing.
+    // Will glue adjacent single-char tokens together if `glue` is set.
+    fn bump(&mut self, glue: bool) -> (Token, Spacing) {
+        let (this_spacing, next_tok) = loop {
+            let (next_tok, is_next_tok_preceded_by_whitespace) = self.string_reader.next_token();
 
-        Ok(TokenTree::Delimited(delim_span, open_delim, tts))
+            if is_next_tok_preceded_by_whitespace {
+                break (Spacing::Alone, next_tok);
+            } else if glue && let Some(glued) = self.token.glue(&next_tok) {
+                self.token = glued;
+            } else {
+                let this_spacing = if next_tok.is_punct() {
+                    Spacing::Joint
+                } else if next_tok.kind == token::Eof {
+                    Spacing::Alone
+                } else {
+                    Spacing::JointHidden
+                };
+                break (this_spacing, next_tok);
+            }
+        };
+        let this_tok = std::mem::replace(&mut self.token, next_tok);
+        (this_tok, this_spacing)
+    }
+
+    fn unclosed_delim_err(&mut self, tts: TokenStream, mut errs: Vec<PErr<'a>>) -> Vec<PErr<'a>> {
+        // If there are unclosed delims, see if there are diff markers and if so, point them
+        // out instead of complaining about the unclosed delims.
+        let mut parser = crate::stream_to_parser(self.string_reader.sess, tts, None);
+        let mut diff_errs = vec![];
+        // Suggest removing a `{` we think appears in an `if`/`while` condition
+        // We want to suggest removing a `{` only if we think we're in an `if`/`while` condition, but
+        // we have no way of tracking this in the lexer itself, so we piggyback on the parser
+        let mut in_cond = false;
+        while parser.token != token::Eof {
+            if let Err(diff_err) = parser.err_diff_marker() {
+                diff_errs.push(diff_err);
+            } else if parser.is_keyword_ahead(0, &[kw::If, kw::While]) {
+                in_cond = true;
+            } else if matches!(
+                parser.token.kind,
+                token::CloseDelim(Delimiter::Brace) | token::FatArrow
+            ) {
+                // end of the `if`/`while` body, or the end of a `match` guard
+                in_cond = false;
+            } else if in_cond && parser.token == token::OpenDelim(Delimiter::Brace) {
+                // Store the `&&` and `let` to use their spans later when creating the diagnostic
+                let maybe_andand = parser.look_ahead(1, |t| t.clone());
+                let maybe_let = parser.look_ahead(2, |t| t.clone());
+                if maybe_andand == token::OpenDelim(Delimiter::Brace) {
+                    // This might be the beginning of the `if`/`while` body (i.e., the end of the condition)
+                    in_cond = false;
+                } else if maybe_andand == token::AndAnd && maybe_let.is_keyword(kw::Let) {
+                    let mut err = parser.struct_span_err(
+                        parser.token.span,
+                        "found a `{` in the middle of a let-chain",
+                    );
+                    err.span_suggestion(
+                        parser.token.span,
+                        "consider removing this brace to parse the `let` as part of the same chain",
+                        "",
+                        Applicability::MachineApplicable,
+                    );
+                    err.span_label(
+                        maybe_andand.span.to(maybe_let.span),
+                        "you might have meant to continue the let-chain here",
+                    );
+                    errs.push(err);
+                }
+            }
+            parser.bump();
+        }
+        if !diff_errs.is_empty() {
+            errs.iter_mut().for_each(|err| {
+                err.delay_as_bug();
+            });
+            return diff_errs;
+        }
+        return errs;
     }
 
     fn close_delim_err(&mut self, delim: Delimiter) -> PErr<'a> {
@@ -225,12 +290,12 @@ impl<'a> TokenTreesReader<'a> {
         // matching opening delimiter).
         let token_str = token_to_string(&self.token);
         let msg = format!("unexpected closing delimiter: `{token_str}`");
-        let mut err = self.string_reader.sess.span_diagnostic.struct_span_err(self.token.span, msg);
+        let mut err = self.string_reader.sess.dcx.struct_span_err(self.token.span, msg);
 
         report_suspicious_mismatch_block(
             &mut err,
             &self.diag_info,
-            &self.string_reader.sess.source_map(),
+            self.string_reader.sess.source_map(),
             delim,
         );
         err.span_label(self.token.span, "unexpected closing delimiter");
diff --git a/compiler/rustc_parse/src/lexer/unescape_error_reporting.rs b/compiler/rustc_parse/src/lexer/unescape_error_reporting.rs
index b659c40b2..775082adb 100644
--- a/compiler/rustc_parse/src/lexer/unescape_error_reporting.rs
+++ b/compiler/rustc_parse/src/lexer/unescape_error_reporting.rs
@@ -3,20 +3,20 @@
 use std::iter::once;
 use std::ops::Range;
 
-use rustc_errors::{Applicability, Handler};
+use rustc_errors::{Applicability, DiagCtxt};
 use rustc_lexer::unescape::{EscapeError, Mode};
 use rustc_span::{BytePos, Span};
 
 use crate::errors::{MoreThanOneCharNote, MoreThanOneCharSugg, NoBraceUnicodeSub, UnescapeError};
 
 pub(crate) fn emit_unescape_error(
-    handler: &Handler,
-    // interior part of the literal, without quotes
+    dcx: &DiagCtxt,
+    // interior part of the literal, between quotes
     lit: &str,
-    // full span of the literal, including quotes
-    span_with_quotes: Span,
-    // interior span of the literal, without quotes
-    span: Span,
+    // full span of the literal, including quotes and any prefix
+    full_lit_span: Span,
+    // span of the error part of the literal
+    err_span: Span,
     mode: Mode,
     // range of the error inside `lit`
     range: Range<usize>,
@@ -24,19 +24,19 @@ pub(crate) fn emit_unescape_error(
 ) {
     debug!(
         "emit_unescape_error: {:?}, {:?}, {:?}, {:?}, {:?}",
-        lit, span_with_quotes, mode, range, error
+        lit, full_lit_span, mode, range, error
     );
     let last_char = || {
         let c = lit[range.clone()].chars().next_back().unwrap();
-        let span = span.with_lo(span.hi() - BytePos(c.len_utf8() as u32));
+        let span = err_span.with_lo(err_span.hi() - BytePos(c.len_utf8() as u32));
         (c, span)
     };
     match error {
         EscapeError::LoneSurrogateUnicodeEscape => {
-            handler.emit_err(UnescapeError::InvalidUnicodeEscape { span, surrogate: true });
+            dcx.emit_err(UnescapeError::InvalidUnicodeEscape { span: err_span, surrogate: true });
         }
         EscapeError::OutOfRangeUnicodeEscape => {
-            handler.emit_err(UnescapeError::InvalidUnicodeEscape { span, surrogate: false });
+            dcx.emit_err(UnescapeError::InvalidUnicodeEscape { span: err_span, surrogate: false });
         }
         EscapeError::MoreThanOneChar => {
             use unicode_normalization::{char::is_combining_mark, UnicodeNormalization};
@@ -49,12 +49,16 @@ pub(crate) fn emit_unescape_error(
                 let normalized = lit.nfc().to_string();
                 if normalized.chars().count() == 1 {
                     let ch = normalized.chars().next().unwrap().escape_default().to_string();
-                    sugg = Some(MoreThanOneCharSugg::NormalizedForm { span, ch, normalized });
+                    sugg = Some(MoreThanOneCharSugg::NormalizedForm {
+                        span: err_span,
+                        ch,
+                        normalized,
+                    });
                 }
                 let escaped_marks =
                     rest.iter().map(|c| c.escape_default().to_string()).collect::<Vec<_>>();
                 note = Some(MoreThanOneCharNote::AllCombining {
-                    span,
+                    span: err_span,
                     chr: format!("{first}"),
                     len: escaped_marks.len(),
                     escaped_marks: escaped_marks.join(""),
@@ -69,10 +73,12 @@ pub(crate) fn emit_unescape_error(
                     .collect();
 
                 if let &[ch] = printable.as_slice() {
-                    sugg =
-                        Some(MoreThanOneCharSugg::RemoveNonPrinting { span, ch: ch.to_string() });
+                    sugg = Some(MoreThanOneCharSugg::RemoveNonPrinting {
+                        span: err_span,
+                        ch: ch.to_string(),
+                    });
                     note = Some(MoreThanOneCharNote::NonPrinting {
-                        span,
+                        span: err_span,
                         escaped: lit.escape_default().to_string(),
                     });
                 }
@@ -91,21 +97,21 @@ pub(crate) fn emit_unescape_error(
                 }
                 let sugg = format!("{prefix}\"{escaped}\"");
                 MoreThanOneCharSugg::Quotes {
-                    span: span_with_quotes,
+                    span: full_lit_span,
                     is_byte: mode == Mode::Byte,
                     sugg,
                 }
             });
-            handler.emit_err(UnescapeError::MoreThanOneChar {
-                span: span_with_quotes,
+            dcx.emit_err(UnescapeError::MoreThanOneChar {
+                span: full_lit_span,
                 note,
                 suggestion: sugg,
             });
         }
         EscapeError::EscapeOnlyChar => {
             let (c, char_span) = last_char();
-            handler.emit_err(UnescapeError::EscapeOnlyChar {
-                span,
+            dcx.emit_err(UnescapeError::EscapeOnlyChar {
+                span: err_span,
                 char_span,
                 escaped_sugg: c.escape_default().to_string(),
                 escaped_msg: escaped_char(c),
@@ -114,11 +120,11 @@ pub(crate) fn emit_unescape_error(
         }
         EscapeError::BareCarriageReturn => {
             let double_quotes = mode.in_double_quotes();
-            handler.emit_err(UnescapeError::BareCr { span, double_quotes });
+            dcx.emit_err(UnescapeError::BareCr { span: err_span, double_quotes });
         }
         EscapeError::BareCarriageReturnInRawString => {
             assert!(mode.in_double_quotes());
-            handler.emit_err(UnescapeError::BareCrRawString(span));
+            dcx.emit_err(UnescapeError::BareCrRawString(err_span));
         }
         EscapeError::InvalidEscape => {
             let (c, span) = last_char();
@@ -129,7 +135,7 @@ pub(crate) fn emit_unescape_error(
                 "unknown character escape"
             };
             let ec = escaped_char(c);
-            let mut diag = handler.struct_span_err(span, format!("{label}: `{ec}`"));
+            let mut diag = dcx.struct_span_err(span, format!("{label}: `{ec}`"));
             diag.span_label(span, label);
             if c == '{' || c == '}' && matches!(mode, Mode::Str | Mode::RawStr) {
                 diag.help(
@@ -143,7 +149,7 @@ pub(crate) fn emit_unescape_error(
             } else {
                 if mode == Mode::Str || mode == Mode::Char {
                     diag.span_suggestion(
-                        span_with_quotes,
+                        full_lit_span,
                         "if you meant to write a literal backslash (perhaps escaping in a regular expression), consider a raw string literal",
                         format!("r\"{lit}\""),
                         Applicability::MaybeIncorrect,
@@ -158,13 +164,13 @@ pub(crate) fn emit_unescape_error(
             diag.emit();
         }
         EscapeError::TooShortHexEscape => {
-            handler.emit_err(UnescapeError::TooShortHexEscape(span));
+            dcx.emit_err(UnescapeError::TooShortHexEscape(err_span));
         }
         EscapeError::InvalidCharInHexEscape | EscapeError::InvalidCharInUnicodeEscape => {
             let (c, span) = last_char();
             let is_hex = error == EscapeError::InvalidCharInHexEscape;
             let ch = escaped_char(c);
-            handler.emit_err(UnescapeError::InvalidCharInEscape { span, is_hex, ch });
+            dcx.emit_err(UnescapeError::InvalidCharInEscape { span, is_hex, ch });
         }
         EscapeError::NonAsciiCharInByte => {
             let (c, span) = last_char();
@@ -174,7 +180,7 @@ pub(crate) fn emit_unescape_error(
                 Mode::RawByteStr => "raw byte string literal",
                 _ => panic!("non-is_byte literal paired with NonAsciiCharInByte"),
             };
-            let mut err = handler.struct_span_err(span, format!("non-ASCII character in {desc}"));
+            let mut err = dcx.struct_span_err(span, format!("non-ASCII character in {desc}"));
             let postfix = if unicode_width::UnicodeWidthChar::width(c).unwrap_or(1) == 0 {
                 format!(" but is {c:?}")
             } else {
@@ -210,20 +216,20 @@ pub(crate) fn emit_unescape_error(
             err.emit();
         }
         EscapeError::OutOfRangeHexEscape => {
-            handler.emit_err(UnescapeError::OutOfRangeHexEscape(span));
+            dcx.emit_err(UnescapeError::OutOfRangeHexEscape(err_span));
         }
         EscapeError::LeadingUnderscoreUnicodeEscape => {
             let (c, span) = last_char();
-            handler.emit_err(UnescapeError::LeadingUnderscoreUnicodeEscape {
+            dcx.emit_err(UnescapeError::LeadingUnderscoreUnicodeEscape {
                 span,
                 ch: escaped_char(c),
             });
         }
         EscapeError::OverlongUnicodeEscape => {
-            handler.emit_err(UnescapeError::OverlongUnicodeEscape(span));
+            dcx.emit_err(UnescapeError::OverlongUnicodeEscape(err_span));
         }
         EscapeError::UnclosedUnicodeEscape => {
-            handler.emit_err(UnescapeError::UnclosedUnicodeEscape(span, span.shrink_to_hi()));
+            dcx.emit_err(UnescapeError::UnclosedUnicodeEscape(err_span, err_span.shrink_to_hi()));
         }
         EscapeError::NoBraceInUnicodeEscape => {
             let mut suggestion = "\\u{".to_owned();
@@ -238,34 +244,34 @@ pub(crate) fn emit_unescape_error(
             let (label, sub) = if suggestion_len > 0 {
                 suggestion.push('}');
                 let hi = char_span.lo() + BytePos(suggestion_len as u32);
-                (None, NoBraceUnicodeSub::Suggestion { span: span.with_hi(hi), suggestion })
+                (None, NoBraceUnicodeSub::Suggestion { span: err_span.with_hi(hi), suggestion })
             } else {
-                (Some(span), NoBraceUnicodeSub::Help)
+                (Some(err_span), NoBraceUnicodeSub::Help)
             };
-            handler.emit_err(UnescapeError::NoBraceInUnicodeEscape { span, label, sub });
+            dcx.emit_err(UnescapeError::NoBraceInUnicodeEscape { span: err_span, label, sub });
         }
         EscapeError::UnicodeEscapeInByte => {
-            handler.emit_err(UnescapeError::UnicodeEscapeInByte(span));
+            dcx.emit_err(UnescapeError::UnicodeEscapeInByte(err_span));
         }
         EscapeError::EmptyUnicodeEscape => {
-            handler.emit_err(UnescapeError::EmptyUnicodeEscape(span));
+            dcx.emit_err(UnescapeError::EmptyUnicodeEscape(err_span));
         }
         EscapeError::ZeroChars => {
-            handler.emit_err(UnescapeError::ZeroChars(span));
+            dcx.emit_err(UnescapeError::ZeroChars(err_span));
         }
         EscapeError::LoneSlash => {
-            handler.emit_err(UnescapeError::LoneSlash(span));
+            dcx.emit_err(UnescapeError::LoneSlash(err_span));
         }
         EscapeError::UnskippedWhitespaceWarning => {
             let (c, char_span) = last_char();
-            handler.emit_warning(UnescapeError::UnskippedWhitespace {
-                span,
+            dcx.emit_warning(UnescapeError::UnskippedWhitespace {
+                span: err_span,
                 ch: escaped_char(c),
                 char_span,
             });
         }
         EscapeError::MultipleSkippedLinesWarning => {
-            handler.emit_warning(UnescapeError::MultipleSkippedLinesWarning(span));
+            dcx.emit_warning(UnescapeError::MultipleSkippedLinesWarning(err_span));
         }
     }
 }
diff --git a/compiler/rustc_parse/src/lexer/unicode_chars.rs b/compiler/rustc_parse/src/lexer/unicode_chars.rs
index bbfb160eb..dac7569e3 100644
--- a/compiler/rustc_parse/src/lexer/unicode_chars.rs
+++ b/compiler/rustc_parse/src/lexer/unicode_chars.rs
@@ -350,8 +350,7 @@ pub(super) fn check_for_substitution(
 
     let Some((_, ascii_name, token)) = ASCII_ARRAY.iter().find(|&&(s, _, _)| s == ascii_str) else {
         let msg = format!("substitution character not found for '{ch}'");
-        reader.sess.span_diagnostic.span_bug_no_panic(span, msg);
-        return (None, None);
+        reader.sess.dcx.span_bug(span, msg);
     };
 
     // special help suggestion for "directed" double quotes
diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs
index c012a8663..82b0ff70c 100644
--- a/compiler/rustc_parse/src/lib.rs
+++ b/compiler/rustc_parse/src/lib.rs
@@ -20,8 +20,6 @@ use rustc_ast::{AttrItem, Attribute, MetaItem};
 use rustc_ast_pretty::pprust;
 use rustc_data_structures::sync::Lrc;
 use rustc_errors::{Diagnostic, FatalError, Level, PResult};
-use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
-use rustc_fluent_macro::fluent_messages;
 use rustc_session::parse::ParseSess;
 use rustc_span::{FileName, SourceFile, Span};
 
@@ -37,7 +35,7 @@ pub mod validate_attr;
 
 mod errors;
 
-fluent_messages! { "../messages.ftl" }
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
 
 // A bunch of utility functions of the form `parse_<thing>_from_<source>`
 // where <thing> includes crate, expr, item, stmt, tts, and one that
@@ -53,8 +51,8 @@ macro_rules! panictry_buffer {
         match $e {
             Ok(e) => e,
             Err(errs) => {
-                for mut e in errs {
-                    $handler.emit_diagnostic(&mut e);
+                for e in errs {
+                    $handler.emit_diagnostic(e);
                 }
                 FatalError.raise()
             }
@@ -102,7 +100,7 @@ pub fn parse_stream_from_source_str(
 
 /// Creates a new parser from a source string.
 pub fn new_parser_from_source_str(sess: &ParseSess, name: FileName, source: String) -> Parser<'_> {
-    panictry_buffer!(&sess.span_diagnostic, maybe_new_parser_from_source_str(sess, name, source))
+    panictry_buffer!(&sess.dcx, maybe_new_parser_from_source_str(sess, name, source))
 }
 
 /// Creates a new parser from a source string. Returns any buffered errors from lexing the initial
@@ -123,7 +121,7 @@ pub fn new_parser_from_file<'a>(sess: &'a ParseSess, path: &Path, sp: Option<Spa
 
 /// Given a session and a `source_file`, returns a parser.
 fn source_file_to_parser(sess: &ParseSess, source_file: Lrc<SourceFile>) -> Parser<'_> {
-    panictry_buffer!(&sess.span_diagnostic, maybe_source_file_to_parser(sess, source_file))
+    panictry_buffer!(&sess.dcx, maybe_source_file_to_parser(sess, source_file))
 }
 
 /// Given a session and a `source_file`, return a parser. Returns any buffered errors from lexing the
@@ -167,8 +165,8 @@ fn try_file_to_source_file(
 fn file_to_source_file(sess: &ParseSess, path: &Path, spanopt: Option<Span>) -> Lrc<SourceFile> {
     match try_file_to_source_file(sess, path, spanopt) {
         Ok(source_file) => source_file,
-        Err(mut d) => {
-            sess.span_diagnostic.emit_diagnostic(&mut d);
+        Err(d) => {
+            sess.dcx.emit_diagnostic(d);
             FatalError.raise();
         }
     }
@@ -180,7 +178,7 @@ pub fn source_file_to_stream(
     source_file: Lrc<SourceFile>,
     override_span: Option<Span>,
 ) -> TokenStream {
-    panictry_buffer!(&sess.span_diagnostic, maybe_file_to_stream(sess, source_file, override_span))
+    panictry_buffer!(&sess.dcx, maybe_file_to_stream(sess, source_file, override_span))
 }
 
 /// Given a source file, produces a sequence of token trees. Returns any buffered errors from
@@ -191,7 +189,7 @@ pub fn maybe_file_to_stream(
     override_span: Option<Span>,
 ) -> Result<TokenStream, Vec<Diagnostic>> {
     let src = source_file.src.as_ref().unwrap_or_else(|| {
-        sess.span_diagnostic.bug(format!(
+        sess.dcx.bug(format!(
             "cannot lex `source_file` without source: {}",
             sess.source_map().filename_for_diagnostics(&source_file.name)
         ));
diff --git a/compiler/rustc_parse/src/parser/attr.rs b/compiler/rustc_parse/src/parser/attr.rs
index 104de47b9..56e52baf9 100644
--- a/compiler/rustc_parse/src/parser/attr.rs
+++ b/compiler/rustc_parse/src/parser/attr.rs
@@ -7,7 +7,6 @@ use rustc_ast::attr;
 use rustc_ast::token::{self, Delimiter, Nonterminal};
 use rustc_errors::{error_code, Diagnostic, IntoDiagnostic, PResult};
 use rustc_span::{sym, BytePos, Span};
-use std::convert::TryInto;
 use thin_vec::ThinVec;
 use tracing::debug;
 
@@ -56,7 +55,7 @@ impl<'a> Parser<'a> {
             } else if let token::DocComment(comment_kind, attr_style, data) = self.token.kind {
                 if attr_style != ast::AttrStyle::Outer {
                     let span = self.token.span;
-                    let mut err = self.sess.span_diagnostic.struct_span_err_with_code(
+                    let mut err = self.dcx().struct_span_err_with_code(
                         span,
                         fluent::parse_inner_doc_comment_not_permitted,
                         error_code!(E0753),
@@ -249,7 +248,7 @@ impl<'a> Parser<'a> {
     /// The delimiters or `=` are still put into the resulting token stream.
     pub fn parse_attr_item(&mut self, capture_tokens: bool) -> PResult<'a, ast::AttrItem> {
         let item = match &self.token.kind {
-            token::Interpolated(nt) => match &**nt {
+            token::Interpolated(nt) => match &nt.0 {
                 Nonterminal::NtMeta(item) => Some(item.clone().into_inner()),
                 _ => None,
             },
@@ -369,7 +368,7 @@ impl<'a> Parser<'a> {
     /// ```
     pub fn parse_meta_item(&mut self) -> PResult<'a, ast::MetaItem> {
         let nt_meta = match &self.token.kind {
-            token::Interpolated(nt) => match &**nt {
+            token::Interpolated(nt) => match &nt.0 {
                 token::NtMeta(e) => Some(e.clone()),
                 _ => None,
             },
@@ -418,7 +417,7 @@ impl<'a> Parser<'a> {
         }
 
         Err(InvalidMetaItem { span: self.token.span, token: self.token.clone() }
-            .into_diagnostic(&self.sess.span_diagnostic))
+            .into_diagnostic(self.dcx()))
     }
 }
 
diff --git a/compiler/rustc_parse/src/parser/attr_wrapper.rs b/compiler/rustc_parse/src/parser/attr_wrapper.rs
index c4e8d9006..2307f4cff 100644
--- a/compiler/rustc_parse/src/parser/attr_wrapper.rs
+++ b/compiler/rustc_parse/src/parser/attr_wrapper.rs
@@ -1,7 +1,7 @@
 use super::{Capturing, FlatToken, ForceCollect, Parser, ReplaceRange, TokenCursor, TrailingToken};
 use rustc_ast::token::{self, Delimiter, Token, TokenKind};
-use rustc_ast::tokenstream::{AttrTokenStream, AttributesData, ToAttrTokenStream};
-use rustc_ast::tokenstream::{AttrTokenTree, DelimSpan, LazyAttrTokenStream, Spacing};
+use rustc_ast::tokenstream::{AttrTokenStream, AttrTokenTree, AttributesData, DelimSpacing};
+use rustc_ast::tokenstream::{DelimSpan, LazyAttrTokenStream, Spacing, ToAttrTokenStream};
 use rustc_ast::{self as ast};
 use rustc_ast::{AttrVec, Attribute, HasAttrs, HasTokens};
 use rustc_errors::PResult;
@@ -41,7 +41,7 @@ impl AttrWrapper {
     }
 
     pub(crate) fn take_for_recovery(self, sess: &ParseSess) -> AttrVec {
-        sess.span_diagnostic.delay_span_bug(
+        sess.dcx.span_delayed_bug(
             self.attrs.get(0).map(|attr| attr.span).unwrap_or(DUMMY_SP),
             "AttrVec is taken for recovery but no error is produced",
         );
@@ -266,9 +266,7 @@ impl<'a> Parser<'a> {
             if let Some(attr_range) = self.capture_state.inner_attr_ranges.remove(&inner_attr.id) {
                 inner_attr_replace_ranges.push(attr_range);
             } else {
-                self.sess
-                    .span_diagnostic
-                    .delay_span_bug(inner_attr.span, "Missing token range for attribute");
+                self.dcx().span_delayed_bug(inner_attr.span, "Missing token range for attribute");
             }
         }
 
@@ -390,7 +388,7 @@ fn make_token_stream(
     #[derive(Debug)]
     struct FrameData {
         // This is `None` for the first frame, `Some` for all others.
-        open_delim_sp: Option<(Delimiter, Span)>,
+        open_delim_sp: Option<(Delimiter, Span, Spacing)>,
         inner: Vec<AttrTokenTree>,
     }
     let mut stack = vec![FrameData { open_delim_sp: None, inner: vec![] }];
@@ -398,21 +396,23 @@ fn make_token_stream(
     while let Some((token, spacing)) = token_and_spacing {
         match token {
             FlatToken::Token(Token { kind: TokenKind::OpenDelim(delim), span }) => {
-                stack.push(FrameData { open_delim_sp: Some((delim, span)), inner: vec![] });
+                stack
+                    .push(FrameData { open_delim_sp: Some((delim, span, spacing)), inner: vec![] });
             }
             FlatToken::Token(Token { kind: TokenKind::CloseDelim(delim), span }) => {
                 let frame_data = stack
                     .pop()
                     .unwrap_or_else(|| panic!("Token stack was empty for token: {token:?}"));
 
-                let (open_delim, open_sp) = frame_data.open_delim_sp.unwrap();
+                let (open_delim, open_sp, open_spacing) = frame_data.open_delim_sp.unwrap();
                 assert_eq!(
                     open_delim, delim,
                     "Mismatched open/close delims: open={open_delim:?} close={span:?}"
                 );
                 let dspan = DelimSpan::from_pair(open_sp, span);
+                let dspacing = DelimSpacing::new(open_spacing, spacing);
                 let stream = AttrTokenStream::new(frame_data.inner);
-                let delimited = AttrTokenTree::Delimited(dspan, delim, stream);
+                let delimited = AttrTokenTree::Delimited(dspan, dspacing, delim, stream);
                 stack
                     .last_mut()
                     .unwrap_or_else(|| panic!("Bottom token frame is missing for token: {token:?}"))
diff --git a/compiler/rustc_parse/src/parser/diagnostics.rs b/compiler/rustc_parse/src/parser/diagnostics.rs
index 2a8eb6edd..c077e0a83 100644
--- a/compiler/rustc_parse/src/parser/diagnostics.rs
+++ b/compiler/rustc_parse/src/parser/diagnostics.rs
@@ -10,31 +10,32 @@ use crate::errors::{
     ConstGenericWithoutBracesSugg, DocCommentDoesNotDocumentAnything, DocCommentOnParamType,
     DoubleColonInBound, ExpectedIdentifier, ExpectedSemi, ExpectedSemiSugg,
     GenericParamsWithoutAngleBrackets, GenericParamsWithoutAngleBracketsSugg,
-    HelpIdentifierStartsWithNumber, InInTypo, IncorrectAwait, IncorrectSemicolon,
-    IncorrectUseOfAwait, ParenthesesInForHead, ParenthesesInForHeadSugg,
-    PatternMethodParamWithoutBody, QuestionMarkInType, QuestionMarkInTypeSugg, SelfParamNotFirst,
-    StructLiteralBodyWithoutPath, StructLiteralBodyWithoutPathSugg, StructLiteralNeedingParens,
-    StructLiteralNeedingParensSugg, SuggAddMissingLetStmt, SuggEscapeIdentifier, SuggRemoveComma,
-    TernaryOperator, UnexpectedConstInGenericParam, UnexpectedConstParamDeclaration,
+    HelpIdentifierStartsWithNumber, HelpUseLatestEdition, InInTypo, IncorrectAwait,
+    IncorrectSemicolon, IncorrectUseOfAwait, PatternMethodParamWithoutBody, QuestionMarkInType,
+    QuestionMarkInTypeSugg, SelfParamNotFirst, StructLiteralBodyWithoutPath,
+    StructLiteralBodyWithoutPathSugg, StructLiteralNeedingParens, StructLiteralNeedingParensSugg,
+    SuggAddMissingLetStmt, SuggEscapeIdentifier, SuggRemoveComma, TernaryOperator,
+    UnexpectedConstInGenericParam, UnexpectedConstParamDeclaration,
     UnexpectedConstParamDeclarationSugg, UnmatchedAngleBrackets, UseEqInstead, WrapType,
 };
-
 use crate::fluent_generated as fluent;
 use crate::parser;
+use crate::parser::attr::InnerAttrPolicy;
 use rustc_ast as ast;
 use rustc_ast::ptr::P;
 use rustc_ast::token::{self, Delimiter, Lit, LitKind, TokenKind};
+use rustc_ast::tokenstream::AttrTokenTree;
 use rustc_ast::util::parser::AssocOp;
 use rustc_ast::{
     AngleBracketedArg, AngleBracketedArgs, AnonConst, AttrVec, BinOpKind, BindingAnnotation, Block,
-    BlockCheckMode, Expr, ExprKind, GenericArg, Generics, Item, ItemKind, Param, Pat, PatKind,
-    Path, PathSegment, QSelf, Ty, TyKind,
+    BlockCheckMode, Expr, ExprKind, GenericArg, Generics, HasTokens, Item, ItemKind, Param, Pat,
+    PatKind, Path, PathSegment, QSelf, Ty, TyKind,
 };
 use rustc_ast_pretty::pprust;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_errors::{
-    pluralize, AddToDiagnostic, Applicability, Diagnostic, DiagnosticBuilder, DiagnosticMessage,
-    ErrorGuaranteed, FatalError, Handler, IntoDiagnostic, MultiSpan, PResult,
+    pluralize, AddToDiagnostic, Applicability, DiagCtxt, Diagnostic, DiagnosticBuilder,
+    DiagnosticMessage, ErrorGuaranteed, FatalError, IntoDiagnostic, MultiSpan, PResult,
 };
 use rustc_session::errors::ExprParenthesesNeeded;
 use rustc_span::source_map::Spanned;
@@ -245,15 +246,15 @@ impl<'a> Parser<'a> {
         sp: S,
         m: impl Into<DiagnosticMessage>,
     ) -> DiagnosticBuilder<'a, ErrorGuaranteed> {
-        self.sess.span_diagnostic.struct_span_err(sp, m)
+        self.dcx().struct_span_err(sp, m)
     }
 
-    pub fn span_bug<S: Into<MultiSpan>>(&self, sp: S, m: impl Into<String>) -> ! {
-        self.sess.span_diagnostic.span_bug(sp, m)
+    pub fn span_bug<S: Into<MultiSpan>>(&self, sp: S, msg: impl Into<DiagnosticMessage>) -> ! {
+        self.dcx().span_bug(sp, msg)
     }
 
-    pub(super) fn diagnostic(&self) -> &'a Handler {
-        &self.sess.span_diagnostic
+    pub(super) fn dcx(&self) -> &'a DiagCtxt {
+        &self.sess.dcx
     }
 
     /// Replace `self` with `snapshot.parser`.
@@ -283,7 +284,7 @@ impl<'a> Parser<'a> {
                 span: self.prev_token.span,
                 missing_comma: None,
             }
-            .into_diagnostic(&self.sess.span_diagnostic));
+            .into_diagnostic(self.dcx()));
         }
 
         let valid_follow = &[
@@ -346,7 +347,7 @@ impl<'a> Parser<'a> {
             suggest_remove_comma,
             help_cannot_start_number,
         };
-        let mut err = err.into_diagnostic(&self.sess.span_diagnostic);
+        let mut err = err.into_diagnostic(self.dcx());
 
         // if the token we have is a `<`
         // it *might* be a misplaced generic
@@ -506,7 +507,9 @@ impl<'a> Parser<'a> {
         if expected.contains(&TokenType::Token(token::Semi)) {
             // If the user is trying to write a ternary expression, recover it and
             // return an Err to prevent a cascade of irrelevant diagnostics
-            if self.prev_token == token::Question && let Err(e) = self.maybe_recover_from_ternary_operator() {
+            if self.prev_token == token::Question
+                && let Err(e) = self.maybe_recover_from_ternary_operator()
+            {
                 return Err(e);
             }
 
@@ -637,6 +640,28 @@ impl<'a> Parser<'a> {
             }
         }
 
+        // Try to detect an intended c-string literal while using a pre-2021 edition. The heuristic
+        // here is to identify a cooked, uninterpolated `c` id immediately followed by a string, or
+        // a cooked, uninterpolated `cr` id immediately followed by a string or a `#`, in an edition
+        // where c-string literals are not allowed. There is the very slight possibility of a false
+        // positive for a `cr#` that wasn't intended to start a c-string literal, but identifying
+        // that in the parser requires unbounded lookahead, so we only add a hint to the existing
+        // error rather than replacing it entirely.
+        if ((self.prev_token.kind == TokenKind::Ident(sym::c, false)
+            && matches!(&self.token.kind, TokenKind::Literal(token::Lit { kind: token::Str, .. })))
+            || (self.prev_token.kind == TokenKind::Ident(sym::cr, false)
+                && matches!(
+                    &self.token.kind,
+                    TokenKind::Literal(token::Lit { kind: token::Str, .. }) | token::Pound
+                )))
+            && self.prev_token.span.hi() == self.token.span.lo()
+            && !self.token.span.at_least_rust_2021()
+        {
+            err.note("you may be trying to write a c-string literal");
+            err.note("c-string literals require Rust 2021 or later");
+            HelpUseLatestEdition::new().add_to_diagnostic(&mut err);
+        }
+
         // `pub` may be used for an item or `pub(crate)`
         if self.prev_token.is_ident_named(sym::public)
             && (self.token.can_begin_item()
@@ -670,15 +695,6 @@ impl<'a> Parser<'a> {
             );
         }
 
-        // Add suggestion for a missing closing angle bracket if '>' is included in expected_tokens
-        // there are unclosed angle brackets
-        if self.unmatched_angle_bracket_count > 0
-            && self.token.kind == TokenKind::Eq
-            && expected.iter().any(|tok| matches!(tok, TokenType::Token(TokenKind::Gt)))
-        {
-            err.span_label(self.prev_token.span, "maybe try to close unmatched angle bracket");
-        }
-
         let sp = if self.token == token::Eof {
             // This is EOF; don't want to point at the following char, but rather the last token.
             self.prev_token.span
@@ -720,6 +736,95 @@ impl<'a> Parser<'a> {
         Err(err)
     }
 
+    pub(super) fn attr_on_non_tail_expr(&self, expr: &Expr) {
+        // Missing semicolon typo error.
+        let span = self.prev_token.span.shrink_to_hi();
+        let mut err = self.sess.create_err(ExpectedSemi {
+            span,
+            token: self.token.clone(),
+            unexpected_token_label: Some(self.token.span),
+            sugg: ExpectedSemiSugg::AddSemi(span),
+        });
+        let attr_span = match &expr.attrs[..] {
+            [] => unreachable!(),
+            [only] => only.span,
+            [first, rest @ ..] => {
+                for attr in rest {
+                    err.span_label(attr.span, "");
+                }
+                first.span
+            }
+        };
+        err.span_label(
+            attr_span,
+            format!(
+                "only `;` terminated statements or tail expressions are allowed after {}",
+                if expr.attrs.len() == 1 { "this attribute" } else { "these attributes" },
+            ),
+        );
+        if self.token == token::Pound
+            && self.look_ahead(1, |t| t.kind == token::OpenDelim(Delimiter::Bracket))
+        {
+            // We have
+            // #[attr]
+            // expr
+            // #[not_attr]
+            // other_expr
+            err.span_label(span, "expected `;` here");
+            err.multipart_suggestion(
+                "alternatively, consider surrounding the expression with a block",
+                vec![
+                    (expr.span.shrink_to_lo(), "{ ".to_string()),
+                    (expr.span.shrink_to_hi(), " }".to_string()),
+                ],
+                Applicability::MachineApplicable,
+            );
+            let mut snapshot = self.create_snapshot_for_diagnostic();
+            if let [attr] = &expr.attrs[..]
+                && let ast::AttrKind::Normal(attr_kind) = &attr.kind
+                && let [segment] = &attr_kind.item.path.segments[..]
+                && segment.ident.name == sym::cfg
+                && let Some(args_span) = attr_kind.item.args.span()
+                && let Ok(next_attr) = snapshot.parse_attribute(InnerAttrPolicy::Forbidden(None))
+                && let ast::AttrKind::Normal(next_attr_kind) = next_attr.kind
+                && let Some(next_attr_args_span) = next_attr_kind.item.args.span()
+                && let [next_segment] = &next_attr_kind.item.path.segments[..]
+                && segment.ident.name == sym::cfg
+                && let Ok(next_expr) = snapshot.parse_expr()
+            {
+                // We have for sure
+                // #[cfg(..)]
+                // expr
+                // #[cfg(..)]
+                // other_expr
+                // So we suggest using `if cfg!(..) { expr } else if cfg!(..) { other_expr }`.
+                let margin = self.sess.source_map().span_to_margin(next_expr.span).unwrap_or(0);
+                let sugg = vec![
+                    (attr.span.with_hi(segment.span().hi()), "if cfg!".to_string()),
+                    (args_span.shrink_to_hi().with_hi(attr.span.hi()), " {".to_string()),
+                    (expr.span.shrink_to_lo(), "    ".to_string()),
+                    (
+                        next_attr.span.with_hi(next_segment.span().hi()),
+                        "} else if cfg!".to_string(),
+                    ),
+                    (
+                        next_attr_args_span.shrink_to_hi().with_hi(next_attr.span.hi()),
+                        " {".to_string(),
+                    ),
+                    (next_expr.span.shrink_to_lo(), "    ".to_string()),
+                    (next_expr.span.shrink_to_hi(), format!("\n{}}}", " ".repeat(margin))),
+                ];
+                err.multipart_suggestion(
+                    "it seems like you are trying to provide different expressions depending on \
+                     `cfg`, consider using `if cfg!(..)`",
+                    sugg,
+                    Applicability::MachineApplicable,
+                );
+            }
+        }
+        err.emit();
+    }
+
     fn check_too_many_raw_str_terminators(&mut self, err: &mut Diagnostic) -> bool {
         let sm = self.sess.source_map();
         match (&self.prev_token.kind, &self.token.kind) {
@@ -1182,11 +1287,11 @@ impl<'a> Parser<'a> {
                 (BinOpKind::Ge, AssocOp::GreaterEqual | AssocOp::Greater) => {
                     let expr_to_str = |e: &Expr| {
                         self.span_to_snippet(e.span)
-                            .unwrap_or_else(|_| pprust::expr_to_string(&e))
+                            .unwrap_or_else(|_| pprust::expr_to_string(e))
                     };
                     err.chaining_sugg = Some(ComparisonOperatorsCannotBeChainedSugg::SplitComparison {
                         span: inner_op.span.shrink_to_hi(),
-                        middle_term: expr_to_str(&r1),
+                        middle_term: expr_to_str(r1),
                     });
                     false // Keep the current parse behavior, where the AST is `(x < y) < z`.
                 }
@@ -1327,7 +1432,7 @@ impl<'a> Parser<'a> {
                                 // Not entirely sure now, but we bubble the error up with the
                                 // suggestion.
                                 self.restore_snapshot(snapshot);
-                                Err(err.into_diagnostic(&self.sess.span_diagnostic))
+                                Err(err.into_diagnostic(self.dcx()))
                             }
                         }
                     } else if token::OpenDelim(Delimiter::Parenthesis) == self.token.kind {
@@ -1342,7 +1447,7 @@ impl<'a> Parser<'a> {
                         }
                         // Consume the fn call arguments.
                         match self.consume_fn_args() {
-                            Err(()) => Err(err.into_diagnostic(&self.sess.span_diagnostic)),
+                            Err(()) => Err(err.into_diagnostic(self.dcx())),
                             Ok(()) => {
                                 self.sess.emit_err(err);
                                 // FIXME: actually check that the two expressions in the binop are
@@ -1368,7 +1473,7 @@ impl<'a> Parser<'a> {
                             mk_err_expr(self, inner_op.span.to(self.prev_token.span))
                         } else {
                             // These cases cause too many knock-down errors, bail out (#61329).
-                            Err(err.into_diagnostic(&self.sess.span_diagnostic))
+                            Err(err.into_diagnostic(self.dcx()))
                         }
                     };
                 }
@@ -1407,7 +1512,7 @@ impl<'a> Parser<'a> {
 
     pub(super) fn maybe_report_ambiguous_plus(&mut self, impl_dyn_multi: bool, ty: &Ty) {
         if impl_dyn_multi {
-            self.sess.emit_err(AmbiguousPlus { sum_ty: pprust::ty_to_string(&ty), span: ty.span });
+            self.sess.emit_err(AmbiguousPlus { sum_ty: pprust::ty_to_string(ty), span: ty.span });
         }
     }
 
@@ -1840,7 +1945,7 @@ impl<'a> Parser<'a> {
         self.sess.emit_err(IncorrectAwait {
             span,
             sugg_span: (span, applicability),
-            expr: self.span_to_snippet(expr.span).unwrap_or_else(|_| pprust::expr_to_string(&expr)),
+            expr: self.span_to_snippet(expr.span).unwrap_or_else(|_| pprust::expr_to_string(expr)),
             question_mark: if is_question { "?" } else { "" },
         });
 
@@ -1895,54 +2000,37 @@ impl<'a> Parser<'a> {
         }
     }
 
-    /// Recovers a situation like `for ( $pat in $expr )`
-    /// and suggest writing `for $pat in $expr` instead.
-    ///
-    /// This should be called before parsing the `$block`.
-    pub(super) fn recover_parens_around_for_head(
+    /// When trying to close a generics list and encountering code like
+    /// ```text
+    /// impl<S: Into<std::borrow::Cow<'static, str>> From<S> for Canonical {}
+    ///                                          // ^ missing > here
+    /// ```
+    /// we provide a structured suggestion on the error from `expect_gt`.
+    pub(super) fn expect_gt_or_maybe_suggest_closing_generics(
         &mut self,
-        pat: P<Pat>,
-        begin_paren: Option<Span>,
-    ) -> P<Pat> {
-        match (&self.token.kind, begin_paren) {
-            (token::CloseDelim(Delimiter::Parenthesis), Some(begin_par_sp)) => {
-                self.bump();
-
-                let sm = self.sess.source_map();
-                let left = begin_par_sp;
-                let right = self.prev_token.span;
-                let left_snippet = if let Ok(snip) = sm.span_to_prev_source(left)
-                    && !snip.ends_with(' ')
-                {
-                    " ".to_string()
-                } else {
-                    "".to_string()
-                };
-
-                let right_snippet = if let Ok(snip) = sm.span_to_next_source(right)
-                    && !snip.starts_with(' ')
-                {
-                    " ".to_string()
-                } else {
-                    "".to_string()
-                };
-
-                self.sess.emit_err(ParenthesesInForHead {
-                    span: vec![left, right],
-                    // With e.g. `for (x) in y)` this would replace `(x) in y)`
-                    // with `x) in y)` which is syntactically invalid.
-                    // However, this is prevented before we get here.
-                    sugg: ParenthesesInForHeadSugg { left, right, left_snippet, right_snippet },
-                });
-
-                // Unwrap `(pat)` into `pat` to avoid the `unused_parens` lint.
-                pat.and_then(|pat| match pat.kind {
-                    PatKind::Paren(pat) => pat,
-                    _ => P(pat),
+        params: &[ast::GenericParam],
+    ) -> PResult<'a, ()> {
+        let Err(mut err) = self.expect_gt() else {
+            return Ok(());
+        };
+        // Attempt to find places where a missing `>` might belong.
+        if let [.., ast::GenericParam { bounds, .. }] = params
+            && let Some(poly) = bounds
+                .iter()
+                .filter_map(|bound| match bound {
+                    ast::GenericBound::Trait(poly, _) => Some(poly),
+                    _ => None,
                 })
-            }
-            _ => pat,
+                .last()
+        {
+            err.span_suggestion_verbose(
+                poly.span.shrink_to_hi(),
+                "you might have meant to end the type parameters here",
+                ">",
+                Applicability::MaybeIncorrect,
+            );
         }
+        Err(err)
     }
 
     pub(super) fn recover_seq_parse_error(
@@ -2250,6 +2338,59 @@ impl<'a> Parser<'a> {
             err.subdiagnostic(ExprParenthesesNeeded::surrounding(*sp));
         }
         err.span_label(span, "expected expression");
+
+        // Walk the chain of macro expansions for the current token to point at how the original
+        // code was interpreted. This helps the user realize when a macro argument of one type is
+        // later reinterpreted as a different type, like `$x:expr` being reinterpreted as `$x:pat`
+        // in a subsequent macro invocation (#71039).
+        let mut tok = self.token.clone();
+        let mut labels = vec![];
+        while let TokenKind::Interpolated(node) = &tok.kind {
+            let tokens = node.0.tokens();
+            labels.push(node.clone());
+            if let Some(tokens) = tokens
+                && let tokens = tokens.to_attr_token_stream()
+                && let tokens = tokens.0.deref()
+                && let [AttrTokenTree::Token(token, _)] = &tokens[..]
+            {
+                tok = token.clone();
+            } else {
+                break;
+            }
+        }
+        let mut iter = labels.into_iter().peekable();
+        let mut show_link = false;
+        while let Some(node) = iter.next() {
+            let descr = node.0.descr();
+            if let Some(next) = iter.peek() {
+                let next_descr = next.0.descr();
+                if next_descr != descr {
+                    err.span_label(next.1, format!("this macro fragment matcher is {next_descr}"));
+                    err.span_label(node.1, format!("this macro fragment matcher is {descr}"));
+                    err.span_label(
+                        next.0.use_span(),
+                        format!("this is expected to be {next_descr}"),
+                    );
+                    err.span_label(
+                        node.0.use_span(),
+                        format!(
+                            "this is interpreted as {}, but it is expected to be {}",
+                            next_descr, descr,
+                        ),
+                    );
+                    show_link = true;
+                } else {
+                    err.span_label(node.1, "");
+                }
+            }
+        }
+        if show_link {
+            err.note(
+                "when forwarding a matched fragment to another macro-by-example, matchers in the \
+                 second macro will see an opaque AST of the fragment type, not the underlying \
+                 tokens",
+            );
+        }
         err
     }
 
@@ -2420,8 +2561,7 @@ impl<'a> Parser<'a> {
             Ok(Some(GenericArg::Const(self.parse_const_arg()?)))
         } else {
             let after_kw_const = self.token.span;
-            self.recover_const_arg(after_kw_const, err.into_diagnostic(&self.sess.span_diagnostic))
-                .map(Some)
+            self.recover_const_arg(after_kw_const, err.into_diagnostic(self.dcx())).map(Some)
         }
     }
 
@@ -2721,7 +2861,6 @@ impl<'a> Parser<'a> {
     pub(crate) fn maybe_recover_unexpected_comma(
         &mut self,
         lo: Span,
-        is_mac_invoc: bool,
         rt: CommaRecoveryMode,
     ) -> PResult<'a, ()> {
         if self.token != token::Comma {
@@ -2742,28 +2881,24 @@ impl<'a> Parser<'a> {
         let seq_span = lo.to(self.prev_token.span);
         let mut err = self.struct_span_err(comma_span, "unexpected `,` in pattern");
         if let Ok(seq_snippet) = self.span_to_snippet(seq_span) {
-            if is_mac_invoc {
-                err.note(fluent::parse_macro_expands_to_match_arm);
-            } else {
-                err.multipart_suggestion(
-                    format!(
-                        "try adding parentheses to match on a tuple{}",
-                        if let CommaRecoveryMode::LikelyTuple = rt { "" } else { "..." },
-                    ),
-                    vec![
-                        (seq_span.shrink_to_lo(), "(".to_string()),
-                        (seq_span.shrink_to_hi(), ")".to_string()),
-                    ],
+            err.multipart_suggestion(
+                format!(
+                    "try adding parentheses to match on a tuple{}",
+                    if let CommaRecoveryMode::LikelyTuple = rt { "" } else { "..." },
+                ),
+                vec![
+                    (seq_span.shrink_to_lo(), "(".to_string()),
+                    (seq_span.shrink_to_hi(), ")".to_string()),
+                ],
+                Applicability::MachineApplicable,
+            );
+            if let CommaRecoveryMode::EitherTupleOrPipe = rt {
+                err.span_suggestion(
+                    seq_span,
+                    "...or a vertical bar to match on multiple alternatives",
+                    seq_snippet.replace(',', " |"),
                     Applicability::MachineApplicable,
                 );
-                if let CommaRecoveryMode::EitherTupleOrPipe = rt {
-                    err.span_suggestion(
-                        seq_span,
-                        "...or a vertical bar to match on multiple alternatives",
-                        seq_snippet.replace(',', " |"),
-                        Applicability::MachineApplicable,
-                    );
-                }
             }
         }
         Err(err)
@@ -2784,7 +2919,7 @@ impl<'a> Parser<'a> {
                         span: path.span.shrink_to_hi(),
                         between: between_span,
                     }
-                    .into_diagnostic(&self.sess.span_diagnostic));
+                    .into_diagnostic(self.dcx()));
                 }
             }
         }
diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs
index 235b28b6e..cd3e8b92f 100644
--- a/compiler/rustc_parse/src/parser/expr.rs
+++ b/compiler/rustc_parse/src/parser/expr.rs
@@ -1,3 +1,4 @@
+// ignore-tidy-filelength
 use super::diagnostics::SnapshotParser;
 use super::pat::{CommaRecoveryMode, Expected, RecoverColon, RecoverComma};
 use super::ty::{AllowPlus, RecoverQPath, RecoverReturnSign};
@@ -9,7 +10,7 @@ use super::{
 use crate::errors;
 use crate::maybe_recover_from_interpolated_ty_qpath;
 use ast::mut_visit::{noop_visit_expr, MutVisitor};
-use ast::{GenBlockKind, Path, PathSegment};
+use ast::{CoroutineKind, GenBlockKind, Pat, Path, PathSegment};
 use core::mem;
 use rustc_ast::ptr::P;
 use rustc_ast::token::{self, Delimiter, Token, TokenKind};
@@ -20,7 +21,7 @@ use rustc_ast::util::parser::{prec_let_scrutinee_needs_par, AssocOp, Fixity};
 use rustc_ast::visit::Visitor;
 use rustc_ast::{self as ast, AttrStyle, AttrVec, CaptureBy, ExprField, UnOp, DUMMY_NODE_ID};
 use rustc_ast::{AnonConst, BinOp, BinOpKind, FnDecl, FnRetTy, MacCall, Param, Ty, TyKind};
-use rustc_ast::{Arm, Async, BlockCheckMode, Expr, ExprKind, Label, Movability, RangeLimits};
+use rustc_ast::{Arm, BlockCheckMode, Expr, ExprKind, Label, Movability, RangeLimits};
 use rustc_ast::{ClosureBinder, MetaItemLit, StmtKind};
 use rustc_ast_pretty::pprust;
 use rustc_data_structures::stack::ensure_sufficient_stack;
@@ -46,7 +47,7 @@ use thin_vec::{thin_vec, ThinVec};
 macro_rules! maybe_whole_expr {
     ($p:expr) => {
         if let token::Interpolated(nt) = &$p.token.kind {
-            match &**nt {
+            match &nt.0 {
                 token::NtExpr(e) | token::NtLiteral(e) => {
                     let e = e.clone();
                     $p.bump();
@@ -1060,7 +1061,7 @@ impl<'a> Parser<'a> {
         match &*components {
             // 1e2
             [IdentLike(i)] => {
-                DestructuredFloat::Single(Symbol::intern(&i), span)
+                DestructuredFloat::Single(Symbol::intern(i), span)
             }
             // 1.
             [IdentLike(i), Punct('.')] => {
@@ -1072,7 +1073,7 @@ impl<'a> Parser<'a> {
                 } else {
                     (span, span)
                 };
-                let symbol = Symbol::intern(&i);
+                let symbol = Symbol::intern(i);
                 DestructuredFloat::TrailingDot(symbol, ident_span, dot_span)
             }
             // 1.2 | 1.2e3
@@ -1088,8 +1089,8 @@ impl<'a> Parser<'a> {
                 } else {
                     (span, span, span)
                 };
-                let symbol1 = Symbol::intern(&i1);
-                let symbol2 = Symbol::intern(&i2);
+                let symbol1 = Symbol::intern(i1);
+                let symbol2 = Symbol::intern(i2);
                 DestructuredFloat::MiddleDot(symbol1, ident1_span, dot_span, symbol2, ident2_span)
             }
             // 1e+ | 1e- (recovered)
@@ -1268,7 +1269,7 @@ impl<'a> Parser<'a> {
                                         .collect(),
                                 },
                             }
-                            .into_diagnostic(&self.sess.span_diagnostic);
+                            .into_diagnostic(self.dcx());
                             replacement_err.emit();
 
                             let old_err = mem::replace(err, replacement_err);
@@ -1439,22 +1440,25 @@ impl<'a> Parser<'a> {
             } else if this.eat_keyword(kw::Underscore) {
                 Ok(this.mk_expr(this.prev_token.span, ExprKind::Underscore))
             } else if this.token.uninterpolated_span().at_least_rust_2018() {
-                // `Span:.at_least_rust_2018()` is somewhat expensive; don't get it repeatedly.
-                if this.check_keyword(kw::Async) {
-                    if this.is_gen_block(kw::Async) {
-                        // Check for `async {` and `async move {`.
+                // `Span::at_least_rust_2018()` is somewhat expensive; don't get it repeatedly.
+                if this.token.uninterpolated_span().at_least_rust_2024()
+                    // check for `gen {}` and `gen move {}`
+                    // or `async gen {}` and `async gen move {}`
+                    && (this.is_gen_block(kw::Gen, 0)
+                        || (this.check_keyword(kw::Async) && this.is_gen_block(kw::Gen, 1)))
+                {
+                    // FIXME: (async) gen closures aren't yet parsed.
+                    this.parse_gen_block()
+                } else if this.check_keyword(kw::Async) {
+                    // FIXME(gen_blocks): Parse `gen async` and suggest swap
+                    if this.is_gen_block(kw::Async, 0) {
+                        // Check for `async {` and `async move {`,
                         this.parse_gen_block()
                     } else {
                         this.parse_expr_closure()
                     }
-                } else if this.eat_keyword(kw::Await) {
+                } else if this.eat_keyword_noexpect(kw::Await) {
                     this.recover_incorrect_await_syntax(lo, this.prev_token.span)
-                } else if this.token.uninterpolated_span().at_least_rust_2024() {
-                    if this.is_gen_block(kw::Gen) {
-                        this.parse_gen_block()
-                    } else {
-                        this.parse_expr_lit()
-                    }
                 } else {
                     this.parse_expr_lit()
                 }
@@ -1689,8 +1693,7 @@ impl<'a> Parser<'a> {
         mk_lit_char: impl FnOnce(Symbol, Span) -> L,
         err: impl FnOnce(&Self) -> DiagnosticBuilder<'a, ErrorGuaranteed>,
     ) -> L {
-        if let Some(mut diag) =
-            self.sess.span_diagnostic.steal_diagnostic(lifetime.span, StashKey::LifetimeIsChar)
+        if let Some(mut diag) = self.dcx().steal_diagnostic(lifetime.span, StashKey::LifetimeIsChar)
         {
             diag.span_suggestion_verbose(
                 lifetime.span.shrink_to_hi(),
@@ -1880,8 +1883,8 @@ impl<'a> Parser<'a> {
         self.bump(); // `#`
 
         let Some((ident, false)) = self.token.ident() else {
-            let err = errors::ExpectedBuiltinIdent { span: self.token.span }
-                .into_diagnostic(&self.sess.span_diagnostic);
+            let err =
+                errors::ExpectedBuiltinIdent { span: self.token.span }.into_diagnostic(self.dcx());
             return Err(err);
         };
         self.sess.gated_spans.gate(sym::builtin_syntax, ident.span);
@@ -1892,7 +1895,7 @@ impl<'a> Parser<'a> {
             Ok(res)
         } else {
             let err = errors::UnknownBuiltinConstruct { span: lo.to(ident.span), name: ident.name }
-                .into_diagnostic(&self.sess.span_diagnostic);
+                .into_diagnostic(self.dcx());
             return Err(err);
         };
         self.expect(&TokenKind::CloseDelim(Delimiter::Parenthesis))?;
@@ -1952,11 +1955,11 @@ impl<'a> Parser<'a> {
         mk_lit_char: impl FnOnce(Symbol, Span) -> L,
     ) -> PResult<'a, L> {
         if let token::Interpolated(nt) = &self.token.kind
-            && let token::NtExpr(e) | token::NtLiteral(e) = &**nt
+            && let token::NtExpr(e) | token::NtLiteral(e) = &nt.0
             && matches!(e.kind, ExprKind::Err)
         {
             let mut err = errors::InvalidInterpolatedExpression { span: self.token.span }
-                .into_diagnostic(&self.sess.span_diagnostic);
+                .into_diagnostic(self.dcx());
             err.downgrade_to_delayed_bug();
             return Err(err);
         }
@@ -2052,7 +2055,7 @@ impl<'a> Parser<'a> {
                     Err(err) => {
                         let span = token.uninterpolated_span();
                         self.bump();
-                        report_lit_error(&self.sess, err, lit, span);
+                        report_lit_error(self.sess, err, lit, span);
                         // Pack possible quotes and prefixes from the original literal into
                         // the error literal's symbol so they can be pretty-printed faithfully.
                         let suffixless_lit = token::Lit::new(lit.kind, lit.symbol, None);
@@ -2168,7 +2171,7 @@ impl<'a> Parser<'a> {
                     return Err(errors::MissingSemicolonBeforeArray {
                         open_delim: open_delim_span,
                         semicolon: prev_span.shrink_to_hi(),
-                    }.into_diagnostic(&self.sess.span_diagnostic));
+                    }.into_diagnostic(self.dcx()));
                 }
                 Ok(_) => (),
                 Err(err) => err.cancel(),
@@ -2233,10 +2236,10 @@ impl<'a> Parser<'a> {
         let movability =
             if self.eat_keyword(kw::Static) { Movability::Static } else { Movability::Movable };
 
-        let asyncness = if self.token.uninterpolated_span().at_least_rust_2018() {
-            self.parse_asyncness(Case::Sensitive)
+        let coroutine_kind = if self.token.uninterpolated_span().at_least_rust_2018() {
+            self.parse_coroutine_kind(Case::Sensitive)
         } else {
-            Async::No
+            None
         };
 
         let capture_clause = self.parse_capture_clause()?;
@@ -2260,13 +2263,21 @@ impl<'a> Parser<'a> {
             }
         };
 
-        if let Async::Yes { span, .. } = asyncness {
-            // Feature-gate `async ||` closures.
-            self.sess.gated_spans.gate(sym::async_closure, span);
+        match coroutine_kind {
+            Some(CoroutineKind::Async { span, .. }) => {
+                // Feature-gate `async ||` closures.
+                self.sess.gated_spans.gate(sym::async_closure, span);
+            }
+            Some(CoroutineKind::Gen { span, .. }) | Some(CoroutineKind::AsyncGen { span, .. }) => {
+                // Feature-gate `gen ||` and `async gen ||` closures.
+                // FIXME(gen_blocks): This perhaps should be a different gate.
+                self.sess.gated_spans.gate(sym::gen_blocks, span);
+            }
+            None => {}
         }
 
         if self.token.kind == TokenKind::Semi
-            && matches!(self.token_cursor.stack.last(), Some((_, Delimiter::Parenthesis, _)))
+            && matches!(self.token_cursor.stack.last(), Some((.., Delimiter::Parenthesis)))
             && self.may_recover()
         {
             // It is likely that the closure body is a block but where the
@@ -2283,7 +2294,7 @@ impl<'a> Parser<'a> {
                 binder,
                 capture_clause,
                 constness,
-                asyncness,
+                coroutine_kind,
                 movability,
                 fn_decl,
                 body,
@@ -2308,7 +2319,7 @@ impl<'a> Parser<'a> {
             if self.check_keyword(kw::Async) {
                 let move_async_span = self.token.span.with_lo(self.prev_token.span.data().lo);
                 Err(errors::AsyncMoveOrderIncorrect { span: move_async_span }
-                    .into_diagnostic(&self.sess.span_diagnostic))
+                    .into_diagnostic(self.dcx()))
             } else {
                 Ok(CaptureBy::Value { move_kw: move_kw_span })
             }
@@ -2477,7 +2488,7 @@ impl<'a> Parser<'a> {
         let mut cond =
             self.parse_expr_res(Restrictions::NO_STRUCT_LITERAL | Restrictions::ALLOW_LET, None)?;
 
-        CondChecker { parser: self, forbid_let_reason: None }.visit_expr(&mut cond);
+        CondChecker::new(self).visit_expr(&mut cond);
 
         if let ExprKind::Let(_, _, _, None) = cond.kind {
             // Remove the last feature gating of a `let` expression since it's stable.
@@ -2493,10 +2504,12 @@ impl<'a> Parser<'a> {
             let err = errors::ExpectedExpressionFoundLet {
                 span: self.token.span,
                 reason: ForbiddenLetReason::OtherForbidden,
+                missing_let: None,
+                comparison: None,
             };
             if self.prev_token.kind == token::BinOp(token::Or) {
                 // This was part of a closure, the that part of the parser recover.
-                return Err(err.into_diagnostic(&self.sess.span_diagnostic));
+                return Err(err.into_diagnostic(self.dcx()));
             } else {
                 Some(self.sess.emit_err(err))
             }
@@ -2606,30 +2619,72 @@ impl<'a> Parser<'a> {
         }
     }
 
-    /// Parses `for <src_pat> in <src_expr> <src_loop_block>` (`for` token already eaten).
-    fn parse_expr_for(&mut self, opt_label: Option<Label>, lo: Span) -> PResult<'a, P<Expr>> {
-        // Record whether we are about to parse `for (`.
-        // This is used below for recovery in case of `for ( $stuff ) $block`
-        // in which case we will suggest `for $stuff $block`.
-        let begin_paren = match self.token.kind {
-            token::OpenDelim(Delimiter::Parenthesis) => Some(self.token.span),
-            _ => None,
+    fn parse_for_head(&mut self) -> PResult<'a, (P<Pat>, P<Expr>)> {
+        let begin_paren = if self.token.kind == token::OpenDelim(Delimiter::Parenthesis) {
+            // Record whether we are about to parse `for (`.
+            // This is used below for recovery in case of `for ( $stuff ) $block`
+            // in which case we will suggest `for $stuff $block`.
+            let start_span = self.token.span;
+            let left = self.prev_token.span.between(self.look_ahead(1, |t| t.span));
+            Some((start_span, left))
+        } else {
+            None
+        };
+        // Try to parse the pattern `for ($PAT) in $EXPR`.
+        let pat = match (
+            self.parse_pat_allow_top_alt(
+                None,
+                RecoverComma::Yes,
+                RecoverColon::Yes,
+                CommaRecoveryMode::LikelyTuple,
+            ),
+            begin_paren,
+        ) {
+            (Ok(pat), _) => pat, // Happy path.
+            (Err(err), Some((start_span, left))) if self.eat_keyword(kw::In) => {
+                // We know for sure we have seen `for ($SOMETHING in`. In the happy path this would
+                // happen right before the return of this method.
+                let expr = match self.parse_expr_res(Restrictions::NO_STRUCT_LITERAL, None) {
+                    Ok(expr) => expr,
+                    Err(expr_err) => {
+                        // We don't know what followed the `in`, so cancel and bubble up the
+                        // original error.
+                        expr_err.cancel();
+                        return Err(err);
+                    }
+                };
+                return if self.token.kind == token::CloseDelim(Delimiter::Parenthesis) {
+                    // We know for sure we have seen `for ($SOMETHING in $EXPR)`, so we recover the
+                    // parser state and emit a targetted suggestion.
+                    let span = vec![start_span, self.token.span];
+                    let right = self.prev_token.span.between(self.look_ahead(1, |t| t.span));
+                    self.bump(); // )
+                    err.cancel();
+                    self.sess.emit_err(errors::ParenthesesInForHead {
+                        span,
+                        // With e.g. `for (x) in y)` this would replace `(x) in y)`
+                        // with `x) in y)` which is syntactically invalid.
+                        // However, this is prevented before we get here.
+                        sugg: errors::ParenthesesInForHeadSugg { left, right },
+                    });
+                    Ok((self.mk_pat(start_span.to(right), ast::PatKind::Wild), expr))
+                } else {
+                    Err(err) // Some other error, bubble up.
+                };
+            }
+            (Err(err), _) => return Err(err), // Some other error, bubble up.
         };
-
-        let pat = self.parse_pat_allow_top_alt(
-            None,
-            RecoverComma::Yes,
-            RecoverColon::Yes,
-            CommaRecoveryMode::LikelyTuple,
-        )?;
         if !self.eat_keyword(kw::In) {
             self.error_missing_in_for_loop();
         }
         self.check_for_for_in_in_typo(self.prev_token.span);
         let expr = self.parse_expr_res(Restrictions::NO_STRUCT_LITERAL, None)?;
+        Ok((pat, expr))
+    }
 
-        let pat = self.recover_parens_around_for_head(pat, begin_paren);
-
+    /// Parses `for <src_pat> in <src_expr> <src_loop_block>` (`for` token already eaten).
+    fn parse_expr_for(&mut self, opt_label: Option<Label>, lo: Span) -> PResult<'a, P<Expr>> {
+        let (pat, expr) = self.parse_for_head()?;
         // Recover from missing expression in `for` loop
         if matches!(expr.kind, ExprKind::Block(..))
             && !matches!(self.token.kind, token::OpenDelim(Delimiter::Brace))
@@ -2850,167 +2905,167 @@ impl<'a> Parser<'a> {
     }
 
     pub(super) fn parse_arm(&mut self) -> PResult<'a, Arm> {
-        // Used to check the `let_chains` and `if_let_guard` features mostly by scanning
-        // `&&` tokens.
-        fn check_let_expr(expr: &Expr) -> (bool, bool) {
-            match &expr.kind {
-                ExprKind::Binary(BinOp { node: BinOpKind::And, .. }, lhs, rhs) => {
-                    let lhs_rslt = check_let_expr(lhs);
-                    let rhs_rslt = check_let_expr(rhs);
-                    (lhs_rslt.0 || rhs_rslt.0, false)
-                }
-                ExprKind::Let(..) => (true, true),
-                _ => (false, true),
-            }
-        }
         let attrs = self.parse_outer_attributes()?;
         self.collect_tokens_trailing_token(attrs, ForceCollect::No, |this, attrs| {
             let lo = this.token.span;
-            let pat = this.parse_pat_allow_top_alt(
-                None,
-                RecoverComma::Yes,
-                RecoverColon::Yes,
-                CommaRecoveryMode::EitherTupleOrPipe,
-            )?;
-            let guard = if this.eat_keyword(kw::If) {
-                let if_span = this.prev_token.span;
-                let mut cond = this.parse_match_guard_condition()?;
-
-                CondChecker { parser: this, forbid_let_reason: None }.visit_expr(&mut cond);
-
-                let (has_let_expr, does_not_have_bin_op) = check_let_expr(&cond);
-                if has_let_expr {
-                    if does_not_have_bin_op {
-                        // Remove the last feature gating of a `let` expression since it's stable.
-                        this.sess.gated_spans.ungate_last(sym::let_chains, cond.span);
-                    }
-                    let span = if_span.to(cond.span);
-                    this.sess.gated_spans.gate(sym::if_let_guard, span);
-                }
-                Some(cond)
+            let (pat, guard) = this.parse_match_arm_pat_and_guard()?;
+
+            let span_before_body = this.prev_token.span;
+            let arm_body;
+            let is_fat_arrow = this.check(&token::FatArrow);
+            let is_almost_fat_arrow = TokenKind::FatArrow
+                .similar_tokens()
+                .is_some_and(|similar_tokens| similar_tokens.contains(&this.token.kind));
+            let mut result = if !is_fat_arrow && !is_almost_fat_arrow {
+                // A pattern without a body, allowed for never patterns.
+                arm_body = None;
+                this.expect_one_of(&[token::Comma], &[token::CloseDelim(Delimiter::Brace)]).map(
+                    |x| {
+                        // Don't gate twice
+                        if !pat.contains_never_pattern() {
+                            this.sess.gated_spans.gate(sym::never_patterns, pat.span);
+                        }
+                        x
+                    },
+                )
             } else {
-                None
-            };
-            let arrow_span = this.token.span;
-            if let Err(mut err) = this.expect(&token::FatArrow) {
-                // We might have a `=>` -> `=` or `->` typo (issue #89396).
-                if TokenKind::FatArrow
-                    .similar_tokens()
-                    .is_some_and(|similar_tokens| similar_tokens.contains(&this.token.kind))
-                {
-                    err.span_suggestion(
-                        this.token.span,
-                        "use a fat arrow to start a match arm",
-                        "=>",
-                        Applicability::MachineApplicable,
-                    );
-                    err.emit();
-                    this.bump();
-                } else if matches!(
-                    (&this.prev_token.kind, &this.token.kind),
-                    (token::DotDotEq, token::Gt)
-                ) {
-                    // `error_inclusive_range_match_arrow` handles cases like `0..=> {}`,
-                    // so we suppress the error here
-                    err.delay_as_bug();
-                    this.bump();
-                } else {
-                    return Err(err);
+                if let Err(mut err) = this.expect(&token::FatArrow) {
+                    // We might have a `=>` -> `=` or `->` typo (issue #89396).
+                    if is_almost_fat_arrow {
+                        err.span_suggestion(
+                            this.token.span,
+                            "use a fat arrow to start a match arm",
+                            "=>",
+                            Applicability::MachineApplicable,
+                        );
+                        if matches!(
+                            (&this.prev_token.kind, &this.token.kind),
+                            (token::DotDotEq, token::Gt)
+                        ) {
+                            // `error_inclusive_range_match_arrow` handles cases like `0..=> {}`,
+                            // so we suppress the error here
+                            err.delay_as_bug();
+                        } else {
+                            err.emit();
+                        }
+                        this.bump();
+                    } else {
+                        return Err(err);
+                    }
                 }
-            }
-            let arm_start_span = this.token.span;
+                let arrow_span = this.prev_token.span;
+                let arm_start_span = this.token.span;
 
-            let expr = this.parse_expr_res(Restrictions::STMT_EXPR, None).map_err(|mut err| {
-                err.span_label(arrow_span, "while parsing the `match` arm starting here");
-                err
-            })?;
+                let expr =
+                    this.parse_expr_res(Restrictions::STMT_EXPR, None).map_err(|mut err| {
+                        err.span_label(arrow_span, "while parsing the `match` arm starting here");
+                        err
+                    })?;
 
-            let require_comma = classify::expr_requires_semi_to_be_stmt(&expr)
-                && this.token != token::CloseDelim(Delimiter::Brace);
-
-            let hi = this.prev_token.span;
-
-            if require_comma {
-                let sm = this.sess.source_map();
-                if let Some(body) = this.parse_arm_body_missing_braces(&expr, arrow_span) {
-                    let span = body.span;
-                    return Ok((
-                        ast::Arm {
-                            attrs,
-                            pat,
-                            guard,
-                            body,
-                            span,
-                            id: DUMMY_NODE_ID,
-                            is_placeholder: false,
-                        },
-                        TrailingToken::None,
-                    ));
-                }
-                this.expect_one_of(&[token::Comma], &[token::CloseDelim(Delimiter::Brace)])
-                    .or_else(|mut err| {
-                        if this.token == token::FatArrow {
-                            if let Ok(expr_lines) = sm.span_to_lines(expr.span)
-                                && let Ok(arm_start_lines) = sm.span_to_lines(arm_start_span)
-                                && arm_start_lines.lines[0].end_col == expr_lines.lines[0].end_col
-                                && expr_lines.lines.len() == 2
-                            {
-                                // We check whether there's any trailing code in the parse span,
-                                // if there isn't, we very likely have the following:
-                                //
-                                // X |     &Y => "y"
-                                //   |        --    - missing comma
-                                //   |        |
-                                //   |        arrow_span
-                                // X |     &X => "x"
-                                //   |      - ^^ self.token.span
-                                //   |      |
-                                //   |      parsed until here as `"y" & X`
-                                err.span_suggestion_short(
-                                    arm_start_span.shrink_to_hi(),
-                                    "missing a comma here to end this `match` arm",
-                                    ",",
-                                    Applicability::MachineApplicable,
+                let require_comma = classify::expr_requires_semi_to_be_stmt(&expr)
+                    && this.token != token::CloseDelim(Delimiter::Brace);
+
+                if !require_comma {
+                    arm_body = Some(expr);
+                    this.eat(&token::Comma);
+                    Ok(false)
+                } else if let Some(body) = this.parse_arm_body_missing_braces(&expr, arrow_span) {
+                    arm_body = Some(body);
+                    Ok(true)
+                } else {
+                    let expr_span = expr.span;
+                    arm_body = Some(expr);
+                    this.expect_one_of(&[token::Comma], &[token::CloseDelim(Delimiter::Brace)])
+                        .map_err(|mut err| {
+                            if this.token == token::FatArrow {
+                                let sm = this.sess.source_map();
+                                if let Ok(expr_lines) = sm.span_to_lines(expr_span)
+                                    && let Ok(arm_start_lines) = sm.span_to_lines(arm_start_span)
+                                    && arm_start_lines.lines[0].end_col
+                                        == expr_lines.lines[0].end_col
+                                    && expr_lines.lines.len() == 2
+                                {
+                                    // We check whether there's any trailing code in the parse span,
+                                    // if there isn't, we very likely have the following:
+                                    //
+                                    // X |     &Y => "y"
+                                    //   |        --    - missing comma
+                                    //   |        |
+                                    //   |        arrow_span
+                                    // X |     &X => "x"
+                                    //   |      - ^^ self.token.span
+                                    //   |      |
+                                    //   |      parsed until here as `"y" & X`
+                                    err.span_suggestion_short(
+                                        arm_start_span.shrink_to_hi(),
+                                        "missing a comma here to end this `match` arm",
+                                        ",",
+                                        Applicability::MachineApplicable,
+                                    );
+                                }
+                            } else {
+                                err.span_label(
+                                    arrow_span,
+                                    "while parsing the `match` arm starting here",
                                 );
-                                return Err(err);
-                            }
-                        } else {
-                            // FIXME(compiler-errors): We could also recover `; PAT =>` here
-
-                            // Try to parse a following `PAT =>`, if successful
-                            // then we should recover.
-                            let mut snapshot = this.create_snapshot_for_diagnostic();
-                            let pattern_follows = snapshot
-                                .parse_pat_allow_top_alt(
-                                    None,
-                                    RecoverComma::Yes,
-                                    RecoverColon::Yes,
-                                    CommaRecoveryMode::EitherTupleOrPipe,
-                                )
-                                .map_err(|err| err.cancel())
-                                .is_ok();
-                            if pattern_follows && snapshot.check(&TokenKind::FatArrow) {
-                                err.cancel();
-                                this.sess.emit_err(errors::MissingCommaAfterMatchArm {
-                                    span: hi.shrink_to_hi(),
-                                });
-                                return Ok(true);
                             }
-                        }
-                        err.span_label(arrow_span, "while parsing the `match` arm starting here");
-                        Err(err)
-                    })?;
-            } else {
-                this.eat(&token::Comma);
+                            err
+                        })
+                }
+            };
+
+            let hi_span = arm_body.as_ref().map_or(span_before_body, |body| body.span);
+            let arm_span = lo.to(hi_span);
+
+            // We want to recover:
+            // X |     Some(_) => foo()
+            //   |                     - missing comma
+            // X |     None => "x"
+            //   |     ^^^^ self.token.span
+            // as well as:
+            // X |     Some(!)
+            //   |            - missing comma
+            // X |     None => "x"
+            //   |     ^^^^ self.token.span
+            // But we musn't recover
+            // X |     pat[0] => {}
+            //   |        ^ self.token.span
+            let recover_missing_comma = arm_body.is_some() || pat.could_be_never_pattern();
+            if recover_missing_comma {
+                result = result.or_else(|err| {
+                    // FIXME(compiler-errors): We could also recover `; PAT =>` here
+
+                    // Try to parse a following `PAT =>`, if successful
+                    // then we should recover.
+                    let mut snapshot = this.create_snapshot_for_diagnostic();
+                    let pattern_follows = snapshot
+                        .parse_pat_allow_top_alt(
+                            None,
+                            RecoverComma::Yes,
+                            RecoverColon::Yes,
+                            CommaRecoveryMode::EitherTupleOrPipe,
+                        )
+                        .map_err(|err| err.cancel())
+                        .is_ok();
+                    if pattern_follows && snapshot.check(&TokenKind::FatArrow) {
+                        err.cancel();
+                        this.sess.emit_err(errors::MissingCommaAfterMatchArm {
+                            span: arm_span.shrink_to_hi(),
+                        });
+                        return Ok(true);
+                    }
+                    Err(err)
+                });
             }
+            result?;
 
             Ok((
                 ast::Arm {
                     attrs,
                     pat,
                     guard,
-                    body: expr,
-                    span: lo.to(hi),
+                    body: arm_body,
+                    span: arm_span,
                     id: DUMMY_NODE_ID,
                     is_placeholder: false,
                 },
@@ -3019,6 +3074,90 @@ impl<'a> Parser<'a> {
         })
     }
 
+    fn parse_match_arm_guard(&mut self) -> PResult<'a, Option<P<Expr>>> {
+        // Used to check the `let_chains` and `if_let_guard` features mostly by scanning
+        // `&&` tokens.
+        fn check_let_expr(expr: &Expr) -> (bool, bool) {
+            match &expr.kind {
+                ExprKind::Binary(BinOp { node: BinOpKind::And, .. }, lhs, rhs) => {
+                    let lhs_rslt = check_let_expr(lhs);
+                    let rhs_rslt = check_let_expr(rhs);
+                    (lhs_rslt.0 || rhs_rslt.0, false)
+                }
+                ExprKind::Let(..) => (true, true),
+                _ => (false, true),
+            }
+        }
+        if !self.eat_keyword(kw::If) {
+            // No match arm guard present.
+            return Ok(None);
+        }
+
+        let if_span = self.prev_token.span;
+        let mut cond = self.parse_match_guard_condition()?;
+
+        CondChecker::new(self).visit_expr(&mut cond);
+
+        let (has_let_expr, does_not_have_bin_op) = check_let_expr(&cond);
+        if has_let_expr {
+            if does_not_have_bin_op {
+                // Remove the last feature gating of a `let` expression since it's stable.
+                self.sess.gated_spans.ungate_last(sym::let_chains, cond.span);
+            }
+            let span = if_span.to(cond.span);
+            self.sess.gated_spans.gate(sym::if_let_guard, span);
+        }
+        Ok(Some(cond))
+    }
+
+    fn parse_match_arm_pat_and_guard(&mut self) -> PResult<'a, (P<Pat>, Option<P<Expr>>)> {
+        if self.token.kind == token::OpenDelim(Delimiter::Parenthesis) {
+            // Detect and recover from `($pat if $cond) => $arm`.
+            let left = self.token.span;
+            match self.parse_pat_allow_top_alt(
+                None,
+                RecoverComma::Yes,
+                RecoverColon::Yes,
+                CommaRecoveryMode::EitherTupleOrPipe,
+            ) {
+                Ok(pat) => Ok((pat, self.parse_match_arm_guard()?)),
+                Err(err)
+                    if let prev_sp = self.prev_token.span
+                        && let true = self.eat_keyword(kw::If) =>
+                {
+                    // We know for certain we've found `($pat if` so far.
+                    let mut cond = match self.parse_match_guard_condition() {
+                        Ok(cond) => cond,
+                        Err(cond_err) => {
+                            cond_err.cancel();
+                            return Err(err);
+                        }
+                    };
+                    err.cancel();
+                    CondChecker::new(self).visit_expr(&mut cond);
+                    self.eat_to_tokens(&[&token::CloseDelim(Delimiter::Parenthesis)]);
+                    self.expect(&token::CloseDelim(Delimiter::Parenthesis))?;
+                    let right = self.prev_token.span;
+                    self.sess.emit_err(errors::ParenthesesInMatchPat {
+                        span: vec![left, right],
+                        sugg: errors::ParenthesesInMatchPatSugg { left, right },
+                    });
+                    Ok((self.mk_pat(left.to(prev_sp), ast::PatKind::Wild), Some(cond)))
+                }
+                Err(err) => Err(err),
+            }
+        } else {
+            // Regular parser flow:
+            let pat = self.parse_pat_allow_top_alt(
+                None,
+                RecoverComma::Yes,
+                RecoverColon::Yes,
+                CommaRecoveryMode::EitherTupleOrPipe,
+            )?;
+            Ok((pat, self.parse_match_arm_guard()?))
+        }
+    }
+
     fn parse_match_guard_condition(&mut self) -> PResult<'a, P<Expr>> {
         self.parse_expr_res(Restrictions::ALLOW_LET | Restrictions::IN_IF_GUARD, None).map_err(
             |mut err| {
@@ -3054,8 +3193,7 @@ impl<'a> Parser<'a> {
     fn parse_try_block(&mut self, span_lo: Span) -> PResult<'a, P<Expr>> {
         let (attrs, body) = self.parse_inner_attrs_and_block()?;
         if self.eat_keyword(kw::Catch) {
-            Err(errors::CatchAfterTry { span: self.prev_token.span }
-                .into_diagnostic(&self.sess.span_diagnostic))
+            Err(errors::CatchAfterTry { span: self.prev_token.span }.into_diagnostic(self.dcx()))
         } else {
             let span = span_lo.to(body.span);
             self.sess.gated_spans.gate(sym::try_blocks, span);
@@ -3086,34 +3224,45 @@ impl<'a> Parser<'a> {
     fn parse_gen_block(&mut self) -> PResult<'a, P<Expr>> {
         let lo = self.token.span;
         let kind = if self.eat_keyword(kw::Async) {
-            GenBlockKind::Async
+            if self.eat_keyword(kw::Gen) { GenBlockKind::AsyncGen } else { GenBlockKind::Async }
         } else {
             assert!(self.eat_keyword(kw::Gen));
-            self.sess.gated_spans.gate(sym::gen_blocks, lo.to(self.token.span));
             GenBlockKind::Gen
         };
+        match kind {
+            GenBlockKind::Async => {
+                // `async` blocks are stable
+            }
+            GenBlockKind::Gen | GenBlockKind::AsyncGen => {
+                self.sess.gated_spans.gate(sym::gen_blocks, lo.to(self.prev_token.span));
+            }
+        }
         let capture_clause = self.parse_capture_clause()?;
         let (attrs, body) = self.parse_inner_attrs_and_block()?;
         let kind = ExprKind::Gen(capture_clause, body, kind);
         Ok(self.mk_expr_with_attrs(lo.to(self.prev_token.span), kind, attrs))
     }
 
-    fn is_gen_block(&self, kw: Symbol) -> bool {
-        self.token.is_keyword(kw)
+    fn is_gen_block(&self, kw: Symbol, lookahead: usize) -> bool {
+        self.is_keyword_ahead(lookahead, &[kw])
             && ((
                 // `async move {`
-                self.is_keyword_ahead(1, &[kw::Move])
-                    && self.look_ahead(2, |t| {
+                self.is_keyword_ahead(lookahead + 1, &[kw::Move])
+                    && self.look_ahead(lookahead + 2, |t| {
                         *t == token::OpenDelim(Delimiter::Brace) || t.is_whole_block()
                     })
             ) || (
                 // `async {`
-                self.look_ahead(1, |t| {
+                self.look_ahead(lookahead + 1, |t| {
                     *t == token::OpenDelim(Delimiter::Brace) || t.is_whole_block()
                 })
             ))
     }
 
+    pub(super) fn is_async_gen_block(&self) -> bool {
+        self.token.is_keyword(kw::Async) && self.is_gen_block(kw::Gen, 1)
+    }
+
     fn is_certainly_not_a_block(&self) -> bool {
         self.look_ahead(1, |t| t.is_ident())
             && (
@@ -3206,7 +3355,7 @@ impl<'a> Parser<'a> {
                     if let Some((ident, _)) = self.token.ident()
                         && !self.token.is_reserved_ident()
                         && self.look_ahead(1, |t| {
-                            AssocOp::from_token(&t).is_some()
+                            AssocOp::from_token(t).is_some()
                                 || matches!(t.kind, token::OpenDelim(_))
                                 || t.kind == token::Dot
                         })
@@ -3386,7 +3535,7 @@ impl<'a> Parser<'a> {
                     ident_span: this.token.span,
                     token: this.look_ahead(1, |t| t.clone()),
                 }
-                .into_diagnostic(&self.sess.span_diagnostic));
+                .into_diagnostic(&self.sess.dcx));
             }
             let (ident, expr) = if is_shorthand {
                 // Mimic `x: x` for the `x` field shorthand.
@@ -3551,6 +3700,14 @@ pub(crate) enum ForbiddenLetReason {
 struct CondChecker<'a> {
     parser: &'a Parser<'a>,
     forbid_let_reason: Option<ForbiddenLetReason>,
+    missing_let: Option<errors::MaybeMissingLet>,
+    comparison: Option<errors::MaybeComparison>,
+}
+
+impl<'a> CondChecker<'a> {
+    fn new(parser: &'a Parser<'a>) -> Self {
+        CondChecker { parser, forbid_let_reason: None, missing_let: None, comparison: None }
+    }
 }
 
 impl MutVisitor for CondChecker<'_> {
@@ -3561,11 +3718,13 @@ impl MutVisitor for CondChecker<'_> {
         match e.kind {
             ExprKind::Let(_, _, _, ref mut is_recovered @ None) => {
                 if let Some(reason) = self.forbid_let_reason {
-                    *is_recovered = Some(
-                        self.parser
-                            .sess
-                            .emit_err(errors::ExpectedExpressionFoundLet { span, reason }),
-                    );
+                    *is_recovered =
+                        Some(self.parser.sess.emit_err(errors::ExpectedExpressionFoundLet {
+                            span,
+                            reason,
+                            missing_let: self.missing_let,
+                            comparison: self.comparison,
+                        }));
                 } else {
                     self.parser.sess.gated_spans.gate(sym::let_chains, span);
                 }
@@ -3589,9 +3748,28 @@ impl MutVisitor for CondChecker<'_> {
                 noop_visit_expr(e, self);
                 self.forbid_let_reason = forbid_let_reason;
             }
+            ExprKind::Assign(ref lhs, _, span) => {
+                let forbid_let_reason = self.forbid_let_reason;
+                self.forbid_let_reason = Some(OtherForbidden);
+                let missing_let = self.missing_let;
+                if let ExprKind::Binary(_, _, rhs) = &lhs.kind
+                    && let ExprKind::Path(_, _)
+                    | ExprKind::Struct(_)
+                    | ExprKind::Call(_, _)
+                    | ExprKind::Array(_) = rhs.kind
+                {
+                    self.missing_let =
+                        Some(errors::MaybeMissingLet { span: rhs.span.shrink_to_lo() });
+                }
+                let comparison = self.comparison;
+                self.comparison = Some(errors::MaybeComparison { span: span.shrink_to_hi() });
+                noop_visit_expr(e, self);
+                self.forbid_let_reason = forbid_let_reason;
+                self.missing_let = missing_let;
+                self.comparison = comparison;
+            }
             ExprKind::Unary(_, _)
             | ExprKind::Await(_, _)
-            | ExprKind::Assign(_, _, _)
             | ExprKind::AssignOp(_, _, _)
             | ExprKind::Range(_, _, _)
             | ExprKind::Try(_)
diff --git a/compiler/rustc_parse/src/parser/generics.rs b/compiler/rustc_parse/src/parser/generics.rs
index 242c9d332..20f67b284 100644
--- a/compiler/rustc_parse/src/parser/generics.rs
+++ b/compiler/rustc_parse/src/parser/generics.rs
@@ -279,7 +279,7 @@ impl<'a> Parser<'a> {
         let span_lo = self.token.span;
         let (params, span) = if self.eat_lt() {
             let params = self.parse_generic_params()?;
-            self.expect_gt()?;
+            self.expect_gt_or_maybe_suggest_closing_generics(&params)?;
             (params, span_lo.to(self.prev_token.span))
         } else {
             (ThinVec::new(), self.prev_token.span.shrink_to_hi())
diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs
index 801860c21..09ee042ef 100644
--- a/compiler/rustc_parse/src/parser/item.rs
+++ b/compiler/rustc_parse/src/parser/item.rs
@@ -3,18 +3,12 @@ use super::ty::{AllowPlus, RecoverQPath, RecoverReturnSign};
 use super::{AttrWrapper, FollowedByType, ForceCollect, Parser, PathStyle, TrailingToken};
 use crate::errors::{self, MacroExpandsToAdtField};
 use crate::fluent_generated as fluent;
-use ast::StaticItem;
 use rustc_ast::ast::*;
 use rustc_ast::ptr::P;
 use rustc_ast::token::{self, Delimiter, TokenKind};
 use rustc_ast::tokenstream::{DelimSpan, TokenStream, TokenTree};
 use rustc_ast::util::case::Case;
-use rustc_ast::MacCall;
-use rustc_ast::{self as ast, AttrVec, Attribute, DUMMY_NODE_ID};
-use rustc_ast::{Async, Const, Defaultness, IsAuto, Mutability, Unsafe, UseTree, UseTreeKind};
-use rustc_ast::{BindingAnnotation, Block, FnDecl, FnSig, Param, SelfKind};
-use rustc_ast::{EnumDef, FieldDef, Generics, TraitRef, Ty, TyKind, Variant, VariantData};
-use rustc_ast::{FnHeader, ForeignItem, Path, PathSegment, Visibility, VisibilityKind};
+use rustc_ast::{self as ast};
 use rustc_ast_pretty::pprust;
 use rustc_errors::{
     struct_span_err, Applicability, DiagnosticBuilder, ErrorGuaranteed, IntoDiagnostic, PResult,
@@ -123,7 +117,7 @@ impl<'a> Parser<'a> {
         // Don't use `maybe_whole` so that we have precise control
         // over when we bump the parser
         if let token::Interpolated(nt) = &self.token.kind
-            && let token::NtItem(item) = &**nt
+            && let token::NtItem(item) = &nt.0
         {
             let mut item = item.clone();
             self.bump();
@@ -444,11 +438,7 @@ impl<'a> Parser<'a> {
             None
         };
 
-        if let Some(err) = err {
-            Err(err.into_diagnostic(&self.sess.span_diagnostic))
-        } else {
-            Ok(())
-        }
+        if let Some(err) = err { Err(err.into_diagnostic(self.dcx())) } else { Ok(()) }
     }
 
     fn parse_item_builtin(&mut self) -> PResult<'a, Option<ItemInfo>> {
@@ -769,7 +759,7 @@ impl<'a> Parser<'a> {
             if self.look_ahead(1, |tok| tok == &token::CloseDelim(Delimiter::Brace)) {
                 // FIXME: merge with `DocCommentDoesNotDocumentAnything` (E0585)
                 struct_span_err!(
-                    self.diagnostic(),
+                    self.dcx(),
                     self.token.span,
                     E0584,
                     "found a documentation comment that doesn't document anything",
@@ -933,7 +923,7 @@ impl<'a> Parser<'a> {
         );
         let where_predicates_split = before_where_clause.predicates.len();
         let mut predicates = before_where_clause.predicates;
-        predicates.extend(after_where_clause.predicates.into_iter());
+        predicates.extend(after_where_clause.predicates);
         let where_clause = WhereClause {
             has_where_token: before_where_clause.has_where_token
                 || after_where_clause.has_where_token,
@@ -1143,9 +1133,11 @@ impl<'a> Parser<'a> {
                     Ok(kind) => kind,
                     Err(kind) => match kind {
                         ItemKind::Const(box ConstItem { ty, expr, .. }) => {
+                            let const_span = Some(span.with_hi(ident.span.lo()))
+                                .filter(|span| span.can_be_used_for_suggestions());
                             self.sess.emit_err(errors::ExternItemCannotBeConst {
                                 ident_span: ident.span,
-                                const_span: span.with_hi(ident.span.lo()),
+                                const_span,
                             });
                             ForeignItemKind::Static(ty, Mutability::Not, expr)
                         }
@@ -1382,8 +1374,7 @@ impl<'a> Parser<'a> {
 
         let span = self.prev_token.span.shrink_to_hi();
         let err: DiagnosticBuilder<'_, ErrorGuaranteed> =
-            errors::MissingConstType { span, colon, kind }
-                .into_diagnostic(&self.sess.span_diagnostic);
+            errors::MissingConstType { span, colon, kind }.into_diagnostic(self.dcx());
         err.stash(span, StashKey::ItemNoType);
 
         // The user intended that the type be inferred,
@@ -1400,7 +1391,7 @@ impl<'a> Parser<'a> {
                 self.bump();
                 self.sess.emit_err(err);
             } else {
-                return Err(err.into_diagnostic(&self.sess.span_diagnostic));
+                return Err(err.into_diagnostic(self.dcx()));
             }
         }
 
@@ -1415,8 +1406,8 @@ impl<'a> Parser<'a> {
             self.bump();
             (thin_vec![], false)
         } else {
-            self.parse_delim_comma_seq(Delimiter::Brace, |p| p.parse_enum_variant()).map_err(
-                |mut err| {
+            self.parse_delim_comma_seq(Delimiter::Brace, |p| p.parse_enum_variant(id.span))
+                .map_err(|mut err| {
                     err.span_label(id.span, "while parsing this enum");
                     if self.token == token::Colon {
                         let snapshot = self.create_snapshot_for_diagnostic();
@@ -1436,20 +1427,22 @@ impl<'a> Parser<'a> {
                         }
                         self.restore_snapshot(snapshot);
                     }
-                    self.recover_stmt();
+                    self.eat_to_tokens(&[&token::CloseDelim(Delimiter::Brace)]);
+                    self.bump(); // }
                     err
-                },
-            )?
+                })?
         };
 
         let enum_definition = EnumDef { variants: variants.into_iter().flatten().collect() };
         Ok((id, ItemKind::Enum(enum_definition, generics)))
     }
 
-    fn parse_enum_variant(&mut self) -> PResult<'a, Option<Variant>> {
+    fn parse_enum_variant(&mut self, span: Span) -> PResult<'a, Option<Variant>> {
         self.recover_diff_marker();
         let variant_attrs = self.parse_outer_attributes()?;
         self.recover_diff_marker();
+        let help = "enum variants can be `Variant`, `Variant = <integer>`, \
+                    `Variant(Type, ..., TypeN)` or `Variant { fields: Types }`";
         self.collect_tokens_trailing_token(
             variant_attrs,
             ForceCollect::No,
@@ -1476,10 +1469,39 @@ impl<'a> Parser<'a> {
                 let struct_def = if this.check(&token::OpenDelim(Delimiter::Brace)) {
                     // Parse a struct variant.
                     let (fields, recovered) =
-                        this.parse_record_struct_body("struct", ident.span, false)?;
-                    VariantData::Struct(fields, recovered)
+                        match this.parse_record_struct_body("struct", ident.span, false) {
+                            Ok((fields, recovered)) => (fields, recovered),
+                            Err(mut err) => {
+                                if this.token == token::Colon {
+                                    // We handle `enum` to `struct` suggestion in the caller.
+                                    return Err(err);
+                                }
+                                this.eat_to_tokens(&[&token::CloseDelim(Delimiter::Brace)]);
+                                this.bump(); // }
+                                err.span_label(span, "while parsing this enum");
+                                err.help(help);
+                                err.emit();
+                                (thin_vec![], true)
+                            }
+                        };
+                    VariantData::Struct { fields, recovered }
                 } else if this.check(&token::OpenDelim(Delimiter::Parenthesis)) {
-                    VariantData::Tuple(this.parse_tuple_struct_body()?, DUMMY_NODE_ID)
+                    let body = match this.parse_tuple_struct_body() {
+                        Ok(body) => body,
+                        Err(mut err) => {
+                            if this.token == token::Colon {
+                                // We handle `enum` to `struct` suggestion in the caller.
+                                return Err(err);
+                            }
+                            this.eat_to_tokens(&[&token::CloseDelim(Delimiter::Parenthesis)]);
+                            this.bump(); // )
+                            err.span_label(span, "while parsing this enum");
+                            err.help(help);
+                            err.emit();
+                            thin_vec![]
+                        }
+                    };
+                    VariantData::Tuple(body, DUMMY_NODE_ID)
                 } else {
                     VariantData::Unit(DUMMY_NODE_ID)
                 };
@@ -1500,8 +1522,9 @@ impl<'a> Parser<'a> {
 
                 Ok((Some(vr), TrailingToken::MaybeComma))
             },
-        ).map_err(|mut err| {
-            err.help("enum variants can be `Variant`, `Variant = <integer>`, `Variant(Type, ..., TypeN)` or `Variant { fields: Types }`");
+        )
+        .map_err(|mut err| {
+            err.help(help);
             err
         })
     }
@@ -1546,7 +1569,7 @@ impl<'a> Parser<'a> {
                     class_name.span,
                     generics.where_clause.has_where_token,
                 )?;
-                VariantData::Struct(fields, recovered)
+                VariantData::Struct { fields, recovered }
             }
         // No `where` so: `struct Foo<T>;`
         } else if self.eat(&token::Semi) {
@@ -1558,7 +1581,7 @@ impl<'a> Parser<'a> {
                 class_name.span,
                 generics.where_clause.has_where_token,
             )?;
-            VariantData::Struct(fields, recovered)
+            VariantData::Struct { fields, recovered }
         // Tuple-style struct definition with optional where-clause.
         } else if self.token == token::OpenDelim(Delimiter::Parenthesis) {
             let body = VariantData::Tuple(self.parse_tuple_struct_body()?, DUMMY_NODE_ID);
@@ -1568,7 +1591,7 @@ impl<'a> Parser<'a> {
         } else {
             let err =
                 errors::UnexpectedTokenAfterStructName::new(self.token.span, self.token.clone());
-            return Err(err.into_diagnostic(&self.sess.span_diagnostic));
+            return Err(err.into_diagnostic(self.dcx()));
         };
 
         Ok((class_name, ItemKind::Struct(vdata, generics)))
@@ -1587,14 +1610,14 @@ impl<'a> Parser<'a> {
                 class_name.span,
                 generics.where_clause.has_where_token,
             )?;
-            VariantData::Struct(fields, recovered)
+            VariantData::Struct { fields, recovered }
         } else if self.token == token::OpenDelim(Delimiter::Brace) {
             let (fields, recovered) = self.parse_record_struct_body(
                 "union",
                 class_name.span,
                 generics.where_clause.has_where_token,
             )?;
-            VariantData::Struct(fields, recovered)
+            VariantData::Struct { fields, recovered }
         } else {
             let token_str = super::token_descr(&self.token);
             let msg = format!("expected `where` or `{{` after union name, found {token_str}");
@@ -1764,7 +1787,7 @@ impl<'a> Parser<'a> {
                         let sp = previous_span.shrink_to_hi();
                         err.missing_comma = Some(sp);
                     }
-                    return Err(err.into_diagnostic(&self.sess.span_diagnostic));
+                    return Err(err.into_diagnostic(self.dcx()));
                 }
             }
             _ => {
@@ -1814,7 +1837,7 @@ impl<'a> Parser<'a> {
                     // Make sure an error was emitted (either by recovering an angle bracket,
                     // or by finding an identifier as the next token), since we're
                     // going to continue parsing
-                    assert!(self.sess.span_diagnostic.has_errors().is_some());
+                    assert!(self.dcx().has_errors().is_some());
                 } else {
                     return Err(err);
                 }
@@ -2271,7 +2294,7 @@ impl<'a> Parser<'a> {
             } else {
                 &[token::Semi, token::OpenDelim(Delimiter::Brace)]
             };
-            if let Err(mut err) = self.expected_one_of_not_found(&[], &expected) {
+            if let Err(mut err) = self.expected_one_of_not_found(&[], expected) {
                 if self.token.kind == token::CloseDelim(Delimiter::Brace) {
                     // The enclosing `mod`, `trait` or `impl` is being closed, so keep the `fn` in
                     // the AST for typechecking.
@@ -2330,8 +2353,10 @@ impl<'a> Parser<'a> {
                             || case == Case::Insensitive
                                 && t.is_non_raw_ident_where(|i| quals.iter().any(|qual| qual.as_str() == i.name.as_str().to_lowercase()))
                         )
-                        // Rule out unsafe extern block.
-                        && !self.is_unsafe_foreign_mod())
+                        // Rule out `unsafe extern {`.
+                        && !self.is_unsafe_foreign_mod()
+                        // Rule out `async gen {` and `async gen move {`
+                        && !self.is_async_gen_block())
                 })
             // `extern ABI fn`
             || self.check_keyword_case(kw::Extern, case)
@@ -2363,10 +2388,7 @@ impl<'a> Parser<'a> {
         let constness = self.parse_constness(case);
 
         let async_start_sp = self.token.span;
-        let asyncness = self.parse_asyncness(case);
-
-        let _gen_start_sp = self.token.span;
-        let genness = self.parse_genness(case);
+        let coroutine_kind = self.parse_coroutine_kind(case);
 
         let unsafe_start_sp = self.token.span;
         let unsafety = self.parse_unsafety(case);
@@ -2374,7 +2396,7 @@ impl<'a> Parser<'a> {
         let ext_start_sp = self.token.span;
         let ext = self.parse_extern(case);
 
-        if let Async::Yes { span, .. } = asyncness {
+        if let Some(CoroutineKind::Async { span, .. }) = coroutine_kind {
             if span.is_rust_2015() {
                 self.sess.emit_err(errors::AsyncFnIn2015 {
                     span,
@@ -2383,8 +2405,11 @@ impl<'a> Parser<'a> {
             }
         }
 
-        if let Gen::Yes { span, .. } = genness {
-            self.sess.emit_err(errors::GenFn { span });
+        match coroutine_kind {
+            Some(CoroutineKind::Gen { span, .. }) | Some(CoroutineKind::AsyncGen { span, .. }) => {
+                self.sess.gated_spans.gate(sym::gen_blocks, span);
+            }
+            Some(CoroutineKind::Async { .. }) | None => {}
         }
 
         if !self.eat_keyword_case(kw::Fn, case) {
@@ -2403,7 +2428,7 @@ impl<'a> Parser<'a> {
 
                     // We may be able to recover
                     let mut recover_constness = constness;
-                    let mut recover_asyncness = asyncness;
+                    let mut recover_coroutine_kind = coroutine_kind;
                     let mut recover_unsafety = unsafety;
                     // This will allow the machine fix to directly place the keyword in the correct place or to indicate
                     // that the keyword is already present and the second instance should be removed.
@@ -2416,14 +2441,28 @@ impl<'a> Parser<'a> {
                             }
                         }
                     } else if self.check_keyword(kw::Async) {
-                        match asyncness {
-                            Async::Yes { span, .. } => Some(WrongKw::Duplicated(span)),
-                            Async::No => {
-                                recover_asyncness = Async::Yes {
+                        match coroutine_kind {
+                            Some(CoroutineKind::Async { span, .. }) => {
+                                Some(WrongKw::Duplicated(span))
+                            }
+                            Some(CoroutineKind::AsyncGen { span, .. }) => {
+                                Some(WrongKw::Duplicated(span))
+                            }
+                            Some(CoroutineKind::Gen { .. }) => {
+                                recover_coroutine_kind = Some(CoroutineKind::AsyncGen {
+                                    span: self.token.span,
+                                    closure_id: DUMMY_NODE_ID,
+                                    return_impl_trait_id: DUMMY_NODE_ID,
+                                });
+                                // FIXME(gen_blocks): This span is wrong, didn't want to think about it.
+                                Some(WrongKw::Misplaced(unsafe_start_sp))
+                            }
+                            None => {
+                                recover_coroutine_kind = Some(CoroutineKind::Async {
                                     span: self.token.span,
                                     closure_id: DUMMY_NODE_ID,
                                     return_impl_trait_id: DUMMY_NODE_ID,
-                                };
+                                });
                                 Some(WrongKw::Misplaced(unsafe_start_sp))
                             }
                         }
@@ -2504,6 +2543,8 @@ impl<'a> Parser<'a> {
                         }
                     }
 
+                    // FIXME(gen_blocks): add keyword recovery logic for genness
+
                     if wrong_kw.is_some()
                         && self.may_recover()
                         && self.look_ahead(1, |tok| tok.is_keyword_case(kw::Fn, case))
@@ -2515,7 +2556,7 @@ impl<'a> Parser<'a> {
                         return Ok(FnHeader {
                             constness: recover_constness,
                             unsafety: recover_unsafety,
-                            asyncness: recover_asyncness,
+                            coroutine_kind: recover_coroutine_kind,
                             ext,
                         });
                     }
@@ -2525,7 +2566,7 @@ impl<'a> Parser<'a> {
             }
         }
 
-        Ok(FnHeader { constness, unsafety, asyncness, ext })
+        Ok(FnHeader { constness, unsafety, coroutine_kind, ext })
     }
 
     /// Parses the parameter list and result type of a function declaration.
@@ -2750,7 +2791,7 @@ impl<'a> Parser<'a> {
 
     fn is_named_param(&self) -> bool {
         let offset = match &self.token.kind {
-            token::Interpolated(nt) => match **nt {
+            token::Interpolated(nt) => match &nt.0 {
                 token::NtPat(..) => return self.look_ahead(1, |t| t == &token::Colon),
                 _ => 0,
             },
diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs
index 1a7ae4069..b91432f10 100644
--- a/compiler/rustc_parse/src/parser/mod.rs
+++ b/compiler/rustc_parse/src/parser/mod.rs
@@ -11,7 +11,6 @@ mod stmt;
 mod ty;
 
 use crate::lexer::UnmatchedDelim;
-use ast::Gen;
 pub use attr_wrapper::AttrWrapper;
 pub use diagnostics::AttemptLocalParseRecovery;
 pub(crate) use expr::ForbiddenLetReason;
@@ -21,13 +20,14 @@ pub use path::PathStyle;
 
 use rustc_ast::ptr::P;
 use rustc_ast::token::{self, Delimiter, Nonterminal, Token, TokenKind};
-use rustc_ast::tokenstream::{AttributesData, DelimSpan, Spacing};
+use rustc_ast::tokenstream::{AttributesData, DelimSpacing, DelimSpan, Spacing};
 use rustc_ast::tokenstream::{TokenStream, TokenTree, TokenTreeCursor};
 use rustc_ast::util::case::Case;
 use rustc_ast::AttrId;
+use rustc_ast::CoroutineKind;
 use rustc_ast::DUMMY_NODE_ID;
 use rustc_ast::{self as ast, AnonConst, Const, DelimArgs, Extern};
-use rustc_ast::{Async, AttrArgs, AttrArgsEq, Expr, ExprKind, Mutability, StrLit};
+use rustc_ast::{AttrArgs, AttrArgsEq, Expr, ExprKind, Mutability, StrLit};
 use rustc_ast::{HasAttrs, HasTokens, Unsafe, Visibility, VisibilityKind};
 use rustc_ast_pretty::pprust;
 use rustc_data_structures::fx::FxHashMap;
@@ -93,7 +93,7 @@ pub enum TrailingToken {
 macro_rules! maybe_whole {
     ($p:expr, $constructor:ident, |$x:ident| $e:expr) => {
         if let token::Interpolated(nt) = &$p.token.kind {
-            if let token::$constructor(x) = &**nt {
+            if let token::$constructor(x) = &nt.0 {
                 let $x = x.clone();
                 $p.bump();
                 return Ok($e);
@@ -107,15 +107,15 @@ macro_rules! maybe_whole {
 macro_rules! maybe_recover_from_interpolated_ty_qpath {
     ($self: expr, $allow_qpath_recovery: expr) => {
         if $allow_qpath_recovery
-                    && $self.may_recover()
-                    && $self.look_ahead(1, |t| t == &token::ModSep)
-                    && let token::Interpolated(nt) = &$self.token.kind
-                    && let token::NtTy(ty) = &**nt
-                {
-                    let ty = ty.clone();
-                    $self.bump();
-                    return $self.maybe_recover_from_bad_qpath_stage_2($self.prev_token.span, ty);
-                }
+            && $self.may_recover()
+            && $self.look_ahead(1, |t| t == &token::ModSep)
+            && let token::Interpolated(nt) = &$self.token.kind
+            && let token::NtTy(ty) = &nt.0
+        {
+            let ty = ty.clone();
+            $self.bump();
+            return $self.maybe_recover_from_bad_qpath_stage_2($self.prev_token.span, ty);
+        }
     };
 }
 
@@ -130,7 +130,7 @@ pub struct Parser<'a> {
     pub sess: &'a ParseSess,
     /// The current token.
     pub token: Token,
-    /// The spacing for the current token
+    /// The spacing for the current token.
     pub token_spacing: Spacing,
     /// The previous token.
     pub prev_token: Token,
@@ -240,7 +240,7 @@ struct TokenCursor {
     // Token streams surrounding the current one. The delimiters for stack[n]'s
     // tokens are in `stack[n-1]`. `stack[0]` (when present) has no delimiters
     // because it's the outermost token stream which never has delimiters.
-    stack: Vec<(TokenTreeCursor, Delimiter, DelimSpan)>,
+    stack: Vec<(TokenTreeCursor, DelimSpan, DelimSpacing, Delimiter)>,
 }
 
 impl TokenCursor {
@@ -264,24 +264,31 @@ impl TokenCursor {
                         ));
                         return (token.clone(), spacing);
                     }
-                    &TokenTree::Delimited(sp, delim, ref tts) => {
+                    &TokenTree::Delimited(sp, spacing, delim, ref tts) => {
                         let trees = tts.clone().into_trees();
-                        self.stack.push((mem::replace(&mut self.tree_cursor, trees), delim, sp));
+                        self.stack.push((
+                            mem::replace(&mut self.tree_cursor, trees),
+                            sp,
+                            spacing,
+                            delim,
+                        ));
                         if delim != Delimiter::Invisible {
-                            return (Token::new(token::OpenDelim(delim), sp.open), Spacing::Alone);
+                            return (Token::new(token::OpenDelim(delim), sp.open), spacing.open);
                         }
                         // No open delimiter to return; continue on to the next iteration.
                     }
                 };
-            } else if let Some((tree_cursor, delim, span)) = self.stack.pop() {
+            } else if let Some((tree_cursor, span, spacing, delim)) = self.stack.pop() {
                 // We have exhausted this token stream. Move back to its parent token stream.
                 self.tree_cursor = tree_cursor;
                 if delim != Delimiter::Invisible {
-                    return (Token::new(token::CloseDelim(delim), span.close), Spacing::Alone);
+                    return (Token::new(token::CloseDelim(delim), span.close), spacing.close);
                 }
                 // No close delimiter to return; continue on to the next iteration.
             } else {
-                // We have exhausted the outermost token stream.
+                // We have exhausted the outermost token stream. The use of
+                // `Spacing::Alone` is arbitrary and immaterial, because the
+                // `Eof` token's spacing is never used.
                 return (Token::new(token::Eof, DUMMY_SP), Spacing::Alone);
             }
         }
@@ -367,12 +374,14 @@ impl TokenDescription {
 pub(super) fn token_descr(token: &Token) -> String {
     let name = pprust::token_to_string(token).to_string();
 
-    let kind = TokenDescription::from_token(token).map(|kind| match kind {
-        TokenDescription::ReservedIdentifier => "reserved identifier",
-        TokenDescription::Keyword => "keyword",
-        TokenDescription::ReservedKeyword => "reserved keyword",
-        TokenDescription::DocComment => "doc comment",
-    });
+    let kind = match (TokenDescription::from_token(token), &token.kind) {
+        (Some(TokenDescription::ReservedIdentifier), _) => Some("reserved identifier"),
+        (Some(TokenDescription::Keyword), _) => Some("keyword"),
+        (Some(TokenDescription::ReservedKeyword), _) => Some("reserved keyword"),
+        (Some(TokenDescription::DocComment), _) => Some("doc comment"),
+        (None, TokenKind::Interpolated(node)) => Some(node.0.descr()),
+        (None, _) => None,
+    };
 
     if let Some(kind) = kind { format!("{kind} `{name}`") } else { format!("`{name}`") }
 }
@@ -662,7 +671,7 @@ impl<'a> Parser<'a> {
     fn check_inline_const(&self, dist: usize) -> bool {
         self.is_keyword_ahead(dist, &[kw::Const])
             && self.look_ahead(dist + 1, |t| match &t.kind {
-                token::Interpolated(nt) => matches!(**nt, token::NtBlock(..)),
+                token::Interpolated(nt) => matches!(&nt.0, token::NtBlock(..)),
                 token::OpenDelim(Delimiter::Brace) => true,
                 _ => false,
             })
@@ -697,8 +706,8 @@ impl<'a> Parser<'a> {
                 // is not needed (we'll capture the entire 'glued' token),
                 // and `bump` will set this field to `None`
                 self.break_last_token = true;
-                // Use the spacing of the glued token as the spacing
-                // of the unglued second token.
+                // Use the spacing of the glued token as the spacing of the
+                // unglued second token.
                 self.bump_with((Token::new(second, second_span), self.token_spacing));
                 true
             }
@@ -830,8 +839,8 @@ impl<'a> Parser<'a> {
                             // https://github.com/rust-lang/rust/issues/72373
                             if self.prev_token.is_ident() && self.token.kind == token::DotDot {
                                 let msg = format!(
-                                    "if you meant to bind the contents of \
-                                    the rest of the array pattern into `{}`, use `@`",
+                                    "if you meant to bind the contents of the rest of the array \
+                                     pattern into `{}`, use `@`",
                                     pprust::token_to_string(&self.prev_token)
                                 );
                                 expect_err
@@ -873,6 +882,9 @@ impl<'a> Parser<'a> {
                                     if self.token == token::Colon {
                                         // we will try to recover in `maybe_recover_struct_lit_bad_delims`
                                         return Err(expect_err);
+                                    } else if let [token::CloseDelim(Delimiter::Parenthesis)] = kets
+                                    {
+                                        return Err(expect_err);
                                     } else {
                                         expect_err.emit();
                                         break;
@@ -1063,7 +1075,7 @@ impl<'a> Parser<'a> {
             return looker(&self.token);
         }
 
-        if let Some(&(_, delim, span)) = self.token_cursor.stack.last()
+        if let Some(&(_, span, _, delim)) = self.token_cursor.stack.last()
             && delim != Delimiter::Invisible
         {
             // We are not in the outermost token stream, and the token stream
@@ -1072,7 +1084,7 @@ impl<'a> Parser<'a> {
             let tree_cursor = &self.token_cursor.tree_cursor;
             let all_normal = (0..dist).all(|i| {
                 let token = tree_cursor.look_ahead(i);
-                !matches!(token, Some(TokenTree::Delimited(_, Delimiter::Invisible, _)))
+                !matches!(token, Some(TokenTree::Delimited(.., Delimiter::Invisible, _)))
             });
             if all_normal {
                 // There were no skipped delimiters. Do lookahead by plain indexing.
@@ -1081,7 +1093,7 @@ impl<'a> Parser<'a> {
                         // Indexing stayed within the current token stream.
                         match tree {
                             TokenTree::Token(token, _) => looker(token),
-                            TokenTree::Delimited(dspan, delim, _) => {
+                            TokenTree::Delimited(dspan, _, delim, _) => {
                                 looker(&Token::new(token::OpenDelim(*delim), dspan.open))
                             }
                         }
@@ -1115,27 +1127,42 @@ impl<'a> Parser<'a> {
     }
 
     /// Returns whether any of the given keywords are `dist` tokens ahead of the current one.
-    fn is_keyword_ahead(&self, dist: usize, kws: &[Symbol]) -> bool {
+    pub(crate) fn is_keyword_ahead(&self, dist: usize, kws: &[Symbol]) -> bool {
         self.look_ahead(dist, |t| kws.iter().any(|&kw| t.is_keyword(kw)))
     }
 
     /// Parses asyncness: `async` or nothing.
-    fn parse_asyncness(&mut self, case: Case) -> Async {
+    fn parse_coroutine_kind(&mut self, case: Case) -> Option<CoroutineKind> {
+        let span = self.token.uninterpolated_span();
         if self.eat_keyword_case(kw::Async, case) {
-            let span = self.prev_token.uninterpolated_span();
-            Async::Yes { span, closure_id: DUMMY_NODE_ID, return_impl_trait_id: DUMMY_NODE_ID }
-        } else {
-            Async::No
-        }
-    }
-
-    /// Parses genness: `gen` or nothing.
-    fn parse_genness(&mut self, case: Case) -> Gen {
-        if self.token.span.at_least_rust_2024() && self.eat_keyword_case(kw::Gen, case) {
-            let span = self.prev_token.uninterpolated_span();
-            Gen::Yes { span, closure_id: DUMMY_NODE_ID, return_impl_trait_id: DUMMY_NODE_ID }
+            // FIXME(gen_blocks): Do we want to unconditionally parse `gen` and then
+            // error if edition <= 2024, like we do with async and edition <= 2018?
+            if self.token.uninterpolated_span().at_least_rust_2024()
+                && self.eat_keyword_case(kw::Gen, case)
+            {
+                let gen_span = self.prev_token.uninterpolated_span();
+                Some(CoroutineKind::AsyncGen {
+                    span: span.to(gen_span),
+                    closure_id: DUMMY_NODE_ID,
+                    return_impl_trait_id: DUMMY_NODE_ID,
+                })
+            } else {
+                Some(CoroutineKind::Async {
+                    span,
+                    closure_id: DUMMY_NODE_ID,
+                    return_impl_trait_id: DUMMY_NODE_ID,
+                })
+            }
+        } else if self.token.uninterpolated_span().at_least_rust_2024()
+            && self.eat_keyword_case(kw::Gen, case)
+        {
+            Some(CoroutineKind::Gen {
+                span,
+                closure_id: DUMMY_NODE_ID,
+                return_impl_trait_id: DUMMY_NODE_ID,
+            })
         } else {
-            Gen::No
+            None
         }
     }
 
@@ -1244,7 +1271,7 @@ impl<'a> Parser<'a> {
             || self.check(&token::OpenDelim(Delimiter::Brace));
 
         delimited.then(|| {
-            let TokenTree::Delimited(dspan, delim, tokens) = self.parse_token_tree() else {
+            let TokenTree::Delimited(dspan, _, delim, tokens) = self.parse_token_tree() else {
                 unreachable!()
             };
             DelimArgs { dspan, delim, tokens }
@@ -1268,7 +1295,7 @@ impl<'a> Parser<'a> {
             token::OpenDelim(..) => {
                 // Grab the tokens within the delimiters.
                 let stream = self.token_cursor.tree_cursor.stream.clone();
-                let (_, delim, span) = *self.token_cursor.stack.last().unwrap();
+                let (_, span, spacing, delim) = *self.token_cursor.stack.last().unwrap();
 
                 // Advance the token cursor through the entire delimited
                 // sequence. After getting the `OpenDelim` we are *within* the
@@ -1288,12 +1315,13 @@ impl<'a> Parser<'a> {
 
                 // Consume close delimiter
                 self.bump();
-                TokenTree::Delimited(span, delim, stream)
+                TokenTree::Delimited(span, spacing, delim, stream)
             }
             token::CloseDelim(_) | token::Eof => unreachable!(),
             _ => {
+                let prev_spacing = self.token_spacing;
                 self.bump();
-                TokenTree::Token(self.prev_token.clone(), Spacing::Alone)
+                TokenTree::Token(self.prev_token.clone(), prev_spacing)
             }
         }
     }
@@ -1479,7 +1507,7 @@ pub(crate) fn make_unclosed_delims_error(
         opening_candidate: unmatched.candidate_span,
         unclosed: unmatched.unclosed_span,
     }
-    .into_diagnostic(&sess.span_diagnostic);
+    .into_diagnostic(&sess.dcx);
     Some(err)
 }
 
diff --git a/compiler/rustc_parse/src/parser/nonterminal.rs b/compiler/rustc_parse/src/parser/nonterminal.rs
index 025b0615a..301a88cd0 100644
--- a/compiler/rustc_parse/src/parser/nonterminal.rs
+++ b/compiler/rustc_parse/src/parser/nonterminal.rs
@@ -50,12 +50,12 @@ impl<'a> Parser<'a> {
             NonterminalKind::Literal => token.can_begin_literal_maybe_minus(),
             NonterminalKind::Vis => match token.kind {
                 // The follow-set of :vis + "priv" keyword + interpolated
-                token::Comma | token::Ident(..) | token::Interpolated(..) => true,
+                token::Comma | token::Ident(..) | token::Interpolated(_) => true,
                 _ => token.can_begin_type(),
             },
             NonterminalKind::Block => match &token.kind {
                 token::OpenDelim(Delimiter::Brace) => true,
-                token::Interpolated(nt) => match **nt {
+                token::Interpolated(nt) => match &nt.0 {
                     NtBlock(_) | NtLifetime(_) | NtStmt(_) | NtExpr(_) | NtLiteral(_) => true,
                     NtItem(_) | NtPat(_) | NtTy(_) | NtIdent(..) | NtMeta(_) | NtPath(_)
                     | NtVis(_) => false,
@@ -64,7 +64,7 @@ impl<'a> Parser<'a> {
             },
             NonterminalKind::Path | NonterminalKind::Meta => match &token.kind {
                 token::ModSep | token::Ident(..) => true,
-                token::Interpolated(nt) => may_be_ident(nt),
+                token::Interpolated(nt) => may_be_ident(&nt.0),
                 _ => false,
             },
             NonterminalKind::PatParam { .. } | NonterminalKind::PatWithOr => {
@@ -75,7 +75,7 @@ impl<'a> Parser<'a> {
                 token::BinOp(token::And) |                  // reference
                 token::BinOp(token::Minus) |                // negative literal
                 token::AndAnd |                             // double reference
-                token::Literal(..) |                        // literal
+                token::Literal(_) |                        // literal
                 token::DotDot |                             // range pattern (future compat)
                 token::DotDotDot |                          // range pattern (future compat)
                 token::ModSep |                             // path
@@ -83,14 +83,14 @@ impl<'a> Parser<'a> {
                 token::BinOp(token::Shl) => true,           // path (double UFCS)
                 // leading vert `|` or-pattern
                 token::BinOp(token::Or) => matches!(kind, NonterminalKind::PatWithOr),
-                token::Interpolated(nt) => may_be_ident(nt),
+                token::Interpolated(nt) => may_be_ident(&nt.0),
                 _ => false,
             }
             }
             NonterminalKind::Lifetime => match &token.kind {
                 token::Lifetime(_) => true,
                 token::Interpolated(nt) => {
-                    matches!(**nt, NtLifetime(_))
+                    matches!(&nt.0, NtLifetime(_))
                 }
                 _ => false,
             },
@@ -114,8 +114,9 @@ impl<'a> Parser<'a> {
             NonterminalKind::Item => match self.parse_item(ForceCollect::Yes)? {
                 Some(item) => NtItem(item),
                 None => {
-                    return Err(UnexpectedNonterminal::Item(self.token.span)
-                        .into_diagnostic(&self.sess.span_diagnostic));
+                    return Err(
+                        UnexpectedNonterminal::Item(self.token.span).into_diagnostic(self.dcx())
+                    );
                 }
             },
             NonterminalKind::Block => {
@@ -127,7 +128,7 @@ impl<'a> Parser<'a> {
                 Some(s) => NtStmt(P(s)),
                 None => {
                     return Err(UnexpectedNonterminal::Statement(self.token.span)
-                        .into_diagnostic(&self.sess.span_diagnostic));
+                        .into_diagnostic(self.dcx()));
                 }
             },
             NonterminalKind::PatParam { .. } | NonterminalKind::PatWithOr => {
@@ -163,7 +164,7 @@ impl<'a> Parser<'a> {
                     span: self.token.span,
                     token: self.token.clone(),
                 }
-                .into_diagnostic(&self.sess.span_diagnostic));
+                .into_diagnostic(self.dcx()));
             }
             NonterminalKind::Path => {
                 NtPath(P(self.collect_tokens_no_attrs(|this| this.parse_path(PathStyle::Type))?))
@@ -181,7 +182,7 @@ impl<'a> Parser<'a> {
                         span: self.token.span,
                         token: self.token.clone(),
                     }
-                    .into_diagnostic(&self.sess.span_diagnostic));
+                    .into_diagnostic(self.dcx()));
                 }
             }
         };
@@ -191,7 +192,7 @@ impl<'a> Parser<'a> {
             panic!(
                 "Missing tokens for nt {:?} at {:?}: {:?}",
                 nt,
-                nt.span(),
+                nt.use_span(),
                 pprust::nonterminal_to_string(&nt)
             );
         }
diff --git a/compiler/rustc_parse/src/parser/pat.rs b/compiler/rustc_parse/src/parser/pat.rs
index 0a4c7c17d..80233eddb 100644
--- a/compiler/rustc_parse/src/parser/pat.rs
+++ b/compiler/rustc_parse/src/parser/pat.rs
@@ -5,8 +5,8 @@ use crate::errors::{
     ExpectedCommaAfterPatternField, GenericArgsInPatRequireTurbofishSyntax,
     InclusiveRangeExtraEquals, InclusiveRangeMatchArrow, InclusiveRangeNoEnd, InvalidMutInPattern,
     PatternOnWrongSideOfAt, RefMutOrderIncorrect, RemoveLet, RepeatedMutInPattern,
-    TopLevelOrPatternNotAllowed, TopLevelOrPatternNotAllowedSugg, TrailingVertNotAllowed,
-    UnexpectedLifetimeInPattern, UnexpectedVertVertBeforeFunctionParam,
+    SwitchRefBoxOrder, TopLevelOrPatternNotAllowed, TopLevelOrPatternNotAllowedSugg,
+    TrailingVertNotAllowed, UnexpectedLifetimeInPattern, UnexpectedVertVertBeforeFunctionParam,
     UnexpectedVertVertInPattern,
 };
 use crate::{maybe_recover_from_interpolated_ty_qpath, maybe_whole};
@@ -141,13 +141,21 @@ impl<'a> Parser<'a> {
         };
 
         // Parse the first pattern (`p_0`).
-        let mut first_pat = self.parse_pat_no_top_alt(expected, syntax_loc)?;
-        if rc == RecoverComma::Yes {
-            self.maybe_recover_unexpected_comma(
-                first_pat.span,
-                matches!(first_pat.kind, PatKind::MacCall(_)),
-                rt,
-            )?;
+        let mut first_pat = match self.parse_pat_no_top_alt(expected, syntax_loc) {
+            Ok(pat) => pat,
+            Err(mut err)
+                if self.token.is_reserved_ident()
+                    && !self.token.is_keyword(kw::In)
+                    && !self.token.is_keyword(kw::If) =>
+            {
+                err.emit();
+                self.bump();
+                self.mk_pat(self.token.span, PatKind::Wild)
+            }
+            Err(err) => return Err(err),
+        };
+        if rc == RecoverComma::Yes && !first_pat.could_be_never_pattern() {
+            self.maybe_recover_unexpected_comma(first_pat.span, rt)?;
         }
 
         // If the next token is not a `|`,
@@ -188,8 +196,8 @@ impl<'a> Parser<'a> {
                 err.span_label(lo, WHILE_PARSING_OR_MSG);
                 err
             })?;
-            if rc == RecoverComma::Yes {
-                self.maybe_recover_unexpected_comma(pat.span, false, rt)?;
+            if rc == RecoverComma::Yes && !pat.could_be_never_pattern() {
+                self.maybe_recover_unexpected_comma(pat.span, rt)?;
             }
             pats.push(pat);
         }
@@ -368,12 +376,22 @@ impl<'a> Parser<'a> {
             self.recover_dotdotdot_rest_pat(lo)
         } else if let Some(form) = self.parse_range_end() {
             self.parse_pat_range_to(form)? // `..=X`, `...X`, or `..X`.
+        } else if self.eat(&token::Not) {
+            // Parse `!`
+            self.sess.gated_spans.gate(sym::never_patterns, self.prev_token.span);
+            PatKind::Never
         } else if self.eat_keyword(kw::Underscore) {
-            // Parse _
+            // Parse `_`
             PatKind::Wild
         } else if self.eat_keyword(kw::Mut) {
             self.parse_pat_ident_mut(syntax_loc)?
         } else if self.eat_keyword(kw::Ref) {
+            if self.check_keyword(kw::Box) {
+                // Suggest `box ref`.
+                let span = self.prev_token.span.to(self.token.span);
+                self.bump();
+                self.sess.emit_err(SwitchRefBoxOrder { span });
+            }
             // Parse ref ident @ pat / ref mut ident @ pat
             let mutbl = self.parse_mutability();
             self.parse_pat_ident(BindingAnnotation(ByRef::Yes, mutbl), syntax_loc)?
@@ -541,7 +559,7 @@ impl<'a> Parser<'a> {
         }
 
         self.sess
-            .emit_err(AmbiguousRangePattern { span: pat.span, pat: pprust::pat_to_string(&pat) });
+            .emit_err(AmbiguousRangePattern { span: pat.span, pat: pprust::pat_to_string(pat) });
     }
 
     /// Parse `&pat` / `&mut pat`.
@@ -592,7 +610,7 @@ impl<'a> Parser<'a> {
 
         // Make sure we don't allow e.g. `let mut $p;` where `$p:pat`.
         if let token::Interpolated(nt) = &self.token.kind {
-            if let token::NtPat(_) = **nt {
+            if let token::NtPat(..) = &nt.0 {
                 self.expected_ident_found_err().emit();
             }
         }
@@ -638,13 +656,13 @@ impl<'a> Parser<'a> {
 
     /// Error on `mut $pat` where `$pat` is not an ident.
     fn ban_mut_general_pat(&self, lo: Span, pat: &Pat, changed_any_binding: bool) {
-        let span = lo.to(pat.span);
-        let pat = pprust::pat_to_string(&pat);
-
         self.sess.emit_err(if changed_any_binding {
-            InvalidMutInPattern::NestedIdent { span, pat }
+            InvalidMutInPattern::NestedIdent {
+                span: lo.to(pat.span),
+                pat: pprust::pat_to_string(pat),
+            }
         } else {
-            InvalidMutInPattern::NonIdent { span, pat }
+            InvalidMutInPattern::NonIdent { span: lo.until(pat.span) }
         });
     }
 
@@ -829,7 +847,7 @@ impl<'a> Parser<'a> {
         binding_annotation: BindingAnnotation,
         syntax_loc: Option<PatternLocation>,
     ) -> PResult<'a, PatKind> {
-        let ident = self.parse_ident()?;
+        let ident = self.parse_ident_common(false)?;
 
         if self.may_recover()
             && !matches!(syntax_loc, Some(PatternLocation::FunctionParameter))
@@ -855,7 +873,7 @@ impl<'a> Parser<'a> {
         // will direct us over to `parse_enum_variant()`.
         if self.token == token::OpenDelim(Delimiter::Parenthesis) {
             return Err(EnumPatternInsteadOfIdentifier { span: self.prev_token.span }
-                .into_diagnostic(&self.sess.span_diagnostic));
+                .into_diagnostic(self.dcx()));
         }
 
         Ok(PatKind::Ident(binding_annotation, ident, sub))
@@ -969,7 +987,7 @@ impl<'a> Parser<'a> {
             // check that a comma comes after every field
             if !ate_comma {
                 let mut err = ExpectedCommaAfterPatternField { span: self.token.span }
-                    .into_diagnostic(&self.sess.span_diagnostic);
+                    .into_diagnostic(self.dcx());
                 if let Some(mut delayed) = delayed_err {
                     delayed.emit();
                 }
diff --git a/compiler/rustc_parse/src/parser/path.rs b/compiler/rustc_parse/src/parser/path.rs
index 8626dbe40..3b92a9119 100644
--- a/compiler/rustc_parse/src/parser/path.rs
+++ b/compiler/rustc_parse/src/parser/path.rs
@@ -123,7 +123,7 @@ impl<'a> Parser<'a> {
 
         self.bump(); // colon
 
-        self.diagnostic()
+        self.dcx()
             .struct_span_err(
                 self.prev_token.span,
                 "found single colon before projection in qualified path",
@@ -185,7 +185,7 @@ impl<'a> Parser<'a> {
         });
 
         if let token::Interpolated(nt) = &self.token.kind {
-            if let token::NtTy(ty) = &**nt {
+            if let token::NtTy(ty) = &nt.0 {
                 if let ast::TyKind::Path(None, path) = &ty.kind {
                     let path = path.clone();
                     self.bump();
@@ -326,7 +326,7 @@ impl<'a> Parser<'a> {
                                     .is_nightly_build()
                                     .then_some(()),
                             }
-                            .into_diagnostic(self.diagnostic());
+                            .into_diagnostic(self.dcx());
                         }
                         // Attempt to find places where a missing `>` might belong.
                         else if let Some(arg) = args
diff --git a/compiler/rustc_parse/src/parser/stmt.rs b/compiler/rustc_parse/src/parser/stmt.rs
index aa939a71d..1ee5a96d5 100644
--- a/compiler/rustc_parse/src/parser/stmt.rs
+++ b/compiler/rustc_parse/src/parser/stmt.rs
@@ -53,7 +53,7 @@ impl<'a> Parser<'a> {
         // Don't use `maybe_whole` so that we have precise control
         // over when we bump the parser
         if let token::Interpolated(nt) = &self.token.kind
-            && let token::NtStmt(stmt) = &**nt
+            && let token::NtStmt(stmt) = &nt.0
         {
             let mut stmt = stmt.clone();
             self.bump();
@@ -384,10 +384,10 @@ impl<'a> Parser<'a> {
 
     fn check_let_else_init_bool_expr(&self, init: &ast::Expr) {
         if let ast::ExprKind::Binary(op, ..) = init.kind {
-            if op.node.lazy() {
+            if op.node.is_lazy() {
                 self.sess.emit_err(errors::InvalidExpressionInLetElse {
                     span: init.span,
-                    operator: op.node.to_string(),
+                    operator: op.node.as_str(),
                     sugg: errors::WrapExpressionInParentheses {
                         left: init.span.shrink_to_lo(),
                         right: init.span.shrink_to_hi(),
@@ -567,20 +567,37 @@ impl<'a> Parser<'a> {
                         snapshot.recover_diff_marker();
                     }
                     if self.token == token::Colon {
-                        // if next token is following a colon, it's likely a path
-                        // and we can suggest a path separator
-                        self.bump();
-                        if self.token.span.lo() == self.prev_token.span.hi() {
+                        // if a previous and next token of the current one is
+                        // integer literal (e.g. `1:42`), it's likely a range
+                        // expression for Pythonistas and we can suggest so.
+                        if self.prev_token.is_integer_lit()
+                            && self.may_recover()
+                            && self.look_ahead(1, |token| token.is_integer_lit())
+                        {
+                            // FIXME(hkmatsumoto): Might be better to trigger
+                            // this only when parsing an index expression.
                             err.span_suggestion_verbose(
-                                self.prev_token.span,
-                                "maybe write a path separator here",
-                                "::",
+                                self.token.span,
+                                "you might have meant a range expression",
+                                "..",
                                 Applicability::MaybeIncorrect,
                             );
-                        }
-                        if self.sess.unstable_features.is_nightly_build() {
-                            // FIXME(Nilstrieb): Remove this again after a few months.
-                            err.note("type ascription syntax has been removed, see issue #101728 <https://github.com/rust-lang/rust/issues/101728>");
+                        } else {
+                            // if next token is following a colon, it's likely a path
+                            // and we can suggest a path separator
+                            self.bump();
+                            if self.token.span.lo() == self.prev_token.span.hi() {
+                                err.span_suggestion_verbose(
+                                    self.prev_token.span,
+                                    "maybe write a path separator here",
+                                    "::",
+                                    Applicability::MaybeIncorrect,
+                                );
+                            }
+                            if self.sess.unstable_features.is_nightly_build() {
+                                // FIXME(Nilstrieb): Remove this again after a few months.
+                                err.note("type ascription syntax has been removed, see issue #101728 <https://github.com/rust-lang/rust/issues/101728>");
+                            }
                         }
                     }
 
@@ -619,6 +636,20 @@ impl<'a> Parser<'a> {
         match &mut stmt.kind {
             // Expression without semicolon.
             StmtKind::Expr(expr)
+                if classify::expr_requires_semi_to_be_stmt(expr)
+                    && !expr.attrs.is_empty()
+                    && ![token::Eof, token::Semi, token::CloseDelim(Delimiter::Brace)]
+                        .contains(&self.token.kind) =>
+            {
+                // The user has written `#[attr] expr` which is unsupported. (#106020)
+                self.attr_on_non_tail_expr(&expr);
+                // We already emitted an error, so don't emit another type error
+                let sp = expr.span.to(self.prev_token.span);
+                *expr = self.mk_expr_err(sp);
+            }
+
+            // Expression without semicolon.
+            StmtKind::Expr(expr)
                 if self.token != token::Eof && classify::expr_requires_semi_to_be_stmt(expr) =>
             {
                 // Just check for errors and recover; do not eat semicolon yet.
diff --git a/compiler/rustc_parse/src/parser/ty.rs b/compiler/rustc_parse/src/parser/ty.rs
index be2cbaf30..da8cc05ff 100644
--- a/compiler/rustc_parse/src/parser/ty.rs
+++ b/compiler/rustc_parse/src/parser/ty.rs
@@ -37,7 +37,7 @@ impl BoundModifiers {
             (BoundPolarity::Positive, None) => TraitBoundModifier::None,
             (BoundPolarity::Negative(_), None) => TraitBoundModifier::Negative,
             (BoundPolarity::Maybe(_), None) => TraitBoundModifier::Maybe,
-            (BoundPolarity::Positive, Some(_)) => TraitBoundModifier::MaybeConst,
+            (BoundPolarity::Positive, Some(sp)) => TraitBoundModifier::MaybeConst(sp),
             (BoundPolarity::Negative(_), Some(_)) => TraitBoundModifier::MaybeConstNegative,
             (BoundPolarity::Maybe(_), Some(_)) => TraitBoundModifier::MaybeConstMaybe,
         }
@@ -135,7 +135,7 @@ impl<'a> Parser<'a> {
         )
     }
 
-    /// Parse a type suitable for a field defintion.
+    /// Parse a type suitable for a field definition.
     /// The difference from `parse_ty` is that this version
     /// allows anonymous structs and unions.
     pub fn parse_ty_for_field_def(&mut self) -> PResult<'a, P<Ty>> {
@@ -287,6 +287,7 @@ impl<'a> Parser<'a> {
             // Function pointer type
             self.parse_ty_bare_fn(lo, ThinVec::new(), None, recover_return_sign)?
         } else if self.check_keyword(kw::For) {
+            let for_span = self.token.span;
             // Function pointer type or bound list (trait object type) starting with a poly-trait.
             //   `for<'lt> [unsafe] [extern "ABI"] fn (&'lt S) -> T`
             //   `for<'lt> Trait1<'lt> + Trait2 + 'a`
@@ -299,9 +300,44 @@ impl<'a> Parser<'a> {
                     recover_return_sign,
                 )?
             } else {
-                let path = self.parse_path(PathStyle::Type)?;
-                let parse_plus = allow_plus == AllowPlus::Yes && self.check_plus();
-                self.parse_remaining_bounds_path(lifetime_defs, path, lo, parse_plus)?
+                // Try to recover `for<'a> dyn Trait` or `for<'a> impl Trait`.
+                if self.may_recover()
+                    && (self.eat_keyword_noexpect(kw::Impl) || self.eat_keyword_noexpect(kw::Dyn))
+                {
+                    let kw = self.prev_token.ident().unwrap().0;
+                    let removal_span = kw.span.with_hi(self.token.span.lo());
+                    let path = self.parse_path(PathStyle::Type)?;
+                    let parse_plus = allow_plus == AllowPlus::Yes && self.check_plus();
+                    let kind =
+                        self.parse_remaining_bounds_path(lifetime_defs, path, lo, parse_plus)?;
+                    let mut err = self.sess.create_err(errors::TransposeDynOrImpl {
+                        span: kw.span,
+                        kw: kw.name.as_str(),
+                        sugg: errors::TransposeDynOrImplSugg {
+                            removal_span,
+                            insertion_span: for_span.shrink_to_lo(),
+                            kw: kw.name.as_str(),
+                        },
+                    });
+
+                    // Take the parsed bare trait object and turn it either
+                    // into a `dyn` object or an `impl Trait`.
+                    let kind = match (kind, kw.name) {
+                        (TyKind::TraitObject(bounds, _), kw::Dyn) => {
+                            TyKind::TraitObject(bounds, TraitObjectSyntax::Dyn)
+                        }
+                        (TyKind::TraitObject(bounds, _), kw::Impl) => {
+                            TyKind::ImplTrait(ast::DUMMY_NODE_ID, bounds)
+                        }
+                        _ => return Err(err),
+                    };
+                    err.emit();
+                    kind
+                } else {
+                    let path = self.parse_path(PathStyle::Type)?;
+                    let parse_plus = allow_plus == AllowPlus::Yes && self.check_plus();
+                    self.parse_remaining_bounds_path(lifetime_defs, path, lo, parse_plus)?
+                }
             }
         } else if self.eat_keyword(kw::Impl) {
             self.parse_impl_ty(&mut impl_dyn_multi)?
@@ -562,7 +598,7 @@ impl<'a> Parser<'a> {
             tokens: None,
         };
         let span_start = self.token.span;
-        let ast::FnHeader { ext, unsafety, constness, asyncness } =
+        let ast::FnHeader { ext, unsafety, constness, coroutine_kind } =
             self.parse_fn_front_matter(&inherited_vis, Case::Sensitive)?;
         if self.may_recover() && self.token.kind == TokenKind::Lt {
             self.recover_fn_ptr_with_generics(lo, &mut params, param_insertion_point)?;
@@ -575,9 +611,10 @@ impl<'a> Parser<'a> {
             // cover it.
             self.sess.emit_err(FnPointerCannotBeConst { span: whole_span, qualifier: span });
         }
-        if let ast::Async::Yes { span, .. } = asyncness {
+        if let Some(ast::CoroutineKind::Async { span, .. }) = coroutine_kind {
             self.sess.emit_err(FnPointerCannotBeAsync { span: whole_span, qualifier: span });
         }
+        // FIXME(gen_blocks): emit a similar error for `gen fn()`
         let decl_span = span_start.to(self.token.span);
         Ok(TyKind::BareFn(P(BareFnTy { ext, unsafety, generic_params: params, decl, decl_span })))
     }
diff --git a/compiler/rustc_parse/src/validate_attr.rs b/compiler/rustc_parse/src/validate_attr.rs
index f73965982..9fea38266 100644
--- a/compiler/rustc_parse/src/validate_attr.rs
+++ b/compiler/rustc_parse/src/validate_attr.rs
@@ -6,9 +6,9 @@ use rustc_ast::token::Delimiter;
 use rustc_ast::tokenstream::DelimSpan;
 use rustc_ast::MetaItemKind;
 use rustc_ast::{self as ast, AttrArgs, AttrArgsEq, Attribute, DelimArgs, MetaItem};
-use rustc_ast_pretty::pprust;
 use rustc_errors::{Applicability, FatalError, PResult};
 use rustc_feature::{AttributeTemplate, BuiltinAttribute, BUILTIN_ATTRIBUTE_MAP};
+use rustc_session::errors::report_lit_error;
 use rustc_session::lint::builtin::ILL_FORMED_ATTRIBUTE_INPUT;
 use rustc_session::parse::ParseSess;
 use rustc_span::{sym, Span, Symbol};
@@ -51,29 +51,45 @@ pub fn parse_meta<'a>(sess: &'a ParseSess, attr: &Attribute) -> PResult<'a, Meta
                 MetaItemKind::List(nmis)
             }
             AttrArgs::Eq(_, AttrArgsEq::Ast(expr)) => {
-                if let ast::ExprKind::Lit(token_lit) = expr.kind
-                    && let Ok(lit) = ast::MetaItemLit::from_token_lit(token_lit, expr.span)
-                {
-                    if token_lit.suffix.is_some() {
-                        let mut err = sess.span_diagnostic.struct_span_err(
-                            expr.span,
-                            "suffixed literals are not allowed in attributes",
-                        );
-                        err.help(
-                            "instead of using a suffixed literal (`1u8`, `1.0f32`, etc.), \
-                            use an unsuffixed version (`1`, `1.0`, etc.)",
-                        );
-                        return Err(err);
-                    } else {
-                        MetaItemKind::NameValue(lit)
-                    }
+                if let ast::ExprKind::Lit(token_lit) = expr.kind {
+                    let res = ast::MetaItemLit::from_token_lit(token_lit, expr.span);
+                    let res = match res {
+                        Ok(lit) => {
+                            if token_lit.suffix.is_some() {
+                                let mut err = sess.dcx.struct_span_err(
+                                    expr.span,
+                                    "suffixed literals are not allowed in attributes",
+                                );
+                                err.help(
+                                    "instead of using a suffixed literal (`1u8`, `1.0f32`, etc.), \
+                                    use an unsuffixed version (`1`, `1.0`, etc.)",
+                                );
+                                return Err(err);
+                            } else {
+                                MetaItemKind::NameValue(lit)
+                            }
+                        }
+                        Err(err) => {
+                            report_lit_error(sess, err, token_lit, expr.span);
+                            let lit = ast::MetaItemLit {
+                                symbol: token_lit.symbol,
+                                suffix: token_lit.suffix,
+                                kind: ast::LitKind::Err,
+                                span: expr.span,
+                            };
+                            MetaItemKind::NameValue(lit)
+                        }
+                    };
+                    res
                 } else {
-                    // The non-error case can happen with e.g. `#[foo = 1+1]`. The error case can
-                    // happen with e.g. `#[foo = include_str!("nonexistent-file.rs")]`; in that
-                    // case we delay the error because an earlier error will have already been
-                    // reported.
-                    let msg = format!("unexpected expression: `{}`", pprust::expr_to_string(expr));
-                    let mut err = sess.span_diagnostic.struct_span_err(expr.span, msg);
+                    // Example cases:
+                    // - `#[foo = 1+1]`: results in `ast::ExprKind::BinOp`.
+                    // - `#[foo = include_str!("nonexistent-file.rs")]`:
+                    //   results in `ast::ExprKind::Err`. In that case we delay
+                    //   the error because an earlier error will have already
+                    //   been reported.
+                    let msg = format!("attribute value must be a literal");
+                    let mut err = sess.dcx.struct_span_err(expr.span, msg);
                     if let ast::ExprKind::Err = expr.kind {
                         err.downgrade_to_delayed_bug();
                     }
@@ -186,10 +202,11 @@ fn emit_malformed_attribute(
         msg.push_str(&format!("`{code}`"));
         suggestions.push(code);
     }
+    suggestions.sort();
     if should_warn(name) {
-        sess.buffer_lint(&ILL_FORMED_ATTRIBUTE_INPUT, span, ast::CRATE_NODE_ID, msg);
+        sess.buffer_lint(ILL_FORMED_ATTRIBUTE_INPUT, span, ast::CRATE_NODE_ID, msg);
     } else {
-        sess.span_diagnostic
+        sess.dcx
             .struct_span_err(span, error_msg)
             .span_suggestions(
                 span,
@@ -198,7 +215,7 @@ fn emit_malformed_attribute(
                 } else {
                     "the following are the possible correct uses"
                 },
-                suggestions.into_iter(),
+                suggestions,
                 Applicability::HasPlaceholders,
             )
             .emit();
diff --git a/compiler/rustc_parse_format/src/lib.rs b/compiler/rustc_parse_format/src/lib.rs
index 7b6153eea..e886db3da 100644
--- a/compiler/rustc_parse_format/src/lib.rs
+++ b/compiler/rustc_parse_format/src/lib.rs
@@ -9,13 +9,10 @@
     html_playground_url = "https://play.rust-lang.org/",
     test(attr(deny(warnings)))
 )]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
-#![cfg_attr(not(bootstrap), allow(internal_features))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
-// We want to be able to build this crate with a stable compiler, so no
-// `#![feature]` attributes should be added.
+// WARNING: We want to be able to build this crate with a stable compiler,
+//          so no `#![feature]` attributes should be added!
 
 use rustc_lexer::unescape;
 pub use Alignment::*;
diff --git a/compiler/rustc_passes/Cargo.toml b/compiler/rustc_passes/Cargo.toml
index 40c3811e0..80e6c104b 100644
--- a/compiler/rustc_passes/Cargo.toml
+++ b/compiler/rustc_passes/Cargo.toml
@@ -5,7 +5,7 @@ edition = "2021"
 
 [dependencies]
 # tidy-alphabetical-start
-itertools = "0.10.1"
+itertools = "0.11"
 rustc_ast = { path = "../rustc_ast" }
 rustc_ast_pretty = { path = "../rustc_ast_pretty" }
 rustc_attr = { path = "../rustc_attr" }
@@ -19,7 +19,6 @@ rustc_index = { path = "../rustc_index" }
 rustc_lexer = { path = "../rustc_lexer" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_middle = { path = "../rustc_middle" }
-rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
diff --git a/compiler/rustc_passes/src/check_attr.rs b/compiler/rustc_passes/src/check_attr.rs
index c5767fd90..c5073048b 100644
--- a/compiler/rustc_passes/src/check_attr.rs
+++ b/compiler/rustc_passes/src/check_attr.rs
@@ -130,59 +130,59 @@ impl CheckAttrVisitor<'_> {
                     &mut specified_inline,
                     &mut doc_aliases,
                 ),
-                sym::no_link => self.check_no_link(hir_id, &attr, span, target),
-                sym::export_name => self.check_export_name(hir_id, &attr, span, target),
+                sym::no_link => self.check_no_link(hir_id, attr, span, target),
+                sym::export_name => self.check_export_name(hir_id, attr, span, target),
                 sym::rustc_layout_scalar_valid_range_start
                 | sym::rustc_layout_scalar_valid_range_end => {
-                    self.check_rustc_layout_scalar_valid_range(&attr, span, target)
+                    self.check_rustc_layout_scalar_valid_range(attr, span, target)
                 }
                 sym::allow_internal_unstable => {
-                    self.check_allow_internal_unstable(hir_id, &attr, span, target, &attrs)
+                    self.check_allow_internal_unstable(hir_id, attr, span, target, attrs)
                 }
-                sym::debugger_visualizer => self.check_debugger_visualizer(&attr, target),
+                sym::debugger_visualizer => self.check_debugger_visualizer(attr, target),
                 sym::rustc_allow_const_fn_unstable => {
-                    self.check_rustc_allow_const_fn_unstable(hir_id, &attr, span, target)
+                    self.check_rustc_allow_const_fn_unstable(hir_id, attr, span, target)
                 }
                 sym::rustc_std_internal_symbol => {
-                    self.check_rustc_std_internal_symbol(&attr, span, target)
+                    self.check_rustc_std_internal_symbol(attr, span, target)
                 }
                 sym::naked => self.check_naked(hir_id, attr, span, target),
                 sym::rustc_never_returns_null_ptr => {
                     self.check_applied_to_fn_or_method(hir_id, attr, span, target)
                 }
                 sym::rustc_legacy_const_generics => {
-                    self.check_rustc_legacy_const_generics(hir_id, &attr, span, target, item)
+                    self.check_rustc_legacy_const_generics(hir_id, attr, span, target, item)
                 }
                 sym::rustc_lint_query_instability => {
-                    self.check_rustc_lint_query_instability(hir_id, &attr, span, target)
+                    self.check_rustc_lint_query_instability(hir_id, attr, span, target)
                 }
                 sym::rustc_lint_diagnostics => {
-                    self.check_rustc_lint_diagnostics(hir_id, &attr, span, target)
+                    self.check_rustc_lint_diagnostics(hir_id, attr, span, target)
                 }
-                sym::rustc_lint_opt_ty => self.check_rustc_lint_opt_ty(&attr, span, target),
+                sym::rustc_lint_opt_ty => self.check_rustc_lint_opt_ty(attr, span, target),
                 sym::rustc_lint_opt_deny_field_access => {
-                    self.check_rustc_lint_opt_deny_field_access(&attr, span, target)
+                    self.check_rustc_lint_opt_deny_field_access(attr, span, target)
                 }
                 sym::rustc_clean
                 | sym::rustc_dirty
                 | sym::rustc_if_this_changed
-                | sym::rustc_then_this_would_need => self.check_rustc_dirty_clean(&attr),
+                | sym::rustc_then_this_would_need => self.check_rustc_dirty_clean(attr),
                 sym::rustc_coinductive
                 | sym::rustc_must_implement_one_of
                 | sym::rustc_deny_explicit_impl
-                | sym::const_trait => self.check_must_be_applied_to_trait(&attr, span, target),
+                | sym::const_trait => self.check_must_be_applied_to_trait(attr, span, target),
                 sym::cmse_nonsecure_entry => {
                     self.check_cmse_nonsecure_entry(hir_id, attr, span, target)
                 }
                 sym::collapse_debuginfo => self.check_collapse_debuginfo(attr, span, target),
-                sym::must_not_suspend => self.check_must_not_suspend(&attr, span, target),
-                sym::must_use => self.check_must_use(hir_id, &attr, target),
-                sym::rustc_pass_by_value => self.check_pass_by_value(&attr, span, target),
+                sym::must_not_suspend => self.check_must_not_suspend(attr, span, target),
+                sym::must_use => self.check_must_use(hir_id, attr, target),
+                sym::rustc_pass_by_value => self.check_pass_by_value(attr, span, target),
                 sym::rustc_allow_incoherent_impl => {
-                    self.check_allow_incoherent_impl(&attr, span, target)
+                    self.check_allow_incoherent_impl(attr, span, target)
                 }
                 sym::rustc_has_incoherent_inherent_impls => {
-                    self.check_has_incoherent_inherent_impls(&attr, span, target)
+                    self.check_has_incoherent_inherent_impls(attr, span, target)
                 }
                 sym::ffi_pure => self.check_ffi_pure(attr.span, attrs, target),
                 sym::ffi_const => self.check_ffi_const(attr.span, target),
@@ -192,9 +192,9 @@ impl CheckAttrVisitor<'_> {
                 | sym::unstable
                 | sym::stable
                 | sym::rustc_allowed_through_unstable_modules
-                | sym::rustc_promotable => self.check_stability_promotable(&attr, span, target),
-                sym::link_ordinal => self.check_link_ordinal(&attr, span, target),
-                sym::rustc_confusables => self.check_confusables(&attr, target),
+                | sym::rustc_promotable => self.check_stability_promotable(attr, span, target),
+                sym::link_ordinal => self.check_link_ordinal(attr, span, target),
+                sym::rustc_confusables => self.check_confusables(attr, target),
                 sym::rustc_safe_intrinsic => {
                     self.check_rustc_safe_intrinsic(hir_id, attr, span, target)
                 }
@@ -604,7 +604,7 @@ impl CheckAttrVisitor<'_> {
                     && !self.tcx.sess.target.is_like_wasm
                     && !self.tcx.sess.opts.actually_rustdoc
                 {
-                    let hir::Node::Item(item) = self.tcx.hir().get(hir_id) else {
+                    let hir::Node::Item(item) = self.tcx.hir_node(hir_id) else {
                         unreachable!();
                     };
                     let hir::ItemKind::Fn(sig, _, _) = item.kind else {
@@ -820,11 +820,11 @@ impl CheckAttrVisitor<'_> {
             self.doc_attr_str_error(meta, "keyword");
             return false;
         }
-        match self.tcx.hir().find(hir_id).and_then(|node| match node {
+        match self.tcx.opt_hir_node(hir_id).and_then(|node| match node {
             hir::Node::Item(item) => Some(&item.kind),
             _ => None,
         }) {
-            Some(ItemKind::Mod(ref module)) => {
+            Some(ItemKind::Mod(module)) => {
                 if !module.item_ids.is_empty() {
                     self.tcx.sess.emit_err(errors::DocKeywordEmptyMod { span: meta.span() });
                     return false;
@@ -846,11 +846,11 @@ impl CheckAttrVisitor<'_> {
     }
 
     fn check_doc_fake_variadic(&self, meta: &NestedMetaItem, hir_id: HirId) -> bool {
-        match self.tcx.hir().find(hir_id).and_then(|node| match node {
+        match self.tcx.opt_hir_node(hir_id).and_then(|node| match node {
             hir::Node::Item(item) => Some(&item.kind),
             _ => None,
         }) {
-            Some(ItemKind::Impl(ref i)) => {
+            Some(ItemKind::Impl(i)) => {
                 let is_valid = matches!(&i.self_ty.kind, hir::TyKind::Tup([_]))
                     || if let hir::TyKind::BareFn(bare_fn_ty) = &i.self_ty.kind {
                         bare_fn_ty.decl.inputs.len() == 1
@@ -1387,7 +1387,7 @@ impl CheckAttrVisitor<'_> {
     /// Checks if `#[link]` is applied to an item other than a foreign module.
     fn check_link(&self, hir_id: HirId, attr: &Attribute, span: Span, target: Target) {
         if target == Target::ForeignMod
-            && let hir::Node::Item(item) = self.tcx.hir().get(hir_id)
+            && let hir::Node::Item(item) = self.tcx.hir_node(hir_id)
             && let Item { kind: ItemKind::ForeignMod { abi, .. }, .. } = item
             && !matches!(abi, Abi::Rust | Abi::RustIntrinsic | Abi::PlatformIntrinsic)
         {
@@ -1456,7 +1456,7 @@ impl CheckAttrVisitor<'_> {
     }
 
     fn is_impl_item(&self, hir_id: HirId) -> bool {
-        matches!(self.tcx.hir().get(hir_id), hir::Node::ImplItem(..))
+        matches!(self.tcx.hir_node(hir_id), hir::Node::ImplItem(..))
     }
 
     /// Checks if `#[export_name]` is applied to a function or static. Returns `true` if valid.
@@ -2074,7 +2074,7 @@ impl CheckAttrVisitor<'_> {
             && let hir::Node::Item(Item {
                 kind: ItemKind::ForeignMod { abi: Abi::RustIntrinsic | Abi::PlatformIntrinsic, .. },
                 ..
-            }) = hir.get(parent)
+            }) = self.tcx.hir_node(parent)
         {
             return true;
         }
@@ -2222,7 +2222,7 @@ impl CheckAttrVisitor<'_> {
         } else {
             // special case when `#[macro_export]` is applied to a macro 2.0
             let (macro_definition, _) =
-                self.tcx.hir().find(hir_id).unwrap().expect_item().expect_macro();
+                self.tcx.opt_hir_node(hir_id).unwrap().expect_item().expect_macro();
             let is_decl_macro = !macro_definition.macro_rules;
 
             if is_decl_macro {
@@ -2395,7 +2395,7 @@ impl<'tcx> Visitor<'tcx> for CheckAttrVisitor<'tcx> {
         // Historically we've run more checks on non-exported than exported macros,
         // so this lets us continue to run them while maintaining backwards compatibility.
         // In the long run, the checks should be harmonized.
-        if let ItemKind::Macro(ref macro_def, _) = item.kind {
+        if let ItemKind::Macro(macro_def, _) = item.kind {
             let def_id = item.owner_id.to_def_id();
             if macro_def.macro_rules && !self.tcx.has_attr(def_id, sym::macro_export) {
                 check_non_exported_macro_for_invalid_attrs(self.tcx, item);
@@ -2443,7 +2443,7 @@ impl<'tcx> Visitor<'tcx> for CheckAttrVisitor<'tcx> {
 
     fn visit_stmt(&mut self, stmt: &'tcx hir::Stmt<'tcx>) {
         // When checking statements ignore expressions, they will be checked later.
-        if let hir::StmtKind::Local(ref l) = stmt.kind {
+        if let hir::StmtKind::Local(l) = stmt.kind {
             self.check_attributes(l.hir_id, stmt.span, Target::Statement, None);
         }
         intravisit::walk_stmt(self, stmt)
diff --git a/compiler/rustc_passes/src/check_const.rs b/compiler/rustc_passes/src/check_const.rs
index 7188c177f..76c746734 100644
--- a/compiler/rustc_passes/src/check_const.rs
+++ b/compiler/rustc_passes/src/check_const.rs
@@ -17,7 +17,7 @@ use rustc_middle::ty::TyCtxt;
 use rustc_session::parse::feature_err;
 use rustc_span::{sym, Span, Symbol};
 
-use crate::errors::{ExprNotAllowedInContext, SkippingConstChecks};
+use crate::errors::SkippingConstChecks;
 
 /// An expression that is not *always* legal in a const context.
 #[derive(Clone, Copy)]
@@ -110,8 +110,8 @@ impl<'tcx> CheckConstVisitor<'tcx> {
 
             // However, we cannot allow stable `const fn`s to use unstable features without an explicit
             // opt-in via `rustc_allow_const_fn_unstable`.
-            let attrs = tcx.hir().attrs(tcx.hir().local_def_id_to_hir_id(def_id));
-            attr::rustc_allow_const_fn_unstable(&tcx.sess, attrs).any(|name| name == feature_gate)
+            let attrs = tcx.hir().attrs(tcx.local_def_id_to_hir_id(def_id));
+            attr::rustc_allow_const_fn_unstable(tcx.sess, attrs).any(|name| name == feature_gate)
         };
 
         match required_gates {
@@ -138,11 +138,10 @@ impl<'tcx> CheckConstVisitor<'tcx> {
 
         match missing_gates.as_slice() {
             [] => {
-                tcx.sess.emit_err(ExprNotAllowedInContext {
+                span_bug!(
                     span,
-                    expr: expr.name(),
-                    context: const_kind.keyword_name(),
-                });
+                    "we should not have reached this point, since `.await` is denied earlier"
+                );
             }
 
             [missing_primary, ref missing_secondary @ ..] => {
diff --git a/compiler/rustc_passes/src/dead.rs b/compiler/rustc_passes/src/dead.rs
index 2e8c58b02..d27079497 100644
--- a/compiler/rustc_passes/src/dead.rs
+++ b/compiler/rustc_passes/src/dead.rs
@@ -32,7 +32,7 @@ use crate::errors::{
 // may need to be marked as live.
 fn should_explore(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
     matches!(
-        tcx.hir().find_by_def_id(def_id),
+        tcx.opt_hir_node_by_def_id(def_id),
         Some(
             Node::Item(..)
                 | Node::ImplItem(..)
@@ -297,7 +297,7 @@ impl<'tcx> MarkSymbolVisitor<'tcx> {
             // tuple struct constructor function
             let id = self.struct_constructors.get(&id).copied().unwrap_or(id);
 
-            if let Some(node) = self.tcx.hir().find_by_def_id(id) {
+            if let Some(node) = self.tcx.opt_hir_node_by_def_id(id) {
                 // When using `#[allow]` or `#[expect]` of `dead_code`, we do a QOL improvement
                 // by declaring fn calls, statics, ... within said items as live, as well as
                 // the item itself, although technically this is not the case.
@@ -314,7 +314,7 @@ impl<'tcx> MarkSymbolVisitor<'tcx> {
                 // for the `#[expect]` case.
                 //
                 // Note that an item can and will be duplicated on the worklist with different
-                // `ComesFromAllowExpect`, particulary if it was added from the
+                // `ComesFromAllowExpect`, particularly if it was added from the
                 // `effective_visibilities` query or from the `#[allow]`/`#[expect]` checks,
                 // this "duplication" is essential as otherwise a function with `#[expect]`
                 // called from a `pub fn` may be falsely reported as not live, falsely
@@ -373,10 +373,10 @@ impl<'tcx> MarkSymbolVisitor<'tcx> {
                     self.repr_has_repr_c = def.repr().c();
                     self.repr_has_repr_simd = def.repr().simd();
 
-                    intravisit::walk_item(self, &item)
+                    intravisit::walk_item(self, item)
                 }
                 hir::ItemKind::ForeignMod { .. } => {}
-                _ => intravisit::walk_item(self, &item),
+                _ => intravisit::walk_item(self, item),
             },
             Node::TraitItem(trait_item) => {
                 intravisit::walk_trait_item(self, trait_item);
@@ -403,7 +403,7 @@ impl<'tcx> MarkSymbolVisitor<'tcx> {
                 intravisit::walk_impl_item(self, impl_item);
             }
             Node::ForeignItem(foreign_item) => {
-                intravisit::walk_foreign_item(self, &foreign_item);
+                intravisit::walk_foreign_item(self, foreign_item);
             }
             _ => {}
         }
@@ -459,9 +459,9 @@ impl<'tcx> Visitor<'tcx> for MarkSymbolVisitor<'tcx> {
                 self.lookup_and_handle_method(expr.hir_id);
             }
             hir::ExprKind::Field(ref lhs, ..) => {
-                self.handle_field_access(&lhs, expr.hir_id);
+                self.handle_field_access(lhs, expr.hir_id);
             }
-            hir::ExprKind::Struct(ref qpath, ref fields, _) => {
+            hir::ExprKind::Struct(qpath, fields, _) => {
                 let res = self.typeck_results().qpath_res(qpath, expr.hir_id);
                 self.handle_res(res);
                 if let ty::Adt(adt, _) = self.typeck_results().expr_ty(expr).kind() {
@@ -493,7 +493,7 @@ impl<'tcx> Visitor<'tcx> for MarkSymbolVisitor<'tcx> {
     fn visit_pat(&mut self, pat: &'tcx hir::Pat<'tcx>) {
         self.in_pat = true;
         match pat.kind {
-            PatKind::Struct(ref path, ref fields, _) => {
+            PatKind::Struct(ref path, fields, _) => {
                 let res = self.typeck_results().qpath_res(path, pat.hir_id);
                 self.handle_field_pattern_match(pat, res, fields);
             }
@@ -501,7 +501,7 @@ impl<'tcx> Visitor<'tcx> for MarkSymbolVisitor<'tcx> {
                 let res = self.typeck_results().qpath_res(qpath, pat.hir_id);
                 self.handle_res(res);
             }
-            PatKind::TupleStruct(ref qpath, ref fields, dotdot) => {
+            PatKind::TupleStruct(ref qpath, fields, dotdot) => {
                 let res = self.typeck_results().qpath_res(qpath, pat.hir_id);
                 self.handle_tuple_field_pattern_match(pat, res, fields, dotdot);
             }
@@ -559,7 +559,7 @@ fn has_allow_dead_code_or_lang_attr(
     }
 
     fn has_allow_expect_dead_code(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
-        let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
+        let hir_id = tcx.local_def_id_to_hir_id(def_id);
         let lint_level = tcx.lint_level_at_node(lint::builtin::DEAD_CODE, hir_id).0;
         matches!(lint_level, lint::Allow | lint::Expect(_))
     }
@@ -805,10 +805,10 @@ impl<'tcx> DeadVisitor<'tcx> {
         };
         let tcx = self.tcx;
 
-        let first_hir_id = tcx.hir().local_def_id_to_hir_id(first_id);
+        let first_hir_id = tcx.local_def_id_to_hir_id(first_id);
         let first_lint_level = tcx.lint_level_at_node(lint::builtin::DEAD_CODE, first_hir_id).0;
         assert!(dead_codes.iter().skip(1).all(|id| {
-            let hir_id = tcx.hir().local_def_id_to_hir_id(*id);
+            let hir_id = tcx.local_def_id_to_hir_id(*id);
             let level = tcx.lint_level_at_node(lint::builtin::DEAD_CODE, hir_id).0;
             level == first_lint_level
         }));
@@ -969,7 +969,7 @@ fn check_mod_deathness(tcx: TyCtxt<'_>, module: LocalModDefId) {
                 let def_id = item.id.owner_id.def_id;
                 if !visitor.is_live_code(def_id) {
                     let name = tcx.item_name(def_id.to_def_id());
-                    let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
+                    let hir_id = tcx.local_def_id_to_hir_id(def_id);
                     let level = tcx.lint_level_at_node(lint::builtin::DEAD_CODE, hir_id).0;
 
                     dead_items.push(DeadItem { def_id, name, level })
@@ -997,7 +997,7 @@ fn check_mod_deathness(tcx: TyCtxt<'_>, module: LocalModDefId) {
                 let def_id = variant.def_id.expect_local();
                 if !live_symbols.contains(&def_id) {
                     // Record to group diagnostics.
-                    let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
+                    let hir_id = tcx.local_def_id_to_hir_id(def_id);
                     let level = tcx.lint_level_at_node(lint::builtin::DEAD_CODE, hir_id).0;
                     dead_variants.push(DeadItem { def_id, name: variant.name, level });
                     continue;
@@ -1009,9 +1009,9 @@ fn check_mod_deathness(tcx: TyCtxt<'_>, module: LocalModDefId) {
                     .iter()
                     .filter_map(|field| {
                         let def_id = field.did.expect_local();
-                        let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
+                        let hir_id = tcx.local_def_id_to_hir_id(def_id);
                         if let ShouldWarnAboutField::Yes(is_pos) =
-                            visitor.should_warn_about_field(&field)
+                            visitor.should_warn_about_field(field)
                         {
                             let level = tcx
                                 .lint_level_at_node(
diff --git a/compiler/rustc_passes/src/entry.rs b/compiler/rustc_passes/src/entry.rs
index 51a64b385..7667fc21e 100644
--- a/compiler/rustc_passes/src/entry.rs
+++ b/compiler/rustc_passes/src/entry.rs
@@ -38,7 +38,7 @@ fn entry_fn(tcx: TyCtxt<'_>, (): ()) -> Option<(DefId, EntryFnType)> {
     }
 
     // If the user wants no main function at all, then stop here.
-    if attr::contains_name(&tcx.hir().attrs(CRATE_HIR_ID), sym::no_main) {
+    if attr::contains_name(tcx.hir().attrs(CRATE_HIR_ID), sym::no_main) {
         return None;
     }
 
@@ -126,7 +126,7 @@ fn configure_main(tcx: TyCtxt<'_>, visitor: &EntryContext<'_>) -> Option<(DefId,
         {
             // non-local main imports are handled below
             if let Some(def_id) = def_id.as_local()
-                && matches!(tcx.hir().find_by_def_id(def_id), Some(Node::ForeignItem(_)))
+                && matches!(tcx.opt_hir_node_by_def_id(def_id), Some(Node::ForeignItem(_)))
             {
                 tcx.sess.emit_err(ExternMain { span: tcx.def_span(def_id) });
                 return None;
diff --git a/compiler/rustc_passes/src/errors.rs b/compiler/rustc_passes/src/errors.rs
index 411c94101..8f8da211d 100644
--- a/compiler/rustc_passes/src/errors.rs
+++ b/compiler/rustc_passes/src/errors.rs
@@ -867,9 +867,9 @@ impl IntoDiagnostic<'_> for InvalidAttrAtCrateLevel {
     #[track_caller]
     fn into_diagnostic(
         self,
-        handler: &'_ rustc_errors::Handler,
+        dcx: &'_ rustc_errors::DiagCtxt,
     ) -> rustc_errors::DiagnosticBuilder<'_, ErrorGuaranteed> {
-        let mut diag = handler.struct_err(fluent::passes_invalid_attr_at_crate_level);
+        let mut diag = dcx.struct_err(fluent::passes_invalid_attr_at_crate_level);
         diag.set_span(self.span);
         diag.set_arg("name", self.name);
         // Only emit an error with a suggestion if we can create a string out
@@ -1005,15 +1005,6 @@ pub struct FeaturePreviouslyDeclared<'a, 'b> {
     pub prev_declared: &'b str,
 }
 
-#[derive(Diagnostic)]
-#[diag(passes_expr_not_allowed_in_context, code = "E0744")]
-pub struct ExprNotAllowedInContext<'a> {
-    #[primary_span]
-    pub span: Span,
-    pub expr: String,
-    pub context: &'a str,
-}
-
 pub struct BreakNonLoop<'a> {
     pub span: Span,
     pub head: Option<Span>,
@@ -1029,9 +1020,9 @@ impl<'a> IntoDiagnostic<'_> for BreakNonLoop<'a> {
     #[track_caller]
     fn into_diagnostic(
         self,
-        handler: &rustc_errors::Handler,
+        dcx: &rustc_errors::DiagCtxt,
     ) -> rustc_errors::DiagnosticBuilder<'_, ErrorGuaranteed> {
-        let mut diag = handler.struct_span_err_with_code(
+        let mut diag = dcx.struct_span_err_with_code(
             self.span,
             fluent::passes_break_non_loop,
             error_code!(E0571),
@@ -1178,9 +1169,9 @@ impl IntoDiagnostic<'_> for NakedFunctionsAsmBlock {
     #[track_caller]
     fn into_diagnostic(
         self,
-        handler: &rustc_errors::Handler,
+        dcx: &rustc_errors::DiagCtxt,
     ) -> rustc_errors::DiagnosticBuilder<'_, ErrorGuaranteed> {
-        let mut diag = handler.struct_span_err_with_code(
+        let mut diag = dcx.struct_span_err_with_code(
             self.span,
             fluent::passes_naked_functions_asm_block,
             error_code!(E0787),
@@ -1294,9 +1285,9 @@ impl<'a> IntoDiagnostic<'a> for NoMainErr {
     #[track_caller]
     fn into_diagnostic(
         self,
-        handler: &'a rustc_errors::Handler,
+        dcx: &'a rustc_errors::DiagCtxt,
     ) -> rustc_errors::DiagnosticBuilder<'a, ErrorGuaranteed> {
-        let mut diag = handler.struct_span_err_with_code(
+        let mut diag = dcx.struct_span_err_with_code(
             DUMMY_SP,
             fluent::passes_no_main_function,
             error_code!(E0601),
@@ -1357,9 +1348,9 @@ impl IntoDiagnostic<'_> for DuplicateLangItem {
     #[track_caller]
     fn into_diagnostic(
         self,
-        handler: &rustc_errors::Handler,
+        dcx: &rustc_errors::DiagCtxt,
     ) -> rustc_errors::DiagnosticBuilder<'_, ErrorGuaranteed> {
-        let mut diag = handler.struct_err_with_code(
+        let mut diag = dcx.struct_err_with_code(
             match self.duplicate {
                 Duplicate::Plain => fluent::passes_duplicate_lang_item,
                 Duplicate::Crate => fluent::passes_duplicate_lang_item_crate,
diff --git a/compiler/rustc_passes/src/hir_id_validator.rs b/compiler/rustc_passes/src/hir_id_validator.rs
index f825363ae..18a80aa34 100644
--- a/compiler/rustc_passes/src/hir_id_validator.rs
+++ b/compiler/rustc_passes/src/hir_id_validator.rs
@@ -31,7 +31,7 @@ pub fn check_crate(tcx: TyCtxt<'_>) {
 
         if !errors.is_empty() {
             let message = errors.iter().fold(String::new(), |s1, s2| s1 + "\n" + s2);
-            tcx.sess.delay_span_bug(rustc_span::DUMMY_SP, message);
+            tcx.sess.span_delayed_bug(rustc_span::DUMMY_SP, message);
         }
     }
 }
diff --git a/compiler/rustc_passes/src/hir_stats.rs b/compiler/rustc_passes/src/hir_stats.rs
index f915c1057..c2392620c 100644
--- a/compiler/rustc_passes/src/hir_stats.rs
+++ b/compiler/rustc_passes/src/hir_stats.rs
@@ -121,6 +121,8 @@ impl<'k> StatCollector<'k> {
     }
 
     fn print(&self, title: &str, prefix: &str) {
+        // We will soon sort, so the initial order does not matter.
+        #[allow(rustc::potential_query_instability)]
         let mut nodes: Vec<_> = self.nodes.iter().collect();
         nodes.sort_by_key(|(_, node)| node.stats.count * node.stats.size);
 
@@ -147,6 +149,8 @@ impl<'k> StatCollector<'k> {
                 to_readable_str(node.stats.size)
             );
             if !node.subnodes.is_empty() {
+                // We will soon sort, so the initial order does not matter.
+                #[allow(rustc::potential_query_instability)]
                 let mut subnodes: Vec<_> = node.subnodes.iter().collect();
                 subnodes.sort_by_key(|(_, subnode)| subnode.count * subnode.size);
 
@@ -286,7 +290,21 @@ impl<'v> hir_visit::Visitor<'v> for StatCollector<'v> {
     fn visit_pat(&mut self, p: &'v hir::Pat<'v>) {
         record_variants!(
             (self, p, p.kind, Id::Node(p.hir_id), hir, Pat, PatKind),
-            [Wild, Binding, Struct, TupleStruct, Or, Path, Tuple, Box, Ref, Lit, Range, Slice]
+            [
+                Wild,
+                Binding,
+                Struct,
+                TupleStruct,
+                Or,
+                Never,
+                Path,
+                Tuple,
+                Box,
+                Ref,
+                Lit,
+                Range,
+                Slice
+            ]
         );
         hir_visit::walk_pat(self, p)
     }
@@ -411,7 +429,7 @@ impl<'v> hir_visit::Visitor<'v> for StatCollector<'v> {
     fn visit_param_bound(&mut self, b: &'v hir::GenericBound<'v>) {
         record_variants!(
             (self, b, b, Id::None, hir, GenericBound, GenericBound),
-            [Trait, LangItemTrait, Outlives]
+            [Trait, Outlives]
         );
         hir_visit::walk_param_bound(self, b)
     }
@@ -554,6 +572,7 @@ impl<'v> ast_visit::Visitor<'v> for StatCollector<'v> {
                 Range,
                 Slice,
                 Rest,
+                Never,
                 Paren,
                 MacCall
             ]
diff --git a/compiler/rustc_passes/src/lang_items.rs b/compiler/rustc_passes/src/lang_items.rs
index 2aec4ea7e..6d14a1409 100644
--- a/compiler/rustc_passes/src/lang_items.rs
+++ b/compiler/rustc_passes/src/lang_items.rs
@@ -7,21 +7,21 @@
 //! * Traits that represent operators; e.g., `Add`, `Sub`, `Index`.
 //! * Functions called by the compiler itself.
 
-use crate::check_attr::target_from_impl_item;
 use crate::errors::{
     DuplicateLangItem, IncorrectTarget, LangItemOnIncorrectTarget, UnknownLangItem,
 };
 use crate::weak_lang_items;
 
-use rustc_hir as hir;
-use rustc_hir::def::DefKind;
+use rustc_ast as ast;
+use rustc_ast::visit;
+use rustc_data_structures::fx::FxHashMap;
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::lang_items::{extract, GenericRequirement};
-use rustc_hir::{LangItem, LanguageItems, Target};
-use rustc_middle::ty::TyCtxt;
+use rustc_hir::{LangItem, LanguageItems, MethodKind, Target};
+use rustc_middle::ty::{ResolverAstLowering, TyCtxt};
 use rustc_session::cstore::ExternCrate;
 use rustc_span::symbol::kw::Empty;
-use rustc_span::{sym, Span};
+use rustc_span::Span;
 
 use rustc_middle::query::Providers;
 
@@ -31,28 +31,55 @@ pub(crate) enum Duplicate {
     CrateDepends,
 }
 
-struct LanguageItemCollector<'tcx> {
+struct LanguageItemCollector<'ast, 'tcx> {
     items: LanguageItems,
     tcx: TyCtxt<'tcx>,
+    resolver: &'ast ResolverAstLowering,
+    // FIXME(#118552): We should probably feed def_span eagerly on def-id creation
+    // so we can avoid constructing this map for local def-ids.
+    item_spans: FxHashMap<DefId, Span>,
+    parent_item: Option<&'ast ast::Item>,
 }
 
-impl<'tcx> LanguageItemCollector<'tcx> {
-    fn new(tcx: TyCtxt<'tcx>) -> LanguageItemCollector<'tcx> {
-        LanguageItemCollector { tcx, items: LanguageItems::new() }
+impl<'ast, 'tcx> LanguageItemCollector<'ast, 'tcx> {
+    fn new(
+        tcx: TyCtxt<'tcx>,
+        resolver: &'ast ResolverAstLowering,
+    ) -> LanguageItemCollector<'ast, 'tcx> {
+        LanguageItemCollector {
+            tcx,
+            resolver,
+            items: LanguageItems::new(),
+            item_spans: FxHashMap::default(),
+            parent_item: None,
+        }
     }
 
-    fn check_for_lang(&mut self, actual_target: Target, def_id: LocalDefId) {
-        let attrs = self.tcx.hir().attrs(self.tcx.hir().local_def_id_to_hir_id(def_id));
-        if let Some((name, span)) = extract(&attrs) {
+    fn check_for_lang(
+        &mut self,
+        actual_target: Target,
+        def_id: LocalDefId,
+        attrs: &'ast [ast::Attribute],
+        item_span: Span,
+        generics: Option<&'ast ast::Generics>,
+    ) {
+        if let Some((name, attr_span)) = extract(attrs) {
             match LangItem::from_name(name) {
                 // Known lang item with attribute on correct target.
                 Some(lang_item) if actual_target == lang_item.target() => {
-                    self.collect_item_extended(lang_item, def_id, span);
+                    self.collect_item_extended(
+                        lang_item,
+                        def_id,
+                        item_span,
+                        attr_span,
+                        generics,
+                        actual_target,
+                    );
                 }
                 // Known lang item with attribute on incorrect target.
                 Some(lang_item) => {
                     self.tcx.sess.emit_err(LangItemOnIncorrectTarget {
-                        span,
+                        span: attr_span,
                         name,
                         expected_target: lang_item.target(),
                         actual_target,
@@ -60,122 +87,131 @@ impl<'tcx> LanguageItemCollector<'tcx> {
                 }
                 // Unknown lang item.
                 _ => {
-                    self.tcx.sess.emit_err(UnknownLangItem { span, name });
+                    self.tcx.sess.emit_err(UnknownLangItem { span: attr_span, name });
                 }
             }
         }
     }
 
-    fn collect_item(&mut self, lang_item: LangItem, item_def_id: DefId) {
+    fn collect_item(&mut self, lang_item: LangItem, item_def_id: DefId, item_span: Option<Span>) {
         // Check for duplicates.
-        if let Some(original_def_id) = self.items.get(lang_item) {
-            if original_def_id != item_def_id {
-                let local_span = self.tcx.hir().span_if_local(item_def_id);
-                let lang_item_name = lang_item.name();
-                let crate_name = self.tcx.crate_name(item_def_id.krate);
-                let mut dependency_of = Empty;
-                let is_local = item_def_id.is_local();
-                let path = if is_local {
-                    String::new()
-                } else {
-                    self.tcx
-                        .crate_extern_paths(item_def_id.krate)
-                        .iter()
-                        .map(|p| p.display().to_string())
-                        .collect::<Vec<_>>()
-                        .join(", ")
-                };
-                let first_defined_span = self.tcx.hir().span_if_local(original_def_id);
-                let mut orig_crate_name = Empty;
-                let mut orig_dependency_of = Empty;
-                let orig_is_local = original_def_id.is_local();
-                let orig_path = if orig_is_local {
-                    String::new()
-                } else {
-                    self.tcx
-                        .crate_extern_paths(original_def_id.krate)
-                        .iter()
-                        .map(|p| p.display().to_string())
-                        .collect::<Vec<_>>()
-                        .join(", ")
-                };
-                if first_defined_span.is_none() {
-                    orig_crate_name = self.tcx.crate_name(original_def_id.krate);
-                    if let Some(ExternCrate { dependency_of: inner_dependency_of, .. }) =
-                        self.tcx.extern_crate(original_def_id)
-                    {
-                        orig_dependency_of = self.tcx.crate_name(*inner_dependency_of);
-                    }
+        if let Some(original_def_id) = self.items.get(lang_item)
+            && original_def_id != item_def_id
+        {
+            let lang_item_name = lang_item.name();
+            let crate_name = self.tcx.crate_name(item_def_id.krate);
+            let mut dependency_of = Empty;
+            let is_local = item_def_id.is_local();
+            let path = if is_local {
+                String::new()
+            } else {
+                self.tcx
+                    .crate_extern_paths(item_def_id.krate)
+                    .iter()
+                    .map(|p| p.display().to_string())
+                    .collect::<Vec<_>>()
+                    .join(", ")
+            };
+
+            let first_defined_span = self.item_spans.get(&original_def_id).copied();
+            let mut orig_crate_name = Empty;
+            let mut orig_dependency_of = Empty;
+            let orig_is_local = original_def_id.is_local();
+            let orig_path = if orig_is_local {
+                String::new()
+            } else {
+                self.tcx
+                    .crate_extern_paths(original_def_id.krate)
+                    .iter()
+                    .map(|p| p.display().to_string())
+                    .collect::<Vec<_>>()
+                    .join(", ")
+            };
+
+            if first_defined_span.is_none() {
+                orig_crate_name = self.tcx.crate_name(original_def_id.krate);
+                if let Some(ExternCrate { dependency_of: inner_dependency_of, .. }) =
+                    self.tcx.extern_crate(original_def_id)
+                {
+                    orig_dependency_of = self.tcx.crate_name(*inner_dependency_of);
                 }
+            }
 
-                let duplicate = if local_span.is_some() {
-                    Duplicate::Plain
-                } else {
-                    match self.tcx.extern_crate(item_def_id) {
-                        Some(ExternCrate { dependency_of: inner_dependency_of, .. }) => {
-                            dependency_of = self.tcx.crate_name(*inner_dependency_of);
-                            Duplicate::CrateDepends
-                        }
-                        _ => Duplicate::Crate,
+            let duplicate = if item_span.is_some() {
+                Duplicate::Plain
+            } else {
+                match self.tcx.extern_crate(item_def_id) {
+                    Some(ExternCrate { dependency_of: inner_dependency_of, .. }) => {
+                        dependency_of = self.tcx.crate_name(*inner_dependency_of);
+                        Duplicate::CrateDepends
                     }
-                };
-
-                self.tcx.sess.emit_err(DuplicateLangItem {
-                    local_span,
-                    lang_item_name,
-                    crate_name,
-                    dependency_of,
-                    is_local,
-                    path,
-                    first_defined_span,
-                    orig_crate_name,
-                    orig_dependency_of,
-                    orig_is_local,
-                    orig_path,
-                    duplicate,
-                });
+                    _ => Duplicate::Crate,
+                }
+            };
+
+            self.tcx.sess.emit_err(DuplicateLangItem {
+                local_span: item_span,
+                lang_item_name,
+                crate_name,
+                dependency_of,
+                is_local,
+                path,
+                first_defined_span,
+                orig_crate_name,
+                orig_dependency_of,
+                orig_is_local,
+                orig_path,
+                duplicate,
+            });
+        } else {
+            // Matched.
+            self.items.set(lang_item, item_def_id);
+            // Collect span for error later
+            if let Some(item_span) = item_span {
+                self.item_spans.insert(item_def_id, item_span);
             }
         }
-
-        // Matched.
-        self.items.set(lang_item, item_def_id);
     }
 
     // Like collect_item() above, but also checks whether the lang item is declared
     // with the right number of generic arguments.
-    fn collect_item_extended(&mut self, lang_item: LangItem, item_def_id: LocalDefId, span: Span) {
+    fn collect_item_extended(
+        &mut self,
+        lang_item: LangItem,
+        item_def_id: LocalDefId,
+        item_span: Span,
+        attr_span: Span,
+        generics: Option<&'ast ast::Generics>,
+        target: Target,
+    ) {
         let name = lang_item.name();
 
-        // Now check whether the lang_item has the expected number of generic
-        // arguments. Generally speaking, binary and indexing operations have
-        // one (for the RHS/index), unary operations have none, the closure
-        // traits have one for the argument list, coroutines have one for the
-        // resume argument, and ordering/equality relations have one for the RHS
-        // Some other types like Box and various functions like drop_in_place
-        // have minimum requirements.
+        if let Some(generics) = generics {
+            // Now check whether the lang_item has the expected number of generic
+            // arguments. Generally speaking, binary and indexing operations have
+            // one (for the RHS/index), unary operations have none, the closure
+            // traits have one for the argument list, coroutines have one for the
+            // resume argument, and ordering/equality relations have one for the RHS
+            // Some other types like Box and various functions like drop_in_place
+            // have minimum requirements.
 
-        if let hir::Node::Item(hir::Item { kind, span: item_span, .. }) =
-            self.tcx.hir().get_by_def_id(item_def_id)
-        {
-            let (actual_num, generics_span) = match kind.generics() {
-                Some(generics) => (
-                    generics
-                        .params
-                        .iter()
-                        .filter(|p| !self.tcx.has_attr(p.def_id, sym::rustc_host))
-                        .count(),
-                    generics.span,
-                ),
-                None => (0, *item_span),
-            };
+            // FIXME: This still doesn't count, e.g., elided lifetimes and APITs.
+            let mut actual_num = generics.params.len();
+            if target.is_associated_item() {
+                actual_num += self
+                    .parent_item
+                    .unwrap()
+                    .opt_generics()
+                    .map_or(0, |generics| generics.params.len());
+            }
 
             let mut at_least = false;
             let required = match lang_item.required_generics() {
                 GenericRequirement::Exact(num) if num != actual_num => Some(num),
                 GenericRequirement::Minimum(num) if actual_num < num => {
                     at_least = true;
-                    Some(num)}
-                ,
+                    Some(num)
+                }
                 // If the number matches, or there is no requirement, handle it normally
                 _ => None,
             };
@@ -185,10 +221,10 @@ impl<'tcx> LanguageItemCollector<'tcx> {
                 // item kind of the target is correct, the target is still wrong
                 // because of the wrong number of generic arguments.
                 self.tcx.sess.emit_err(IncorrectTarget {
-                    span,
-                    generics_span,
+                    span: attr_span,
+                    generics_span: generics.span,
                     name: name.as_str(),
-                    kind: kind.descr(),
+                    kind: target.name(),
                     num,
                     actual_num,
                     at_least,
@@ -199,58 +235,117 @@ impl<'tcx> LanguageItemCollector<'tcx> {
             }
         }
 
-        self.collect_item(lang_item, item_def_id.to_def_id());
+        self.collect_item(lang_item, item_def_id.to_def_id(), Some(item_span));
     }
 }
 
 /// Traverses and collects all the lang items in all crates.
 fn get_lang_items(tcx: TyCtxt<'_>, (): ()) -> LanguageItems {
+    let resolver = tcx.resolver_for_lowering(()).borrow();
+    let (resolver, krate) = &*resolver;
+
     // Initialize the collector.
-    let mut collector = LanguageItemCollector::new(tcx);
+    let mut collector = LanguageItemCollector::new(tcx, resolver);
 
     // Collect lang items in other crates.
     for &cnum in tcx.crates(()).iter() {
         for &(def_id, lang_item) in tcx.defined_lang_items(cnum).iter() {
-            collector.collect_item(lang_item, def_id);
+            collector.collect_item(lang_item, def_id, None);
         }
     }
 
-    // Collect lang items in this crate.
-    let crate_items = tcx.hir_crate_items(());
+    // Collect lang items local to this crate.
+    visit::Visitor::visit_crate(&mut collector, krate);
 
-    for id in crate_items.items() {
-        collector
-            .check_for_lang(Target::from_def_kind(tcx.def_kind(id.owner_id)), id.owner_id.def_id);
+    // Find all required but not-yet-defined lang items.
+    weak_lang_items::check_crate(tcx, &mut collector.items, krate);
 
-        if matches!(tcx.def_kind(id.owner_id), DefKind::Enum) {
-            let item = tcx.hir().item(id);
-            if let hir::ItemKind::Enum(def, ..) = &item.kind {
-                for variant in def.variants {
-                    collector.check_for_lang(Target::Variant, variant.def_id);
-                }
-            }
-        }
-    }
+    // Return all the lang items that were found.
+    collector.items
+}
+
+impl<'ast, 'tcx> visit::Visitor<'ast> for LanguageItemCollector<'ast, 'tcx> {
+    fn visit_item(&mut self, i: &'ast ast::Item) {
+        let target = match &i.kind {
+            ast::ItemKind::ExternCrate(_) => Target::ExternCrate,
+            ast::ItemKind::Use(_) => Target::Use,
+            ast::ItemKind::Static(_) => Target::Static,
+            ast::ItemKind::Const(_) => Target::Const,
+            ast::ItemKind::Fn(_) => Target::Fn,
+            ast::ItemKind::Mod(_, _) => Target::Mod,
+            ast::ItemKind::ForeignMod(_) => Target::ForeignFn,
+            ast::ItemKind::GlobalAsm(_) => Target::GlobalAsm,
+            ast::ItemKind::TyAlias(_) => Target::TyAlias,
+            ast::ItemKind::Enum(_, _) => Target::Enum,
+            ast::ItemKind::Struct(_, _) => Target::Struct,
+            ast::ItemKind::Union(_, _) => Target::Union,
+            ast::ItemKind::Trait(_) => Target::Trait,
+            ast::ItemKind::TraitAlias(_, _) => Target::TraitAlias,
+            ast::ItemKind::Impl(_) => Target::Impl,
+            ast::ItemKind::MacroDef(_) => Target::MacroDef,
+            ast::ItemKind::MacCall(_) => unreachable!("macros should have been expanded"),
+        };
 
-    // FIXME: avoid calling trait_item() when possible
-    for id in crate_items.trait_items() {
-        let item = tcx.hir().trait_item(id);
-        collector.check_for_lang(Target::from_trait_item(item), item.owner_id.def_id)
+        self.check_for_lang(
+            target,
+            self.resolver.node_id_to_def_id[&i.id],
+            &i.attrs,
+            i.span,
+            i.opt_generics(),
+        );
+
+        let parent_item = self.parent_item.replace(i);
+        visit::walk_item(self, i);
+        self.parent_item = parent_item;
     }
 
-    // FIXME: avoid calling impl_item() when possible
-    for id in crate_items.impl_items() {
-        let item = tcx.hir().impl_item(id);
-        collector.check_for_lang(target_from_impl_item(tcx, item), item.owner_id.def_id)
+    fn visit_enum_def(&mut self, enum_definition: &'ast ast::EnumDef) {
+        for variant in &enum_definition.variants {
+            self.check_for_lang(
+                Target::Variant,
+                self.resolver.node_id_to_def_id[&variant.id],
+                &variant.attrs,
+                variant.span,
+                None,
+            );
+        }
+
+        visit::walk_enum_def(self, enum_definition);
     }
 
-    // Extract out the found lang items.
-    let LanguageItemCollector { mut items, .. } = collector;
+    fn visit_assoc_item(&mut self, i: &'ast ast::AssocItem, ctxt: visit::AssocCtxt) {
+        let (target, generics) = match &i.kind {
+            ast::AssocItemKind::Fn(fun) => (
+                match &self.parent_item.unwrap().kind {
+                    ast::ItemKind::Impl(i) => {
+                        if i.of_trait.is_some() {
+                            Target::Method(MethodKind::Trait { body: fun.body.is_some() })
+                        } else {
+                            Target::Method(MethodKind::Inherent)
+                        }
+                    }
+                    ast::ItemKind::Trait(_) => {
+                        Target::Method(MethodKind::Trait { body: fun.body.is_some() })
+                    }
+                    _ => unreachable!(),
+                },
+                &fun.generics,
+            ),
+            ast::AssocItemKind::Const(ct) => (Target::AssocConst, &ct.generics),
+            ast::AssocItemKind::Type(ty) => (Target::AssocTy, &ty.generics),
+            ast::AssocItemKind::MacCall(_) => unreachable!("macros should have been expanded"),
+        };
 
-    // Find all required but not-yet-defined lang items.
-    weak_lang_items::check_crate(tcx, &mut items);
+        self.check_for_lang(
+            target,
+            self.resolver.node_id_to_def_id[&i.id],
+            &i.attrs,
+            i.span,
+            Some(generics),
+        );
 
-    items
+        visit::walk_assoc_item(self, i, ctxt);
+    }
 }
 
 pub fn provide(providers: &mut Providers) {
diff --git a/compiler/rustc_passes/src/lib.rs b/compiler/rustc_passes/src/lib.rs
index 946a9e68d..c969867e8 100644
--- a/compiler/rustc_passes/src/lib.rs
+++ b/compiler/rustc_passes/src/lib.rs
@@ -4,11 +4,10 @@
 //!
 //! This API is completely unstable and subject to change.
 
-#![allow(rustc::potential_query_instability)]
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
-#![cfg_attr(not(bootstrap), allow(internal_features))]
+#![doc(rust_logo)]
+#![feature(rustdoc_internals)]
+#![allow(internal_features)]
 #![feature(iter_intersperse)]
 #![feature(let_chains)]
 #![feature(map_try_insert)]
@@ -23,8 +22,6 @@ extern crate rustc_middle;
 #[macro_use]
 extern crate tracing;
 
-use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
-use rustc_fluent_macro::fluent_messages;
 use rustc_middle::query::Providers;
 
 pub mod abi_test;
@@ -48,7 +45,7 @@ pub mod stability;
 mod upvars;
 mod weak_lang_items;
 
-fluent_messages! { "../messages.ftl" }
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
 
 pub fn provide(providers: &mut Providers) {
     check_attr::provide(providers);
diff --git a/compiler/rustc_passes/src/lib_features.rs b/compiler/rustc_passes/src/lib_features.rs
index 0daa273db..6c0412bed 100644
--- a/compiler/rustc_passes/src/lib_features.rs
+++ b/compiler/rustc_passes/src/lib_features.rs
@@ -8,18 +8,14 @@ use rustc_ast::Attribute;
 use rustc_attr::VERSION_PLACEHOLDER;
 use rustc_hir::intravisit::Visitor;
 use rustc_middle::hir::nested_filter;
-use rustc_middle::middle::lib_features::LibFeatures;
-use rustc_middle::query::Providers;
+use rustc_middle::middle::lib_features::{FeatureStability, LibFeatures};
+use rustc_middle::query::{LocalCrate, Providers};
 use rustc_middle::ty::TyCtxt;
 use rustc_span::symbol::Symbol;
 use rustc_span::{sym, Span};
 
 use crate::errors::{FeaturePreviouslyDeclared, FeatureStableTwice};
 
-fn new_lib_features() -> LibFeatures {
-    LibFeatures { stable: Default::default(), unstable: Default::default() }
-}
-
 pub struct LibFeatureCollector<'tcx> {
     tcx: TyCtxt<'tcx>,
     lib_features: LibFeatures,
@@ -27,10 +23,10 @@ pub struct LibFeatureCollector<'tcx> {
 
 impl<'tcx> LibFeatureCollector<'tcx> {
     fn new(tcx: TyCtxt<'tcx>) -> LibFeatureCollector<'tcx> {
-        LibFeatureCollector { tcx, lib_features: new_lib_features() }
+        LibFeatureCollector { tcx, lib_features: LibFeatures::default() }
     }
 
-    fn extract(&self, attr: &Attribute) -> Option<(Symbol, Option<Symbol>, Span)> {
+    fn extract(&self, attr: &Attribute) -> Option<(Symbol, FeatureStability, Span)> {
         let stab_attrs = [
             sym::stable,
             sym::unstable,
@@ -72,8 +68,11 @@ impl<'tcx> LibFeatureCollector<'tcx> {
                             | sym::rustc_const_unstable
                             | sym::rustc_default_body_unstable
                     );
-                    if since.is_some() || is_unstable {
-                        return Some((feature, since, attr.span));
+                    if is_unstable {
+                        return Some((feature, FeatureStability::Unstable, attr.span));
+                    }
+                    if let Some(since) = since {
+                        return Some((feature, FeatureStability::AcceptedSince(since), attr.span));
                     }
                 }
                 // We need to iterate over the other attributes, because
@@ -86,39 +85,39 @@ impl<'tcx> LibFeatureCollector<'tcx> {
         None
     }
 
-    fn collect_feature(&mut self, feature: Symbol, since: Option<Symbol>, span: Span) {
-        let already_in_stable = self.lib_features.stable.contains_key(&feature);
-        let already_in_unstable = self.lib_features.unstable.contains_key(&feature);
-
-        match (since, already_in_stable, already_in_unstable) {
-            (Some(since), _, false) => {
-                if let Some((prev_since, _)) = self.lib_features.stable.get(&feature) {
-                    if *prev_since != since {
-                        self.tcx.sess.emit_err(FeatureStableTwice {
-                            span,
-                            feature,
-                            since,
-                            prev_since: *prev_since,
-                        });
-                        return;
-                    }
-                }
+    fn collect_feature(&mut self, feature: Symbol, stability: FeatureStability, span: Span) {
+        let existing_stability = self.lib_features.stability.get(&feature).cloned();
 
-                self.lib_features.stable.insert(feature, (since, span));
+        match (stability, existing_stability) {
+            (_, None) => {
+                self.lib_features.stability.insert(feature, (stability, span));
             }
-            (None, false, _) => {
-                self.lib_features.unstable.insert(feature, span);
+            (
+                FeatureStability::AcceptedSince(since),
+                Some((FeatureStability::AcceptedSince(prev_since), _)),
+            ) => {
+                if prev_since != since {
+                    self.tcx.sess.emit_err(FeatureStableTwice { span, feature, since, prev_since });
+                }
+            }
+            (FeatureStability::AcceptedSince(_), Some((FeatureStability::Unstable, _))) => {
+                self.tcx.sess.emit_err(FeaturePreviouslyDeclared {
+                    span,
+                    feature,
+                    declared: "stable",
+                    prev_declared: "unstable",
+                });
             }
-            (Some(_), _, true) | (None, true, _) => {
-                let declared = if since.is_some() { "stable" } else { "unstable" };
-                let prev_declared = if since.is_none() { "stable" } else { "unstable" };
+            (FeatureStability::Unstable, Some((FeatureStability::AcceptedSince(_), _))) => {
                 self.tcx.sess.emit_err(FeaturePreviouslyDeclared {
                     span,
                     feature,
-                    declared,
-                    prev_declared,
+                    declared: "unstable",
+                    prev_declared: "stable",
                 });
             }
+            // duplicate `unstable` feature is ok.
+            (FeatureStability::Unstable, Some((FeatureStability::Unstable, _))) => {}
         }
     }
 }
@@ -137,11 +136,11 @@ impl<'tcx> Visitor<'tcx> for LibFeatureCollector<'tcx> {
     }
 }
 
-fn lib_features(tcx: TyCtxt<'_>, (): ()) -> LibFeatures {
+fn lib_features(tcx: TyCtxt<'_>, LocalCrate: LocalCrate) -> LibFeatures {
     // If `staged_api` is not enabled then we aren't allowed to define lib
     // features; there is no point collecting them.
     if !tcx.features().staged_api {
-        return new_lib_features();
+        return LibFeatures::default();
     }
 
     let mut collector = LibFeatureCollector::new(tcx);
diff --git a/compiler/rustc_passes/src/liveness.rs b/compiler/rustc_passes/src/liveness.rs
index b73fb984c..c9e5eb50b 100644
--- a/compiler/rustc_passes/src/liveness.rs
+++ b/compiler/rustc_passes/src/liveness.rs
@@ -170,7 +170,7 @@ fn check_liveness(tcx: TyCtxt<'_>, def_id: LocalDefId) {
 
     // compute liveness
     let mut lsets = Liveness::new(&mut maps, def_id);
-    let entry_ln = lsets.compute(&body, hir_id);
+    let entry_ln = lsets.compute(body, hir_id);
     lsets.log_liveness(entry_ln, body_id.hir_id);
 
     // check for various error conditions
@@ -366,7 +366,7 @@ impl<'tcx> IrMaps<'tcx> {
 
 impl<'tcx> Visitor<'tcx> for IrMaps<'tcx> {
     fn visit_local(&mut self, local: &'tcx hir::Local<'tcx>) {
-        self.add_from_pat(&local.pat);
+        self.add_from_pat(local.pat);
         if local.els.is_some() {
             self.add_live_node_for_node(local.hir_id, ExprNode(local.span, local.hir_id));
         }
@@ -374,8 +374,8 @@ impl<'tcx> Visitor<'tcx> for IrMaps<'tcx> {
     }
 
     fn visit_arm(&mut self, arm: &'tcx hir::Arm<'tcx>) {
-        self.add_from_pat(&arm.pat);
-        if let Some(hir::Guard::IfLet(ref let_expr)) = arm.guard {
+        self.add_from_pat(arm.pat);
+        if let Some(hir::Guard::IfLet(let_expr)) = arm.guard {
             self.add_from_pat(let_expr.pat);
         }
         intravisit::walk_arm(self, arm);
@@ -400,7 +400,7 @@ impl<'tcx> Visitor<'tcx> for IrMaps<'tcx> {
     fn visit_expr(&mut self, expr: &'tcx Expr<'tcx>) {
         match expr.kind {
             // live nodes required for uses or definitions of variables:
-            hir::ExprKind::Path(hir::QPath::Resolved(_, ref path)) => {
+            hir::ExprKind::Path(hir::QPath::Resolved(_, path)) => {
                 debug!("expr {}: path that leads to {:?}", expr.hir_id, path.res);
                 if let Res::Local(_var_hir_id) = path.res {
                     self.add_live_node_for_node(expr.hir_id, ExprNode(expr.span, expr.hir_id));
@@ -592,7 +592,7 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> {
         match self.successors[ln] {
             Some(successor) => self.assigned_on_entry(successor, var),
             None => {
-                self.ir.tcx.sess.delay_span_bug(DUMMY_SP, "no successor");
+                self.ir.tcx.sess.span_delayed_bug(DUMMY_SP, "no successor");
                 true
             }
         }
@@ -736,7 +736,7 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> {
             }
         }
 
-        let succ = self.propagate_through_expr(&body.value, self.exit_ln);
+        let succ = self.propagate_through_expr(body.value, self.exit_ln);
 
         if self.closure_min_captures.is_none() {
             // Either not a closure, or closure without any captured variables.
@@ -776,7 +776,7 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> {
             if !self.merge_from_succ(self.exit_ln, self.closure_ln) {
                 break;
             }
-            assert_eq!(succ, self.propagate_through_expr(&body.value, self.exit_ln));
+            assert_eq!(succ, self.propagate_through_expr(body.value, self.exit_ln));
         }
 
         succ
@@ -792,7 +792,7 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> {
 
     fn propagate_through_stmt(&mut self, stmt: &hir::Stmt<'_>, succ: LiveNode) -> LiveNode {
         match stmt.kind {
-            hir::StmtKind::Local(ref local) => {
+            hir::StmtKind::Local(local) => {
                 // Note: we mark the variable as defined regardless of whether
                 // there is an initializer. Initially I had thought to only mark
                 // the live variable as defined if it was initialized, and then we
@@ -830,7 +830,7 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> {
                         self.init_from_succ(ln, succ);
                         self.merge_from_succ(ln, else_ln);
                         let succ = self.propagate_through_expr(init, ln);
-                        self.define_bindings_in_pat(&local.pat, succ)
+                        self.define_bindings_in_pat(local.pat, succ)
                     } else {
                         span_bug!(
                             stmt.span,
@@ -839,18 +839,18 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> {
                     }
                 } else {
                     let succ = self.propagate_through_opt_expr(local.init, succ);
-                    self.define_bindings_in_pat(&local.pat, succ)
+                    self.define_bindings_in_pat(local.pat, succ)
                 }
             }
             hir::StmtKind::Item(..) => succ,
             hir::StmtKind::Expr(ref expr) | hir::StmtKind::Semi(ref expr) => {
-                self.propagate_through_expr(&expr, succ)
+                self.propagate_through_expr(expr, succ)
             }
         }
     }
 
     fn propagate_through_exprs(&mut self, exprs: &[Expr<'_>], succ: LiveNode) -> LiveNode {
-        exprs.iter().rev().fold(succ, |succ, expr| self.propagate_through_expr(&expr, succ))
+        exprs.iter().rev().fold(succ, |succ, expr| self.propagate_through_expr(expr, succ))
     }
 
     fn propagate_through_opt_expr(
@@ -866,11 +866,11 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> {
 
         match expr.kind {
             // Interesting cases with control flow or which gen/kill
-            hir::ExprKind::Path(hir::QPath::Resolved(_, ref path)) => {
+            hir::ExprKind::Path(hir::QPath::Resolved(_, path)) => {
                 self.access_path(expr.hir_id, path, succ, ACC_READ | ACC_USE)
             }
 
-            hir::ExprKind::Field(ref e, _) => self.propagate_through_expr(&e, succ),
+            hir::ExprKind::Field(ref e, _) => self.propagate_through_expr(e, succ),
 
             hir::ExprKind::Closure { .. } => {
                 debug!("{:?} is an ExprKind::Closure", expr);
@@ -899,9 +899,9 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> {
 
             // Note that labels have been resolved, so we don't need to look
             // at the label ident
-            hir::ExprKind::Loop(ref blk, ..) => self.propagate_through_loop(expr, &blk, succ),
+            hir::ExprKind::Loop(ref blk, ..) => self.propagate_through_loop(expr, blk, succ),
 
-            hir::ExprKind::Yield(ref e, ..) => {
+            hir::ExprKind::Yield(e, ..) => {
                 let yield_ln = self.live_node(expr.hir_id, expr.span);
                 self.init_from_succ(yield_ln, succ);
                 self.merge_from_succ(yield_ln, self.exit_ln);
@@ -923,11 +923,11 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> {
                 //   (  succ  )
                 //
                 let else_ln = self.propagate_through_opt_expr(else_opt.as_deref(), succ);
-                let then_ln = self.propagate_through_expr(&then, succ);
+                let then_ln = self.propagate_through_expr(then, succ);
                 let ln = self.live_node(expr.hir_id, expr.span);
                 self.init_from_succ(ln, else_ln);
                 self.merge_from_succ(ln, then_ln);
-                self.propagate_through_expr(&cond, ln)
+                self.propagate_through_expr(cond, ln)
             }
 
             hir::ExprKind::Match(ref e, arms, _) => {
@@ -948,7 +948,7 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> {
                 let ln = self.live_node(expr.hir_id, expr.span);
                 self.init_empty(ln, succ);
                 for arm in arms {
-                    let body_succ = self.propagate_through_expr(&arm.body, succ);
+                    let body_succ = self.propagate_through_expr(arm.body, succ);
 
                     let guard_succ = arm.guard.as_ref().map_or(body_succ, |g| match g {
                         hir::Guard::If(e) => self.propagate_through_expr(e, body_succ),
@@ -957,10 +957,10 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> {
                             self.propagate_through_expr(let_expr.init, let_bind)
                         }
                     });
-                    let arm_succ = self.define_bindings_in_pat(&arm.pat, guard_succ);
+                    let arm_succ = self.define_bindings_in_pat(arm.pat, guard_succ);
                     self.merge_from_succ(ln, arm_succ);
                 }
-                self.propagate_through_expr(&e, ln)
+                self.propagate_through_expr(e, ln)
             }
 
             hir::ExprKind::Ret(ref o_e) => {
@@ -968,7 +968,7 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> {
                 self.propagate_through_opt_expr(o_e.as_deref(), self.exit_ln)
             }
 
-            hir::ExprKind::Become(ref e) => {
+            hir::ExprKind::Become(e) => {
                 // Ignore succ and subst exit_ln.
                 self.propagate_through_expr(e, self.exit_ln)
             }
@@ -1007,63 +1007,63 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> {
             hir::ExprKind::Assign(ref l, ref r, _) => {
                 // see comment on places in
                 // propagate_through_place_components()
-                let succ = self.write_place(&l, succ, ACC_WRITE);
-                let succ = self.propagate_through_place_components(&l, succ);
-                self.propagate_through_expr(&r, succ)
+                let succ = self.write_place(l, succ, ACC_WRITE);
+                let succ = self.propagate_through_place_components(l, succ);
+                self.propagate_through_expr(r, succ)
             }
 
             hir::ExprKind::AssignOp(_, ref l, ref r) => {
                 // an overloaded assign op is like a method call
                 if self.typeck_results.is_method_call(expr) {
-                    let succ = self.propagate_through_expr(&l, succ);
-                    self.propagate_through_expr(&r, succ)
+                    let succ = self.propagate_through_expr(l, succ);
+                    self.propagate_through_expr(r, succ)
                 } else {
                     // see comment on places in
                     // propagate_through_place_components()
-                    let succ = self.write_place(&l, succ, ACC_WRITE | ACC_READ);
-                    let succ = self.propagate_through_expr(&r, succ);
-                    self.propagate_through_place_components(&l, succ)
+                    let succ = self.write_place(l, succ, ACC_WRITE | ACC_READ);
+                    let succ = self.propagate_through_expr(r, succ);
+                    self.propagate_through_place_components(l, succ)
                 }
             }
 
             // Uninteresting cases: just propagate in rev exec order
-            hir::ExprKind::Array(ref exprs) => self.propagate_through_exprs(exprs, succ),
+            hir::ExprKind::Array(exprs) => self.propagate_through_exprs(exprs, succ),
 
-            hir::ExprKind::Struct(_, ref fields, ref with_expr) => {
+            hir::ExprKind::Struct(_, fields, ref with_expr) => {
                 let succ = self.propagate_through_opt_expr(with_expr.as_deref(), succ);
                 fields
                     .iter()
                     .rev()
-                    .fold(succ, |succ, field| self.propagate_through_expr(&field.expr, succ))
+                    .fold(succ, |succ, field| self.propagate_through_expr(field.expr, succ))
             }
 
-            hir::ExprKind::Call(ref f, ref args) => {
+            hir::ExprKind::Call(ref f, args) => {
                 let succ = self.check_is_ty_uninhabited(expr, succ);
                 let succ = self.propagate_through_exprs(args, succ);
-                self.propagate_through_expr(&f, succ)
+                self.propagate_through_expr(f, succ)
             }
 
-            hir::ExprKind::MethodCall(.., receiver, ref args, _) => {
+            hir::ExprKind::MethodCall(.., receiver, args, _) => {
                 let succ = self.check_is_ty_uninhabited(expr, succ);
                 let succ = self.propagate_through_exprs(args, succ);
                 self.propagate_through_expr(receiver, succ)
             }
 
-            hir::ExprKind::Tup(ref exprs) => self.propagate_through_exprs(exprs, succ),
+            hir::ExprKind::Tup(exprs) => self.propagate_through_exprs(exprs, succ),
 
             hir::ExprKind::Binary(op, ref l, ref r) if op.node.is_lazy() => {
-                let r_succ = self.propagate_through_expr(&r, succ);
+                let r_succ = self.propagate_through_expr(r, succ);
 
                 let ln = self.live_node(expr.hir_id, expr.span);
                 self.init_from_succ(ln, succ);
                 self.merge_from_succ(ln, r_succ);
 
-                self.propagate_through_expr(&l, ln)
+                self.propagate_through_expr(l, ln)
             }
 
             hir::ExprKind::Index(ref l, ref r, _) | hir::ExprKind::Binary(_, ref l, ref r) => {
-                let r_succ = self.propagate_through_expr(&r, succ);
-                self.propagate_through_expr(&l, r_succ)
+                let r_succ = self.propagate_through_expr(r, succ);
+                self.propagate_through_expr(l, r_succ)
             }
 
             hir::ExprKind::AddrOf(_, _, ref e)
@@ -1071,9 +1071,9 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> {
             | hir::ExprKind::Type(ref e, _)
             | hir::ExprKind::DropTemps(ref e)
             | hir::ExprKind::Unary(_, ref e)
-            | hir::ExprKind::Repeat(ref e, _) => self.propagate_through_expr(&e, succ),
+            | hir::ExprKind::Repeat(ref e, _) => self.propagate_through_expr(e, succ),
 
-            hir::ExprKind::InlineAsm(ref asm) => {
+            hir::ExprKind::InlineAsm(asm) => {
                 // Handle non-returning asm
                 let mut succ = if asm.options.contains(InlineAsmOptions::NORETURN) {
                     self.exit_ln
@@ -1141,7 +1141,7 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> {
 
             // Note that labels have been resolved, so we don't need to look
             // at the label ident
-            hir::ExprKind::Block(ref blk, _) => self.propagate_through_block(&blk, succ),
+            hir::ExprKind::Block(ref blk, _) => self.propagate_through_block(blk, succ),
         }
     }
 
@@ -1197,7 +1197,7 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> {
 
         match expr.kind {
             hir::ExprKind::Path(_) => succ,
-            hir::ExprKind::Field(ref e, _) => self.propagate_through_expr(&e, succ),
+            hir::ExprKind::Field(ref e, _) => self.propagate_through_expr(e, succ),
             _ => self.propagate_through_expr(expr, succ),
         }
     }
@@ -1205,7 +1205,7 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> {
     // see comment on propagate_through_place()
     fn write_place(&mut self, expr: &Expr<'_>, succ: LiveNode, acc: u32) -> LiveNode {
         match expr.kind {
-            hir::ExprKind::Path(hir::QPath::Resolved(_, ref path)) => {
+            hir::ExprKind::Path(hir::QPath::Resolved(_, path)) => {
                 self.access_path(expr.hir_id, path, succ, acc)
             }
 
@@ -1341,7 +1341,7 @@ impl<'a, 'tcx> Liveness<'a, 'tcx> {
 
 impl<'a, 'tcx> Visitor<'tcx> for Liveness<'a, 'tcx> {
     fn visit_local(&mut self, local: &'tcx hir::Local<'tcx>) {
-        self.check_unused_vars_in_pat(&local.pat, None, None, |spans, hir_id, ln, var| {
+        self.check_unused_vars_in_pat(local.pat, None, None, |spans, hir_id, ln, var| {
             if local.init.is_some() {
                 self.warn_about_dead_assign(spans, hir_id, ln, var);
             }
@@ -1356,7 +1356,7 @@ impl<'a, 'tcx> Visitor<'tcx> for Liveness<'a, 'tcx> {
     }
 
     fn visit_arm(&mut self, arm: &'tcx hir::Arm<'tcx>) {
-        self.check_unused_vars_in_pat(&arm.pat, None, None, |_, _, _, _| {});
+        self.check_unused_vars_in_pat(arm.pat, None, None, |_, _, _, _| {});
         intravisit::walk_arm(self, arm);
     }
 }
@@ -1364,16 +1364,16 @@ impl<'a, 'tcx> Visitor<'tcx> for Liveness<'a, 'tcx> {
 fn check_expr<'tcx>(this: &mut Liveness<'_, 'tcx>, expr: &'tcx Expr<'tcx>) {
     match expr.kind {
         hir::ExprKind::Assign(ref l, ..) => {
-            this.check_place(&l);
+            this.check_place(l);
         }
 
         hir::ExprKind::AssignOp(_, ref l, _) => {
             if !this.typeck_results.is_method_call(expr) {
-                this.check_place(&l);
+                this.check_place(l);
             }
         }
 
-        hir::ExprKind::InlineAsm(ref asm) => {
+        hir::ExprKind::InlineAsm(asm) => {
             for (op, _op_sp) in asm.operands {
                 match op {
                     hir::InlineAsmOperand::Out { expr, .. } => {
@@ -1434,7 +1434,7 @@ fn check_expr<'tcx>(this: &mut Liveness<'_, 'tcx>, expr: &'tcx Expr<'tcx>) {
 impl<'tcx> Liveness<'_, 'tcx> {
     fn check_place(&mut self, expr: &'tcx Expr<'tcx>) {
         match expr.kind {
-            hir::ExprKind::Path(hir::QPath::Resolved(_, ref path)) => {
+            hir::ExprKind::Path(hir::QPath::Resolved(_, path)) => {
                 if let Res::Local(var_hid) = path.res {
                     // Assignment to an immutable variable or argument: only legal
                     // if there is no later assignment. If this local is actually
@@ -1507,7 +1507,7 @@ impl<'tcx> Liveness<'_, 'tcx> {
     fn warn_about_unused_args(&self, body: &hir::Body<'_>, entry_ln: LiveNode) {
         for p in body.params {
             self.check_unused_vars_in_pat(
-                &p.pat,
+                p.pat,
                 Some(entry_ln),
                 Some(body),
                 |spans, hir_id, ln, var| {
diff --git a/compiler/rustc_passes/src/loops.rs b/compiler/rustc_passes/src/loops.rs
index 25e131d74..bfaf4a5a9 100644
--- a/compiler/rustc_passes/src/loops.rs
+++ b/compiler/rustc_passes/src/loops.rs
@@ -4,7 +4,6 @@ use rustc_hir as hir;
 use rustc_hir::def_id::{LocalDefId, LocalModDefId};
 use rustc_hir::intravisit::{self, Visitor};
 use rustc_hir::{Destination, Movability, Node};
-use rustc_middle::hir::map::Map;
 use rustc_middle::hir::nested_filter;
 use rustc_middle::query::Providers;
 use rustc_middle::ty::TyCtxt;
@@ -30,16 +29,16 @@ enum Context {
 }
 
 #[derive(Copy, Clone)]
-struct CheckLoopVisitor<'a, 'hir> {
+struct CheckLoopVisitor<'a, 'tcx> {
     sess: &'a Session,
-    hir_map: Map<'hir>,
+    tcx: TyCtxt<'tcx>,
     cx: Context,
 }
 
 fn check_mod_loops(tcx: TyCtxt<'_>, module_def_id: LocalModDefId) {
     tcx.hir().visit_item_likes_in_module(
         module_def_id,
-        &mut CheckLoopVisitor { sess: &tcx.sess, hir_map: tcx.hir(), cx: Normal },
+        &mut CheckLoopVisitor { sess: tcx.sess, tcx, cx: Normal },
     );
 }
 
@@ -51,7 +50,7 @@ impl<'a, 'hir> Visitor<'hir> for CheckLoopVisitor<'a, 'hir> {
     type NestedFilter = nested_filter::OnlyBodies;
 
     fn nested_visit_map(&mut self) -> Self::Map {
-        self.hir_map
+        self.tcx.hir()
     }
 
     fn visit_anon_const(&mut self, c: &'hir hir::AnonConst) {
@@ -84,7 +83,7 @@ impl<'a, 'hir> Visitor<'hir> for CheckLoopVisitor<'a, 'hir> {
     fn visit_expr(&mut self, e: &'hir hir::Expr<'hir>) {
         match e.kind {
             hir::ExprKind::Loop(ref b, _, source, _) => {
-                self.with_context(Loop(source), |v| v.visit_block(&b));
+                self.with_context(Loop(source), |v| v.visit_block(b));
             }
             hir::ExprKind::Closure(&hir::Closure {
                 ref fn_decl,
@@ -98,19 +97,19 @@ impl<'a, 'hir> Visitor<'hir> for CheckLoopVisitor<'a, 'hir> {
                 } else {
                     Closure(fn_decl_span)
                 };
-                self.visit_fn_decl(&fn_decl);
+                self.visit_fn_decl(fn_decl);
                 self.with_context(cx, |v| v.visit_nested_body(body));
             }
             hir::ExprKind::Block(ref b, Some(_label)) => {
-                self.with_context(LabeledBlock, |v| v.visit_block(&b));
+                self.with_context(LabeledBlock, |v| v.visit_block(b));
             }
             hir::ExprKind::Block(ref b, None) if matches!(self.cx, Fn) => {
-                self.with_context(Normal, |v| v.visit_block(&b));
+                self.with_context(Normal, |v| v.visit_block(b));
             }
             hir::ExprKind::Block(ref b, None)
                 if matches!(self.cx, Normal | Constant | UnlabeledBlock(_)) =>
             {
-                self.with_context(UnlabeledBlock(b.span.shrink_to_lo()), |v| v.visit_block(&b));
+                self.with_context(UnlabeledBlock(b.span.shrink_to_lo()), |v| v.visit_block(b));
             }
             hir::ExprKind::Break(break_label, ref opt_expr) => {
                 if let Some(e) = opt_expr {
@@ -136,13 +135,13 @@ impl<'a, 'hir> Visitor<'hir> for CheckLoopVisitor<'a, 'hir> {
                     Err(hir::LoopIdError::UnresolvedLabel) => None,
                 };
 
-                if let Some(Node::Block(_)) = loop_id.and_then(|id| self.hir_map.find(id)) {
+                if let Some(Node::Block(_)) = loop_id.and_then(|id| self.tcx.opt_hir_node(id)) {
                     return;
                 }
 
                 if let Some(break_expr) = opt_expr {
                     let (head, loop_label, loop_kind) = if let Some(loop_id) = loop_id {
-                        match self.hir_map.expect_expr(loop_id).kind {
+                        match self.tcx.hir().expect_expr(loop_id).kind {
                             hir::ExprKind::Loop(_, label, source, sp) => {
                                 (Some(sp), label, Some(source))
                             }
@@ -188,7 +187,7 @@ impl<'a, 'hir> Visitor<'hir> for CheckLoopVisitor<'a, 'hir> {
 
                 match destination.target_id {
                     Ok(loop_id) => {
-                        if let Node::Block(block) = self.hir_map.find(loop_id).unwrap() {
+                        if let Node::Block(block) = self.tcx.opt_hir_node(loop_id).unwrap() {
                             self.sess.emit_err(ContinueLabeledBlock {
                                 span: e.span,
                                 block_span: block.span,
diff --git a/compiler/rustc_passes/src/naked_functions.rs b/compiler/rustc_passes/src/naked_functions.rs
index 7f36c59ad..25637f935 100644
--- a/compiler/rustc_passes/src/naked_functions.rs
+++ b/compiler/rustc_passes/src/naked_functions.rs
@@ -35,7 +35,7 @@ fn check_mod_naked_functions(tcx: TyCtxt<'_>, module_def_id: LocalModDefId) {
             continue;
         }
 
-        let (fn_header, body_id) = match tcx.hir().get_by_def_id(def_id) {
+        let (fn_header, body_id) = match tcx.hir_node_by_def_id(def_id) {
             hir::Node::Item(hir::Item { kind: hir::ItemKind::Fn(sig, _, body_id), .. })
             | hir::Node::TraitItem(hir::TraitItem {
                 kind: hir::TraitItemKind::Fn(sig, hir::TraitFn::Provided(body_id)),
@@ -68,7 +68,7 @@ fn check_inline(tcx: TyCtxt<'_>, def_id: LocalDefId) {
 /// Checks that function uses non-Rust ABI.
 fn check_abi(tcx: TyCtxt<'_>, def_id: LocalDefId, abi: Abi) {
     if abi == Abi::Rust {
-        let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
+        let hir_id = tcx.local_def_id_to_hir_id(def_id);
         let span = tcx.def_span(def_id);
         tcx.emit_spanned_lint(
             UNDEFINED_NAKED_FUNCTION_ABI,
@@ -211,7 +211,7 @@ impl<'tcx> CheckInlineAssembly<'tcx> {
                 self.items.push((ItemKind::NonAsm, span));
             }
 
-            ExprKind::InlineAsm(ref asm) => {
+            ExprKind::InlineAsm(asm) => {
                 self.items.push((ItemKind::Asm, span));
                 self.check_inline_asm(asm, span);
             }
@@ -282,13 +282,13 @@ impl<'tcx> Visitor<'tcx> for CheckInlineAssembly<'tcx> {
             StmtKind::Local(..) => {
                 self.items.push((ItemKind::NonAsm, stmt.span));
             }
-            StmtKind::Expr(ref expr) | StmtKind::Semi(ref expr) => {
+            StmtKind::Expr(expr) | StmtKind::Semi(expr) => {
                 self.check_expr(expr, stmt.span);
             }
         }
     }
 
     fn visit_expr(&mut self, expr: &'tcx hir::Expr<'tcx>) {
-        self.check_expr(&expr, expr.span);
+        self.check_expr(expr, expr.span);
     }
 }
diff --git a/compiler/rustc_passes/src/reachable.rs b/compiler/rustc_passes/src/reachable.rs
index 650bb97c4..f46f831dd 100644
--- a/compiler/rustc_passes/src/reachable.rs
+++ b/compiler/rustc_passes/src/reachable.rs
@@ -115,7 +115,7 @@ impl<'tcx> ReachableContext<'tcx> {
             return false;
         };
 
-        match self.tcx.hir().find_by_def_id(def_id) {
+        match self.tcx.opt_hir_node_by_def_id(def_id) {
             Some(Node::Item(item)) => match item.kind {
                 hir::ItemKind::Fn(..) => item_might_be_inlined(self.tcx, def_id.into()),
                 _ => false,
@@ -146,7 +146,7 @@ impl<'tcx> ReachableContext<'tcx> {
                 continue;
             }
 
-            if let Some(ref item) = self.tcx.hir().find_by_def_id(search_item) {
+            if let Some(ref item) = self.tcx.opt_hir_node_by_def_id(search_item) {
                 self.propagate_node(item, search_item);
             }
         }
@@ -260,9 +260,7 @@ impl<'tcx> ReachableContext<'tcx> {
             _ => {
                 bug!(
                     "found unexpected node kind in worklist: {} ({:?})",
-                    self.tcx
-                        .hir()
-                        .node_to_string(self.tcx.hir().local_def_id_to_hir_id(search_item)),
+                    self.tcx.hir().node_to_string(self.tcx.local_def_id_to_hir_id(search_item)),
                     node,
                 );
             }
diff --git a/compiler/rustc_passes/src/stability.rs b/compiler/rustc_passes/src/stability.rs
index 6a2498f3f..676622cef 100644
--- a/compiler/rustc_passes/src/stability.rs
+++ b/compiler/rustc_passes/src/stability.rs
@@ -9,11 +9,12 @@ use rustc_attr::{
 use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexMap};
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
-use rustc_hir::def_id::{LocalDefId, LocalModDefId, CRATE_DEF_ID};
+use rustc_hir::def_id::{LocalDefId, LocalModDefId, CRATE_DEF_ID, LOCAL_CRATE};
 use rustc_hir::hir_id::CRATE_HIR_ID;
 use rustc_hir::intravisit::{self, Visitor};
 use rustc_hir::{FieldDef, Item, ItemKind, TraitRef, Ty, TyKind, Variant};
 use rustc_middle::hir::nested_filter;
+use rustc_middle::middle::lib_features::{FeatureStability, LibFeatures};
 use rustc_middle::middle::privacy::EffectiveVisibilities;
 use rustc_middle::middle::stability::{AllowUnstable, DeprecationEntry, Index};
 use rustc_middle::query::Providers;
@@ -110,7 +111,7 @@ impl<'a, 'tcx> Annotator<'a, 'tcx> {
     ) where
         F: FnOnce(&mut Self),
     {
-        let attrs = self.tcx.hir().attrs(self.tcx.hir().local_def_id_to_hir_id(def_id));
+        let attrs = self.tcx.hir().attrs(self.tcx.local_def_id_to_hir_id(def_id));
         debug!("annotate(id = {:?}, attrs = {:?})", def_id, attrs);
 
         let depr = attr::find_deprecation(self.tcx.sess, self.tcx.features(), attrs);
@@ -119,7 +120,7 @@ impl<'a, 'tcx> Annotator<'a, 'tcx> {
             is_deprecated = true;
 
             if matches!(kind, AnnotationKind::Prohibited | AnnotationKind::DeprecationProhibited) {
-                let hir_id = self.tcx.hir().local_def_id_to_hir_id(def_id);
+                let hir_id = self.tcx.local_def_id_to_hir_id(def_id);
                 self.tcx.emit_spanned_lint(
                     USELESS_DEPRECATED,
                     hir_id,
@@ -157,9 +158,9 @@ impl<'a, 'tcx> Annotator<'a, 'tcx> {
             return;
         }
 
-        let stab = attr::find_stability(&self.tcx.sess, attrs, item_sp);
-        let const_stab = attr::find_const_stability(&self.tcx.sess, attrs, item_sp);
-        let body_stab = attr::find_body_stability(&self.tcx.sess, attrs);
+        let stab = attr::find_stability(self.tcx.sess, attrs, item_sp);
+        let const_stab = attr::find_const_stability(self.tcx.sess, attrs, item_sp);
+        let body_stab = attr::find_body_stability(self.tcx.sess, attrs);
         let mut const_span = None;
 
         let const_stab = const_stab.map(|(const_stab, const_span_node)| {
@@ -196,7 +197,10 @@ impl<'a, 'tcx> Annotator<'a, 'tcx> {
             }
         }
 
-        if let Some((depr, span)) = &depr && depr.is_since_rustc_version() && stab.is_none() {
+        if let Some((depr, span)) = &depr
+            && depr.is_since_rustc_version()
+            && stab.is_none()
+        {
             self.tcx.sess.emit_err(errors::DeprecatedAttribute { span: *span });
         }
 
@@ -717,8 +721,8 @@ impl<'tcx> Visitor<'tcx> for Checker<'tcx> {
                 let features = self.tcx.features();
                 if features.staged_api {
                     let attrs = self.tcx.hir().attrs(item.hir_id());
-                    let stab = attr::find_stability(&self.tcx.sess, attrs, item.span);
-                    let const_stab = attr::find_const_stability(&self.tcx.sess, attrs, item.span);
+                    let stab = attr::find_stability(self.tcx.sess, attrs, item.span);
+                    let const_stab = attr::find_const_stability(self.tcx.sess, attrs, item.span);
 
                     // If this impl block has an #[unstable] attribute, give an
                     // error if all involved types and traits are stable, because
@@ -975,29 +979,27 @@ pub fn check_unused_or_stable_features(tcx: TyCtxt<'_>) {
         tcx: TyCtxt<'tcx>,
         remaining_lib_features: &mut FxIndexMap<&Symbol, Span>,
         remaining_implications: &mut FxHashMap<Symbol, Symbol>,
-        defined_features: &[(Symbol, Option<Symbol>)],
+        defined_features: &LibFeatures,
         all_implications: &FxHashMap<Symbol, Symbol>,
     ) {
-        for (feature, since) in defined_features {
-            if let Some(since) = since
+        for (feature, since) in defined_features.to_vec() {
+            if let FeatureStability::AcceptedSince(since) = since
                 && let Some(span) = remaining_lib_features.get(&feature)
             {
                 // Warn if the user has enabled an already-stable lib feature.
                 if let Some(implies) = all_implications.get(&feature) {
-                    unnecessary_partially_stable_feature_lint(
-                        tcx, *span, *feature, *implies, *since,
-                    );
+                    unnecessary_partially_stable_feature_lint(tcx, *span, feature, *implies, since);
                 } else {
-                    unnecessary_stable_feature_lint(tcx, *span, *feature, *since);
+                    unnecessary_stable_feature_lint(tcx, *span, feature, since);
                 }
             }
-            remaining_lib_features.remove(feature);
+            remaining_lib_features.remove(&feature);
 
             // `feature` is the feature doing the implying, but `implied_by` is the feature with
             // the attribute that establishes this relationship. `implied_by` is guaranteed to be a
             // feature defined in the local crate because `remaining_implications` is only the
             // implications from this crate.
-            remaining_implications.remove(feature);
+            remaining_implications.remove(&feature);
 
             if remaining_lib_features.is_empty() && remaining_implications.is_empty() {
                 break;
@@ -1006,12 +1008,11 @@ pub fn check_unused_or_stable_features(tcx: TyCtxt<'_>) {
     }
 
     // All local crate implications need to have the feature that implies it confirmed to exist.
-    let mut remaining_implications =
-        tcx.stability_implications(rustc_hir::def_id::LOCAL_CRATE).clone();
+    let mut remaining_implications = tcx.stability_implications(LOCAL_CRATE).clone();
 
     // We always collect the lib features declared in the current crate, even if there are
     // no unknown features, because the collection also does feature attribute validation.
-    let local_defined_features = tcx.lib_features(()).to_vec();
+    let local_defined_features = tcx.lib_features(LOCAL_CRATE);
     if !remaining_lib_features.is_empty() || !remaining_implications.is_empty() {
         // Loading the implications of all crates is unavoidable to be able to emit the partial
         // stabilization diagnostic, but it can be avoided when there are no
@@ -1025,7 +1026,7 @@ pub fn check_unused_or_stable_features(tcx: TyCtxt<'_>) {
             tcx,
             &mut remaining_lib_features,
             &mut remaining_implications,
-            local_defined_features.as_slice(),
+            local_defined_features,
             &all_implications,
         );
 
@@ -1037,7 +1038,7 @@ pub fn check_unused_or_stable_features(tcx: TyCtxt<'_>) {
                 tcx,
                 &mut remaining_lib_features,
                 &mut remaining_implications,
-                tcx.defined_lib_features(cnum).to_vec().as_slice(),
+                tcx.lib_features(cnum),
                 &all_implications,
             );
         }
@@ -1047,14 +1048,16 @@ pub fn check_unused_or_stable_features(tcx: TyCtxt<'_>) {
         tcx.sess.emit_err(errors::UnknownFeature { span, feature: *feature });
     }
 
+    // We only use the hash map contents to emit errors, and the order of
+    // emitted errors do not affect query stability.
+    #[allow(rustc::potential_query_instability)]
     for (implied_by, feature) in remaining_implications {
-        let local_defined_features = tcx.lib_features(());
-        let span = *local_defined_features
-            .stable
+        let local_defined_features = tcx.lib_features(LOCAL_CRATE);
+        let span = local_defined_features
+            .stability
             .get(&feature)
-            .map(|(_, span)| span)
-            .or_else(|| local_defined_features.unstable.get(&feature))
-            .expect("feature that implied another does not exist");
+            .expect("feature that implied another does not exist")
+            .1;
         tcx.sess.emit_err(errors::ImpliedFeatureNotExist { span, feature, implied_by });
     }
 
diff --git a/compiler/rustc_passes/src/weak_lang_items.rs b/compiler/rustc_passes/src/weak_lang_items.rs
index 9a6fb88c2..b226c65e9 100644
--- a/compiler/rustc_passes/src/weak_lang_items.rs
+++ b/compiler/rustc_passes/src/weak_lang_items.rs
@@ -1,5 +1,7 @@
 //! Validity checking for weak lang items
 
+use rustc_ast as ast;
+use rustc_ast::visit;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_hir::lang_items::{self, LangItem};
 use rustc_hir::weak_lang_items::WEAK_LANG_ITEMS;
@@ -11,7 +13,7 @@ use crate::errors::{MissingLangItem, MissingPanicHandler, UnknownExternLangItem}
 
 /// Checks the crate for usage of weak lang items, returning a vector of all the
 /// language items required by this crate, but not defined yet.
-pub fn check_crate(tcx: TyCtxt<'_>, items: &mut lang_items::LanguageItems) {
+pub fn check_crate(tcx: TyCtxt<'_>, items: &mut lang_items::LanguageItems, krate: &ast::Crate) {
     // These are never called by user code, they're generated by the compiler.
     // They will never implicitly be added to the `missing` array unless we do
     // so here.
@@ -22,24 +24,30 @@ pub fn check_crate(tcx: TyCtxt<'_>, items: &mut lang_items::LanguageItems) {
         items.missing.push(LangItem::EhCatchTypeinfo);
     }
 
-    let crate_items = tcx.hir_crate_items(());
-    for id in crate_items.foreign_items() {
-        let attrs = tcx.hir().attrs(id.hir_id());
-        if let Some((lang_item, _)) = lang_items::extract(attrs) {
+    visit::Visitor::visit_crate(&mut WeakLangItemVisitor { tcx, items }, krate);
+
+    verify(tcx, items);
+}
+
+struct WeakLangItemVisitor<'a, 'tcx> {
+    tcx: TyCtxt<'tcx>,
+    items: &'a mut lang_items::LanguageItems,
+}
+
+impl<'ast> visit::Visitor<'ast> for WeakLangItemVisitor<'_, '_> {
+    fn visit_foreign_item(&mut self, i: &'ast ast::ForeignItem) {
+        if let Some((lang_item, _)) = lang_items::extract(&i.attrs) {
             if let Some(item) = LangItem::from_name(lang_item)
                 && item.is_weak()
             {
-                if items.get(item).is_none() {
-                    items.missing.push(item);
+                if self.items.get(item).is_none() {
+                    self.items.missing.push(item);
                 }
             } else {
-                let span = tcx.def_span(id.owner_id);
-                tcx.sess.emit_err(UnknownExternLangItem { span, lang_item });
+                self.tcx.sess.emit_err(UnknownExternLangItem { span: i.span, lang_item });
             }
         }
     }
-
-    verify(tcx, items);
 }
 
 fn verify(tcx: TyCtxt<'_>, items: &lang_items::LanguageItems) {
diff --git a/compiler/rustc_pattern_analysis/Cargo.toml b/compiler/rustc_pattern_analysis/Cargo.toml
new file mode 100644
index 000000000..908d00cf1
--- /dev/null
+++ b/compiler/rustc_pattern_analysis/Cargo.toml
@@ -0,0 +1,45 @@
+[package]
+name = "rustc_pattern_analysis"
+version = "0.0.0"
+edition = "2021"
+
+[dependencies]
+# tidy-alphabetical-start
+rustc_apfloat = "0.2.0"
+rustc_arena = { path = "../rustc_arena", optional = true }
+rustc_data_structures = { path = "../rustc_data_structures", optional = true }
+rustc_errors = { path = "../rustc_errors", optional = true }
+rustc_fluent_macro = { path = "../rustc_fluent_macro", optional = true }
+rustc_hir = { path = "../rustc_hir", optional = true }
+rustc_index = { path = "../rustc_index", default-features = false }
+rustc_macros = { path = "../rustc_macros", optional = true }
+rustc_middle = { path = "../rustc_middle", optional = true }
+rustc_session = { path = "../rustc_session", optional = true }
+rustc_span = { path = "../rustc_span", optional = true }
+rustc_target = { path = "../rustc_target", optional = true }
+smallvec = { version = "1.8.1", features = ["union"] }
+tracing = "0.1"
+typed-arena = { version = "2.0.2", optional = true }
+# tidy-alphabetical-end
+
+[features]
+default = ["rustc"]
+# It's not possible to only enable the `typed_arena` dependency when the `rustc` feature is off, so
+# we use another feature instead. The crate won't compile if one of these isn't enabled.
+rustc = [
+    "dep:rustc_arena",
+    "dep:rustc_data_structures",
+    "dep:rustc_errors",
+    "dep:rustc_fluent_macro",
+    "dep:rustc_hir",
+    "dep:rustc_macros",
+    "dep:rustc_middle",
+    "dep:rustc_session",
+    "dep:rustc_span",
+    "dep:rustc_target",
+    "smallvec/may_dangle",
+    "rustc_index/nightly",
+]
+stable = [
+    "dep:typed-arena",
+]
diff --git a/compiler/rustc_pattern_analysis/messages.ftl b/compiler/rustc_pattern_analysis/messages.ftl
new file mode 100644
index 000000000..827928f97
--- /dev/null
+++ b/compiler/rustc_pattern_analysis/messages.ftl
@@ -0,0 +1,19 @@
+pattern_analysis_non_exhaustive_omitted_pattern = some variants are not matched explicitly
+    .help = ensure that all variants are matched explicitly by adding the suggested match arms
+    .note = the matched value is of type `{$scrut_ty}` and the `non_exhaustive_omitted_patterns` attribute was found
+
+pattern_analysis_non_exhaustive_omitted_pattern_lint_on_arm = the lint level must be set on the whole match
+    .help = it no longer has any effect to set the lint level on an individual match arm
+    .label = remove this attribute
+    .suggestion = set the lint level on the whole match
+
+pattern_analysis_overlapping_range_endpoints = multiple patterns overlap on their endpoints
+    .label = ... with this range
+    .note = you likely meant to write mutually exclusive ranges
+
+pattern_analysis_uncovered = {$count ->
+        [1] pattern `{$witness_1}`
+        [2] patterns `{$witness_1}` and `{$witness_2}`
+        [3] patterns `{$witness_1}`, `{$witness_2}` and `{$witness_3}`
+        *[other] patterns `{$witness_1}`, `{$witness_2}`, `{$witness_3}` and {$remainder} more
+    } not covered
diff --git a/compiler/rustc_pattern_analysis/src/constructor.rs b/compiler/rustc_pattern_analysis/src/constructor.rs
new file mode 100644
index 000000000..af0a7497a
--- /dev/null
+++ b/compiler/rustc_pattern_analysis/src/constructor.rs
@@ -0,0 +1,1021 @@
+//! As explained in [`crate::usefulness`], values and patterns are made from constructors applied to
+//! fields. This file defines a `Constructor` enum and various operations to manipulate them.
+//!
+//! There are two important bits of core logic in this file: constructor inclusion and constructor
+//! splitting. Constructor inclusion, i.e. whether a constructor is included in/covered by another,
+//! is straightforward and defined in [`Constructor::is_covered_by`].
+//!
+//! Constructor splitting is mentioned in [`crate::usefulness`] but not detailed. We describe it
+//! precisely here.
+//!
+//!
+//!
+//! # Constructor grouping and splitting
+//!
+//! As explained in the corresponding section in [`crate::usefulness`], to make usefulness tractable
+//! we need to group together constructors that have the same effect when they are used to
+//! specialize the matrix.
+//!
+//! Example:
+//! ```compile_fail,E0004
+//! match (0, false) {
+//!     (0 ..=100, true) => {}
+//!     (50..=150, false) => {}
+//!     (0 ..=200, _) => {}
+//! }
+//! ```
+//!
+//! In this example we can restrict specialization to 5 cases: `0..50`, `50..=100`, `101..=150`,
+//! `151..=200` and `200..`.
+//!
+//! In [`crate::usefulness`], we had said that `specialize` only takes value-only constructors. We
+//! now relax this restriction: we allow `specialize` to take constructors like `0..50` as long as
+//! we're careful to only do that with constructors that make sense. For example, `specialize(0..50,
+//! (0..=100, true))` is sensible, but `specialize(50..=200, (0..=100, true))` is not.
+//!
+//! Constructor splitting looks at the constructors in the first column of the matrix and constructs
+//! such a sensible set of constructors. Formally, we want to find a smallest disjoint set of
+//! constructors:
+//! - Whose union covers the whole type, and
+//! - That have no non-trivial intersection with any of the constructors in the column (i.e. they're
+//!     each either disjoint with or covered by any given column constructor).
+//!
+//! We compute this in two steps: first [`TypeCx::ctors_for_ty`] determines the
+//! set of all possible constructors for the type. Then [`ConstructorSet::split`] looks at the
+//! column of constructors and splits the set into groups accordingly. The precise invariants of
+//! [`ConstructorSet::split`] is described in [`SplitConstructorSet`].
+//!
+//! Constructor splitting has two interesting special cases: integer range splitting (see
+//! [`IntRange::split`]) and slice splitting (see [`Slice::split`]).
+//!
+//!
+//!
+//! # The `Missing` constructor
+//!
+//! We detail a special case of constructor splitting that is a bit subtle. Take the following:
+//!
+//! ```
+//! enum Direction { North, South, East, West }
+//! # let wind = (Direction::North, 0u8);
+//! match wind {
+//!     (Direction::North, 50..) => {}
+//!     (_, _) => {}
+//! }
+//! ```
+//!
+//! Here we expect constructor splitting to output two cases: `North`, and "everything else". This
+//! "everything else" is represented by [`Constructor::Missing`]. Unlike other constructors, it's a
+//! bit contextual: to know the exact list of constructors it represents we have to look at the
+//! column. In practice however we don't need to, because by construction it only matches rows that
+//! have wildcards. This is how this constructor is special: the only constructor that covers it is
+//! `Wildcard`.
+//!
+//! The only place where we care about which constructors `Missing` represents is in diagnostics
+//! (see `crate::usefulness::WitnessMatrix::apply_constructor`).
+//!
+//! We choose whether to specialize with `Missing` in
+//! `crate::usefulness::compute_exhaustiveness_and_usefulness`.
+//!
+//!
+//!
+//! ## Empty types, empty constructors, and the `exhaustive_patterns` feature
+//!
+//! An empty type is a type that has no valid value, like `!`, `enum Void {}`, or `Result<!, !>`.
+//! They require careful handling.
+//!
+//! First, for soundness reasons related to the possible existence of invalid values, by default we
+//! don't treat empty types as empty. We force them to be matched with wildcards. Except if the
+//! `exhaustive_patterns` feature is turned on, in which case we do treat them as empty. And also
+//! except if the type has no constructors (like `enum Void {}` but not like `Result<!, !>`), we
+//! specifically allow `match void {}` to be exhaustive. There are additionally considerations of
+//! place validity that are handled in `crate::usefulness`. Yes this is a bit tricky.
+//!
+//! The second thing is that regardless of the above, it is always allowed to use all the
+//! constructors of a type. For example, all the following is ok:
+//!
+//! ```rust,ignore(example)
+//! # #![feature(never_type)]
+//! # #![feature(exhaustive_patterns)]
+//! fn foo(x: Option<!>) {
+//!   match x {
+//!     None => {}
+//!     Some(_) => {}
+//!   }
+//! }
+//! fn bar(x: &[!]) -> u32 {
+//!   match x {
+//!     [] => 1,
+//!     [_] => 2,
+//!     [_, _] => 3,
+//!   }
+//! }
+//! ```
+//!
+//! Moreover, take the following:
+//!
+//! ```rust
+//! # #![feature(never_type)]
+//! # #![feature(exhaustive_patterns)]
+//! # let x = None::<!>;
+//! match x {
+//!   None => {}
+//! }
+//! ```
+//!
+//! On a normal type, we would identify `Some` as missing and tell the user. If `x: Option<!>`
+//! however (and `exhaustive_patterns` is on), it's ok to omit `Some`. When listing the constructors
+//! of a type, we must therefore track which can be omitted.
+//!
+//! Let's call "empty" a constructor that matches no valid value for the type, like `Some` for the
+//! type `Option<!>`. What this all means is that `ConstructorSet` must know which constructors are
+//! empty. The difference between empty and nonempty constructors is that empty constructors need
+//! not be present for the match to be exhaustive.
+//!
+//! A final remark: empty constructors of arity 0 break specialization, we must avoid them. The
+//! reason is that if we specialize by them, nothing remains to witness the emptiness; the rest of
+//! the algorithm can't distinguish them from a nonempty constructor. The only known case where this
+//! could happen is the `[..]` pattern on `[!; N]` with `N > 0` so we must take care to not emit it.
+//!
+//! This is all handled by [`TypeCx::ctors_for_ty`] and
+//! [`ConstructorSet::split`]. The invariants of [`SplitConstructorSet`] are also of interest.
+//!
+//!
+//!
+//! ## Opaque patterns
+//!
+//! Some patterns, such as constants that are not allowed to be matched structurally, cannot be
+//! inspected, which we handle with `Constructor::Opaque`. Since we know nothing of these patterns,
+//! we assume they never cover each other. In order to respect the invariants of
+//! [`SplitConstructorSet`], we give each `Opaque` constructor a unique id so we can recognize it.
+
+use std::cmp::{self, max, min, Ordering};
+use std::fmt;
+use std::iter::once;
+
+use smallvec::SmallVec;
+
+use rustc_apfloat::ieee::{DoubleS, IeeeFloat, SingleS};
+use rustc_index::bit_set::{BitSet, GrowableBitSet};
+use rustc_index::IndexVec;
+
+use self::Constructor::*;
+use self::MaybeInfiniteInt::*;
+use self::SliceKind::*;
+
+use crate::usefulness::PlaceCtxt;
+use crate::TypeCx;
+
+/// Whether we have seen a constructor in the column or not.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+enum Presence {
+    Unseen,
+    Seen,
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub enum RangeEnd {
+    Included,
+    Excluded,
+}
+
+impl fmt::Display for RangeEnd {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.write_str(match self {
+            RangeEnd::Included => "..=",
+            RangeEnd::Excluded => "..",
+        })
+    }
+}
+
+/// A possibly infinite integer. Values are encoded such that the ordering on `u128` matches the
+/// natural order on the original type. For example, `-128i8` is encoded as `0` and `127i8` as
+/// `255`. See `signed_bias` for details.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub enum MaybeInfiniteInt {
+    NegInfinity,
+    /// Encoded value. DO NOT CONSTRUCT BY HAND; use `new_finite`.
+    #[non_exhaustive]
+    Finite(u128),
+    /// The integer after `u128::MAX`. We need it to represent `x..=u128::MAX` as an exclusive range.
+    JustAfterMax,
+    PosInfinity,
+}
+
+impl MaybeInfiniteInt {
+    pub fn new_finite_uint(bits: u128) -> Self {
+        Finite(bits)
+    }
+    pub fn new_finite_int(bits: u128, size: u64) -> Self {
+        // Perform a shift if the underlying types are signed, which makes the interval arithmetic
+        // type-independent.
+        let bias = 1u128 << (size - 1);
+        Finite(bits ^ bias)
+    }
+
+    pub fn as_finite_uint(self) -> Option<u128> {
+        match self {
+            Finite(bits) => Some(bits),
+            _ => None,
+        }
+    }
+    pub fn as_finite_int(self, size: u64) -> Option<u128> {
+        // We decode the shift.
+        match self {
+            Finite(bits) => {
+                let bias = 1u128 << (size - 1);
+                Some(bits ^ bias)
+            }
+            _ => None,
+        }
+    }
+
+    /// Note: this will not turn a finite value into an infinite one or vice-versa.
+    pub fn minus_one(self) -> Self {
+        match self {
+            Finite(n) => match n.checked_sub(1) {
+                Some(m) => Finite(m),
+                None => panic!("Called `MaybeInfiniteInt::minus_one` on 0"),
+            },
+            JustAfterMax => Finite(u128::MAX),
+            x => x,
+        }
+    }
+    /// Note: this will not turn a finite value into an infinite one or vice-versa.
+    pub fn plus_one(self) -> Self {
+        match self {
+            Finite(n) => match n.checked_add(1) {
+                Some(m) => Finite(m),
+                None => JustAfterMax,
+            },
+            JustAfterMax => panic!("Called `MaybeInfiniteInt::plus_one` on u128::MAX+1"),
+            x => x,
+        }
+    }
+}
+
+/// An exclusive interval, used for precise integer exhaustiveness checking. `IntRange`s always
+/// store a contiguous range.
+///
+/// `IntRange` is never used to encode an empty range or a "range" that wraps around the (offset)
+/// space: i.e., `range.lo < range.hi`.
+#[derive(Clone, Copy, PartialEq, Eq)]
+pub struct IntRange {
+    pub lo: MaybeInfiniteInt, // Must not be `PosInfinity`.
+    pub hi: MaybeInfiniteInt, // Must not be `NegInfinity`.
+}
+
+impl IntRange {
+    /// Best effort; will not know that e.g. `255u8..` is a singleton.
+    pub fn is_singleton(&self) -> bool {
+        // Since `lo` and `hi` can't be the same `Infinity` and `plus_one` never changes from finite
+        // to infinite, this correctly only detects ranges that contain exacly one `Finite(x)`.
+        self.lo.plus_one() == self.hi
+    }
+
+    #[inline]
+    pub fn from_singleton(x: MaybeInfiniteInt) -> IntRange {
+        IntRange { lo: x, hi: x.plus_one() }
+    }
+
+    #[inline]
+    pub fn from_range(lo: MaybeInfiniteInt, mut hi: MaybeInfiniteInt, end: RangeEnd) -> IntRange {
+        if end == RangeEnd::Included {
+            hi = hi.plus_one();
+        }
+        if lo >= hi {
+            // This should have been caught earlier by E0030.
+            panic!("malformed range pattern: {lo:?}..{hi:?}");
+        }
+        IntRange { lo, hi }
+    }
+
+    fn is_subrange(&self, other: &Self) -> bool {
+        other.lo <= self.lo && self.hi <= other.hi
+    }
+
+    fn intersection(&self, other: &Self) -> Option<Self> {
+        if self.lo < other.hi && other.lo < self.hi {
+            Some(IntRange { lo: max(self.lo, other.lo), hi: min(self.hi, other.hi) })
+        } else {
+            None
+        }
+    }
+
+    /// Partition a range of integers into disjoint subranges. This does constructor splitting for
+    /// integer ranges as explained at the top of the file.
+    ///
+    /// This returns an output that covers `self`. The output is split so that the only
+    /// intersections between an output range and a column range are inclusions. No output range
+    /// straddles the boundary of one of the inputs.
+    ///
+    /// Additionally, we track for each output range whether it is covered by one of the column ranges or not.
+    ///
+    /// The following input:
+    /// ```text
+    ///   (--------------------------) // `self`
+    /// (------) (----------)    (-)
+    ///     (------) (--------)
+    /// ```
+    /// is first intersected with `self`:
+    /// ```text
+    ///   (--------------------------) // `self`
+    ///   (----) (----------)    (-)
+    ///     (------) (--------)
+    /// ```
+    /// and then iterated over as follows:
+    /// ```text
+    ///   (-(--)-(-)-(------)-)--(-)-
+    /// ```
+    /// where each sequence of dashes is an output range, and dashes outside parentheses are marked
+    /// as `Presence::Missing`.
+    ///
+    /// ## `isize`/`usize`
+    ///
+    /// Whereas a wildcard of type `i32` stands for the range `i32::MIN..=i32::MAX`, a `usize`
+    /// wildcard stands for `0..PosInfinity` and a `isize` wildcard stands for
+    /// `NegInfinity..PosInfinity`. In other words, as far as `IntRange` is concerned, there are
+    /// values before `isize::MIN` and after `usize::MAX`/`isize::MAX`.
+    /// This is to avoid e.g. `0..(u32::MAX as usize)` from being exhaustive on one architecture and
+    /// not others. This was decided in <https://github.com/rust-lang/rfcs/pull/2591>.
+    ///
+    /// These infinities affect splitting subtly: it is possible to get `NegInfinity..0` and
+    /// `usize::MAX+1..PosInfinity` in the output. Diagnostics must be careful to handle these
+    /// fictitious ranges sensibly.
+    fn split(
+        &self,
+        column_ranges: impl Iterator<Item = IntRange>,
+    ) -> impl Iterator<Item = (Presence, IntRange)> {
+        // The boundaries of ranges in `column_ranges` intersected with `self`.
+        // We do parenthesis matching for input ranges. A boundary counts as +1 if it starts
+        // a range and -1 if it ends it. When the count is > 0 between two boundaries, we
+        // are within an input range.
+        let mut boundaries: Vec<(MaybeInfiniteInt, isize)> = column_ranges
+            .filter_map(|r| self.intersection(&r))
+            .flat_map(|r| [(r.lo, 1), (r.hi, -1)])
+            .collect();
+        // We sort by boundary, and for each boundary we sort the "closing parentheses" first. The
+        // order of +1/-1 for a same boundary value is actually irrelevant, because we only look at
+        // the accumulated count between distinct boundary values.
+        boundaries.sort_unstable();
+
+        // Accumulate parenthesis counts.
+        let mut paren_counter = 0isize;
+        // Gather pairs of adjacent boundaries.
+        let mut prev_bdy = self.lo;
+        boundaries
+            .into_iter()
+            // End with the end of the range. The count is ignored.
+            .chain(once((self.hi, 0)))
+            // List pairs of adjacent boundaries and the count between them.
+            .map(move |(bdy, delta)| {
+                // `delta` affects the count as we cross `bdy`, so the relevant count between
+                // `prev_bdy` and `bdy` is untouched by `delta`.
+                let ret = (prev_bdy, paren_counter, bdy);
+                prev_bdy = bdy;
+                paren_counter += delta;
+                ret
+            })
+            // Skip empty ranges.
+            .filter(|&(prev_bdy, _, bdy)| prev_bdy != bdy)
+            // Convert back to ranges.
+            .map(move |(prev_bdy, paren_count, bdy)| {
+                use Presence::*;
+                let presence = if paren_count > 0 { Seen } else { Unseen };
+                let range = IntRange { lo: prev_bdy, hi: bdy };
+                (presence, range)
+            })
+    }
+}
+
+/// Note: this will render signed ranges incorrectly. To render properly, convert to a pattern
+/// first.
+impl fmt::Debug for IntRange {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        if let Finite(lo) = self.lo {
+            write!(f, "{lo}")?;
+        }
+        write!(f, "{}", RangeEnd::Excluded)?;
+        if let Finite(hi) = self.hi {
+            write!(f, "{hi}")?;
+        }
+        Ok(())
+    }
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub enum SliceKind {
+    /// Patterns of length `n` (`[x, y]`).
+    FixedLen(usize),
+    /// Patterns using the `..` notation (`[x, .., y]`).
+    /// Captures any array constructor of `length >= i + j`.
+    /// In the case where `array_len` is `Some(_)`,
+    /// this indicates that we only care about the first `i` and the last `j` values of the array,
+    /// and everything in between is a wildcard `_`.
+    VarLen(usize, usize),
+}
+
+impl SliceKind {
+    fn arity(self) -> usize {
+        match self {
+            FixedLen(length) => length,
+            VarLen(prefix, suffix) => prefix + suffix,
+        }
+    }
+
+    /// Whether this pattern includes patterns of length `other_len`.
+    fn covers_length(self, other_len: usize) -> bool {
+        match self {
+            FixedLen(len) => len == other_len,
+            VarLen(prefix, suffix) => prefix + suffix <= other_len,
+        }
+    }
+}
+
+/// A constructor for array and slice patterns.
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub struct Slice {
+    /// `None` if the matched value is a slice, `Some(n)` if it is an array of size `n`.
+    pub(crate) array_len: Option<usize>,
+    /// The kind of pattern it is: fixed-length `[x, y]` or variable length `[x, .., y]`.
+    pub(crate) kind: SliceKind,
+}
+
+impl Slice {
+    pub fn new(array_len: Option<usize>, kind: SliceKind) -> Self {
+        let kind = match (array_len, kind) {
+            // If the middle `..` has length 0, we effectively have a fixed-length pattern.
+            (Some(len), VarLen(prefix, suffix)) if prefix + suffix == len => FixedLen(len),
+            (Some(len), VarLen(prefix, suffix)) if prefix + suffix > len => panic!(
+                "Slice pattern of length {} longer than its array length {len}",
+                prefix + suffix
+            ),
+            _ => kind,
+        };
+        Slice { array_len, kind }
+    }
+
+    pub(crate) fn arity(self) -> usize {
+        self.kind.arity()
+    }
+
+    /// See `Constructor::is_covered_by`
+    fn is_covered_by(self, other: Self) -> bool {
+        other.kind.covers_length(self.arity())
+    }
+
+    /// This computes constructor splitting for variable-length slices, as explained at the top of
+    /// the file.
+    ///
+    /// A slice pattern `[x, .., y]` behaves like the infinite or-pattern `[x, y] | [x, _, y] | [x,
+    /// _, _, y] | etc`. The corresponding value constructors are fixed-length array constructors of
+    /// corresponding lengths. We obviously can't list this infinitude of constructors.
+    /// Thankfully, it turns out that for each finite set of slice patterns, all sufficiently large
+    /// array lengths are equivalent.
+    ///
+    /// Let's look at an example, where we are trying to split the last pattern:
+    /// ```
+    /// # fn foo(x: &[bool]) {
+    /// match x {
+    ///     [true, true, ..] => {}
+    ///     [.., false, false] => {}
+    ///     [..] => {}
+    /// }
+    /// # }
+    /// ```
+    /// Here are the results of specialization for the first few lengths:
+    /// ```
+    /// # fn foo(x: &[bool]) { match x {
+    /// // length 0
+    /// [] => {}
+    /// // length 1
+    /// [_] => {}
+    /// // length 2
+    /// [true, true] => {}
+    /// [false, false] => {}
+    /// [_, _] => {}
+    /// // length 3
+    /// [true, true,  _    ] => {}
+    /// [_,    false, false] => {}
+    /// [_,    _,     _    ] => {}
+    /// // length 4
+    /// [true, true, _,     _    ] => {}
+    /// [_,    _,    false, false] => {}
+    /// [_,    _,    _,     _    ] => {}
+    /// // length 5
+    /// [true, true, _, _,     _    ] => {}
+    /// [_,    _,    _, false, false] => {}
+    /// [_,    _,    _, _,     _    ] => {}
+    /// # _ => {}
+    /// # }}
+    /// ```
+    ///
+    /// We see that above length 4, we are simply inserting columns full of wildcards in the middle.
+    /// This means that specialization and witness computation with slices of length `l >= 4` will
+    /// give equivalent results regardless of `l`. This applies to any set of slice patterns: there
+    /// will be a length `L` above which all lengths behave the same. This is exactly what we need
+    /// for constructor splitting.
+    ///
+    /// A variable-length slice pattern covers all lengths from its arity up to infinity. As we just
+    /// saw, we can split this in two: lengths below `L` are treated individually with a
+    /// fixed-length slice each; lengths above `L` are grouped into a single variable-length slice
+    /// constructor.
+    ///
+    /// For each variable-length slice pattern `p` with a prefix of length `plₚ` and suffix of
+    /// length `slₚ`, only the first `plₚ` and the last `slₚ` elements are examined. Therefore, as
+    /// long as `L` is positive (to avoid concerns about empty types), all elements after the
+    /// maximum prefix length and before the maximum suffix length are not examined by any
+    /// variable-length pattern, and therefore can be ignored. This gives us a way to compute `L`.
+    ///
+    /// Additionally, if fixed-length patterns exist, we must pick an `L` large enough to miss them,
+    /// so we can pick `L = max(max(FIXED_LEN)+1, max(PREFIX_LEN) + max(SUFFIX_LEN))`.
+    /// `max_slice` below will be made to have this arity `L`.
+    ///
+    /// If `self` is fixed-length, it is returned as-is.
+    ///
+    /// Additionally, we track for each output slice whether it is covered by one of the column slices or not.
+    fn split(
+        self,
+        column_slices: impl Iterator<Item = Slice>,
+    ) -> impl Iterator<Item = (Presence, Slice)> {
+        // Range of lengths below `L`.
+        let smaller_lengths;
+        let arity = self.arity();
+        let mut max_slice = self.kind;
+        // Tracks the smallest variable-length slice we've seen. Any slice arity above it is
+        // therefore `Presence::Seen` in the column.
+        let mut min_var_len = usize::MAX;
+        // Tracks the fixed-length slices we've seen, to mark them as `Presence::Seen`.
+        let mut seen_fixed_lens = GrowableBitSet::new_empty();
+        match &mut max_slice {
+            VarLen(max_prefix_len, max_suffix_len) => {
+                // A length larger than any fixed-length slice encountered.
+                // We start at 1 in case the subtype is empty because in that case the zero-length
+                // slice must be treated separately from the rest.
+                let mut fixed_len_upper_bound = 1;
+                // We grow `max_slice` to be larger than all slices encountered, as described above.
+                // `L` is `max_slice.arity()`. For diagnostics, we keep the prefix and suffix
+                // lengths separate.
+                for slice in column_slices {
+                    match slice.kind {
+                        FixedLen(len) => {
+                            fixed_len_upper_bound = cmp::max(fixed_len_upper_bound, len + 1);
+                            seen_fixed_lens.insert(len);
+                        }
+                        VarLen(prefix, suffix) => {
+                            *max_prefix_len = cmp::max(*max_prefix_len, prefix);
+                            *max_suffix_len = cmp::max(*max_suffix_len, suffix);
+                            min_var_len = cmp::min(min_var_len, prefix + suffix);
+                        }
+                    }
+                }
+                // If `fixed_len_upper_bound >= L`, we set `L` to `fixed_len_upper_bound`.
+                if let Some(delta) =
+                    fixed_len_upper_bound.checked_sub(*max_prefix_len + *max_suffix_len)
+                {
+                    *max_prefix_len += delta
+                }
+
+                // We cap the arity of `max_slice` at the array size.
+                match self.array_len {
+                    Some(len) if max_slice.arity() >= len => max_slice = FixedLen(len),
+                    _ => {}
+                }
+
+                smaller_lengths = match self.array_len {
+                    // The only admissible fixed-length slice is one of the array size. Whether `max_slice`
+                    // is fixed-length or variable-length, it will be the only relevant slice to output
+                    // here.
+                    Some(_) => 0..0, // empty range
+                    // We need to cover all arities in the range `(arity..infinity)`. We split that
+                    // range into two: lengths smaller than `max_slice.arity()` are treated
+                    // independently as fixed-lengths slices, and lengths above are captured by
+                    // `max_slice`.
+                    None => self.arity()..max_slice.arity(),
+                };
+            }
+            FixedLen(_) => {
+                // No need to split here. We only track presence.
+                for slice in column_slices {
+                    match slice.kind {
+                        FixedLen(len) => {
+                            if len == arity {
+                                seen_fixed_lens.insert(len);
+                            }
+                        }
+                        VarLen(prefix, suffix) => {
+                            min_var_len = cmp::min(min_var_len, prefix + suffix);
+                        }
+                    }
+                }
+                smaller_lengths = 0..0;
+            }
+        };
+
+        smaller_lengths.map(FixedLen).chain(once(max_slice)).map(move |kind| {
+            let arity = kind.arity();
+            let seen = if min_var_len <= arity || seen_fixed_lens.contains(arity) {
+                Presence::Seen
+            } else {
+                Presence::Unseen
+            };
+            (seen, Slice::new(self.array_len, kind))
+        })
+    }
+}
+
+/// A globally unique id to distinguish `Opaque` patterns.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct OpaqueId(u32);
+
+impl OpaqueId {
+    pub fn new() -> Self {
+        use std::sync::atomic::{AtomicU32, Ordering};
+        static OPAQUE_ID: AtomicU32 = AtomicU32::new(0);
+        OpaqueId(OPAQUE_ID.fetch_add(1, Ordering::SeqCst))
+    }
+}
+
+/// A value can be decomposed into a constructor applied to some fields. This struct represents
+/// the constructor. See also `Fields`.
+///
+/// `pat_constructor` retrieves the constructor corresponding to a pattern.
+/// `specialize_constructor` returns the list of fields corresponding to a pattern, given a
+/// constructor. `Constructor::apply` reconstructs the pattern from a pair of `Constructor` and
+/// `Fields`.
+#[derive(Clone, Debug, PartialEq)]
+pub enum Constructor<Cx: TypeCx> {
+    /// Tuples and structs.
+    Struct,
+    /// Enum variants.
+    Variant(Cx::VariantIdx),
+    /// References
+    Ref,
+    /// Array and slice patterns.
+    Slice(Slice),
+    /// Union field accesses.
+    UnionField,
+    /// Booleans
+    Bool(bool),
+    /// Ranges of integer literal values (`2`, `2..=5` or `2..5`).
+    IntRange(IntRange),
+    /// Ranges of floating-point literal values (`2.0..=5.2`).
+    F32Range(IeeeFloat<SingleS>, IeeeFloat<SingleS>, RangeEnd),
+    F64Range(IeeeFloat<DoubleS>, IeeeFloat<DoubleS>, RangeEnd),
+    /// String literals. Strings are not quite the same as `&[u8]` so we treat them separately.
+    Str(Cx::StrLit),
+    /// Constants that must not be matched structurally. They are treated as black boxes for the
+    /// purposes of exhaustiveness: we must not inspect them, and they don't count towards making a
+    /// match exhaustive.
+    /// Carries an id that must be unique within a match. We need this to ensure the invariants of
+    /// [`SplitConstructorSet`].
+    Opaque(OpaqueId),
+    /// Or-pattern.
+    Or,
+    /// Wildcard pattern.
+    Wildcard,
+    /// Fake extra constructor for enums that aren't allowed to be matched exhaustively. Also used
+    /// for those types for which we cannot list constructors explicitly, like `f64` and `str`.
+    NonExhaustive,
+    /// Fake extra constructor for variants that should not be mentioned in diagnostics.
+    /// We use this for variants behind an unstable gate as well as
+    /// `#[doc(hidden)]` ones.
+    Hidden,
+    /// Fake extra constructor for constructors that are not seen in the matrix, as explained at the
+    /// top of the file.
+    Missing,
+}
+
+impl<Cx: TypeCx> Constructor<Cx> {
+    pub(crate) fn is_non_exhaustive(&self) -> bool {
+        matches!(self, NonExhaustive)
+    }
+
+    pub(crate) fn as_variant(&self) -> Option<Cx::VariantIdx> {
+        match self {
+            Variant(i) => Some(*i),
+            _ => None,
+        }
+    }
+    fn as_bool(&self) -> Option<bool> {
+        match self {
+            Bool(b) => Some(*b),
+            _ => None,
+        }
+    }
+    pub(crate) fn as_int_range(&self) -> Option<&IntRange> {
+        match self {
+            IntRange(range) => Some(range),
+            _ => None,
+        }
+    }
+    fn as_slice(&self) -> Option<Slice> {
+        match self {
+            Slice(slice) => Some(*slice),
+            _ => None,
+        }
+    }
+
+    /// The number of fields for this constructor. This must be kept in sync with
+    /// `Fields::wildcards`.
+    pub(crate) fn arity(&self, pcx: &PlaceCtxt<'_, '_, Cx>) -> usize {
+        pcx.ctor_arity(self)
+    }
+
+    /// Returns whether `self` is covered by `other`, i.e. whether `self` is a subset of `other`.
+    /// For the simple cases, this is simply checking for equality. For the "grouped" constructors,
+    /// this checks for inclusion.
+    // We inline because this has a single call site in `Matrix::specialize_constructor`.
+    #[inline]
+    pub(crate) fn is_covered_by<'p>(&self, pcx: &PlaceCtxt<'_, 'p, Cx>, other: &Self) -> bool {
+        match (self, other) {
+            (Wildcard, _) => pcx
+                .mcx
+                .tycx
+                .bug(format_args!("Constructor splitting should not have returned `Wildcard`")),
+            // Wildcards cover anything
+            (_, Wildcard) => true,
+            // Only a wildcard pattern can match these special constructors.
+            (Missing { .. } | NonExhaustive | Hidden, _) => false,
+
+            (Struct, Struct) => true,
+            (Ref, Ref) => true,
+            (UnionField, UnionField) => true,
+            (Variant(self_id), Variant(other_id)) => self_id == other_id,
+            (Bool(self_b), Bool(other_b)) => self_b == other_b,
+
+            (IntRange(self_range), IntRange(other_range)) => self_range.is_subrange(other_range),
+            (F32Range(self_from, self_to, self_end), F32Range(other_from, other_to, other_end)) => {
+                self_from.ge(other_from)
+                    && match self_to.partial_cmp(other_to) {
+                        Some(Ordering::Less) => true,
+                        Some(Ordering::Equal) => other_end == self_end,
+                        _ => false,
+                    }
+            }
+            (F64Range(self_from, self_to, self_end), F64Range(other_from, other_to, other_end)) => {
+                self_from.ge(other_from)
+                    && match self_to.partial_cmp(other_to) {
+                        Some(Ordering::Less) => true,
+                        Some(Ordering::Equal) => other_end == self_end,
+                        _ => false,
+                    }
+            }
+            (Str(self_val), Str(other_val)) => {
+                // FIXME Once valtrees are available we can directly use the bytes
+                // in the `Str` variant of the valtree for the comparison here.
+                self_val == other_val
+            }
+            (Slice(self_slice), Slice(other_slice)) => self_slice.is_covered_by(*other_slice),
+
+            // Opaque constructors don't interact with anything unless they come from the
+            // syntactically identical pattern.
+            (Opaque(self_id), Opaque(other_id)) => self_id == other_id,
+            (Opaque(..), _) | (_, Opaque(..)) => false,
+
+            _ => pcx.mcx.tycx.bug(format_args!(
+                "trying to compare incompatible constructors {self:?} and {other:?}"
+            )),
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+pub enum VariantVisibility {
+    /// Variant that doesn't fit the other cases, i.e. most variants.
+    Visible,
+    /// Variant behind an unstable gate or with the `#[doc(hidden)]` attribute. It will not be
+    /// mentioned in diagnostics unless the user mentioned it first.
+    Hidden,
+    /// Variant that matches no value. E.g. `Some::<Option<!>>` if the `exhaustive_patterns` feature
+    /// is enabled. Like `Hidden`, it will not be mentioned in diagnostics unless the user mentioned
+    /// it first.
+    Empty,
+}
+
+/// Describes the set of all constructors for a type. For details, in particular about the emptiness
+/// of constructors, see the top of the file.
+///
+/// In terms of division of responsibility, [`ConstructorSet::split`] handles all of the
+/// `exhaustive_patterns` feature.
+#[derive(Debug)]
+pub enum ConstructorSet<Cx: TypeCx> {
+    /// The type is a tuple or struct. `empty` tracks whether the type is empty.
+    Struct { empty: bool },
+    /// This type has the following list of constructors. If `variants` is empty and
+    /// `non_exhaustive` is false, don't use this; use `NoConstructors` instead.
+    Variants { variants: IndexVec<Cx::VariantIdx, VariantVisibility>, non_exhaustive: bool },
+    /// The type is `&T`.
+    Ref,
+    /// The type is a union.
+    Union,
+    /// Booleans.
+    Bool,
+    /// The type is spanned by integer values. The range or ranges give the set of allowed values.
+    /// The second range is only useful for `char`.
+    Integers { range_1: IntRange, range_2: Option<IntRange> },
+    /// The type is matched by slices. `array_len` is the compile-time length of the array, if
+    /// known. If `subtype_is_empty`, all constructors are empty except possibly the zero-length
+    /// slice `[]`.
+    Slice { array_len: Option<usize>, subtype_is_empty: bool },
+    /// The constructors cannot be listed, and the type cannot be matched exhaustively. E.g. `str`,
+    /// floats.
+    Unlistable,
+    /// The type has no constructors (not even empty ones). This is `!` and empty enums.
+    NoConstructors,
+}
+
+/// Describes the result of analyzing the constructors in a column of a match.
+///
+/// `present` is morally the set of constructors present in the column, and `missing` is the set of
+/// constructors that exist in the type but are not present in the column.
+///
+/// More formally, if we discard wildcards from the column, this respects the following constraints:
+/// 1. the union of `present`, `missing` and `missing_empty` covers all the constructors of the type
+/// 2. each constructor in `present` is covered by something in the column
+/// 3. no constructor in `missing` or `missing_empty` is covered by anything in the column
+/// 4. each constructor in the column is equal to the union of one or more constructors in `present`
+/// 5. `missing` does not contain empty constructors (see discussion about emptiness at the top of
+///    the file);
+/// 6. `missing_empty` contains only empty constructors
+/// 7. constructors in `present`, `missing` and `missing_empty` are split for the column; in other
+///    words, they are either fully included in or fully disjoint from each constructor in the
+///    column. In yet other words, there are no non-trivial intersections like between `0..10` and
+///    `5..15`.
+///
+/// We must be particularly careful with weird constructors like `Opaque`: they're not formally part
+/// of the `ConstructorSet` for the type, yet if we forgot to include them in `present` we would be
+/// ignoring any row with `Opaque`s in the algorithm. Hence the importance of point 4.
+#[derive(Debug)]
+pub(crate) struct SplitConstructorSet<Cx: TypeCx> {
+    pub(crate) present: SmallVec<[Constructor<Cx>; 1]>,
+    pub(crate) missing: Vec<Constructor<Cx>>,
+    pub(crate) missing_empty: Vec<Constructor<Cx>>,
+}
+
+impl<Cx: TypeCx> ConstructorSet<Cx> {
+    /// This analyzes a column of constructors to 1/ determine which constructors of the type (if
+    /// any) are missing; 2/ split constructors to handle non-trivial intersections e.g. on ranges
+    /// or slices. This can get subtle; see [`SplitConstructorSet`] for details of this operation
+    /// and its invariants.
+    #[instrument(level = "debug", skip(self, pcx, ctors), ret)]
+    pub(crate) fn split<'a>(
+        &self,
+        pcx: &PlaceCtxt<'_, '_, Cx>,
+        ctors: impl Iterator<Item = &'a Constructor<Cx>> + Clone,
+    ) -> SplitConstructorSet<Cx>
+    where
+        Cx: 'a,
+    {
+        let mut present: SmallVec<[_; 1]> = SmallVec::new();
+        // Empty constructors found missing.
+        let mut missing_empty = Vec::new();
+        // Nonempty constructors found missing.
+        let mut missing = Vec::new();
+        // Constructors in `ctors`, except wildcards and opaques.
+        let mut seen = Vec::new();
+        for ctor in ctors.cloned() {
+            match ctor {
+                Opaque(..) => present.push(ctor),
+                Wildcard => {} // discard wildcards
+                _ => seen.push(ctor),
+            }
+        }
+
+        match self {
+            ConstructorSet::Struct { empty } => {
+                if !seen.is_empty() {
+                    present.push(Struct);
+                } else if *empty {
+                    missing_empty.push(Struct);
+                } else {
+                    missing.push(Struct);
+                }
+            }
+            ConstructorSet::Ref => {
+                if !seen.is_empty() {
+                    present.push(Ref);
+                } else {
+                    missing.push(Ref);
+                }
+            }
+            ConstructorSet::Union => {
+                if !seen.is_empty() {
+                    present.push(UnionField);
+                } else {
+                    missing.push(UnionField);
+                }
+            }
+            ConstructorSet::Variants { variants, non_exhaustive } => {
+                let mut seen_set: BitSet<_> = BitSet::new_empty(variants.len());
+                for idx in seen.iter().map(|c| c.as_variant().unwrap()) {
+                    seen_set.insert(idx);
+                }
+                let mut skipped_a_hidden_variant = false;
+
+                for (idx, visibility) in variants.iter_enumerated() {
+                    let ctor = Variant(idx);
+                    if seen_set.contains(idx) {
+                        present.push(ctor);
+                    } else {
+                        // We only put visible variants directly into `missing`.
+                        match visibility {
+                            VariantVisibility::Visible => missing.push(ctor),
+                            VariantVisibility::Hidden => skipped_a_hidden_variant = true,
+                            VariantVisibility::Empty => missing_empty.push(ctor),
+                        }
+                    }
+                }
+
+                if skipped_a_hidden_variant {
+                    missing.push(Hidden);
+                }
+                if *non_exhaustive {
+                    missing.push(NonExhaustive);
+                }
+            }
+            ConstructorSet::Bool => {
+                let mut seen_false = false;
+                let mut seen_true = false;
+                for b in seen.iter().map(|ctor| ctor.as_bool().unwrap()) {
+                    if b {
+                        seen_true = true;
+                    } else {
+                        seen_false = true;
+                    }
+                }
+                if seen_false {
+                    present.push(Bool(false));
+                } else {
+                    missing.push(Bool(false));
+                }
+                if seen_true {
+                    present.push(Bool(true));
+                } else {
+                    missing.push(Bool(true));
+                }
+            }
+            ConstructorSet::Integers { range_1, range_2 } => {
+                let seen_ranges: Vec<_> =
+                    seen.iter().map(|ctor| *ctor.as_int_range().unwrap()).collect();
+                for (seen, splitted_range) in range_1.split(seen_ranges.iter().cloned()) {
+                    match seen {
+                        Presence::Unseen => missing.push(IntRange(splitted_range)),
+                        Presence::Seen => present.push(IntRange(splitted_range)),
+                    }
+                }
+                if let Some(range_2) = range_2 {
+                    for (seen, splitted_range) in range_2.split(seen_ranges.into_iter()) {
+                        match seen {
+                            Presence::Unseen => missing.push(IntRange(splitted_range)),
+                            Presence::Seen => present.push(IntRange(splitted_range)),
+                        }
+                    }
+                }
+            }
+            ConstructorSet::Slice { array_len, subtype_is_empty } => {
+                let seen_slices = seen.iter().map(|c| c.as_slice().unwrap());
+                let base_slice = Slice::new(*array_len, VarLen(0, 0));
+                for (seen, splitted_slice) in base_slice.split(seen_slices) {
+                    let ctor = Slice(splitted_slice);
+                    match seen {
+                        Presence::Seen => present.push(ctor),
+                        Presence::Unseen => {
+                            if *subtype_is_empty && splitted_slice.arity() != 0 {
+                                // We have subpatterns of an empty type, so the constructor is
+                                // empty.
+                                missing_empty.push(ctor);
+                            } else {
+                                missing.push(ctor);
+                            }
+                        }
+                    }
+                }
+            }
+            ConstructorSet::Unlistable => {
+                // Since we can't list constructors, we take the ones in the column. This might list
+                // some constructors several times but there's not much we can do.
+                present.extend(seen);
+                missing.push(NonExhaustive);
+            }
+            ConstructorSet::NoConstructors => {
+                // In a `MaybeInvalid` place even an empty pattern may be reachable. We therefore
+                // add a dummy empty constructor here, which will be ignored if the place is
+                // `ValidOnly`.
+                missing_empty.push(NonExhaustive);
+            }
+        }
+
+        // We have now grouped all the constructors into 3 buckets: present, missing, missing_empty.
+        // In the absence of the `exhaustive_patterns` feature however, we don't count nested empty
+        // types as empty. Only non-nested `!` or `enum Foo {}` are considered empty.
+        if !pcx.mcx.tycx.is_exhaustive_patterns_feature_on()
+            && !(pcx.is_scrutinee && matches!(self, Self::NoConstructors))
+        {
+            // Treat all missing constructors as nonempty.
+            // This clears `missing_empty`.
+            missing.append(&mut missing_empty);
+        }
+
+        SplitConstructorSet { present, missing, missing_empty }
+    }
+}
diff --git a/compiler/rustc_pattern_analysis/src/errors.rs b/compiler/rustc_pattern_analysis/src/errors.rs
new file mode 100644
index 000000000..88770b0c4
--- /dev/null
+++ b/compiler/rustc_pattern_analysis/src/errors.rs
@@ -0,0 +1,95 @@
+use rustc_errors::{AddToDiagnostic, Diagnostic, SubdiagnosticMessage};
+use rustc_macros::{LintDiagnostic, Subdiagnostic};
+use rustc_middle::thir::Pat;
+use rustc_middle::ty::Ty;
+use rustc_span::Span;
+
+use crate::rustc::{RustcMatchCheckCtxt, WitnessPat};
+
+#[derive(Subdiagnostic)]
+#[label(pattern_analysis_uncovered)]
+pub struct Uncovered<'tcx> {
+    #[primary_span]
+    span: Span,
+    count: usize,
+    witness_1: Pat<'tcx>,
+    witness_2: Pat<'tcx>,
+    witness_3: Pat<'tcx>,
+    remainder: usize,
+}
+
+impl<'tcx> Uncovered<'tcx> {
+    pub fn new<'p>(
+        span: Span,
+        cx: &RustcMatchCheckCtxt<'p, 'tcx>,
+        witnesses: Vec<WitnessPat<'p, 'tcx>>,
+    ) -> Self {
+        let witness_1 = cx.hoist_witness_pat(witnesses.get(0).unwrap());
+        Self {
+            span,
+            count: witnesses.len(),
+            // Substitute dummy values if witnesses is smaller than 3. These will never be read.
+            witness_2: witnesses
+                .get(1)
+                .map(|w| cx.hoist_witness_pat(w))
+                .unwrap_or_else(|| witness_1.clone()),
+            witness_3: witnesses
+                .get(2)
+                .map(|w| cx.hoist_witness_pat(w))
+                .unwrap_or_else(|| witness_1.clone()),
+            witness_1,
+            remainder: witnesses.len().saturating_sub(3),
+        }
+    }
+}
+
+#[derive(LintDiagnostic)]
+#[diag(pattern_analysis_overlapping_range_endpoints)]
+#[note]
+pub struct OverlappingRangeEndpoints<'tcx> {
+    #[label]
+    pub range: Span,
+    #[subdiagnostic]
+    pub overlap: Vec<Overlap<'tcx>>,
+}
+
+pub struct Overlap<'tcx> {
+    pub span: Span,
+    pub range: Pat<'tcx>,
+}
+
+impl<'tcx> AddToDiagnostic for Overlap<'tcx> {
+    fn add_to_diagnostic_with<F>(self, diag: &mut Diagnostic, _: F)
+    where
+        F: Fn(&mut Diagnostic, SubdiagnosticMessage) -> SubdiagnosticMessage,
+    {
+        let Overlap { span, range } = self;
+
+        // FIXME(mejrs) unfortunately `#[derive(LintDiagnostic)]`
+        // does not support `#[subdiagnostic(eager)]`...
+        let message = format!("this range overlaps on `{range}`...");
+        diag.span_label(span, message);
+    }
+}
+
+#[derive(LintDiagnostic)]
+#[diag(pattern_analysis_non_exhaustive_omitted_pattern)]
+#[help]
+#[note]
+pub(crate) struct NonExhaustiveOmittedPattern<'tcx> {
+    pub scrut_ty: Ty<'tcx>,
+    #[subdiagnostic]
+    pub uncovered: Uncovered<'tcx>,
+}
+
+#[derive(LintDiagnostic)]
+#[diag(pattern_analysis_non_exhaustive_omitted_pattern_lint_on_arm)]
+#[help]
+pub(crate) struct NonExhaustiveOmittedPatternLintOnArm {
+    #[label]
+    pub lint_span: Span,
+    #[suggestion(code = "#[{lint_level}({lint_name})]\n", applicability = "maybe-incorrect")]
+    pub suggest_lint_on_match: Option<Span>,
+    pub lint_level: &'static str,
+    pub lint_name: &'static str,
+}
diff --git a/compiler/rustc_pattern_analysis/src/lib.rs b/compiler/rustc_pattern_analysis/src/lib.rs
new file mode 100644
index 000000000..785a60e99
--- /dev/null
+++ b/compiler/rustc_pattern_analysis/src/lib.rs
@@ -0,0 +1,135 @@
+//! Analysis of patterns, notably match exhaustiveness checking.
+
+pub mod constructor;
+#[cfg(feature = "rustc")]
+pub mod errors;
+#[cfg(feature = "rustc")]
+pub(crate) mod lints;
+pub mod pat;
+#[cfg(feature = "rustc")]
+pub mod rustc;
+pub mod usefulness;
+
+#[macro_use]
+extern crate tracing;
+#[cfg(feature = "rustc")]
+#[macro_use]
+extern crate rustc_middle;
+
+#[cfg(feature = "rustc")]
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
+
+use std::fmt;
+
+use rustc_index::Idx;
+#[cfg(feature = "rustc")]
+use rustc_middle::ty::Ty;
+
+use crate::constructor::{Constructor, ConstructorSet};
+#[cfg(feature = "rustc")]
+use crate::lints::{
+    lint_nonexhaustive_missing_variants, lint_overlapping_range_endpoints, PatternColumn,
+};
+use crate::pat::DeconstructedPat;
+#[cfg(feature = "rustc")]
+use crate::rustc::RustcMatchCheckCtxt;
+#[cfg(feature = "rustc")]
+use crate::usefulness::{compute_match_usefulness, ValidityConstraint};
+
+// It's not possible to only enable the `typed_arena` dependency when the `rustc` feature is off, so
+// we use another feature instead. The crate won't compile if one of these isn't enabled.
+#[cfg(feature = "rustc")]
+pub(crate) use rustc_arena::TypedArena;
+#[cfg(feature = "stable")]
+pub(crate) use typed_arena::Arena as TypedArena;
+
+pub trait Captures<'a> {}
+impl<'a, T: ?Sized> Captures<'a> for T {}
+
+/// Context that provides type information about constructors.
+///
+/// Most of the crate is parameterized on a type that implements this trait.
+pub trait TypeCx: Sized + Clone + fmt::Debug {
+    /// The type of a pattern.
+    type Ty: Copy + Clone + fmt::Debug; // FIXME: remove Copy
+    /// The index of an enum variant.
+    type VariantIdx: Clone + Idx;
+    /// A string literal
+    type StrLit: Clone + PartialEq + fmt::Debug;
+    /// Extra data to store in a match arm.
+    type ArmData: Copy + Clone + fmt::Debug;
+    /// Extra data to store in a pattern. `Default` needed when we create fictitious wildcard
+    /// patterns during analysis.
+    type PatData: Clone + Default;
+
+    fn is_opaque_ty(ty: Self::Ty) -> bool;
+    fn is_exhaustive_patterns_feature_on(&self) -> bool;
+
+    /// The number of fields for this constructor.
+    fn ctor_arity(&self, ctor: &Constructor<Self>, ty: Self::Ty) -> usize;
+
+    /// The types of the fields for this constructor. The result must have a length of
+    /// `ctor_arity()`.
+    fn ctor_sub_tys(&self, ctor: &Constructor<Self>, ty: Self::Ty) -> &[Self::Ty];
+
+    /// The set of all the constructors for `ty`.
+    ///
+    /// This must follow the invariants of `ConstructorSet`
+    fn ctors_for_ty(&self, ty: Self::Ty) -> ConstructorSet<Self>;
+
+    /// Best-effort `Debug` implementation.
+    fn debug_pat(f: &mut fmt::Formatter<'_>, pat: &DeconstructedPat<'_, Self>) -> fmt::Result;
+
+    /// Raise a bug.
+    fn bug(&self, fmt: fmt::Arguments<'_>) -> !;
+}
+
+/// Context that provides information global to a match.
+#[derive(Clone)]
+pub struct MatchCtxt<'a, 'p, Cx: TypeCx> {
+    /// The context for type information.
+    pub tycx: &'a Cx,
+    /// An arena to store the wildcards we produce during analysis.
+    pub wildcard_arena: &'a TypedArena<DeconstructedPat<'p, Cx>>,
+}
+
+impl<'a, 'p, Cx: TypeCx> Copy for MatchCtxt<'a, 'p, Cx> {}
+
+/// The arm of a match expression.
+#[derive(Clone, Debug)]
+pub struct MatchArm<'p, Cx: TypeCx> {
+    pub pat: &'p DeconstructedPat<'p, Cx>,
+    pub has_guard: bool,
+    pub arm_data: Cx::ArmData,
+}
+
+impl<'p, Cx: TypeCx> Copy for MatchArm<'p, Cx> {}
+
+/// The entrypoint for this crate. Computes whether a match is exhaustive and which of its arms are
+/// useful, and runs some lints.
+#[cfg(feature = "rustc")]
+pub fn analyze_match<'p, 'tcx>(
+    tycx: &RustcMatchCheckCtxt<'p, 'tcx>,
+    arms: &[rustc::MatchArm<'p, 'tcx>],
+    scrut_ty: Ty<'tcx>,
+) -> rustc::UsefulnessReport<'p, 'tcx> {
+    // Arena to store the extra wildcards we construct during analysis.
+    let wildcard_arena = tycx.pattern_arena;
+    let scrut_validity = ValidityConstraint::from_bool(tycx.known_valid_scrutinee);
+    let cx = MatchCtxt { tycx, wildcard_arena };
+
+    let report = compute_match_usefulness(cx, arms, scrut_ty, scrut_validity);
+
+    let pat_column = PatternColumn::new(arms);
+
+    // Lint on ranges that overlap on their endpoints, which is likely a mistake.
+    lint_overlapping_range_endpoints(cx, &pat_column);
+
+    // Run the non_exhaustive_omitted_patterns lint. Only run on refutable patterns to avoid hitting
+    // `if let`s. Only run if the match is exhaustive otherwise the error is redundant.
+    if tycx.refutable && report.non_exhaustiveness_witnesses.is_empty() {
+        lint_nonexhaustive_missing_variants(cx, arms, &pat_column, scrut_ty)
+    }
+
+    report
+}
diff --git a/compiler/rustc_pattern_analysis/src/lints.rs b/compiler/rustc_pattern_analysis/src/lints.rs
new file mode 100644
index 000000000..072ef4836
--- /dev/null
+++ b/compiler/rustc_pattern_analysis/src/lints.rs
@@ -0,0 +1,296 @@
+use smallvec::SmallVec;
+
+use rustc_data_structures::captures::Captures;
+use rustc_middle::ty::{self, Ty};
+use rustc_session::lint;
+use rustc_session::lint::builtin::NON_EXHAUSTIVE_OMITTED_PATTERNS;
+use rustc_span::Span;
+
+use crate::constructor::{IntRange, MaybeInfiniteInt};
+use crate::errors::{
+    NonExhaustiveOmittedPattern, NonExhaustiveOmittedPatternLintOnArm, Overlap,
+    OverlappingRangeEndpoints, Uncovered,
+};
+use crate::rustc::{
+    Constructor, DeconstructedPat, MatchArm, MatchCtxt, PlaceCtxt, RustcMatchCheckCtxt,
+    SplitConstructorSet, WitnessPat,
+};
+use crate::TypeCx;
+
+/// A column of patterns in the matrix, where a column is the intuitive notion of "subpatterns that
+/// inspect the same subvalue/place".
+/// This is used to traverse patterns column-by-column for lints. Despite similarities with the
+/// algorithm in [`crate::usefulness`], this does a different traversal. Notably this is linear in
+/// the depth of patterns, whereas `compute_exhaustiveness_and_usefulness` is worst-case exponential
+/// (exhaustiveness is NP-complete). The core difference is that we treat sub-columns separately.
+///
+/// This must not contain an or-pattern. `specialize` takes care to expand them.
+///
+/// This is not used in the main algorithm; only in lints.
+#[derive(Debug)]
+pub(crate) struct PatternColumn<'a, 'p, 'tcx> {
+    patterns: Vec<&'a DeconstructedPat<'p, 'tcx>>,
+}
+
+impl<'a, 'p, 'tcx> PatternColumn<'a, 'p, 'tcx> {
+    pub(crate) fn new(arms: &[MatchArm<'p, 'tcx>]) -> Self {
+        let mut patterns = Vec::with_capacity(arms.len());
+        for arm in arms {
+            if arm.pat.is_or_pat() {
+                patterns.extend(arm.pat.flatten_or_pat())
+            } else {
+                patterns.push(arm.pat)
+            }
+        }
+        Self { patterns }
+    }
+
+    fn is_empty(&self) -> bool {
+        self.patterns.is_empty()
+    }
+    fn head_ty(&self) -> Option<Ty<'tcx>> {
+        if self.patterns.len() == 0 {
+            return None;
+        }
+        // If the type is opaque and it is revealed anywhere in the column, we take the revealed
+        // version. Otherwise we could encounter constructors for the revealed type and crash.
+        let first_ty = self.patterns[0].ty();
+        if RustcMatchCheckCtxt::is_opaque_ty(first_ty) {
+            for pat in &self.patterns {
+                let ty = pat.ty();
+                if !RustcMatchCheckCtxt::is_opaque_ty(ty) {
+                    return Some(ty);
+                }
+            }
+        }
+        Some(first_ty)
+    }
+
+    /// Do constructor splitting on the constructors of the column.
+    fn analyze_ctors(&self, pcx: &PlaceCtxt<'_, 'p, 'tcx>) -> SplitConstructorSet<'p, 'tcx> {
+        let column_ctors = self.patterns.iter().map(|p| p.ctor());
+        pcx.ctors_for_ty().split(pcx, column_ctors)
+    }
+
+    fn iter<'b>(&'b self) -> impl Iterator<Item = &'a DeconstructedPat<'p, 'tcx>> + Captures<'b> {
+        self.patterns.iter().copied()
+    }
+
+    /// Does specialization: given a constructor, this takes the patterns from the column that match
+    /// the constructor, and outputs their fields.
+    /// This returns one column per field of the constructor. They usually all have the same length
+    /// (the number of patterns in `self` that matched `ctor`), except that we expand or-patterns
+    /// which may change the lengths.
+    fn specialize(
+        &self,
+        pcx: &PlaceCtxt<'a, 'p, 'tcx>,
+        ctor: &Constructor<'p, 'tcx>,
+    ) -> Vec<PatternColumn<'a, 'p, 'tcx>> {
+        let arity = ctor.arity(pcx);
+        if arity == 0 {
+            return Vec::new();
+        }
+
+        // We specialize the column by `ctor`. This gives us `arity`-many columns of patterns. These
+        // columns may have different lengths in the presence of or-patterns (this is why we can't
+        // reuse `Matrix`).
+        let mut specialized_columns: Vec<_> =
+            (0..arity).map(|_| Self { patterns: Vec::new() }).collect();
+        let relevant_patterns =
+            self.patterns.iter().filter(|pat| ctor.is_covered_by(pcx, pat.ctor()));
+        for pat in relevant_patterns {
+            let specialized = pat.specialize(pcx, ctor);
+            for (subpat, column) in specialized.iter().zip(&mut specialized_columns) {
+                if subpat.is_or_pat() {
+                    column.patterns.extend(subpat.flatten_or_pat())
+                } else {
+                    column.patterns.push(subpat)
+                }
+            }
+        }
+
+        assert!(
+            !specialized_columns[0].is_empty(),
+            "ctor {ctor:?} was listed as present but isn't;
+            there is an inconsistency between `Constructor::is_covered_by` and `ConstructorSet::split`"
+        );
+        specialized_columns
+    }
+}
+
+/// Traverse the patterns to collect any variants of a non_exhaustive enum that fail to be mentioned
+/// in a given column.
+#[instrument(level = "debug", skip(cx), ret)]
+fn collect_nonexhaustive_missing_variants<'a, 'p, 'tcx>(
+    cx: MatchCtxt<'a, 'p, 'tcx>,
+    column: &PatternColumn<'a, 'p, 'tcx>,
+) -> Vec<WitnessPat<'p, 'tcx>> {
+    let Some(ty) = column.head_ty() else {
+        return Vec::new();
+    };
+    let pcx = &PlaceCtxt::new_dummy(cx, ty);
+
+    let set = column.analyze_ctors(pcx);
+    if set.present.is_empty() {
+        // We can't consistently handle the case where no constructors are present (since this would
+        // require digging deep through any type in case there's a non_exhaustive enum somewhere),
+        // so for consistency we refuse to handle the top-level case, where we could handle it.
+        return vec![];
+    }
+
+    let mut witnesses = Vec::new();
+    if cx.tycx.is_foreign_non_exhaustive_enum(ty) {
+        witnesses.extend(
+            set.missing
+                .into_iter()
+                // This will list missing visible variants.
+                .filter(|c| !matches!(c, Constructor::Hidden | Constructor::NonExhaustive))
+                .map(|missing_ctor| WitnessPat::wild_from_ctor(pcx, missing_ctor)),
+        )
+    }
+
+    // Recurse into the fields.
+    for ctor in set.present {
+        let specialized_columns = column.specialize(pcx, &ctor);
+        let wild_pat = WitnessPat::wild_from_ctor(pcx, ctor);
+        for (i, col_i) in specialized_columns.iter().enumerate() {
+            // Compute witnesses for each column.
+            let wits_for_col_i = collect_nonexhaustive_missing_variants(cx, col_i);
+            // For each witness, we build a new pattern in the shape of `ctor(_, _, wit, _, _)`,
+            // adding enough wildcards to match `arity`.
+            for wit in wits_for_col_i {
+                let mut pat = wild_pat.clone();
+                pat.fields[i] = wit;
+                witnesses.push(pat);
+            }
+        }
+    }
+    witnesses
+}
+
+pub(crate) fn lint_nonexhaustive_missing_variants<'a, 'p, 'tcx>(
+    cx: MatchCtxt<'a, 'p, 'tcx>,
+    arms: &[MatchArm<'p, 'tcx>],
+    pat_column: &PatternColumn<'a, 'p, 'tcx>,
+    scrut_ty: Ty<'tcx>,
+) {
+    let rcx: &RustcMatchCheckCtxt<'_, '_> = cx.tycx;
+    if !matches!(
+        rcx.tcx.lint_level_at_node(NON_EXHAUSTIVE_OMITTED_PATTERNS, rcx.match_lint_level).0,
+        rustc_session::lint::Level::Allow
+    ) {
+        let witnesses = collect_nonexhaustive_missing_variants(cx, pat_column);
+        if !witnesses.is_empty() {
+            // Report that a match of a `non_exhaustive` enum marked with `non_exhaustive_omitted_patterns`
+            // is not exhaustive enough.
+            //
+            // NB: The partner lint for structs lives in `compiler/rustc_hir_analysis/src/check/pat.rs`.
+            rcx.tcx.emit_spanned_lint(
+                NON_EXHAUSTIVE_OMITTED_PATTERNS,
+                rcx.match_lint_level,
+                rcx.scrut_span,
+                NonExhaustiveOmittedPattern {
+                    scrut_ty,
+                    uncovered: Uncovered::new(rcx.scrut_span, rcx, witnesses),
+                },
+            );
+        }
+    } else {
+        // We used to allow putting the `#[allow(non_exhaustive_omitted_patterns)]` on a match
+        // arm. This no longer makes sense so we warn users, to avoid silently breaking their
+        // usage of the lint.
+        for arm in arms {
+            let (lint_level, lint_level_source) =
+                rcx.tcx.lint_level_at_node(NON_EXHAUSTIVE_OMITTED_PATTERNS, arm.arm_data);
+            if !matches!(lint_level, rustc_session::lint::Level::Allow) {
+                let decorator = NonExhaustiveOmittedPatternLintOnArm {
+                    lint_span: lint_level_source.span(),
+                    suggest_lint_on_match: rcx.whole_match_span.map(|span| span.shrink_to_lo()),
+                    lint_level: lint_level.as_str(),
+                    lint_name: "non_exhaustive_omitted_patterns",
+                };
+
+                use rustc_errors::DecorateLint;
+                let mut err = rcx.tcx.sess.struct_span_warn(*arm.pat.data(), "");
+                err.set_primary_message(decorator.msg());
+                decorator.decorate_lint(&mut err);
+                err.emit();
+            }
+        }
+    }
+}
+
+/// Traverse the patterns to warn the user about ranges that overlap on their endpoints.
+#[instrument(level = "debug", skip(cx))]
+pub(crate) fn lint_overlapping_range_endpoints<'a, 'p, 'tcx>(
+    cx: MatchCtxt<'a, 'p, 'tcx>,
+    column: &PatternColumn<'a, 'p, 'tcx>,
+) {
+    let Some(ty) = column.head_ty() else {
+        return;
+    };
+    let pcx = &PlaceCtxt::new_dummy(cx, ty);
+    let rcx: &RustcMatchCheckCtxt<'_, '_> = cx.tycx;
+
+    let set = column.analyze_ctors(pcx);
+
+    if matches!(ty.kind(), ty::Char | ty::Int(_) | ty::Uint(_)) {
+        let emit_lint = |overlap: &IntRange, this_span: Span, overlapped_spans: &[Span]| {
+            let overlap_as_pat = rcx.hoist_pat_range(overlap, ty);
+            let overlaps: Vec<_> = overlapped_spans
+                .iter()
+                .copied()
+                .map(|span| Overlap { range: overlap_as_pat.clone(), span })
+                .collect();
+            rcx.tcx.emit_spanned_lint(
+                lint::builtin::OVERLAPPING_RANGE_ENDPOINTS,
+                rcx.match_lint_level,
+                this_span,
+                OverlappingRangeEndpoints { overlap: overlaps, range: this_span },
+            );
+        };
+
+        // If two ranges overlapped, the split set will contain their intersection as a singleton.
+        let split_int_ranges = set.present.iter().filter_map(|c| c.as_int_range());
+        for overlap_range in split_int_ranges.clone() {
+            if overlap_range.is_singleton() {
+                let overlap: MaybeInfiniteInt = overlap_range.lo;
+                // Ranges that look like `lo..=overlap`.
+                let mut prefixes: SmallVec<[_; 1]> = Default::default();
+                // Ranges that look like `overlap..=hi`.
+                let mut suffixes: SmallVec<[_; 1]> = Default::default();
+                // Iterate on patterns that contained `overlap`.
+                for pat in column.iter() {
+                    let this_span = *pat.data();
+                    let Constructor::IntRange(this_range) = pat.ctor() else { continue };
+                    if this_range.is_singleton() {
+                        // Don't lint when one of the ranges is a singleton.
+                        continue;
+                    }
+                    if this_range.lo == overlap {
+                        // `this_range` looks like `overlap..=this_range.hi`; it overlaps with any
+                        // ranges that look like `lo..=overlap`.
+                        if !prefixes.is_empty() {
+                            emit_lint(overlap_range, this_span, &prefixes);
+                        }
+                        suffixes.push(this_span)
+                    } else if this_range.hi == overlap.plus_one() {
+                        // `this_range` looks like `this_range.lo..=overlap`; it overlaps with any
+                        // ranges that look like `overlap..=hi`.
+                        if !suffixes.is_empty() {
+                            emit_lint(overlap_range, this_span, &suffixes);
+                        }
+                        prefixes.push(this_span)
+                    }
+                }
+            }
+        }
+    } else {
+        // Recurse into the fields.
+        for ctor in set.present {
+            for col in column.specialize(pcx, &ctor) {
+                lint_overlapping_range_endpoints(cx, &col);
+            }
+        }
+    }
+}
diff --git a/compiler/rustc_pattern_analysis/src/pat.rs b/compiler/rustc_pattern_analysis/src/pat.rs
new file mode 100644
index 000000000..0cc8477b7
--- /dev/null
+++ b/compiler/rustc_pattern_analysis/src/pat.rs
@@ -0,0 +1,197 @@
+//! As explained in [`crate::usefulness`], values and patterns are made from constructors applied to
+//! fields. This file defines types that represent patterns in this way.
+use std::cell::Cell;
+use std::fmt;
+
+use smallvec::{smallvec, SmallVec};
+
+use crate::constructor::{Constructor, Slice, SliceKind};
+use crate::usefulness::PlaceCtxt;
+use crate::{Captures, TypeCx};
+
+use self::Constructor::*;
+
+/// Values and patterns can be represented as a constructor applied to some fields. This represents
+/// a pattern in this form.
+/// This also uses interior mutability to keep track of whether the pattern has been found reachable
+/// during analysis. For this reason they cannot be cloned.
+/// A `DeconstructedPat` will almost always come from user input; the only exception are some
+/// `Wildcard`s introduced during specialization.
+///
+/// Note that the number of fields may not match the fields declared in the original struct/variant.
+/// This happens if a private or `non_exhaustive` field is uninhabited, because the code mustn't
+/// observe that it is uninhabited. In that case that field is not included in `fields`. Care must
+/// be taken when converting to/from `thir::Pat`.
+pub struct DeconstructedPat<'p, Cx: TypeCx> {
+    ctor: Constructor<Cx>,
+    fields: &'p [DeconstructedPat<'p, Cx>],
+    ty: Cx::Ty,
+    data: Cx::PatData,
+    /// Whether removing this arm would change the behavior of the match expression.
+    useful: Cell<bool>,
+}
+
+impl<'p, Cx: TypeCx> DeconstructedPat<'p, Cx> {
+    pub fn wildcard(ty: Cx::Ty, data: Cx::PatData) -> Self {
+        Self::new(Wildcard, &[], ty, data)
+    }
+
+    pub fn new(
+        ctor: Constructor<Cx>,
+        fields: &'p [DeconstructedPat<'p, Cx>],
+        ty: Cx::Ty,
+        data: Cx::PatData,
+    ) -> Self {
+        DeconstructedPat { ctor, fields, ty, data, useful: Cell::new(false) }
+    }
+
+    pub(crate) fn is_or_pat(&self) -> bool {
+        matches!(self.ctor, Or)
+    }
+    /// Expand this (possibly-nested) or-pattern into its alternatives.
+    pub(crate) fn flatten_or_pat(&self) -> SmallVec<[&Self; 1]> {
+        if self.is_or_pat() {
+            self.iter_fields().flat_map(|p| p.flatten_or_pat()).collect()
+        } else {
+            smallvec![self]
+        }
+    }
+
+    pub fn ctor(&self) -> &Constructor<Cx> {
+        &self.ctor
+    }
+    pub fn ty(&self) -> Cx::Ty {
+        self.ty
+    }
+    pub fn data(&self) -> &Cx::PatData {
+        &self.data
+    }
+
+    pub fn iter_fields<'a>(
+        &'a self,
+    ) -> impl Iterator<Item = &'p DeconstructedPat<'p, Cx>> + Captures<'a> {
+        self.fields.iter()
+    }
+
+    /// Specialize this pattern with a constructor.
+    /// `other_ctor` can be different from `self.ctor`, but must be covered by it.
+    pub(crate) fn specialize<'a>(
+        &self,
+        pcx: &PlaceCtxt<'a, 'p, Cx>,
+        other_ctor: &Constructor<Cx>,
+    ) -> SmallVec<[&'a DeconstructedPat<'p, Cx>; 2]> {
+        let wildcard_sub_tys = || {
+            let tys = pcx.ctor_sub_tys(other_ctor);
+            tys.iter()
+                .map(|ty| DeconstructedPat::wildcard(*ty, Cx::PatData::default()))
+                .map(|pat| pcx.mcx.wildcard_arena.alloc(pat) as &_)
+                .collect()
+        };
+        match (&self.ctor, other_ctor) {
+            // Return a wildcard for each field of `other_ctor`.
+            (Wildcard, _) => wildcard_sub_tys(),
+            // The only non-trivial case: two slices of different arity. `other_slice` is
+            // guaranteed to have a larger arity, so we fill the middle part with enough
+            // wildcards to reach the length of the new, larger slice.
+            (
+                &Slice(self_slice @ Slice { kind: SliceKind::VarLen(prefix, suffix), .. }),
+                &Slice(other_slice),
+            ) if self_slice.arity() != other_slice.arity() => {
+                // Start with a slice of wildcards of the appropriate length.
+                let mut fields: SmallVec<[_; 2]> = wildcard_sub_tys();
+                // Fill in the fields from both ends.
+                let new_arity = fields.len();
+                for i in 0..prefix {
+                    fields[i] = &self.fields[i];
+                }
+                for i in 0..suffix {
+                    fields[new_arity - 1 - i] = &self.fields[self.fields.len() - 1 - i];
+                }
+                fields
+            }
+            _ => self.fields.iter().collect(),
+        }
+    }
+
+    /// We keep track for each pattern if it was ever useful during the analysis. This is used with
+    /// `redundant_subpatterns` to report redundant subpatterns arising from or patterns.
+    pub(crate) fn set_useful(&self) {
+        self.useful.set(true)
+    }
+    pub(crate) fn is_useful(&self) -> bool {
+        if self.useful.get() {
+            true
+        } else if self.is_or_pat() && self.iter_fields().any(|f| f.is_useful()) {
+            // We always expand or patterns in the matrix, so we will never see the actual
+            // or-pattern (the one with constructor `Or`) in the column. As such, it will not be
+            // marked as useful itself, only its children will. We recover this information here.
+            self.set_useful();
+            true
+        } else {
+            false
+        }
+    }
+
+    /// Report the subpatterns that were not useful, if any.
+    pub(crate) fn redundant_subpatterns(&self) -> Vec<&Self> {
+        let mut subpats = Vec::new();
+        self.collect_redundant_subpatterns(&mut subpats);
+        subpats
+    }
+    fn collect_redundant_subpatterns<'a>(&'a self, subpats: &mut Vec<&'a Self>) {
+        // We don't look at subpatterns if we already reported the whole pattern as redundant.
+        if !self.is_useful() {
+            subpats.push(self);
+        } else {
+            for p in self.iter_fields() {
+                p.collect_redundant_subpatterns(subpats);
+            }
+        }
+    }
+}
+
+/// This is mostly copied from the `Pat` impl. This is best effort and not good enough for a
+/// `Display` impl.
+impl<'p, Cx: TypeCx> fmt::Debug for DeconstructedPat<'p, Cx> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        Cx::debug_pat(f, self)
+    }
+}
+
+/// Same idea as `DeconstructedPat`, except this is a fictitious pattern built up for diagnostics
+/// purposes. As such they don't use interning and can be cloned.
+#[derive(Debug, Clone)]
+pub struct WitnessPat<Cx: TypeCx> {
+    ctor: Constructor<Cx>,
+    pub(crate) fields: Vec<WitnessPat<Cx>>,
+    ty: Cx::Ty,
+}
+
+impl<Cx: TypeCx> WitnessPat<Cx> {
+    pub(crate) fn new(ctor: Constructor<Cx>, fields: Vec<Self>, ty: Cx::Ty) -> Self {
+        Self { ctor, fields, ty }
+    }
+    pub(crate) fn wildcard(ty: Cx::Ty) -> Self {
+        Self::new(Wildcard, Vec::new(), ty)
+    }
+
+    /// Construct a pattern that matches everything that starts with this constructor.
+    /// For example, if `ctor` is a `Constructor::Variant` for `Option::Some`, we get the pattern
+    /// `Some(_)`.
+    pub(crate) fn wild_from_ctor(pcx: &PlaceCtxt<'_, '_, Cx>, ctor: Constructor<Cx>) -> Self {
+        let field_tys = pcx.ctor_sub_tys(&ctor);
+        let fields = field_tys.iter().map(|ty| Self::wildcard(*ty)).collect();
+        Self::new(ctor, fields, pcx.ty)
+    }
+
+    pub fn ctor(&self) -> &Constructor<Cx> {
+        &self.ctor
+    }
+    pub fn ty(&self) -> Cx::Ty {
+        self.ty
+    }
+
+    pub fn iter_fields<'a>(&'a self) -> impl Iterator<Item = &'a WitnessPat<Cx>> {
+        self.fields.iter()
+    }
+}
diff --git a/compiler/rustc_pattern_analysis/src/rustc.rs b/compiler/rustc_pattern_analysis/src/rustc.rs
new file mode 100644
index 000000000..65c90aa9f
--- /dev/null
+++ b/compiler/rustc_pattern_analysis/src/rustc.rs
@@ -0,0 +1,920 @@
+use std::fmt;
+use std::iter::once;
+
+use rustc_arena::{DroplessArena, TypedArena};
+use rustc_data_structures::captures::Captures;
+use rustc_hir::def_id::DefId;
+use rustc_hir::HirId;
+use rustc_index::Idx;
+use rustc_index::IndexVec;
+use rustc_middle::middle::stability::EvalResult;
+use rustc_middle::mir::interpret::Scalar;
+use rustc_middle::mir::{self, Const};
+use rustc_middle::thir::{FieldPat, Pat, PatKind, PatRange, PatRangeBoundary};
+use rustc_middle::ty::layout::IntegerExt;
+use rustc_middle::ty::{self, Ty, TyCtxt, VariantDef};
+use rustc_span::{Span, DUMMY_SP};
+use rustc_target::abi::{FieldIdx, Integer, VariantIdx, FIRST_VARIANT};
+use smallvec::SmallVec;
+
+use crate::constructor::{
+    IntRange, MaybeInfiniteInt, OpaqueId, RangeEnd, Slice, SliceKind, VariantVisibility,
+};
+use crate::TypeCx;
+
+use crate::constructor::Constructor::*;
+
+// Re-export rustc-specific versions of all these types.
+pub type Constructor<'p, 'tcx> = crate::constructor::Constructor<RustcMatchCheckCtxt<'p, 'tcx>>;
+pub type ConstructorSet<'p, 'tcx> =
+    crate::constructor::ConstructorSet<RustcMatchCheckCtxt<'p, 'tcx>>;
+pub type DeconstructedPat<'p, 'tcx> =
+    crate::pat::DeconstructedPat<'p, RustcMatchCheckCtxt<'p, 'tcx>>;
+pub type MatchArm<'p, 'tcx> = crate::MatchArm<'p, RustcMatchCheckCtxt<'p, 'tcx>>;
+pub type MatchCtxt<'a, 'p, 'tcx> = crate::MatchCtxt<'a, 'p, RustcMatchCheckCtxt<'p, 'tcx>>;
+pub(crate) type PlaceCtxt<'a, 'p, 'tcx> =
+    crate::usefulness::PlaceCtxt<'a, 'p, RustcMatchCheckCtxt<'p, 'tcx>>;
+pub(crate) type SplitConstructorSet<'p, 'tcx> =
+    crate::constructor::SplitConstructorSet<RustcMatchCheckCtxt<'p, 'tcx>>;
+pub type Usefulness<'p, 'tcx> = crate::usefulness::Usefulness<'p, RustcMatchCheckCtxt<'p, 'tcx>>;
+pub type UsefulnessReport<'p, 'tcx> =
+    crate::usefulness::UsefulnessReport<'p, RustcMatchCheckCtxt<'p, 'tcx>>;
+pub type WitnessPat<'p, 'tcx> = crate::pat::WitnessPat<RustcMatchCheckCtxt<'p, 'tcx>>;
+
+#[derive(Clone)]
+pub struct RustcMatchCheckCtxt<'p, 'tcx> {
+    pub tcx: TyCtxt<'tcx>,
+    /// The module in which the match occurs. This is necessary for
+    /// checking inhabited-ness of types because whether a type is (visibly)
+    /// inhabited can depend on whether it was defined in the current module or
+    /// not. E.g., `struct Foo { _private: ! }` cannot be seen to be empty
+    /// outside its module and should not be matchable with an empty match statement.
+    pub module: DefId,
+    pub param_env: ty::ParamEnv<'tcx>,
+    pub pattern_arena: &'p TypedArena<DeconstructedPat<'p, 'tcx>>,
+    pub dropless_arena: &'p DroplessArena,
+    /// Lint level at the match.
+    pub match_lint_level: HirId,
+    /// The span of the whole match, if applicable.
+    pub whole_match_span: Option<Span>,
+    /// Span of the scrutinee.
+    pub scrut_span: Span,
+    /// Only produce `NON_EXHAUSTIVE_OMITTED_PATTERNS` lint on refutable patterns.
+    pub refutable: bool,
+    /// Whether the data at the scrutinee is known to be valid. This is false if the scrutinee comes
+    /// from a union field, a pointer deref, or a reference deref (pending opsem decisions).
+    pub known_valid_scrutinee: bool,
+}
+
+impl<'p, 'tcx> fmt::Debug for RustcMatchCheckCtxt<'p, 'tcx> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("RustcMatchCheckCtxt").finish()
+    }
+}
+
+impl<'p, 'tcx> RustcMatchCheckCtxt<'p, 'tcx> {
+    pub(crate) fn is_uninhabited(&self, ty: Ty<'tcx>) -> bool {
+        !ty.is_inhabited_from(self.tcx, self.module, self.param_env)
+    }
+
+    /// Returns whether the given type is an enum from another crate declared `#[non_exhaustive]`.
+    pub fn is_foreign_non_exhaustive_enum(&self, ty: Ty<'tcx>) -> bool {
+        match ty.kind() {
+            ty::Adt(def, ..) => {
+                def.is_enum() && def.is_variant_list_non_exhaustive() && !def.did().is_local()
+            }
+            _ => false,
+        }
+    }
+
+    /// Whether the range denotes the fictitious values before `isize::MIN` or after
+    /// `usize::MAX`/`isize::MAX` (see doc of [`IntRange::split`] for why these exist).
+    pub fn is_range_beyond_boundaries(&self, range: &IntRange, ty: Ty<'tcx>) -> bool {
+        ty.is_ptr_sized_integral() && {
+            // The two invalid ranges are `NegInfinity..isize::MIN` (represented as
+            // `NegInfinity..0`), and `{u,i}size::MAX+1..PosInfinity`. `hoist_pat_range_bdy`
+            // converts `MAX+1` to `PosInfinity`, and we couldn't have `PosInfinity` in `range.lo`
+            // otherwise.
+            let lo = self.hoist_pat_range_bdy(range.lo, ty);
+            matches!(lo, PatRangeBoundary::PosInfinity)
+                || matches!(range.hi, MaybeInfiniteInt::Finite(0))
+        }
+    }
+
+    // In the cases of either a `#[non_exhaustive]` field list or a non-public field, we hide
+    // uninhabited fields in order not to reveal the uninhabitedness of the whole variant.
+    // This lists the fields we keep along with their types.
+    pub(crate) fn list_variant_nonhidden_fields<'a>(
+        &'a self,
+        ty: Ty<'tcx>,
+        variant: &'a VariantDef,
+    ) -> impl Iterator<Item = (FieldIdx, Ty<'tcx>)> + Captures<'p> + Captures<'a> {
+        let cx = self;
+        let ty::Adt(adt, args) = ty.kind() else { bug!() };
+        // Whether we must not match the fields of this variant exhaustively.
+        let is_non_exhaustive = variant.is_field_list_non_exhaustive() && !adt.did().is_local();
+
+        variant.fields.iter().enumerate().filter_map(move |(i, field)| {
+            let ty = field.ty(cx.tcx, args);
+            // `field.ty()` doesn't normalize after substituting.
+            let ty = cx.tcx.normalize_erasing_regions(cx.param_env, ty);
+            let is_visible = adt.is_enum() || field.vis.is_accessible_from(cx.module, cx.tcx);
+            let is_uninhabited = cx.tcx.features().exhaustive_patterns && cx.is_uninhabited(ty);
+
+            if is_uninhabited && (!is_visible || is_non_exhaustive) {
+                None
+            } else {
+                Some((FieldIdx::new(i), ty))
+            }
+        })
+    }
+
+    pub(crate) fn variant_index_for_adt(
+        ctor: &Constructor<'p, 'tcx>,
+        adt: ty::AdtDef<'tcx>,
+    ) -> VariantIdx {
+        match *ctor {
+            Variant(idx) => idx,
+            Struct | UnionField => {
+                assert!(!adt.is_enum());
+                FIRST_VARIANT
+            }
+            _ => bug!("bad constructor {:?} for adt {:?}", ctor, adt),
+        }
+    }
+
+    /// Returns the types of the fields for a given constructor. The result must have a length of
+    /// `ctor.arity()`.
+    #[instrument(level = "trace", skip(self))]
+    pub(crate) fn ctor_sub_tys(&self, ctor: &Constructor<'p, 'tcx>, ty: Ty<'tcx>) -> &[Ty<'tcx>] {
+        let cx = self;
+        match ctor {
+            Struct | Variant(_) | UnionField => match ty.kind() {
+                ty::Tuple(fs) => cx.dropless_arena.alloc_from_iter(fs.iter()),
+                ty::Adt(adt, args) => {
+                    if adt.is_box() {
+                        // The only legal patterns of type `Box` (outside `std`) are `_` and box
+                        // patterns. If we're here we can assume this is a box pattern.
+                        cx.dropless_arena.alloc_from_iter(once(args.type_at(0)))
+                    } else {
+                        let variant =
+                            &adt.variant(RustcMatchCheckCtxt::variant_index_for_adt(&ctor, *adt));
+                        let tys = cx.list_variant_nonhidden_fields(ty, variant).map(|(_, ty)| ty);
+                        cx.dropless_arena.alloc_from_iter(tys)
+                    }
+                }
+                _ => bug!("Unexpected type for constructor `{ctor:?}`: {ty:?}"),
+            },
+            Ref => match ty.kind() {
+                ty::Ref(_, rty, _) => cx.dropless_arena.alloc_from_iter(once(*rty)),
+                _ => bug!("Unexpected type for `Ref` constructor: {ty:?}"),
+            },
+            Slice(slice) => match *ty.kind() {
+                ty::Slice(ty) | ty::Array(ty, _) => {
+                    let arity = slice.arity();
+                    cx.dropless_arena.alloc_from_iter((0..arity).map(|_| ty))
+                }
+                _ => bug!("bad slice pattern {:?} {:?}", ctor, ty),
+            },
+            Bool(..)
+            | IntRange(..)
+            | F32Range(..)
+            | F64Range(..)
+            | Str(..)
+            | Opaque(..)
+            | NonExhaustive
+            | Hidden
+            | Missing { .. }
+            | Wildcard => &[],
+            Or => {
+                bug!("called `Fields::wildcards` on an `Or` ctor")
+            }
+        }
+    }
+
+    /// The number of fields for this constructor.
+    pub(crate) fn ctor_arity(&self, ctor: &Constructor<'p, 'tcx>, ty: Ty<'tcx>) -> usize {
+        match ctor {
+            Struct | Variant(_) | UnionField => match ty.kind() {
+                ty::Tuple(fs) => fs.len(),
+                ty::Adt(adt, ..) => {
+                    if adt.is_box() {
+                        // The only legal patterns of type `Box` (outside `std`) are `_` and box
+                        // patterns. If we're here we can assume this is a box pattern.
+                        1
+                    } else {
+                        let variant =
+                            &adt.variant(RustcMatchCheckCtxt::variant_index_for_adt(&ctor, *adt));
+                        self.list_variant_nonhidden_fields(ty, variant).count()
+                    }
+                }
+                _ => bug!("Unexpected type for constructor `{ctor:?}`: {ty:?}"),
+            },
+            Ref => 1,
+            Slice(slice) => slice.arity(),
+            Bool(..)
+            | IntRange(..)
+            | F32Range(..)
+            | F64Range(..)
+            | Str(..)
+            | Opaque(..)
+            | NonExhaustive
+            | Hidden
+            | Missing { .. }
+            | Wildcard => 0,
+            Or => bug!("The `Or` constructor doesn't have a fixed arity"),
+        }
+    }
+
+    /// Creates a set that represents all the constructors of `ty`.
+    ///
+    /// See [`crate::constructor`] for considerations of emptiness.
+    #[instrument(level = "debug", skip(self), ret)]
+    pub fn ctors_for_ty(&self, ty: Ty<'tcx>) -> ConstructorSet<'p, 'tcx> {
+        let cx = self;
+        let make_uint_range = |start, end| {
+            IntRange::from_range(
+                MaybeInfiniteInt::new_finite_uint(start),
+                MaybeInfiniteInt::new_finite_uint(end),
+                RangeEnd::Included,
+            )
+        };
+        // This determines the set of all possible constructors for the type `ty`. For numbers,
+        // arrays and slices we use ranges and variable-length slices when appropriate.
+        match ty.kind() {
+            ty::Bool => ConstructorSet::Bool,
+            ty::Char => {
+                // The valid Unicode Scalar Value ranges.
+                ConstructorSet::Integers {
+                    range_1: make_uint_range('\u{0000}' as u128, '\u{D7FF}' as u128),
+                    range_2: Some(make_uint_range('\u{E000}' as u128, '\u{10FFFF}' as u128)),
+                }
+            }
+            &ty::Int(ity) => {
+                let range = if ty.is_ptr_sized_integral() {
+                    // The min/max values of `isize` are not allowed to be observed.
+                    IntRange {
+                        lo: MaybeInfiniteInt::NegInfinity,
+                        hi: MaybeInfiniteInt::PosInfinity,
+                    }
+                } else {
+                    let size = Integer::from_int_ty(&cx.tcx, ity).size().bits();
+                    let min = 1u128 << (size - 1);
+                    let max = min - 1;
+                    let min = MaybeInfiniteInt::new_finite_int(min, size);
+                    let max = MaybeInfiniteInt::new_finite_int(max, size);
+                    IntRange::from_range(min, max, RangeEnd::Included)
+                };
+                ConstructorSet::Integers { range_1: range, range_2: None }
+            }
+            &ty::Uint(uty) => {
+                let range = if ty.is_ptr_sized_integral() {
+                    // The max value of `usize` is not allowed to be observed.
+                    let lo = MaybeInfiniteInt::new_finite_uint(0);
+                    IntRange { lo, hi: MaybeInfiniteInt::PosInfinity }
+                } else {
+                    let size = Integer::from_uint_ty(&cx.tcx, uty).size();
+                    let max = size.truncate(u128::MAX);
+                    make_uint_range(0, max)
+                };
+                ConstructorSet::Integers { range_1: range, range_2: None }
+            }
+            ty::Slice(sub_ty) => ConstructorSet::Slice {
+                array_len: None,
+                subtype_is_empty: cx.is_uninhabited(*sub_ty),
+            },
+            ty::Array(sub_ty, len) => {
+                // We treat arrays of a constant but unknown length like slices.
+                ConstructorSet::Slice {
+                    array_len: len.try_eval_target_usize(cx.tcx, cx.param_env).map(|l| l as usize),
+                    subtype_is_empty: cx.is_uninhabited(*sub_ty),
+                }
+            }
+            ty::Adt(def, args) if def.is_enum() => {
+                let is_declared_nonexhaustive = cx.is_foreign_non_exhaustive_enum(ty);
+                if def.variants().is_empty() && !is_declared_nonexhaustive {
+                    ConstructorSet::NoConstructors
+                } else {
+                    let mut variants =
+                        IndexVec::from_elem(VariantVisibility::Visible, def.variants());
+                    for (idx, v) in def.variants().iter_enumerated() {
+                        let variant_def_id = def.variant(idx).def_id;
+                        // Visibly uninhabited variants.
+                        let is_inhabited = v
+                            .inhabited_predicate(cx.tcx, *def)
+                            .instantiate(cx.tcx, args)
+                            .apply(cx.tcx, cx.param_env, cx.module);
+                        // Variants that depend on a disabled unstable feature.
+                        let is_unstable = matches!(
+                            cx.tcx.eval_stability(variant_def_id, None, DUMMY_SP, None),
+                            EvalResult::Deny { .. }
+                        );
+                        // Foreign `#[doc(hidden)]` variants.
+                        let is_doc_hidden =
+                            cx.tcx.is_doc_hidden(variant_def_id) && !variant_def_id.is_local();
+                        let visibility = if !is_inhabited {
+                            // FIXME: handle empty+hidden
+                            VariantVisibility::Empty
+                        } else if is_unstable || is_doc_hidden {
+                            VariantVisibility::Hidden
+                        } else {
+                            VariantVisibility::Visible
+                        };
+                        variants[idx] = visibility;
+                    }
+
+                    ConstructorSet::Variants { variants, non_exhaustive: is_declared_nonexhaustive }
+                }
+            }
+            ty::Adt(def, _) if def.is_union() => ConstructorSet::Union,
+            ty::Adt(..) | ty::Tuple(..) => ConstructorSet::Struct { empty: cx.is_uninhabited(ty) },
+            ty::Ref(..) => ConstructorSet::Ref,
+            ty::Never => ConstructorSet::NoConstructors,
+            // This type is one for which we cannot list constructors, like `str` or `f64`.
+            // FIXME(Nadrieril): which of these are actually allowed?
+            ty::Float(_)
+            | ty::Str
+            | ty::Foreign(_)
+            | ty::RawPtr(_)
+            | ty::FnDef(_, _)
+            | ty::FnPtr(_)
+            | ty::Dynamic(_, _, _)
+            | ty::Closure(_, _)
+            | ty::Coroutine(_, _, _)
+            | ty::Alias(_, _)
+            | ty::Param(_)
+            | ty::Error(_) => ConstructorSet::Unlistable,
+            ty::CoroutineWitness(_, _) | ty::Bound(_, _) | ty::Placeholder(_) | ty::Infer(_) => {
+                bug!("Encountered unexpected type in `ConstructorSet::for_ty`: {ty:?}")
+            }
+        }
+    }
+
+    pub(crate) fn lower_pat_range_bdy(
+        &self,
+        bdy: PatRangeBoundary<'tcx>,
+        ty: Ty<'tcx>,
+    ) -> MaybeInfiniteInt {
+        match bdy {
+            PatRangeBoundary::NegInfinity => MaybeInfiniteInt::NegInfinity,
+            PatRangeBoundary::Finite(value) => {
+                let bits = value.eval_bits(self.tcx, self.param_env);
+                match *ty.kind() {
+                    ty::Int(ity) => {
+                        let size = Integer::from_int_ty(&self.tcx, ity).size().bits();
+                        MaybeInfiniteInt::new_finite_int(bits, size)
+                    }
+                    _ => MaybeInfiniteInt::new_finite_uint(bits),
+                }
+            }
+            PatRangeBoundary::PosInfinity => MaybeInfiniteInt::PosInfinity,
+        }
+    }
+
+    /// Note: the input patterns must have been lowered through
+    /// `rustc_mir_build::thir::pattern::check_match::MatchVisitor::lower_pattern`.
+    pub fn lower_pat(&self, pat: &Pat<'tcx>) -> DeconstructedPat<'p, 'tcx> {
+        let singleton = |pat| std::slice::from_ref(self.pattern_arena.alloc(pat));
+        let cx = self;
+        let ctor;
+        let fields: &[_];
+        match &pat.kind {
+            PatKind::AscribeUserType { subpattern, .. }
+            | PatKind::InlineConstant { subpattern, .. } => return self.lower_pat(subpattern),
+            PatKind::Binding { subpattern: Some(subpat), .. } => return self.lower_pat(subpat),
+            PatKind::Binding { subpattern: None, .. } | PatKind::Wild => {
+                ctor = Wildcard;
+                fields = &[];
+            }
+            PatKind::Deref { subpattern } => {
+                fields = singleton(self.lower_pat(subpattern));
+                ctor = match pat.ty.kind() {
+                    // This is a box pattern.
+                    ty::Adt(adt, ..) if adt.is_box() => Struct,
+                    ty::Ref(..) => Ref,
+                    _ => bug!("pattern has unexpected type: pat: {:?}, ty: {:?}", pat, pat.ty),
+                };
+            }
+            PatKind::Leaf { subpatterns } | PatKind::Variant { subpatterns, .. } => {
+                match pat.ty.kind() {
+                    ty::Tuple(fs) => {
+                        ctor = Struct;
+                        let mut wilds: SmallVec<[_; 2]> =
+                            fs.iter().map(|ty| DeconstructedPat::wildcard(ty, pat.span)).collect();
+                        for pat in subpatterns {
+                            wilds[pat.field.index()] = self.lower_pat(&pat.pattern);
+                        }
+                        fields = cx.pattern_arena.alloc_from_iter(wilds);
+                    }
+                    ty::Adt(adt, args) if adt.is_box() => {
+                        // The only legal patterns of type `Box` (outside `std`) are `_` and box
+                        // patterns. If we're here we can assume this is a box pattern.
+                        // FIXME(Nadrieril): A `Box` can in theory be matched either with `Box(_,
+                        // _)` or a box pattern. As a hack to avoid an ICE with the former, we
+                        // ignore other fields than the first one. This will trigger an error later
+                        // anyway.
+                        // See https://github.com/rust-lang/rust/issues/82772,
+                        // explanation: https://github.com/rust-lang/rust/pull/82789#issuecomment-796921977
+                        // The problem is that we can't know from the type whether we'll match
+                        // normally or through box-patterns. We'll have to figure out a proper
+                        // solution when we introduce generalized deref patterns. Also need to
+                        // prevent mixing of those two options.
+                        let pattern = subpatterns.into_iter().find(|pat| pat.field.index() == 0);
+                        let pat = if let Some(pat) = pattern {
+                            self.lower_pat(&pat.pattern)
+                        } else {
+                            DeconstructedPat::wildcard(args.type_at(0), pat.span)
+                        };
+                        ctor = Struct;
+                        fields = singleton(pat);
+                    }
+                    ty::Adt(adt, _) => {
+                        ctor = match pat.kind {
+                            PatKind::Leaf { .. } if adt.is_union() => UnionField,
+                            PatKind::Leaf { .. } => Struct,
+                            PatKind::Variant { variant_index, .. } => Variant(variant_index),
+                            _ => bug!(),
+                        };
+                        let variant =
+                            &adt.variant(RustcMatchCheckCtxt::variant_index_for_adt(&ctor, *adt));
+                        // For each field in the variant, we store the relevant index into `self.fields` if any.
+                        let mut field_id_to_id: Vec<Option<usize>> =
+                            (0..variant.fields.len()).map(|_| None).collect();
+                        let tys = cx
+                            .list_variant_nonhidden_fields(pat.ty, variant)
+                            .enumerate()
+                            .map(|(i, (field, ty))| {
+                                field_id_to_id[field.index()] = Some(i);
+                                ty
+                            });
+                        let mut wilds: SmallVec<[_; 2]> =
+                            tys.map(|ty| DeconstructedPat::wildcard(ty, pat.span)).collect();
+                        for pat in subpatterns {
+                            if let Some(i) = field_id_to_id[pat.field.index()] {
+                                wilds[i] = self.lower_pat(&pat.pattern);
+                            }
+                        }
+                        fields = cx.pattern_arena.alloc_from_iter(wilds);
+                    }
+                    _ => bug!("pattern has unexpected type: pat: {:?}, ty: {:?}", pat, pat.ty),
+                }
+            }
+            PatKind::Constant { value } => {
+                match pat.ty.kind() {
+                    ty::Bool => {
+                        ctor = match value.try_eval_bool(cx.tcx, cx.param_env) {
+                            Some(b) => Bool(b),
+                            None => Opaque(OpaqueId::new()),
+                        };
+                        fields = &[];
+                    }
+                    ty::Char | ty::Int(_) | ty::Uint(_) => {
+                        ctor = match value.try_eval_bits(cx.tcx, cx.param_env) {
+                            Some(bits) => {
+                                let x = match *pat.ty.kind() {
+                                    ty::Int(ity) => {
+                                        let size = Integer::from_int_ty(&cx.tcx, ity).size().bits();
+                                        MaybeInfiniteInt::new_finite_int(bits, size)
+                                    }
+                                    _ => MaybeInfiniteInt::new_finite_uint(bits),
+                                };
+                                IntRange(IntRange::from_singleton(x))
+                            }
+                            None => Opaque(OpaqueId::new()),
+                        };
+                        fields = &[];
+                    }
+                    ty::Float(ty::FloatTy::F32) => {
+                        ctor = match value.try_eval_bits(cx.tcx, cx.param_env) {
+                            Some(bits) => {
+                                use rustc_apfloat::Float;
+                                let value = rustc_apfloat::ieee::Single::from_bits(bits);
+                                F32Range(value, value, RangeEnd::Included)
+                            }
+                            None => Opaque(OpaqueId::new()),
+                        };
+                        fields = &[];
+                    }
+                    ty::Float(ty::FloatTy::F64) => {
+                        ctor = match value.try_eval_bits(cx.tcx, cx.param_env) {
+                            Some(bits) => {
+                                use rustc_apfloat::Float;
+                                let value = rustc_apfloat::ieee::Double::from_bits(bits);
+                                F64Range(value, value, RangeEnd::Included)
+                            }
+                            None => Opaque(OpaqueId::new()),
+                        };
+                        fields = &[];
+                    }
+                    ty::Ref(_, t, _) if t.is_str() => {
+                        // We want a `&str` constant to behave like a `Deref` pattern, to be compatible
+                        // with other `Deref` patterns. This could have been done in `const_to_pat`,
+                        // but that causes issues with the rest of the matching code.
+                        // So here, the constructor for a `"foo"` pattern is `&` (represented by
+                        // `Ref`), and has one field. That field has constructor `Str(value)` and no
+                        // subfields.
+                        // Note: `t` is `str`, not `&str`.
+                        let subpattern = DeconstructedPat::new(Str(*value), &[], *t, pat.span);
+                        ctor = Ref;
+                        fields = singleton(subpattern)
+                    }
+                    // All constants that can be structurally matched have already been expanded
+                    // into the corresponding `Pat`s by `const_to_pat`. Constants that remain are
+                    // opaque.
+                    _ => {
+                        ctor = Opaque(OpaqueId::new());
+                        fields = &[];
+                    }
+                }
+            }
+            PatKind::Range(patrange) => {
+                let PatRange { lo, hi, end, .. } = patrange.as_ref();
+                let end = match end {
+                    rustc_hir::RangeEnd::Included => RangeEnd::Included,
+                    rustc_hir::RangeEnd::Excluded => RangeEnd::Excluded,
+                };
+                let ty = pat.ty;
+                ctor = match ty.kind() {
+                    ty::Char | ty::Int(_) | ty::Uint(_) => {
+                        let lo = cx.lower_pat_range_bdy(*lo, ty);
+                        let hi = cx.lower_pat_range_bdy(*hi, ty);
+                        IntRange(IntRange::from_range(lo, hi, end))
+                    }
+                    ty::Float(fty) => {
+                        use rustc_apfloat::Float;
+                        let lo = lo.as_finite().map(|c| c.eval_bits(cx.tcx, cx.param_env));
+                        let hi = hi.as_finite().map(|c| c.eval_bits(cx.tcx, cx.param_env));
+                        match fty {
+                            ty::FloatTy::F32 => {
+                                use rustc_apfloat::ieee::Single;
+                                let lo = lo.map(Single::from_bits).unwrap_or(-Single::INFINITY);
+                                let hi = hi.map(Single::from_bits).unwrap_or(Single::INFINITY);
+                                F32Range(lo, hi, end)
+                            }
+                            ty::FloatTy::F64 => {
+                                use rustc_apfloat::ieee::Double;
+                                let lo = lo.map(Double::from_bits).unwrap_or(-Double::INFINITY);
+                                let hi = hi.map(Double::from_bits).unwrap_or(Double::INFINITY);
+                                F64Range(lo, hi, end)
+                            }
+                        }
+                    }
+                    _ => bug!("invalid type for range pattern: {}", ty),
+                };
+                fields = &[];
+            }
+            PatKind::Array { prefix, slice, suffix } | PatKind::Slice { prefix, slice, suffix } => {
+                let array_len = match pat.ty.kind() {
+                    ty::Array(_, length) => {
+                        Some(length.eval_target_usize(cx.tcx, cx.param_env) as usize)
+                    }
+                    ty::Slice(_) => None,
+                    _ => span_bug!(pat.span, "bad ty {:?} for slice pattern", pat.ty),
+                };
+                let kind = if slice.is_some() {
+                    SliceKind::VarLen(prefix.len(), suffix.len())
+                } else {
+                    SliceKind::FixedLen(prefix.len() + suffix.len())
+                };
+                ctor = Slice(Slice::new(array_len, kind));
+                fields = cx.pattern_arena.alloc_from_iter(
+                    prefix.iter().chain(suffix.iter()).map(|p| self.lower_pat(&*p)),
+                )
+            }
+            PatKind::Or { .. } => {
+                ctor = Or;
+                let pats = expand_or_pat(pat);
+                fields =
+                    cx.pattern_arena.alloc_from_iter(pats.into_iter().map(|p| self.lower_pat(p)))
+            }
+            PatKind::Never => {
+                // FIXME(never_patterns): handle `!` in exhaustiveness. This is a sane default
+                // in the meantime.
+                ctor = Wildcard;
+                fields = &[];
+            }
+            PatKind::Error(_) => {
+                ctor = Opaque(OpaqueId::new());
+                fields = &[];
+            }
+        }
+        DeconstructedPat::new(ctor, fields, pat.ty, pat.span)
+    }
+
+    /// Convert back to a `thir::PatRangeBoundary` for diagnostic purposes.
+    /// Note: it is possible to get `isize/usize::MAX+1` here, as explained in the doc for
+    /// [`IntRange::split`]. This cannot be represented as a `Const`, so we represent it with
+    /// `PosInfinity`.
+    pub(crate) fn hoist_pat_range_bdy(
+        &self,
+        miint: MaybeInfiniteInt,
+        ty: Ty<'tcx>,
+    ) -> PatRangeBoundary<'tcx> {
+        use MaybeInfiniteInt::*;
+        let tcx = self.tcx;
+        match miint {
+            NegInfinity => PatRangeBoundary::NegInfinity,
+            Finite(_) => {
+                let size = ty.primitive_size(tcx);
+                let bits = match *ty.kind() {
+                    ty::Int(_) => miint.as_finite_int(size.bits()).unwrap(),
+                    _ => miint.as_finite_uint().unwrap(),
+                };
+                match Scalar::try_from_uint(bits, size) {
+                    Some(scalar) => {
+                        let value = mir::Const::from_scalar(tcx, scalar, ty);
+                        PatRangeBoundary::Finite(value)
+                    }
+                    // The value doesn't fit. Since `x >= 0` and 0 always encodes the minimum value
+                    // for a type, the problem isn't that the value is too small. So it must be too
+                    // large.
+                    None => PatRangeBoundary::PosInfinity,
+                }
+            }
+            JustAfterMax | PosInfinity => PatRangeBoundary::PosInfinity,
+        }
+    }
+
+    /// Convert back to a `thir::Pat` for diagnostic purposes.
+    pub(crate) fn hoist_pat_range(&self, range: &IntRange, ty: Ty<'tcx>) -> Pat<'tcx> {
+        use MaybeInfiniteInt::*;
+        let cx = self;
+        let kind = if matches!((range.lo, range.hi), (NegInfinity, PosInfinity)) {
+            PatKind::Wild
+        } else if range.is_singleton() {
+            let lo = cx.hoist_pat_range_bdy(range.lo, ty);
+            let value = lo.as_finite().unwrap();
+            PatKind::Constant { value }
+        } else {
+            // We convert to an inclusive range for diagnostics.
+            let mut end = rustc_hir::RangeEnd::Included;
+            let mut lo = cx.hoist_pat_range_bdy(range.lo, ty);
+            if matches!(lo, PatRangeBoundary::PosInfinity) {
+                // The only reason to get `PosInfinity` here is the special case where
+                // `hoist_pat_range_bdy` found `{u,i}size::MAX+1`. So the range denotes the
+                // fictitious values after `{u,i}size::MAX` (see [`IntRange::split`] for why we do
+                // this). We show this to the user as `usize::MAX..` which is slightly incorrect but
+                // probably clear enough.
+                let c = ty.numeric_max_val(cx.tcx).unwrap();
+                let value = mir::Const::from_ty_const(c, cx.tcx);
+                lo = PatRangeBoundary::Finite(value);
+            }
+            let hi = if matches!(range.hi, Finite(0)) {
+                // The range encodes `..ty::MIN`, so we can't convert it to an inclusive range.
+                end = rustc_hir::RangeEnd::Excluded;
+                range.hi
+            } else {
+                range.hi.minus_one()
+            };
+            let hi = cx.hoist_pat_range_bdy(hi, ty);
+            PatKind::Range(Box::new(PatRange { lo, hi, end, ty }))
+        };
+
+        Pat { ty, span: DUMMY_SP, kind }
+    }
+    /// Convert back to a `thir::Pat` for diagnostic purposes. This panics for patterns that don't
+    /// appear in diagnostics, like float ranges.
+    pub fn hoist_witness_pat(&self, pat: &WitnessPat<'p, 'tcx>) -> Pat<'tcx> {
+        let cx = self;
+        let is_wildcard = |pat: &Pat<'_>| matches!(pat.kind, PatKind::Wild);
+        let mut subpatterns = pat.iter_fields().map(|p| Box::new(cx.hoist_witness_pat(p)));
+        let kind = match pat.ctor() {
+            Bool(b) => PatKind::Constant { value: mir::Const::from_bool(cx.tcx, *b) },
+            IntRange(range) => return self.hoist_pat_range(range, pat.ty()),
+            Struct | Variant(_) | UnionField => match pat.ty().kind() {
+                ty::Tuple(..) => PatKind::Leaf {
+                    subpatterns: subpatterns
+                        .enumerate()
+                        .map(|(i, pattern)| FieldPat { field: FieldIdx::new(i), pattern })
+                        .collect(),
+                },
+                ty::Adt(adt_def, _) if adt_def.is_box() => {
+                    // Without `box_patterns`, the only legal pattern of type `Box` is `_` (outside
+                    // of `std`). So this branch is only reachable when the feature is enabled and
+                    // the pattern is a box pattern.
+                    PatKind::Deref { subpattern: subpatterns.next().unwrap() }
+                }
+                ty::Adt(adt_def, args) => {
+                    let variant_index =
+                        RustcMatchCheckCtxt::variant_index_for_adt(&pat.ctor(), *adt_def);
+                    let variant = &adt_def.variant(variant_index);
+                    let subpatterns = cx
+                        .list_variant_nonhidden_fields(pat.ty(), variant)
+                        .zip(subpatterns)
+                        .map(|((field, _ty), pattern)| FieldPat { field, pattern })
+                        .collect();
+
+                    if adt_def.is_enum() {
+                        PatKind::Variant { adt_def: *adt_def, args, variant_index, subpatterns }
+                    } else {
+                        PatKind::Leaf { subpatterns }
+                    }
+                }
+                _ => bug!("unexpected ctor for type {:?} {:?}", pat.ctor(), pat.ty()),
+            },
+            // Note: given the expansion of `&str` patterns done in `expand_pattern`, we should
+            // be careful to reconstruct the correct constant pattern here. However a string
+            // literal pattern will never be reported as a non-exhaustiveness witness, so we
+            // ignore this issue.
+            Ref => PatKind::Deref { subpattern: subpatterns.next().unwrap() },
+            Slice(slice) => {
+                match slice.kind {
+                    SliceKind::FixedLen(_) => PatKind::Slice {
+                        prefix: subpatterns.collect(),
+                        slice: None,
+                        suffix: Box::new([]),
+                    },
+                    SliceKind::VarLen(prefix, _) => {
+                        let mut subpatterns = subpatterns.peekable();
+                        let mut prefix: Vec<_> = subpatterns.by_ref().take(prefix).collect();
+                        if slice.array_len.is_some() {
+                            // Improves diagnostics a bit: if the type is a known-size array, instead
+                            // of reporting `[x, _, .., _, y]`, we prefer to report `[x, .., y]`.
+                            // This is incorrect if the size is not known, since `[_, ..]` captures
+                            // arrays of lengths `>= 1` whereas `[..]` captures any length.
+                            while !prefix.is_empty() && is_wildcard(prefix.last().unwrap()) {
+                                prefix.pop();
+                            }
+                            while subpatterns.peek().is_some()
+                                && is_wildcard(subpatterns.peek().unwrap())
+                            {
+                                subpatterns.next();
+                            }
+                        }
+                        let suffix: Box<[_]> = subpatterns.collect();
+                        let wild = Pat::wildcard_from_ty(pat.ty());
+                        PatKind::Slice {
+                            prefix: prefix.into_boxed_slice(),
+                            slice: Some(Box::new(wild)),
+                            suffix,
+                        }
+                    }
+                }
+            }
+            &Str(value) => PatKind::Constant { value },
+            Wildcard | NonExhaustive | Hidden => PatKind::Wild,
+            Missing { .. } => bug!(
+                "trying to convert a `Missing` constructor into a `Pat`; this is probably a bug,
+                `Missing` should have been processed in `apply_constructors`"
+            ),
+            F32Range(..) | F64Range(..) | Opaque(..) | Or => {
+                bug!("can't convert to pattern: {:?}", pat)
+            }
+        };
+
+        Pat { ty: pat.ty(), span: DUMMY_SP, kind }
+    }
+
+    /// Best-effort `Debug` implementation.
+    pub(crate) fn debug_pat(
+        f: &mut fmt::Formatter<'_>,
+        pat: &crate::pat::DeconstructedPat<'_, Self>,
+    ) -> fmt::Result {
+        let mut first = true;
+        let mut start_or_continue = |s| {
+            if first {
+                first = false;
+                ""
+            } else {
+                s
+            }
+        };
+        let mut start_or_comma = || start_or_continue(", ");
+
+        match pat.ctor() {
+            Struct | Variant(_) | UnionField => match pat.ty().kind() {
+                ty::Adt(def, _) if def.is_box() => {
+                    // Without `box_patterns`, the only legal pattern of type `Box` is `_` (outside
+                    // of `std`). So this branch is only reachable when the feature is enabled and
+                    // the pattern is a box pattern.
+                    let subpattern = pat.iter_fields().next().unwrap();
+                    write!(f, "box {subpattern:?}")
+                }
+                ty::Adt(..) | ty::Tuple(..) => {
+                    let variant =
+                        match pat.ty().kind() {
+                            ty::Adt(adt, _) => Some(adt.variant(
+                                RustcMatchCheckCtxt::variant_index_for_adt(pat.ctor(), *adt),
+                            )),
+                            ty::Tuple(_) => None,
+                            _ => unreachable!(),
+                        };
+
+                    if let Some(variant) = variant {
+                        write!(f, "{}", variant.name)?;
+                    }
+
+                    // Without `cx`, we can't know which field corresponds to which, so we can't
+                    // get the names of the fields. Instead we just display everything as a tuple
+                    // struct, which should be good enough.
+                    write!(f, "(")?;
+                    for p in pat.iter_fields() {
+                        write!(f, "{}", start_or_comma())?;
+                        write!(f, "{p:?}")?;
+                    }
+                    write!(f, ")")
+                }
+                _ => write!(f, "_"),
+            },
+            // Note: given the expansion of `&str` patterns done in `expand_pattern`, we should
+            // be careful to detect strings here. However a string literal pattern will never
+            // be reported as a non-exhaustiveness witness, so we can ignore this issue.
+            Ref => {
+                let subpattern = pat.iter_fields().next().unwrap();
+                write!(f, "&{:?}", subpattern)
+            }
+            Slice(slice) => {
+                let mut subpatterns = pat.iter_fields();
+                write!(f, "[")?;
+                match slice.kind {
+                    SliceKind::FixedLen(_) => {
+                        for p in subpatterns {
+                            write!(f, "{}{:?}", start_or_comma(), p)?;
+                        }
+                    }
+                    SliceKind::VarLen(prefix_len, _) => {
+                        for p in subpatterns.by_ref().take(prefix_len) {
+                            write!(f, "{}{:?}", start_or_comma(), p)?;
+                        }
+                        write!(f, "{}", start_or_comma())?;
+                        write!(f, "..")?;
+                        for p in subpatterns {
+                            write!(f, "{}{:?}", start_or_comma(), p)?;
+                        }
+                    }
+                }
+                write!(f, "]")
+            }
+            Bool(b) => write!(f, "{b}"),
+            // Best-effort, will render signed ranges incorrectly
+            IntRange(range) => write!(f, "{range:?}"),
+            F32Range(lo, hi, end) => write!(f, "{lo}{end}{hi}"),
+            F64Range(lo, hi, end) => write!(f, "{lo}{end}{hi}"),
+            Str(value) => write!(f, "{value}"),
+            Opaque(..) => write!(f, "<constant pattern>"),
+            Or => {
+                for pat in pat.iter_fields() {
+                    write!(f, "{}{:?}", start_or_continue(" | "), pat)?;
+                }
+                Ok(())
+            }
+            Wildcard | Missing { .. } | NonExhaustive | Hidden => write!(f, "_ : {:?}", pat.ty()),
+        }
+    }
+}
+
+impl<'p, 'tcx> TypeCx for RustcMatchCheckCtxt<'p, 'tcx> {
+    type Ty = Ty<'tcx>;
+    type VariantIdx = VariantIdx;
+    type StrLit = Const<'tcx>;
+    type ArmData = HirId;
+    type PatData = Span;
+
+    fn is_exhaustive_patterns_feature_on(&self) -> bool {
+        self.tcx.features().exhaustive_patterns
+    }
+    fn is_opaque_ty(ty: Self::Ty) -> bool {
+        matches!(ty.kind(), ty::Alias(ty::Opaque, ..))
+    }
+
+    fn ctor_arity(&self, ctor: &crate::constructor::Constructor<Self>, ty: Self::Ty) -> usize {
+        self.ctor_arity(ctor, ty)
+    }
+    fn ctor_sub_tys(
+        &self,
+        ctor: &crate::constructor::Constructor<Self>,
+        ty: Self::Ty,
+    ) -> &[Self::Ty] {
+        self.ctor_sub_tys(ctor, ty)
+    }
+    fn ctors_for_ty(&self, ty: Self::Ty) -> crate::constructor::ConstructorSet<Self> {
+        self.ctors_for_ty(ty)
+    }
+
+    fn debug_pat(
+        f: &mut fmt::Formatter<'_>,
+        pat: &crate::pat::DeconstructedPat<'_, Self>,
+    ) -> fmt::Result {
+        Self::debug_pat(f, pat)
+    }
+    fn bug(&self, fmt: fmt::Arguments<'_>) -> ! {
+        span_bug!(self.scrut_span, "{}", fmt)
+    }
+}
+
+/// Recursively expand this pattern into its subpatterns. Only useful for or-patterns.
+fn expand_or_pat<'p, 'tcx>(pat: &'p Pat<'tcx>) -> Vec<&'p Pat<'tcx>> {
+    fn expand<'p, 'tcx>(pat: &'p Pat<'tcx>, vec: &mut Vec<&'p Pat<'tcx>>) {
+        if let PatKind::Or { pats } = &pat.kind {
+            for pat in pats.iter() {
+                expand(pat, vec);
+            }
+        } else {
+            vec.push(pat)
+        }
+    }
+
+    let mut pats = Vec::new();
+    expand(pat, &mut pats);
+    pats
+}
diff --git a/compiler/rustc_pattern_analysis/src/usefulness.rs b/compiler/rustc_pattern_analysis/src/usefulness.rs
new file mode 100644
index 000000000..a9e74eff8
--- /dev/null
+++ b/compiler/rustc_pattern_analysis/src/usefulness.rs
@@ -0,0 +1,1508 @@
+//! # Match exhaustiveness and redundancy algorithm
+//!
+//! This file contains the logic for exhaustiveness and usefulness checking for pattern-matching.
+//! Specifically, given a list of patterns in a match, we can tell whether:
+//! (a) a given pattern is redundant
+//! (b) the patterns cover every possible value for the type (exhaustiveness)
+//!
+//! The algorithm implemented here is inspired from the one described in [this
+//! paper](http://moscova.inria.fr/~maranget/papers/warn/index.html). We have however changed it in
+//! various ways to accommodate the variety of patterns that Rust supports. We thus explain our
+//! version here, without being as precise.
+//!
+//! Fun fact: computing exhaustiveness is NP-complete, because we can encode a SAT problem as an
+//! exhaustiveness problem. See [here](https://niedzejkob.p4.team/rust-np) for the fun details.
+//!
+//!
+//! # Summary
+//!
+//! The algorithm is given as input a list of patterns, one for each arm of a match, and computes
+//! the following:
+//! - a set of values that match none of the patterns (if any),
+//! - for each subpattern (taking into account or-patterns), whether removing it would change
+//!     anything about how the match executes, i.e. whether it is useful/not redundant.
+//!
+//! To a first approximation, the algorithm works by exploring all possible values for the type
+//! being matched on, and determining which arm(s) catch which value. To make this tractable we
+//! cleverly group together values, as we'll see below.
+//!
+//! The entrypoint of this file is the [`compute_match_usefulness`] function, which computes
+//! usefulness for each subpattern and exhaustiveness for the whole match.
+//!
+//! In this page we explain the necessary concepts to understand how the algorithm works.
+//!
+//!
+//! # Usefulness
+//!
+//! The central concept of this file is the notion of "usefulness". Given some patterns `p_1 ..
+//! p_n`, a pattern `q` is said to be *useful* if there is a value that is matched by `q` and by
+//! none of the `p_i`. We write `usefulness(p_1 .. p_n, q)` for a function that returns a list of
+//! such values. The aim of this file is to compute it efficiently.
+//!
+//! This is enough to compute usefulness: a pattern in a `match` expression is redundant iff it is
+//! not useful w.r.t. the patterns above it:
+//! ```compile_fail,E0004
+//! # #![feature(exclusive_range_pattern)]
+//! # fn foo() {
+//! match Some(0u32) {
+//!     Some(0..100) => {},
+//!     Some(90..190) => {}, // useful: `Some(150)` is matched by this but not the branch above
+//!     Some(50..150) => {}, // redundant: all the values this matches are already matched by
+//!                          //   the branches above
+//!     None => {},          // useful: `None` is matched by this but not the branches above
+//! }
+//! # }
+//! ```
+//!
+//! This is also enough to compute exhaustiveness: a match is exhaustive iff the wildcard `_`
+//! pattern is _not_ useful w.r.t. the patterns in the match. The values returned by `usefulness`
+//! are used to tell the user which values are missing.
+//! ```compile_fail,E0004
+//! # fn foo(x: Option<u32>) {
+//! match x {
+//!     None => {},
+//!     Some(0) => {},
+//!     // not exhaustive: `_` is useful because it matches `Some(1)`
+//! }
+//! # }
+//! ```
+//!
+//!
+//! # Constructors and fields
+//!
+//! In the value `Pair(Some(0), true)`, `Pair` is called the constructor of the value, and `Some(0)`
+//! and `true` are its fields. Every matcheable value can be decomposed in this way. Examples of
+//! constructors are: `Some`, `None`, `(,)` (the 2-tuple constructor), `Foo {..}` (the constructor
+//! for a struct `Foo`), and `2` (the constructor for the number `2`).
+//!
+//! Each constructor takes a fixed number of fields; this is called its arity. `Pair` and `(,)` have
+//! arity 2, `Some` has arity 1, `None` and `42` have arity 0. Each type has a known set of
+//! constructors. Some types have many constructors (like `u64`) or even an infinitely many (like
+//! `&str` and `&[T]`).
+//!
+//! Patterns are similar: `Pair(Some(_), _)` has constructor `Pair` and two fields. The difference
+//! is that we get some extra pattern-only constructors, namely: the wildcard `_`, variable
+//! bindings, integer ranges like `0..=10`, and variable-length slices like `[_, .., _]`. We treat
+//! or-patterns separately, see the dedicated section below.
+//!
+//! Now to check if a value `v` matches a pattern `p`, we check if `v`'s constructor matches `p`'s
+//! constructor, then recursively compare their fields if necessary. A few representative examples:
+//!
+//! - `matches!(v, _) := true`
+//! - `matches!((v0,  v1), (p0,  p1)) := matches!(v0, p0) && matches!(v1, p1)`
+//! - `matches!(Foo { bar: v0, baz: v1 }, Foo { bar: p0, baz: p1 }) := matches!(v0, p0) && matches!(v1, p1)`
+//! - `matches!(Ok(v0), Ok(p0)) := matches!(v0, p0)`
+//! - `matches!(Ok(v0), Err(p0)) := false` (incompatible variants)
+//! - `matches!(v, 1..=100) := matches!(v, 1) || ... || matches!(v, 100)`
+//! - `matches!([v0], [p0, .., p1]) := false` (incompatible lengths)
+//! - `matches!([v0, v1, v2], [p0, .., p1]) := matches!(v0, p0) && matches!(v2, p1)`
+//!
+//! Constructors and relevant operations are defined in the [`crate::constructor`] module. A
+//! representation of patterns that uses constructors is available in [`crate::pat`]. The question
+//! of whether a constructor is matched by another one is answered by
+//! [`Constructor::is_covered_by`].
+//!
+//! Note 1: variable bindings (like the `x` in `Some(x)`) match anything, so we treat them as wildcards.
+//! Note 2: this only applies to matcheable values. For example a value of type `Rc<u64>` can't be
+//! deconstructed that way.
+//!
+//!
+//!
+//! # Specialization
+//!
+//! The examples in the previous section motivate the operation at the heart of the algorithm:
+//! "specialization". It captures this idea of "removing one layer of constructor".
+//!
+//! `specialize(c, p)` takes a value-only constructor `c` and a pattern `p`, and returns a
+//! pattern-tuple or nothing. It works as follows:
+//!
+//! - Specializing for the wrong constructor returns nothing
+//!
+//!   - `specialize(None, Some(p0)) := <nothing>`
+//!   - `specialize([,,,], [p0]) := <nothing>`
+//!
+//! - Specializing for the correct constructor returns a tuple of the fields
+//!
+//!   - `specialize(Variant1, Variant1(p0, p1, p2)) := (p0, p1, p2)`
+//!   - `specialize(Foo{ bar, baz, quz }, Foo { bar: p0, baz: p1, .. }) := (p0, p1, _)`
+//!   - `specialize([,,,], [p0, .., p1]) := (p0, _, _, p1)`
+//!
+//! We get the following property: for any values `v_1, .., v_n` of appropriate types, we have:
+//! ```text
+//! matches!(c(v_1, .., v_n), p)
+//! <=> specialize(c, p) returns something
+//!     && matches!((v_1, .., v_n), specialize(c, p))
+//! ```
+//!
+//! We also extend specialization to pattern-tuples by applying it to the first pattern:
+//! `specialize(c, (p_0, .., p_n)) := specialize(c, p_0) ++ (p_1, .., p_m)`
+//! where `++` is concatenation of tuples.
+//!
+//!
+//! The previous property extends to pattern-tuples:
+//! ```text
+//! matches!((c(v_1, .., v_n), w_1, .., w_m), (p_0, p_1, .., p_m))
+//! <=> specialize(c, p_0) does not error
+//!     && matches!((v_1, .., v_n, w_1, .., w_m), specialize(c, (p_0, p_1, .., p_m)))
+//! ```
+//!
+//! Whether specialization returns something or not is given by [`Constructor::is_covered_by`].
+//! Specialization of a pattern is computed in [`DeconstructedPat::specialize`]. Specialization for
+//! a pattern-tuple is computed in [`PatStack::pop_head_constructor`]. Finally, specialization for a
+//! set of pattern-tuples is computed in [`Matrix::specialize_constructor`].
+//!
+//!
+//!
+//! # Undoing specialization
+//!
+//! To construct witnesses we will need an inverse of specialization. If `c` is a constructor of
+//! arity `n`, we define `unspecialize` as:
+//! `unspecialize(c, (p_1, .., p_n, q_1, .., q_m)) := (c(p_1, .., p_n), q_1, .., q_m)`.
+//!
+//! This is done for a single witness-tuple in [`WitnessStack::apply_constructor`], and for a set of
+//! witness-tuples in [`WitnessMatrix::apply_constructor`].
+//!
+//!
+//!
+//! # Computing usefulness
+//!
+//! We now present a naive version of the algorithm for computing usefulness. From now on we operate
+//! on pattern-tuples.
+//!
+//! Let `pt_1, .., pt_n` and `qt` be length-m tuples of patterns for the same type `(T_1, .., T_m)`.
+//! We compute `usefulness(tp_1, .., tp_n, tq)` as follows:
+//!
+//! - Base case: `m == 0`.
+//!     The pattern-tuples are all empty, i.e. they're all `()`. Thus `tq` is useful iff there are
+//!     no rows above it, i.e. if `n == 0`. In that case we return `()` as a witness-tuple of
+//!     usefulness of `tq`.
+//!
+//! - Inductive case: `m > 0`.
+//!     In this naive version, we list all the possible constructors for values of type `T1` (we
+//!     will be more clever in the next section).
+//!
+//!     - For each such constructor `c` for which `specialize(c, tq)` is not nothing:
+//!         - We recursively compute `usefulness(specialize(c, tp_1) ... specialize(c, tp_n), specialize(c, tq))`,
+//!             where we discard any `specialize(c, p_i)` that returns nothing.
+//!         - For each witness-tuple `w` found, we apply `unspecialize(c, w)` to it.
+//!
+//!     - We return the all the witnesses found, if any.
+//!
+//!
+//! Let's take the following example:
+//! ```compile_fail,E0004
+//! # enum Enum { Variant1(()), Variant2(Option<bool>, u32)}
+//! # use Enum::*;
+//! # fn foo(x: Enum) {
+//! match x {
+//!     Variant1(_) => {} // `p1`
+//!     Variant2(None, 0) => {} // `p2`
+//!     Variant2(Some(_), 0) => {} // `q`
+//! }
+//! # }
+//! ```
+//!
+//! To compute the usefulness of `q`, we would proceed as follows:
+//! ```text
+//! Start:
+//!   `tp1 = [Variant1(_)]`
+//!   `tp2 = [Variant2(None, 0)]`
+//!   `tq  = [Variant2(Some(true), 0)]`
+//!
+//!   Constructors are `Variant1` and `Variant2`. Only `Variant2` can specialize `tq`.
+//!   Specialize with `Variant2`:
+//!     `tp2 = [None, 0]`
+//!     `tq  = [Some(true), 0]`
+//!
+//!     Constructors are `None` and `Some`. Only `Some` can specialize `tq`.
+//!     Specialize with `Some`:
+//!       `tq  = [true, 0]`
+//!
+//!       Constructors are `false` and `true`. Only `true` can specialize `tq`.
+//!       Specialize with `true`:
+//!         `tq  = [0]`
+//!
+//!         Constructors are `0`, `1`, .. up to infinity. Only `0` can specialize `tq`.
+//!         Specialize with `0`:
+//!           `tq  = []`
+//!
+//!           m == 0 and n == 0, so `tq` is useful with witness `[]`.
+//!             `witness  = []`
+//!
+//!         Unspecialize with `0`:
+//!           `witness  = [0]`
+//!       Unspecialize with `true`:
+//!         `witness  = [true, 0]`
+//!     Unspecialize with `Some`:
+//!       `witness  = [Some(true), 0]`
+//!   Unspecialize with `Variant2`:
+//!     `witness  = [Variant2(Some(true), 0)]`
+//! ```
+//!
+//! Therefore `usefulness(tp_1, tp_2, tq)` returns the single witness-tuple `[Variant2(Some(true), 0)]`.
+//!
+//!
+//! Computing the set of constructors for a type is done in [`TypeCx::ctors_for_ty`]. See
+//! the following sections for more accurate versions of the algorithm and corresponding links.
+//!
+//!
+//!
+//! # Computing usefulness and exhaustiveness in one go
+//!
+//! The algorithm we have described so far computes usefulness of each pattern in turn, and ends by
+//! checking if `_` is useful to determine exhaustiveness of the whole match. In practice, instead
+//! of doing "for each pattern { for each constructor { ... } }", we do "for each constructor { for
+//! each pattern { ... } }". This allows us to compute everything in one go.
+//!
+//! [`Matrix`] stores the set of pattern-tuples under consideration. We track usefulness of each
+//! row mutably in the matrix as we go along. We ignore witnesses of usefulness of the match rows.
+//! We gather witnesses of the usefulness of `_` in [`WitnessMatrix`]. The algorithm that computes
+//! all this is in [`compute_exhaustiveness_and_usefulness`].
+//!
+//! See the full example at the bottom of this documentation.
+//!
+//!
+//!
+//! # Making usefulness tractable: constructor splitting
+//!
+//! We're missing one last detail: which constructors do we list? Naively listing all value
+//! constructors cannot work for types like `u64` or `&str`, so we need to be more clever. The final
+//! clever idea for this algorithm is that we can group together constructors that behave the same.
+//!
+//! Examples:
+//! ```compile_fail,E0004
+//! match (0, false) {
+//!     (0 ..=100, true) => {}
+//!     (50..=150, false) => {}
+//!     (0 ..=200, _) => {}
+//! }
+//! ```
+//!
+//! In this example, trying any of `0`, `1`, .., `49` will give the same specialized matrix, and
+//! thus the same usefulness/exhaustiveness results. We can thus accelerate the algorithm by
+//! trying them all at once. Here in fact, the only cases we need to consider are: `0..50`,
+//! `50..=100`, `101..=150`,`151..=200` and `201..`.
+//!
+//! ```
+//! enum Direction { North, South, East, West }
+//! # let wind = (Direction::North, 0u8);
+//! match wind {
+//!     (Direction::North, 50..) => {}
+//!     (_, _) => {}
+//! }
+//! ```
+//!
+//! In this example, trying any of `South`, `East`, `West` will give the same specialized matrix. By
+//! the same reasoning, we only need to try two cases: `North`, and "everything else".
+//!
+//! We call _constructor splitting_ the operation that computes such a minimal set of cases to try.
+//! This is done in [`ConstructorSet::split`] and explained in [`crate::constructor`].
+//!
+//!
+//!
+//! # `Missing` and relevancy
+//!
+//! ## Relevant values
+//!
+//! Take the following example:
+//!
+//! ```compile_fail,E0004
+//! # let foo = (true, true);
+//! match foo {
+//!     (true, _) => 1,
+//!     (_, true) => 2,
+//! };
+//! ```
+//!
+//! Consider the value `(true, true)`:
+//! - Row 2 does not distinguish `(true, true)` and `(false, true)`;
+//! - `false` does not show up in the first column of the match, so without knowing anything else we
+//!     can deduce that `(false, true)` matches the same or fewer rows than `(true, true)`.
+//!
+//! Using those two facts together, we deduce that `(true, true)` will not give us more usefulness
+//! information about row 2 than `(false, true)` would. We say that "`(true, true)` is made
+//! irrelevant for row 2 by `(false, true)`". We will use this idea to prune the search tree.
+//!
+//!
+//! ## Computing relevancy
+//!
+//! We now generalize from the above example to approximate relevancy in a simple way. Note that we
+//! will only compute an approximation: we can sometimes determine when a case is irrelevant, but
+//! computing this precisely is at least as hard as computing usefulness.
+//!
+//! Our computation of relevancy relies on the `Missing` constructor. As explained in
+//! [`crate::constructor`], `Missing` represents the constructors not present in a given column. For
+//! example in the following:
+//!
+//! ```compile_fail,E0004
+//! enum Direction { North, South, East, West }
+//! # let wind = (Direction::North, 0u8);
+//! match wind {
+//!     (Direction::North, _) => 1,
+//!     (_, 50..) => 2,
+//! };
+//! ```
+//!
+//! Here `South`, `East` and `West` are missing in the first column, and `0..50`  is missing in the
+//! second. Both of these sets are represented by `Constructor::Missing` in their corresponding
+//! column.
+//!
+//! We then compute relevancy as follows: during the course of the algorithm, for a row `r`:
+//! - if `r` has a wildcard in the first column;
+//! - and some constructors are missing in that column;
+//! - then any `c != Missing` is considered irrelevant for row `r`.
+//!
+//! By this we mean that continuing the algorithm by specializing with `c` is guaranteed not to
+//! contribute more information about the usefulness of row `r` than what we would get by
+//! specializing with `Missing`. The argument is the same as in the previous subsection.
+//!
+//! Once we've specialized by a constructor `c` that is irrelevant for row `r`, we're guaranteed to
+//! only explore values irrelevant for `r`. If we then ever reach a point where we're only exploring
+//! values that are irrelevant to all of the rows (including the virtual wildcard row used for
+//! exhaustiveness), we skip that case entirely.
+//!
+//!
+//! ## Example
+//!
+//! Let's go through a variation on the first example:
+//!
+//! ```compile_fail,E0004
+//! # let foo = (true, true, true);
+//! match foo {
+//!     (true, _, true) => 1,
+//!     (_, true, _) => 2,
+//! };
+//! ```
+//!
+//! ```text
+//!  ┐ Patterns:
+//!  │   1. `[(true, _, true)]`
+//!  │   2. `[(_, true, _)]`
+//!  │   3. `[_]` // virtual extra wildcard row
+//!  │
+//!  │ Specialize with `(,,)`:
+//!  ├─┐ Patterns:
+//!  │ │   1. `[true, _, true]`
+//!  │ │   2. `[_, true, _]`
+//!  │ │   3. `[_, _, _]`
+//!  │ │
+//!  │ │ There are missing constructors in the first column (namely `false`), hence
+//!  │ │ `true` is irrelevant for rows 2 and 3.
+//!  │ │
+//!  │ │ Specialize with `true`:
+//!  │ ├─┐ Patterns:
+//!  │ │ │   1. `[_, true]`
+//!  │ │ │   2. `[true, _]` // now exploring irrelevant cases
+//!  │ │ │   3. `[_, _]`    // now exploring irrelevant cases
+//!  │ │ │
+//!  │ │ │ There are missing constructors in the first column (namely `false`), hence
+//!  │ │ │ `true` is irrelevant for rows 1 and 3.
+//!  │ │ │
+//!  │ │ │ Specialize with `true`:
+//!  │ │ ├─┐ Patterns:
+//!  │ │ │ │   1. `[true]` // now exploring irrelevant cases
+//!  │ │ │ │   2. `[_]`    // now exploring irrelevant cases
+//!  │ │ │ │   3. `[_]`    // now exploring irrelevant cases
+//!  │ │ │ │
+//!  │ │ │ │ The current case is irrelevant for all rows: we backtrack immediately.
+//!  │ │ ├─┘
+//!  │ │ │
+//!  │ │ │ Specialize with `false`:
+//!  │ │ ├─┐ Patterns:
+//!  │ │ │ │   1. `[true]`
+//!  │ │ │ │   3. `[_]`    // now exploring irrelevant cases
+//!  │ │ │ │
+//!  │ │ │ │ Specialize with `true`:
+//!  │ │ │ ├─┐ Patterns:
+//!  │ │ │ │ │   1. `[]`
+//!  │ │ │ │ │   3. `[]`    // now exploring irrelevant cases
+//!  │ │ │ │ │
+//!  │ │ │ │ │ Row 1 is therefore useful.
+//!  │ │ │ ├─┘
+//! <etc...>
+//! ```
+//!
+//! Relevancy allowed us to skip the case `(true, true, _)` entirely. In some cases this pruning can
+//! give drastic speedups. The case this was built for is the following (#118437):
+//!
+//! ```ignore(illustrative)
+//! match foo {
+//!     (true, _, _, _, ..) => 1,
+//!     (_, true, _, _, ..) => 2,
+//!     (_, _, true, _, ..) => 3,
+//!     (_, _, _, true, ..) => 4,
+//!     ...
+//! }
+//! ```
+//!
+//! Without considering relevancy, we would explore all 2^n combinations of the `true` and `Missing`
+//! constructors. Relevancy tells us that e.g. `(true, true, false, false, false, ...)` is
+//! irrelevant for all the rows. This allows us to skip all cases with more than one `true`
+//! constructor, changing the runtime from exponential to linear.
+//!
+//!
+//! ## Relevancy and exhaustiveness
+//!
+//! For exhaustiveness, we do something slightly different w.r.t relevancy: we do not report
+//! witnesses of non-exhaustiveness that are irrelevant for the virtual wildcard row. For example,
+//! in:
+//!
+//! ```ignore(illustrative)
+//! match foo {
+//!     (true, true) => {}
+//! }
+//! ```
+//!
+//! we only report `(false, _)` as missing. This was a deliberate choice made early in the
+//! development of rust, for diagnostic and performance purposes. As showed in the previous section,
+//! ignoring irrelevant cases preserves usefulness, so this choice still correctly computes whether
+//! a match is exhaustive.
+//!
+//!
+//!
+//! # Or-patterns
+//!
+//! What we have described so far works well if there are no or-patterns. To handle them, if the
+//! first pattern of a row in the matrix is an or-pattern, we expand it by duplicating the rest of
+//! the row as necessary. This is handled automatically in [`Matrix`].
+//!
+//! This makes usefulness tracking subtle, because we also want to compute whether an alternative
+//! of an or-pattern is redundant, e.g. in `Some(_) | Some(0)`. We track usefulness of each
+//! subpattern by interior mutability in [`DeconstructedPat`] with `set_useful`/`is_useful`.
+//!
+//! It's unfortunate that we have to use interior mutability, but believe me (Nadrieril), I have
+//! tried [other](https://github.com/rust-lang/rust/pull/80104)
+//! [solutions](https://github.com/rust-lang/rust/pull/80632) and nothing is remotely as simple.
+//!
+//!
+//!
+//! # Constants and opaques
+//!
+//! There are two kinds of constants in patterns:
+//!
+//! * literals (`1`, `true`, `"foo"`)
+//! * named or inline consts (`FOO`, `const { 5 + 6 }`)
+//!
+//! The latter are converted into the corresponding patterns by a previous phase. For example
+//! `const_to_pat(const { [1, 2, 3] })` becomes an `Array(vec![Const(1), Const(2), Const(3)])`
+//! pattern. This gets problematic when comparing the constant via `==` would behave differently
+//! from matching on the constant converted to a pattern. The situation around this is currently
+//! unclear and the lang team is working on clarifying what we want to do there. In any case, there
+//! are constants we will not turn into patterns. We capture these with `Constructor::Opaque`. These
+//! `Opaque` patterns do not participate in exhaustiveness, specialization or overlap checking.
+//!
+//!
+//!
+//! # Usefulness vs reachability, validity, and empty patterns
+//!
+//! This is likely the subtlest aspect of the algorithm. To be fully precise, a match doesn't
+//! operate on a value, it operates on a place. In certain unsafe circumstances, it is possible for
+//! a place to not contain valid data for its type. This has subtle consequences for empty types.
+//! Take the following:
+//!
+//! ```rust
+//! enum Void {}
+//! let x: u8 = 0;
+//! let ptr: *const Void = &x as *const u8 as *const Void;
+//! unsafe {
+//!     match *ptr {
+//!         _ => println!("Reachable!"),
+//!     }
+//! }
+//! ```
+//!
+//! In this example, `ptr` is a valid pointer pointing to a place with invalid data. The `_` pattern
+//! does not look at the contents of `*ptr`, so this is ok and the arm is taken. In other words,
+//! despite the place we are inspecting being of type `Void`, there is a reachable arm. If the
+//! arm had a binding however:
+//!
+//! ```rust
+//! # #[derive(Copy, Clone)]
+//! # enum Void {}
+//! # let x: u8 = 0;
+//! # let ptr: *const Void = &x as *const u8 as *const Void;
+//! # unsafe {
+//! match *ptr {
+//!     _a => println!("Unreachable!"),
+//! }
+//! # }
+//! ```
+//!
+//! Here the binding loads the value of type `Void` from the `*ptr` place. In this example, this
+//! causes UB since the data is not valid. In the general case, this asserts validity of the data at
+//! `*ptr`. Either way, this arm will never be taken.
+//!
+//! Finally, let's consider the empty match `match *ptr {}`. If we consider this exhaustive, then
+//! having invalid data at `*ptr` is invalid. In other words, the empty match is semantically
+//! equivalent to the `_a => ...` match. In the interest of explicitness, we prefer the case with an
+//! arm, hence we won't tell the user to remove the `_a` arm. In other words, the `_a` arm is
+//! unreachable yet not redundant. This is why we lint on redundant arms rather than unreachable
+//! arms, despite the fact that the lint says "unreachable".
+//!
+//! These considerations only affects certain places, namely those that can contain non-valid data
+//! without UB. These are: pointer dereferences, reference dereferences, and union field accesses.
+//! We track in the algorithm whether a given place is known to contain valid data. This is done
+//! first by inspecting the scrutinee syntactically (which gives us `cx.known_valid_scrutinee`), and
+//! then by tracking validity of each column of the matrix (which correspond to places) as we
+//! recurse into subpatterns. That second part is done through [`ValidityConstraint`], most notably
+//! [`ValidityConstraint::specialize`].
+//!
+//! Having said all that, in practice we don't fully follow what's been presented in this section.
+//! Under `exhaustive_patterns`, we allow omitting empty arms even in `!known_valid` places, for
+//! backwards-compatibility until we have a better alternative. Without `exhaustive_patterns`, we
+//! mostly treat empty types as inhabited, except specifically a non-nested `!` or empty enum. In
+//! this specific case we also allow the empty match regardless of place validity, for
+//! backwards-compatibility. Hopefully we can eventually deprecate this.
+//!
+//!
+//!
+//! # Full example
+//!
+//! We illustrate a full run of the algorithm on the following match.
+//!
+//! ```compile_fail,E0004
+//! # struct Pair(Option<u32>, bool);
+//! # fn foo(x: Pair) -> u32 {
+//! match x {
+//!     Pair(Some(0), _) => 1,
+//!     Pair(_, false) => 2,
+//!     Pair(Some(0), false) => 3,
+//! }
+//! # }
+//! ```
+//!
+//! We keep track of the original row for illustration purposes, this is not what the algorithm
+//! actually does (it tracks usefulness as a boolean on each row).
+//!
+//! ```text
+//!  ┐ Patterns:
+//!  │   1. `[Pair(Some(0), _)]`
+//!  │   2. `[Pair(_, false)]`
+//!  │   3. `[Pair(Some(0), false)]`
+//!  │
+//!  │ Specialize with `Pair`:
+//!  ├─┐ Patterns:
+//!  │ │   1. `[Some(0), _]`
+//!  │ │   2. `[_, false]`
+//!  │ │   3. `[Some(0), false]`
+//!  │ │
+//!  │ │ Specialize with `Some`:
+//!  │ ├─┐ Patterns:
+//!  │ │ │   1. `[0, _]`
+//!  │ │ │   2. `[_, false]`
+//!  │ │ │   3. `[0, false]`
+//!  │ │ │
+//!  │ │ │ Specialize with `0`:
+//!  │ │ ├─┐ Patterns:
+//!  │ │ │ │   1. `[_]`
+//!  │ │ │ │   3. `[false]`
+//!  │ │ │ │
+//!  │ │ │ │ Specialize with `true`:
+//!  │ │ │ ├─┐ Patterns:
+//!  │ │ │ │ │   1. `[]`
+//!  │ │ │ │ │
+//!  │ │ │ │ │ We note arm 1 is useful (by `Pair(Some(0), true)`).
+//!  │ │ │ ├─┘
+//!  │ │ │ │
+//!  │ │ │ │ Specialize with `false`:
+//!  │ │ │ ├─┐ Patterns:
+//!  │ │ │ │ │   1. `[]`
+//!  │ │ │ │ │   3. `[]`
+//!  │ │ │ │ │
+//!  │ │ │ │ │ We note arm 1 is useful (by `Pair(Some(0), false)`).
+//!  │ │ │ ├─┘
+//!  │ │ ├─┘
+//!  │ │ │
+//!  │ │ │ Specialize with `1..`:
+//!  │ │ ├─┐ Patterns:
+//!  │ │ │ │   2. `[false]`
+//!  │ │ │ │
+//!  │ │ │ │ Specialize with `true`:
+//!  │ │ │ ├─┐ Patterns:
+//!  │ │ │ │ │   // no rows left
+//!  │ │ │ │ │
+//!  │ │ │ │ │ We have found an unmatched value (`Pair(Some(1..), true)`)! This gives us a witness.
+//!  │ │ │ │ │ New witnesses:
+//!  │ │ │ │ │   `[]`
+//!  │ │ │ ├─┘
+//!  │ │ │ │ Unspecialize new witnesses with `true`:
+//!  │ │ │ │   `[true]`
+//!  │ │ │ │
+//!  │ │ │ │ Specialize with `false`:
+//!  │ │ │ ├─┐ Patterns:
+//!  │ │ │ │ │   2. `[]`
+//!  │ │ │ │ │
+//!  │ │ │ │ │ We note arm 2 is useful (by `Pair(Some(1..), false)`).
+//!  │ │ │ ├─┘
+//!  │ │ │ │
+//!  │ │ │ │ Total witnesses for `1..`:
+//!  │ │ │ │   `[true]`
+//!  │ │ ├─┘
+//!  │ │ │ Unspecialize new witnesses with `1..`:
+//!  │ │ │   `[1.., true]`
+//!  │ │ │
+//!  │ │ │ Total witnesses for `Some`:
+//!  │ │ │   `[1.., true]`
+//!  │ ├─┘
+//!  │ │ Unspecialize new witnesses with `Some`:
+//!  │ │   `[Some(1..), true]`
+//!  │ │
+//!  │ │ Specialize with `None`:
+//!  │ ├─┐ Patterns:
+//!  │ │ │   2. `[false]`
+//!  │ │ │
+//!  │ │ │ Specialize with `true`:
+//!  │ │ ├─┐ Patterns:
+//!  │ │ │ │   // no rows left
+//!  │ │ │ │
+//!  │ │ │ │ We have found an unmatched value (`Pair(None, true)`)! This gives us a witness.
+//!  │ │ │ │ New witnesses:
+//!  │ │ │ │   `[]`
+//!  │ │ ├─┘
+//!  │ │ │ Unspecialize new witnesses with `true`:
+//!  │ │ │   `[true]`
+//!  │ │ │
+//!  │ │ │ Specialize with `false`:
+//!  │ │ ├─┐ Patterns:
+//!  │ │ │ │   2. `[]`
+//!  │ │ │ │
+//!  │ │ │ │ We note arm 2 is useful (by `Pair(None, false)`).
+//!  │ │ ├─┘
+//!  │ │ │
+//!  │ │ │ Total witnesses for `None`:
+//!  │ │ │   `[true]`
+//!  │ ├─┘
+//!  │ │ Unspecialize new witnesses with `None`:
+//!  │ │   `[None, true]`
+//!  │ │
+//!  │ │ Total witnesses for `Pair`:
+//!  │ │   `[Some(1..), true]`
+//!  │ │   `[None, true]`
+//!  ├─┘
+//!  │ Unspecialize new witnesses with `Pair`:
+//!  │   `[Pair(Some(1..), true)]`
+//!  │   `[Pair(None, true)]`
+//!  │
+//!  │ Final witnesses:
+//!  │   `[Pair(Some(1..), true)]`
+//!  │   `[Pair(None, true)]`
+//!  ┘
+//! ```
+//!
+//! We conclude:
+//! - Arm 3 is redundant (it was never marked as useful);
+//! - The match is not exhaustive;
+//! - Adding arms with `Pair(Some(1..), true)` and `Pair(None, true)` would make the match exhaustive.
+//!
+//! Note that when we're deep in the algorithm, we don't know what specialization steps got us here.
+//! We can only figure out what our witnesses correspond to by unspecializing back up the stack.
+//!
+//!
+//! # Tests
+//!
+//! Note: tests specific to this file can be found in:
+//!
+//!   - `ui/pattern/usefulness`
+//!   - `ui/or-patterns`
+//!   - `ui/consts/const_in_pattern`
+//!   - `ui/rfc-2008-non-exhaustive`
+//!   - `ui/half-open-range-patterns`
+//!   - probably many others
+//!
+//! I (Nadrieril) prefer to put new tests in `ui/pattern/usefulness` unless there's a specific
+//! reason not to, for example if they crucially depend on a particular feature like `or_patterns`.
+
+use smallvec::{smallvec, SmallVec};
+use std::fmt;
+
+use crate::constructor::{Constructor, ConstructorSet};
+use crate::pat::{DeconstructedPat, WitnessPat};
+use crate::{Captures, MatchArm, MatchCtxt, TypeCx, TypedArena};
+
+use self::ValidityConstraint::*;
+
+#[cfg(feature = "rustc")]
+use rustc_data_structures::stack::ensure_sufficient_stack;
+#[cfg(not(feature = "rustc"))]
+pub fn ensure_sufficient_stack<R>(f: impl FnOnce() -> R) -> R {
+    f()
+}
+
+/// Context that provides information local to a place under investigation.
+#[derive(Clone)]
+pub(crate) struct PlaceCtxt<'a, 'p, Cx: TypeCx> {
+    pub(crate) mcx: MatchCtxt<'a, 'p, Cx>,
+    /// Type of the place under investigation.
+    pub(crate) ty: Cx::Ty,
+    /// Whether the place is the original scrutinee place, as opposed to a subplace of it.
+    pub(crate) is_scrutinee: bool,
+}
+
+impl<'a, 'p, Cx: TypeCx> PlaceCtxt<'a, 'p, Cx> {
+    /// A `PlaceCtxt` when code other than `is_useful` needs one.
+    #[cfg_attr(not(feature = "rustc"), allow(dead_code))]
+    pub(crate) fn new_dummy(mcx: MatchCtxt<'a, 'p, Cx>, ty: Cx::Ty) -> Self {
+        PlaceCtxt { mcx, ty, is_scrutinee: false }
+    }
+
+    pub(crate) fn ctor_arity(&self, ctor: &Constructor<Cx>) -> usize {
+        self.mcx.tycx.ctor_arity(ctor, self.ty)
+    }
+    pub(crate) fn ctor_sub_tys(&self, ctor: &Constructor<Cx>) -> &[Cx::Ty] {
+        self.mcx.tycx.ctor_sub_tys(ctor, self.ty)
+    }
+    pub(crate) fn ctors_for_ty(&self) -> ConstructorSet<Cx> {
+        self.mcx.tycx.ctors_for_ty(self.ty)
+    }
+}
+
+impl<'a, 'p, Cx: TypeCx> Copy for PlaceCtxt<'a, 'p, Cx> {}
+
+impl<'a, 'p, Cx: TypeCx> fmt::Debug for PlaceCtxt<'a, 'p, Cx> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("PlaceCtxt").field("ty", &self.ty).finish()
+    }
+}
+
+/// Serves two purposes:
+/// - in a wildcard, tracks whether the wildcard matches only valid values (i.e. is a binding `_a`)
+///     or also invalid values (i.e. is a true `_` pattern).
+/// - in the matrix, track whether a given place (aka column) is known to contain a valid value or
+///     not.
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub enum ValidityConstraint {
+    ValidOnly,
+    MaybeInvalid,
+    /// Option for backwards compatibility: the place is not known to be valid but we allow omitting
+    /// `useful && !reachable` arms anyway.
+    MaybeInvalidButAllowOmittingArms,
+}
+
+impl ValidityConstraint {
+    pub fn from_bool(is_valid_only: bool) -> Self {
+        if is_valid_only { ValidOnly } else { MaybeInvalid }
+    }
+
+    fn allow_omitting_side_effecting_arms(self) -> Self {
+        match self {
+            MaybeInvalid | MaybeInvalidButAllowOmittingArms => MaybeInvalidButAllowOmittingArms,
+            // There are no side-effecting empty arms here, nothing to do.
+            ValidOnly => ValidOnly,
+        }
+    }
+
+    fn is_known_valid(self) -> bool {
+        matches!(self, ValidOnly)
+    }
+    fn allows_omitting_empty_arms(self) -> bool {
+        matches!(self, ValidOnly | MaybeInvalidButAllowOmittingArms)
+    }
+
+    /// If the place has validity given by `self` and we read that the value at the place has
+    /// constructor `ctor`, this computes what we can assume about the validity of the constructor
+    /// fields.
+    ///
+    /// Pending further opsem decisions, the current behavior is: validity is preserved, except
+    /// inside `&` and union fields where validity is reset to `MaybeInvalid`.
+    fn specialize<Cx: TypeCx>(self, ctor: &Constructor<Cx>) -> Self {
+        // We preserve validity except when we go inside a reference or a union field.
+        if matches!(ctor, Constructor::Ref | Constructor::UnionField) {
+            // Validity of `x: &T` does not imply validity of `*x: T`.
+            MaybeInvalid
+        } else {
+            self
+        }
+    }
+}
+
+impl fmt::Display for ValidityConstraint {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let s = match self {
+            ValidOnly => "✓",
+            MaybeInvalid | MaybeInvalidButAllowOmittingArms => "?",
+        };
+        write!(f, "{s}")
+    }
+}
+
+/// Represents a pattern-tuple under investigation.
+// The three lifetimes are:
+// - 'a allocated by us
+// - 'p coming from the input
+// - Cx global compilation context
+#[derive(Clone)]
+struct PatStack<'a, 'p, Cx: TypeCx> {
+    // Rows of len 1 are very common, which is why `SmallVec[_; 2]` works well.
+    pats: SmallVec<[&'a DeconstructedPat<'p, Cx>; 2]>,
+    /// Sometimes we know that as far as this row is concerned, the current case is already handled
+    /// by a different, more general, case. When the case is irrelevant for all rows this allows us
+    /// to skip a case entirely. This is purely an optimization. See at the top for details.
+    relevant: bool,
+}
+
+impl<'a, 'p, Cx: TypeCx> PatStack<'a, 'p, Cx> {
+    fn from_pattern(pat: &'a DeconstructedPat<'p, Cx>) -> Self {
+        PatStack { pats: smallvec![pat], relevant: true }
+    }
+
+    fn is_empty(&self) -> bool {
+        self.pats.is_empty()
+    }
+
+    fn len(&self) -> usize {
+        self.pats.len()
+    }
+
+    fn head(&self) -> &'a DeconstructedPat<'p, Cx> {
+        self.pats[0]
+    }
+
+    fn iter<'b>(&'b self) -> impl Iterator<Item = &'a DeconstructedPat<'p, Cx>> + Captures<'b> {
+        self.pats.iter().copied()
+    }
+
+    // Recursively expand the first or-pattern into its subpatterns. Only useful if the pattern is
+    // an or-pattern. Panics if `self` is empty.
+    fn expand_or_pat<'b>(&'b self) -> impl Iterator<Item = PatStack<'a, 'p, Cx>> + Captures<'b> {
+        self.head().flatten_or_pat().into_iter().map(move |pat| {
+            let mut new = self.clone();
+            new.pats[0] = pat;
+            new
+        })
+    }
+
+    /// This computes `specialize(ctor, self)`. See top of the file for explanations.
+    /// Only call if `ctor.is_covered_by(self.head().ctor())` is true.
+    fn pop_head_constructor(
+        &self,
+        pcx: &PlaceCtxt<'a, 'p, Cx>,
+        ctor: &Constructor<Cx>,
+        ctor_is_relevant: bool,
+    ) -> PatStack<'a, 'p, Cx> {
+        // We pop the head pattern and push the new fields extracted from the arguments of
+        // `self.head()`.
+        let mut new_pats = self.head().specialize(pcx, ctor);
+        new_pats.extend_from_slice(&self.pats[1..]);
+        // `ctor` is relevant for this row if it is the actual constructor of this row, or if the
+        // row has a wildcard and `ctor` is relevant for wildcards.
+        let ctor_is_relevant =
+            !matches!(self.head().ctor(), Constructor::Wildcard) || ctor_is_relevant;
+        PatStack { pats: new_pats, relevant: self.relevant && ctor_is_relevant }
+    }
+}
+
+impl<'a, 'p, Cx: TypeCx> fmt::Debug for PatStack<'a, 'p, Cx> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        // We pretty-print similarly to the `Debug` impl of `Matrix`.
+        write!(f, "+")?;
+        for pat in self.iter() {
+            write!(f, " {pat:?} +")?;
+        }
+        Ok(())
+    }
+}
+
+/// A row of the matrix.
+#[derive(Clone)]
+struct MatrixRow<'a, 'p, Cx: TypeCx> {
+    // The patterns in the row.
+    pats: PatStack<'a, 'p, Cx>,
+    /// Whether the original arm had a guard. This is inherited when specializing.
+    is_under_guard: bool,
+    /// When we specialize, we remember which row of the original matrix produced a given row of the
+    /// specialized matrix. When we unspecialize, we use this to propagate usefulness back up the
+    /// callstack.
+    parent_row: usize,
+    /// False when the matrix is just built. This is set to `true` by
+    /// [`compute_exhaustiveness_and_usefulness`] if the arm is found to be useful.
+    /// This is reset to `false` when specializing.
+    useful: bool,
+}
+
+impl<'a, 'p, Cx: TypeCx> MatrixRow<'a, 'p, Cx> {
+    fn is_empty(&self) -> bool {
+        self.pats.is_empty()
+    }
+
+    fn len(&self) -> usize {
+        self.pats.len()
+    }
+
+    fn head(&self) -> &'a DeconstructedPat<'p, Cx> {
+        self.pats.head()
+    }
+
+    fn iter<'b>(&'b self) -> impl Iterator<Item = &'a DeconstructedPat<'p, Cx>> + Captures<'b> {
+        self.pats.iter()
+    }
+
+    // Recursively expand the first or-pattern into its subpatterns. Only useful if the pattern is
+    // an or-pattern. Panics if `self` is empty.
+    fn expand_or_pat<'b>(&'b self) -> impl Iterator<Item = MatrixRow<'a, 'p, Cx>> + Captures<'b> {
+        self.pats.expand_or_pat().map(|patstack| MatrixRow {
+            pats: patstack,
+            parent_row: self.parent_row,
+            is_under_guard: self.is_under_guard,
+            useful: false,
+        })
+    }
+
+    /// This computes `specialize(ctor, self)`. See top of the file for explanations.
+    /// Only call if `ctor.is_covered_by(self.head().ctor())` is true.
+    fn pop_head_constructor(
+        &self,
+        pcx: &PlaceCtxt<'a, 'p, Cx>,
+        ctor: &Constructor<Cx>,
+        ctor_is_relevant: bool,
+        parent_row: usize,
+    ) -> MatrixRow<'a, 'p, Cx> {
+        MatrixRow {
+            pats: self.pats.pop_head_constructor(pcx, ctor, ctor_is_relevant),
+            parent_row,
+            is_under_guard: self.is_under_guard,
+            useful: false,
+        }
+    }
+}
+
+impl<'a, 'p, Cx: TypeCx> fmt::Debug for MatrixRow<'a, 'p, Cx> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        self.pats.fmt(f)
+    }
+}
+
+/// A 2D matrix. Represents a list of pattern-tuples under investigation.
+///
+/// Invariant: each row must have the same length, and each column must have the same type.
+///
+/// Invariant: the first column must not contain or-patterns. This is handled by
+/// [`Matrix::expand_and_push`].
+///
+/// In fact each column corresponds to a place inside the scrutinee of the match. E.g. after
+/// specializing `(,)` and `Some` on a pattern of type `(Option<u32>, bool)`, the first column of
+/// the matrix will correspond to `scrutinee.0.Some.0` and the second column to `scrutinee.1`.
+#[derive(Clone)]
+struct Matrix<'a, 'p, Cx: TypeCx> {
+    /// Vector of rows. The rows must form a rectangular 2D array. Moreover, all the patterns of
+    /// each column must have the same type. Each column corresponds to a place within the
+    /// scrutinee.
+    rows: Vec<MatrixRow<'a, 'p, Cx>>,
+    /// Stores an extra fictitious row full of wildcards. Mostly used to keep track of the type of
+    /// each column. This must obey the same invariants as the real rows.
+    wildcard_row: PatStack<'a, 'p, Cx>,
+    /// Track for each column/place whether it contains a known valid value.
+    place_validity: SmallVec<[ValidityConstraint; 2]>,
+}
+
+impl<'a, 'p, Cx: TypeCx> Matrix<'a, 'p, Cx> {
+    /// Pushes a new row to the matrix. If the row starts with an or-pattern, this recursively
+    /// expands it. Internal method, prefer [`Matrix::new`].
+    fn expand_and_push(&mut self, row: MatrixRow<'a, 'p, Cx>) {
+        if !row.is_empty() && row.head().is_or_pat() {
+            // Expand nested or-patterns.
+            for new_row in row.expand_or_pat() {
+                self.rows.push(new_row);
+            }
+        } else {
+            self.rows.push(row);
+        }
+    }
+
+    /// Build a new matrix from an iterator of `MatchArm`s.
+    fn new(
+        wildcard_arena: &'a TypedArena<DeconstructedPat<'p, Cx>>,
+        arms: &'a [MatchArm<'p, Cx>],
+        scrut_ty: Cx::Ty,
+        scrut_validity: ValidityConstraint,
+    ) -> Self {
+        let wild_pattern =
+            wildcard_arena.alloc(DeconstructedPat::wildcard(scrut_ty, Default::default()));
+        let wildcard_row = PatStack::from_pattern(wild_pattern);
+        let mut matrix = Matrix {
+            rows: Vec::with_capacity(arms.len()),
+            wildcard_row,
+            place_validity: smallvec![scrut_validity],
+        };
+        for (row_id, arm) in arms.iter().enumerate() {
+            let v = MatrixRow {
+                pats: PatStack::from_pattern(arm.pat),
+                parent_row: row_id, // dummy, we won't read it
+                is_under_guard: arm.has_guard,
+                useful: false,
+            };
+            matrix.expand_and_push(v);
+        }
+        matrix
+    }
+
+    fn head_ty(&self) -> Option<Cx::Ty> {
+        if self.column_count() == 0 {
+            return None;
+        }
+
+        let mut ty = self.wildcard_row.head().ty();
+        // If the type is opaque and it is revealed anywhere in the column, we take the revealed
+        // version. Otherwise we could encounter constructors for the revealed type and crash.
+        if Cx::is_opaque_ty(ty) {
+            for pat in self.heads() {
+                let pat_ty = pat.ty();
+                if !Cx::is_opaque_ty(pat_ty) {
+                    ty = pat_ty;
+                    break;
+                }
+            }
+        }
+        Some(ty)
+    }
+    fn column_count(&self) -> usize {
+        self.wildcard_row.len()
+    }
+
+    fn rows<'b>(
+        &'b self,
+    ) -> impl Iterator<Item = &'b MatrixRow<'a, 'p, Cx>> + Clone + DoubleEndedIterator + ExactSizeIterator
+    {
+        self.rows.iter()
+    }
+    fn rows_mut<'b>(
+        &'b mut self,
+    ) -> impl Iterator<Item = &'b mut MatrixRow<'a, 'p, Cx>> + DoubleEndedIterator + ExactSizeIterator
+    {
+        self.rows.iter_mut()
+    }
+
+    /// Iterate over the first pattern of each row.
+    fn heads<'b>(
+        &'b self,
+    ) -> impl Iterator<Item = &'b DeconstructedPat<'p, Cx>> + Clone + Captures<'a> {
+        self.rows().map(|r| r.head())
+    }
+
+    /// This computes `specialize(ctor, self)`. See top of the file for explanations.
+    fn specialize_constructor(
+        &self,
+        pcx: &PlaceCtxt<'a, 'p, Cx>,
+        ctor: &Constructor<Cx>,
+        ctor_is_relevant: bool,
+    ) -> Matrix<'a, 'p, Cx> {
+        let wildcard_row = self.wildcard_row.pop_head_constructor(pcx, ctor, ctor_is_relevant);
+        let new_validity = self.place_validity[0].specialize(ctor);
+        let new_place_validity = std::iter::repeat(new_validity)
+            .take(ctor.arity(pcx))
+            .chain(self.place_validity[1..].iter().copied())
+            .collect();
+        let mut matrix =
+            Matrix { rows: Vec::new(), wildcard_row, place_validity: new_place_validity };
+        for (i, row) in self.rows().enumerate() {
+            if ctor.is_covered_by(pcx, row.head().ctor()) {
+                let new_row = row.pop_head_constructor(pcx, ctor, ctor_is_relevant, i);
+                matrix.expand_and_push(new_row);
+            }
+        }
+        matrix
+    }
+}
+
+/// Pretty-printer for matrices of patterns, example:
+///
+/// ```text
+/// + _     + []                +
+/// + true  + [First]           +
+/// + true  + [Second(true)]    +
+/// + false + [_]               +
+/// + _     + [_, _, tail @ ..] +
+/// | ✓     | ?                 | // column validity
+/// ```
+impl<'a, 'p, Cx: TypeCx> fmt::Debug for Matrix<'a, 'p, Cx> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "\n")?;
+
+        let mut pretty_printed_matrix: Vec<Vec<String>> = self
+            .rows
+            .iter()
+            .map(|row| row.iter().map(|pat| format!("{pat:?}")).collect())
+            .collect();
+        pretty_printed_matrix
+            .push(self.place_validity.iter().map(|validity| format!("{validity}")).collect());
+
+        let column_count = self.column_count();
+        assert!(self.rows.iter().all(|row| row.len() == column_count));
+        assert!(self.place_validity.len() == column_count);
+        let column_widths: Vec<usize> = (0..column_count)
+            .map(|col| pretty_printed_matrix.iter().map(|row| row[col].len()).max().unwrap_or(0))
+            .collect();
+
+        for (row_i, row) in pretty_printed_matrix.into_iter().enumerate() {
+            let is_validity_row = row_i == self.rows.len();
+            let sep = if is_validity_row { "|" } else { "+" };
+            write!(f, "{sep}")?;
+            for (column, pat_str) in row.into_iter().enumerate() {
+                write!(f, " ")?;
+                write!(f, "{:1$}", pat_str, column_widths[column])?;
+                write!(f, " {sep}")?;
+            }
+            if is_validity_row {
+                write!(f, " // column validity")?;
+            }
+            write!(f, "\n")?;
+        }
+        Ok(())
+    }
+}
+
+/// A witness-tuple of non-exhaustiveness for error reporting, represented as a list of patterns (in
+/// reverse order of construction).
+///
+/// This mirrors `PatStack`: they function similarly, except `PatStack` contains user patterns we
+/// are inspecting, and `WitnessStack` contains witnesses we are constructing.
+/// FIXME(Nadrieril): use the same order of patterns for both.
+///
+/// A `WitnessStack` should have the same types and length as the `PatStack`s we are inspecting
+/// (except we store the patterns in reverse order). The same way `PatStack` starts with length 1,
+/// at the end of the algorithm this will have length 1. In the middle of the algorithm, it can
+/// contain multiple patterns.
+///
+/// For example, if we are constructing a witness for the match against
+///
+/// ```compile_fail,E0004
+/// struct Pair(Option<(u32, u32)>, bool);
+/// # fn foo(p: Pair) {
+/// match p {
+///    Pair(None, _) => {}
+///    Pair(_, false) => {}
+/// }
+/// # }
+/// ```
+///
+/// We'll perform the following steps (among others):
+/// ```text
+/// - Start with a matrix representing the match
+///     `PatStack(vec![Pair(None, _)])`
+///     `PatStack(vec![Pair(_, false)])`
+/// - Specialize with `Pair`
+///     `PatStack(vec![None, _])`
+///     `PatStack(vec![_, false])`
+/// - Specialize with `Some`
+///     `PatStack(vec![_, false])`
+/// - Specialize with `_`
+///     `PatStack(vec![false])`
+/// - Specialize with `true`
+///     // no patstacks left
+/// - This is a non-exhaustive match: we have the empty witness stack as a witness.
+///     `WitnessStack(vec![])`
+/// - Apply `true`
+///     `WitnessStack(vec![true])`
+/// - Apply `_`
+///     `WitnessStack(vec![true, _])`
+/// - Apply `Some`
+///     `WitnessStack(vec![true, Some(_)])`
+/// - Apply `Pair`
+///     `WitnessStack(vec![Pair(Some(_), true)])`
+/// ```
+///
+/// The final `Pair(Some(_), true)` is then the resulting witness.
+///
+/// See the top of the file for more detailed explanations and examples.
+#[derive(Debug, Clone)]
+struct WitnessStack<Cx: TypeCx>(Vec<WitnessPat<Cx>>);
+
+impl<Cx: TypeCx> WitnessStack<Cx> {
+    /// Asserts that the witness contains a single pattern, and returns it.
+    fn single_pattern(self) -> WitnessPat<Cx> {
+        assert_eq!(self.0.len(), 1);
+        self.0.into_iter().next().unwrap()
+    }
+
+    /// Reverses specialization by the `Missing` constructor by pushing a whole new pattern.
+    fn push_pattern(&mut self, pat: WitnessPat<Cx>) {
+        self.0.push(pat);
+    }
+
+    /// Reverses specialization. Given a witness obtained after specialization, this constructs a
+    /// new witness valid for before specialization. See the section on `unspecialize` at the top of
+    /// the file.
+    ///
+    /// Examples:
+    /// ```text
+    /// ctor: tuple of 2 elements
+    /// pats: [false, "foo", _, true]
+    /// result: [(false, "foo"), _, true]
+    ///
+    /// ctor: Enum::Variant { a: (bool, &'static str), b: usize}
+    /// pats: [(false, "foo"), _, true]
+    /// result: [Enum::Variant { a: (false, "foo"), b: _ }, true]
+    /// ```
+    fn apply_constructor(&mut self, pcx: &PlaceCtxt<'_, '_, Cx>, ctor: &Constructor<Cx>) {
+        let len = self.0.len();
+        let arity = ctor.arity(pcx);
+        let fields = self.0.drain((len - arity)..).rev().collect();
+        let pat = WitnessPat::new(ctor.clone(), fields, pcx.ty);
+        self.0.push(pat);
+    }
+}
+
+/// Represents a set of pattern-tuples that are witnesses of non-exhaustiveness for error
+/// reporting. This has similar invariants as `Matrix` does.
+///
+/// The `WitnessMatrix` returned by [`compute_exhaustiveness_and_usefulness`] obeys the invariant
+/// that the union of the input `Matrix` and the output `WitnessMatrix` together matches the type
+/// exhaustively.
+///
+/// Just as the `Matrix` starts with a single column, by the end of the algorithm, this has a single
+/// column, which contains the patterns that are missing for the match to be exhaustive.
+#[derive(Debug, Clone)]
+struct WitnessMatrix<Cx: TypeCx>(Vec<WitnessStack<Cx>>);
+
+impl<Cx: TypeCx> WitnessMatrix<Cx> {
+    /// New matrix with no witnesses.
+    fn empty() -> Self {
+        WitnessMatrix(vec![])
+    }
+    /// New matrix with one `()` witness, i.e. with no columns.
+    fn unit_witness() -> Self {
+        WitnessMatrix(vec![WitnessStack(vec![])])
+    }
+
+    /// Whether this has any witnesses.
+    fn is_empty(&self) -> bool {
+        self.0.is_empty()
+    }
+    /// Asserts that there is a single column and returns the patterns in it.
+    fn single_column(self) -> Vec<WitnessPat<Cx>> {
+        self.0.into_iter().map(|w| w.single_pattern()).collect()
+    }
+
+    /// Reverses specialization by the `Missing` constructor by pushing a whole new pattern.
+    fn push_pattern(&mut self, pat: WitnessPat<Cx>) {
+        for witness in self.0.iter_mut() {
+            witness.push_pattern(pat.clone())
+        }
+    }
+
+    /// Reverses specialization by `ctor`. See the section on `unspecialize` at the top of the file.
+    fn apply_constructor(
+        &mut self,
+        pcx: &PlaceCtxt<'_, '_, Cx>,
+        missing_ctors: &[Constructor<Cx>],
+        ctor: &Constructor<Cx>,
+        report_individual_missing_ctors: bool,
+    ) {
+        if self.is_empty() {
+            return;
+        }
+        if matches!(ctor, Constructor::Missing) {
+            // We got the special `Missing` constructor that stands for the constructors not present
+            // in the match.
+            if missing_ctors.is_empty() {
+                // Nothing to report.
+                *self = Self::empty();
+            } else if !report_individual_missing_ctors {
+                // Report `_` as missing.
+                let pat = WitnessPat::wild_from_ctor(pcx, Constructor::Wildcard);
+                self.push_pattern(pat);
+            } else if missing_ctors.iter().any(|c| c.is_non_exhaustive()) {
+                // We need to report a `_` anyway, so listing other constructors would be redundant.
+                // `NonExhaustive` is displayed as `_` just like `Wildcard`, but it will be picked
+                // up by diagnostics to add a note about why `_` is required here.
+                let pat = WitnessPat::wild_from_ctor(pcx, Constructor::NonExhaustive);
+                self.push_pattern(pat);
+            } else {
+                // For each missing constructor `c`, we add a `c(_, _, _)` witness appropriately
+                // filled with wildcards.
+                let mut ret = Self::empty();
+                for ctor in missing_ctors {
+                    let pat = WitnessPat::wild_from_ctor(pcx, ctor.clone());
+                    // Clone `self` and add `c(_, _, _)` to each of its witnesses.
+                    let mut wit_matrix = self.clone();
+                    wit_matrix.push_pattern(pat);
+                    ret.extend(wit_matrix);
+                }
+                *self = ret;
+            }
+        } else {
+            // Any other constructor we unspecialize as expected.
+            for witness in self.0.iter_mut() {
+                witness.apply_constructor(pcx, ctor)
+            }
+        }
+    }
+
+    /// Merges the witnesses of two matrices. Their column types must match.
+    fn extend(&mut self, other: Self) {
+        self.0.extend(other.0)
+    }
+}
+
+/// The core of the algorithm.
+///
+/// This recursively computes witnesses of the non-exhaustiveness of `matrix` (if any). Also tracks
+/// usefulness of each row in the matrix (in `row.useful`). We track usefulness of each
+/// subpattern using interior mutability in `DeconstructedPat`.
+///
+/// The input `Matrix` and the output `WitnessMatrix` together match the type exhaustively.
+///
+/// The key steps are:
+/// - specialization, where we dig into the rows that have a specific constructor and call ourselves
+///     recursively;
+/// - unspecialization, where we lift the results from the previous step into results for this step
+///     (using `apply_constructor` and by updating `row.useful` for each parent row).
+/// This is all explained at the top of the file.
+#[instrument(level = "debug", skip(mcx, is_top_level), ret)]
+fn compute_exhaustiveness_and_usefulness<'a, 'p, Cx: TypeCx>(
+    mcx: MatchCtxt<'a, 'p, Cx>,
+    matrix: &mut Matrix<'a, 'p, Cx>,
+    is_top_level: bool,
+) -> WitnessMatrix<Cx> {
+    debug_assert!(matrix.rows().all(|r| r.len() == matrix.column_count()));
+
+    if !matrix.wildcard_row.relevant && matrix.rows().all(|r| !r.pats.relevant) {
+        // Here we know that nothing will contribute further to exhaustiveness or usefulness. This
+        // is purely an optimization: skipping this check doesn't affect correctness. See the top of
+        // the file for details.
+        return WitnessMatrix::empty();
+    }
+
+    let Some(ty) = matrix.head_ty() else {
+        // The base case: there are no columns in the matrix. We are morally pattern-matching on ().
+        // A row is useful iff it has no (unguarded) rows above it.
+        for row in matrix.rows_mut() {
+            // All rows are useful until they're not.
+            row.useful = true;
+            // When there's an unguarded row, the match is exhaustive and any subsequent row is not
+            // useful.
+            if !row.is_under_guard {
+                return WitnessMatrix::empty();
+            }
+        }
+        // No (unguarded) rows, so the match is not exhaustive. We return a new witness unless
+        // irrelevant.
+        return if matrix.wildcard_row.relevant {
+            WitnessMatrix::unit_witness()
+        } else {
+            // We choose to not report anything here; see at the top for details.
+            WitnessMatrix::empty()
+        };
+    };
+
+    debug!("ty: {ty:?}");
+    let pcx = &PlaceCtxt { mcx, ty, is_scrutinee: is_top_level };
+
+    // Whether the place/column we are inspecting is known to contain valid data.
+    let place_validity = matrix.place_validity[0];
+    // For backwards compability we allow omitting some empty arms that we ideally shouldn't.
+    let place_validity = place_validity.allow_omitting_side_effecting_arms();
+
+    // Analyze the constructors present in this column.
+    let ctors = matrix.heads().map(|p| p.ctor());
+    let ctors_for_ty = pcx.ctors_for_ty();
+    let is_integers = matches!(ctors_for_ty, ConstructorSet::Integers { .. }); // For diagnostics.
+    let split_set = ctors_for_ty.split(pcx, ctors);
+    let all_missing = split_set.present.is_empty();
+
+    // Build the set of constructors we will specialize with. It must cover the whole type.
+    let mut split_ctors = split_set.present;
+    if !split_set.missing.is_empty() {
+        // We need to iterate over a full set of constructors, so we add `Missing` to represent the
+        // missing ones. This is explained under "Constructor Splitting" at the top of this file.
+        split_ctors.push(Constructor::Missing);
+    } else if !split_set.missing_empty.is_empty() && !place_validity.is_known_valid() {
+        // The missing empty constructors are reachable if the place can contain invalid data.
+        split_ctors.push(Constructor::Missing);
+    }
+
+    // Decide what constructors to report.
+    let always_report_all = is_top_level && !is_integers;
+    // Whether we should report "Enum::A and Enum::C are missing" or "_ is missing".
+    let report_individual_missing_ctors = always_report_all || !all_missing;
+    // Which constructors are considered missing. We ensure that `!missing_ctors.is_empty() =>
+    // split_ctors.contains(Missing)`. The converse usually holds except in the
+    // `MaybeInvalidButAllowOmittingArms` backwards-compatibility case.
+    let mut missing_ctors = split_set.missing;
+    if !place_validity.allows_omitting_empty_arms() {
+        missing_ctors.extend(split_set.missing_empty);
+    }
+
+    let mut ret = WitnessMatrix::empty();
+    for ctor in split_ctors {
+        // Dig into rows that match `ctor`.
+        debug!("specialize({:?})", ctor);
+        // `ctor` is *irrelevant* if there's another constructor in `split_ctors` that matches
+        // strictly fewer rows. In that case we can sometimes skip it. See the top of the file for
+        // details.
+        let ctor_is_relevant = matches!(ctor, Constructor::Missing) || missing_ctors.is_empty();
+        let mut spec_matrix = matrix.specialize_constructor(pcx, &ctor, ctor_is_relevant);
+        let mut witnesses = ensure_sufficient_stack(|| {
+            compute_exhaustiveness_and_usefulness(mcx, &mut spec_matrix, false)
+        });
+
+        // Transform witnesses for `spec_matrix` into witnesses for `matrix`.
+        witnesses.apply_constructor(pcx, &missing_ctors, &ctor, report_individual_missing_ctors);
+        // Accumulate the found witnesses.
+        ret.extend(witnesses);
+
+        // A parent row is useful if any of its children is.
+        for child_row in spec_matrix.rows() {
+            let parent_row = &mut matrix.rows[child_row.parent_row];
+            parent_row.useful = parent_row.useful || child_row.useful;
+        }
+    }
+
+    // Record usefulness in the patterns.
+    for row in matrix.rows() {
+        if row.useful {
+            row.head().set_useful();
+        }
+    }
+
+    ret
+}
+
+/// Indicates whether or not a given arm is useful.
+#[derive(Clone, Debug)]
+pub enum Usefulness<'p, Cx: TypeCx> {
+    /// The arm is useful. This additionally carries a set of or-pattern branches that have been
+    /// found to be redundant despite the overall arm being useful. Used only in the presence of
+    /// or-patterns, otherwise it stays empty.
+    Useful(Vec<&'p DeconstructedPat<'p, Cx>>),
+    /// The arm is redundant and can be removed without changing the behavior of the match
+    /// expression.
+    Redundant,
+}
+
+/// The output of checking a match for exhaustiveness and arm usefulness.
+pub struct UsefulnessReport<'p, Cx: TypeCx> {
+    /// For each arm of the input, whether that arm is useful after the arms above it.
+    pub arm_usefulness: Vec<(MatchArm<'p, Cx>, Usefulness<'p, Cx>)>,
+    /// If the match is exhaustive, this is empty. If not, this contains witnesses for the lack of
+    /// exhaustiveness.
+    pub non_exhaustiveness_witnesses: Vec<WitnessPat<Cx>>,
+}
+
+/// Computes whether a match is exhaustive and which of its arms are useful.
+#[instrument(skip(cx, arms), level = "debug")]
+pub fn compute_match_usefulness<'p, Cx: TypeCx>(
+    cx: MatchCtxt<'_, 'p, Cx>,
+    arms: &[MatchArm<'p, Cx>],
+    scrut_ty: Cx::Ty,
+    scrut_validity: ValidityConstraint,
+) -> UsefulnessReport<'p, Cx> {
+    let mut matrix = Matrix::new(cx.wildcard_arena, arms, scrut_ty, scrut_validity);
+    let non_exhaustiveness_witnesses = compute_exhaustiveness_and_usefulness(cx, &mut matrix, true);
+
+    let non_exhaustiveness_witnesses: Vec<_> = non_exhaustiveness_witnesses.single_column();
+    let arm_usefulness: Vec<_> = arms
+        .iter()
+        .copied()
+        .map(|arm| {
+            debug!(?arm);
+            // We warn when a pattern is not useful.
+            let usefulness = if arm.pat.is_useful() {
+                Usefulness::Useful(arm.pat.redundant_subpatterns())
+            } else {
+                Usefulness::Redundant
+            };
+            (arm, usefulness)
+        })
+        .collect();
+    UsefulnessReport { arm_usefulness, non_exhaustiveness_witnesses }
+}
diff --git a/compiler/rustc_privacy/src/lib.rs b/compiler/rustc_privacy/src/lib.rs
index 4bb7e6574..be9c6b725 100644
--- a/compiler/rustc_privacy/src/lib.rs
+++ b/compiler/rustc_privacy/src/lib.rs
@@ -1,7 +1,7 @@
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
-#![cfg_attr(not(bootstrap), allow(internal_features))]
+#![doc(rust_logo)]
+#![feature(rustdoc_internals)]
+#![allow(internal_features)]
 #![feature(associated_type_defaults)]
 #![feature(rustc_private)]
 #![feature(try_blocks)]
@@ -19,18 +19,15 @@ use rustc_ast::MacroDef;
 use rustc_attr as attr;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_data_structures::intern::Interned;
-use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
-use rustc_fluent_macro::fluent_messages;
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
 use rustc_hir::def_id::{DefId, LocalDefId, LocalModDefId, CRATE_DEF_ID};
 use rustc_hir::intravisit::{self, Visitor};
-use rustc_hir::{AssocItemKind, ForeignItemKind, ItemId, Node, PatKind};
+use rustc_hir::{AssocItemKind, ForeignItemKind, ItemId, PatKind};
 use rustc_middle::bug;
 use rustc_middle::hir::nested_filter;
 use rustc_middle::middle::privacy::{EffectiveVisibilities, EffectiveVisibility, Level};
 use rustc_middle::query::Providers;
-use rustc_middle::span_bug;
 use rustc_middle::ty::GenericArgs;
 use rustc_middle::ty::{self, Const, GenericParamDefKind};
 use rustc_middle::ty::{TraitRef, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable, TypeVisitor};
@@ -49,7 +46,7 @@ use errors::{
     UnnamedItemIsPrivate,
 };
 
-fluent_messages! { "../messages.ftl" }
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
 
 ////////////////////////////////////////////////////////////////////////////////
 /// Generic infrastructure used to implement specific visitors below.
@@ -220,20 +217,21 @@ where
                     return ControlFlow::Continue(());
                 }
 
-                let kind = match kind {
-                    ty::Inherent | ty::Projection => "associated type",
-                    ty::Weak => "type alias",
-                    ty::Opaque => unreachable!(),
-                };
                 self.def_id_visitor.visit_def_id(
                     data.def_id,
-                    kind,
+                    match kind {
+                        ty::Inherent | ty::Projection => "associated type",
+                        ty::Weak => "type alias",
+                        ty::Opaque => unreachable!(),
+                    },
                     &LazyDefPathStr { def_id: data.def_id, tcx },
                 )?;
 
                 // This will also visit args if necessary, so we don't need to recurse.
                 return if V::SHALLOW {
                     ControlFlow::Continue(())
+                } else if kind == ty::Projection {
+                    self.visit_projection_ty(data)
                 } else {
                     data.args.iter().try_for_each(|subst| subst.visit_with(self))
                 };
@@ -493,14 +491,14 @@ impl<'tcx> EmbargoVisitor<'tcx> {
         macro_ev: EffectiveVisibility,
     ) {
         // Non-opaque macros cannot make other items more accessible than they already are.
-        let hir_id = self.tcx.hir().local_def_id_to_hir_id(local_def_id);
+        let hir_id = self.tcx.local_def_id_to_hir_id(local_def_id);
         let attrs = self.tcx.hir().attrs(hir_id);
         if attr::find_transparency(attrs, md.macro_rules).0 != Transparency::Opaque {
             return;
         }
 
         let macro_module_def_id = self.tcx.local_parent(local_def_id);
-        if self.tcx.opt_def_kind(macro_module_def_id) != Some(DefKind::Mod) {
+        if self.tcx.def_kind(macro_module_def_id) != DefKind::Mod {
             // The macro's parent doesn't correspond to a `mod`, return early (#63164, #65252).
             return;
         }
@@ -533,44 +531,21 @@ impl<'tcx> EmbargoVisitor<'tcx> {
         macro_ev: EffectiveVisibility,
     ) -> bool {
         if self.macro_reachable.insert((module_def_id, defining_mod)) {
-            self.update_macro_reachable_mod(module_def_id, defining_mod, macro_ev);
+            for child in self.tcx.module_children_local(module_def_id.to_local_def_id()) {
+                if let Res::Def(def_kind, def_id) = child.res
+                    && let Some(def_id) = def_id.as_local()
+                    && child.vis.is_accessible_from(defining_mod, self.tcx)
+                {
+                    let vis = self.tcx.local_visibility(def_id);
+                    self.update_macro_reachable_def(def_id, def_kind, vis, defining_mod, macro_ev);
+                }
+            }
             true
         } else {
             false
         }
     }
 
-    fn update_macro_reachable_mod(
-        &mut self,
-        module_def_id: LocalModDefId,
-        defining_mod: LocalModDefId,
-        macro_ev: EffectiveVisibility,
-    ) {
-        let module = self.tcx.hir().get_module(module_def_id).0;
-        for item_id in module.item_ids {
-            let def_kind = self.tcx.def_kind(item_id.owner_id);
-            let vis = self.tcx.local_visibility(item_id.owner_id.def_id);
-            self.update_macro_reachable_def(
-                item_id.owner_id.def_id,
-                def_kind,
-                vis,
-                defining_mod,
-                macro_ev,
-            );
-        }
-        for child in self.tcx.module_children_local(module_def_id.to_local_def_id()) {
-            // FIXME: Use module children for the logic above too.
-            if !child.reexport_chain.is_empty()
-                && child.vis.is_accessible_from(defining_mod, self.tcx)
-                && let Res::Def(def_kind, def_id) = child.res
-                && let Some(def_id) = def_id.as_local()
-            {
-                let vis = self.tcx.local_visibility(def_id);
-                self.update_macro_reachable_def(def_id, def_kind, vis, defining_mod, macro_ev);
-            }
-        }
-    }
-
     fn update_macro_reachable_def(
         &mut self,
         def_id: LocalDefId,
@@ -655,8 +630,7 @@ impl<'tcx> EmbargoVisitor<'tcx> {
             | DefKind::Field
             | DefKind::GlobalAsm
             | DefKind::Impl { .. }
-            | DefKind::Closure
-            | DefKind::Coroutine => (),
+            | DefKind::Closure => (),
         }
     }
 }
@@ -664,7 +638,7 @@ impl<'tcx> EmbargoVisitor<'tcx> {
 impl<'tcx> Visitor<'tcx> for EmbargoVisitor<'tcx> {
     fn visit_item(&mut self, item: &'tcx hir::Item<'tcx>) {
         if self.impl_trait_pass
-            && let hir::ItemKind::OpaqueTy(ref opaque) = item.kind
+            && let hir::ItemKind::OpaqueTy(opaque) = item.kind
             && !opaque.in_trait
         {
             // FIXME: This is some serious pessimization intended to workaround deficiencies
@@ -688,7 +662,7 @@ impl<'tcx> Visitor<'tcx> for EmbargoVisitor<'tcx> {
             | hir::ItemKind::GlobalAsm(..) => {}
             // The interface is empty, and all nested items are processed by `visit_item`.
             hir::ItemKind::Mod(..) | hir::ItemKind::OpaqueTy(..) => {}
-            hir::ItemKind::Macro(ref macro_def, _) => {
+            hir::ItemKind::Macro(macro_def, _) => {
                 if let Some(item_ev) = item_ev {
                     self.update_reachability_from_macro(item.owner_id.def_id, macro_def, item_ev);
                 }
@@ -727,7 +701,7 @@ impl<'tcx> Visitor<'tcx> for EmbargoVisitor<'tcx> {
                     self.reach(item.owner_id.def_id, item_ev).generics().predicates();
                 }
             }
-            hir::ItemKind::Impl(ref impl_) => {
+            hir::ItemKind::Impl(impl_) => {
                 // Type inference is very smart sometimes. It can make an impl reachable even some
                 // components of its type or trait are unreachable. E.g. methods of
                 // `impl ReachableTrait<UnreachableTy> for ReachableTy<UnreachableTy> { ... }`
@@ -891,21 +865,6 @@ pub struct TestReachabilityVisitor<'tcx, 'a> {
     effective_visibilities: &'a EffectiveVisibilities,
 }
 
-fn vis_to_string<'tcx>(def_id: LocalDefId, vis: ty::Visibility, tcx: TyCtxt<'tcx>) -> String {
-    match vis {
-        ty::Visibility::Restricted(restricted_id) => {
-            if restricted_id.is_top_level_module() {
-                "pub(crate)".to_string()
-            } else if restricted_id == tcx.parent_module_from_def_id(def_id).to_local_def_id() {
-                "pub(self)".to_string()
-            } else {
-                format!("pub({})", tcx.item_name(restricted_id.to_def_id()))
-            }
-        }
-        ty::Visibility::Public => "pub".to_string(),
-    }
-}
-
 impl<'tcx, 'a> TestReachabilityVisitor<'tcx, 'a> {
     fn effective_visibility_diagnostic(&mut self, def_id: LocalDefId) {
         if self.tcx.has_attr(def_id, sym::rustc_effective_visibility) {
@@ -913,7 +872,7 @@ impl<'tcx, 'a> TestReachabilityVisitor<'tcx, 'a> {
             let span = self.tcx.def_span(def_id.to_def_id());
             if let Some(effective_vis) = self.effective_visibilities.effective_vis(def_id) {
                 for level in Level::all_levels() {
-                    let vis_str = vis_to_string(def_id, *effective_vis.at_level(level), self.tcx);
+                    let vis_str = effective_vis.at_level(level).to_string(def_id, self.tcx);
                     if level != Level::Direct {
                         error_msg.push_str(", ");
                     }
@@ -1004,7 +963,7 @@ impl<'tcx> NamePrivacyVisitor<'tcx> {
 
         // definition of the field
         let ident = Ident::new(kw::Empty, use_ctxt);
-        let hir_id = self.tcx.hir().local_def_id_to_hir_id(self.current_item);
+        let hir_id = self.tcx.local_def_id_to_hir_id(self.current_item);
         let def_id = self.tcx.adjust_ident_and_get_scope(ident, def.did(), hir_id).1;
         if !field.vis.is_accessible_from(def_id, self.tcx) {
             self.tcx.sess.emit_err(FieldIsPrivate {
@@ -1271,7 +1230,7 @@ impl<'tcx> Visitor<'tcx> for TypePrivacyVisitor<'tcx> {
                 } else {
                     self.tcx
                         .sess
-                        .delay_span_bug(expr.span, "no type-dependent def for method call");
+                        .span_delayed_bug(expr.span, "no type-dependent def for method call");
                 }
             }
             _ => {}
@@ -1442,7 +1401,7 @@ impl SearchInterfaceForPrivateItemsVisitor<'_> {
         if self.leaks_private_dep(def_id) {
             self.tcx.emit_spanned_lint(
                 lint::builtin::EXPORTED_PRIVATE_DEPENDENCIES,
-                self.tcx.hir().local_def_id_to_hir_id(self.item_def_id),
+                self.tcx.local_def_id_to_hir_id(self.item_def_id),
                 self.tcx.def_span(self.item_def_id.to_def_id()),
                 FromPrivateDependencyInPublicInterface {
                     kind,
@@ -1499,7 +1458,7 @@ impl SearchInterfaceForPrivateItemsVisitor<'_> {
             };
             self.tcx.emit_spanned_lint(
                 lint,
-                self.tcx.hir().local_def_id_to_hir_id(self.item_def_id),
+                self.tcx.local_def_id_to_hir_id(self.item_def_id),
                 span,
                 PrivateInterfacesOrBoundsLint {
                     item_span: span,
@@ -1509,11 +1468,11 @@ impl SearchInterfaceForPrivateItemsVisitor<'_> {
                         tcx: self.tcx,
                     })
                         .into(),
-                    item_vis_descr: &vis_to_string(self.item_def_id, reachable_at_vis, self.tcx),
+                    item_vis_descr: &reachable_at_vis.to_string(self.item_def_id, self.tcx),
                     ty_span: vis_span,
                     ty_kind: kind,
                     ty_descr: descr.into(),
-                    ty_vis_descr: &vis_to_string(local_def_id, vis, self.tcx),
+                    ty_vis_descr: &vis.to_string(local_def_id, self.tcx),
                 },
             );
         }
@@ -1582,7 +1541,7 @@ impl<'tcx> PrivateItemsInPublicInterfacesChecker<'tcx, '_> {
         let reachable_at_vis = effective_vis.at_level(Level::Reachable);
 
         if reachable_at_vis.is_public() && reexported_at_vis != reachable_at_vis {
-            let hir_id = self.tcx.hir().local_def_id_to_hir_id(def_id);
+            let hir_id = self.tcx.local_def_id_to_hir_id(def_id);
             let span = self.tcx.def_span(def_id.to_def_id());
             self.tcx.emit_spanned_lint(
                 lint::builtin::UNNAMEABLE_TYPES,
@@ -1592,8 +1551,8 @@ impl<'tcx> PrivateItemsInPublicInterfacesChecker<'tcx, '_> {
                     span,
                     kind: self.tcx.def_descr(def_id.to_def_id()),
                     descr: (&LazyDefPathStr { def_id: def_id.to_def_id(), tcx: self.tcx }).into(),
-                    reachable_vis: &vis_to_string(def_id, *reachable_at_vis, self.tcx),
-                    reexported_vis: &vis_to_string(def_id, *reexported_at_vis, self.tcx),
+                    reachable_vis: &reachable_at_vis.to_string(def_id, self.tcx),
+                    reexported_vis: &reexported_at_vis.to_string(def_id, self.tcx),
                 },
             );
         }
@@ -1741,7 +1700,7 @@ impl<'tcx> PrivateItemsInPublicInterfacesChecker<'tcx, '_> {
             // Subitems of trait impls have inherited publicity.
             DefKind::Impl { .. } => {
                 let item = tcx.hir().item(id);
-                if let hir::ItemKind::Impl(ref impl_) = item.kind {
+                if let hir::ItemKind::Impl(impl_) = item.kind {
                     let impl_vis = ty::Visibility::of_impl::<false>(
                         item.owner_id.def_id,
                         tcx,
@@ -1756,7 +1715,7 @@ impl<'tcx> PrivateItemsInPublicInterfacesChecker<'tcx, '_> {
                     //     fn from(_: Priv) -> Pub {...}
                     // }
                     //
-                    // lints shouldn't be emmited even if `from` effective visibility
+                    // lints shouldn't be emitted even if `from` effective visibility
                     // is larger than `Priv` nominal visibility and if `Priv` can leak
                     // in some scenarios due to type inference.
                     let impl_ev = EffectiveVisibility::of_impl::<false>(
@@ -1806,7 +1765,6 @@ impl<'tcx> PrivateItemsInPublicInterfacesChecker<'tcx, '_> {
 
 pub fn provide(providers: &mut Providers) {
     *providers = Providers {
-        visibility,
         effective_visibilities,
         check_private_in_public,
         check_mod_privacy,
@@ -1814,56 +1772,6 @@ pub fn provide(providers: &mut Providers) {
     };
 }
 
-fn visibility(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::Visibility<DefId> {
-    local_visibility(tcx, def_id).to_def_id()
-}
-
-fn local_visibility(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::Visibility {
-    match tcx.resolutions(()).visibilities.get(&def_id) {
-        Some(vis) => *vis,
-        None => {
-            let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
-            match tcx.hir().get(hir_id) {
-                // Unique types created for closures participate in type privacy checking.
-                // They have visibilities inherited from the module they are defined in.
-                Node::Expr(hir::Expr { kind: hir::ExprKind::Closure{..}, .. })
-                // - AST lowering creates dummy `use` items which don't
-                //   get their entries in the resolver's visibility table.
-                // - AST lowering also creates opaque type items with inherited visibilities.
-                //   Visibility on them should have no effect, but to avoid the visibility
-                //   query failing on some items, we provide it for opaque types as well.
-                | Node::Item(hir::Item {
-                    kind: hir::ItemKind::Use(_, hir::UseKind::ListStem)
-                        | hir::ItemKind::OpaqueTy(..),
-                    ..
-                }) => ty::Visibility::Restricted(tcx.parent_module(hir_id).to_local_def_id()),
-                // Visibilities of trait impl items are inherited from their traits
-                // and are not filled in resolve.
-                Node::ImplItem(impl_item) => {
-                    match tcx.hir().get_by_def_id(tcx.hir().get_parent_item(hir_id).def_id) {
-                        Node::Item(hir::Item {
-                            kind: hir::ItemKind::Impl(hir::Impl { of_trait: Some(tr), .. }),
-                            ..
-                        }) => tr.path.res.opt_def_id().map_or_else(
-                            || {
-                                tcx.sess.delay_span_bug(tr.path.span, "trait without a def-id");
-                                ty::Visibility::Public
-                            },
-                            |def_id| tcx.visibility(def_id).expect_local(),
-                        ),
-                        _ => span_bug!(impl_item.span, "the parent is not a trait impl"),
-                    }
-                }
-                _ => span_bug!(
-                    tcx.def_span(def_id),
-                    "visibility table unexpectedly missing a def-id: {:?}",
-                    def_id,
-                ),
-            }
-        }
-    }
-}
-
 fn check_mod_privacy(tcx: TyCtxt<'_>, module_def_id: LocalModDefId) {
     // Check privacy of names not checked in previous compilation stages.
     let mut visitor = NamePrivacyVisitor {
diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs
index a1465dabe..59812efc3 100644
--- a/compiler/rustc_query_impl/src/lib.rs
+++ b/compiler/rustc_query_impl/src/lib.rs
@@ -1,8 +1,8 @@
 //! Support for serializing the dep-graph and reloading it.
 
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
+#![doc(rust_logo)]
+#![feature(rustdoc_internals)]
 // this shouldn't be necessary, but the check for `&mut _` is too naive and denies returning a function pointer that takes a mut ref
 #![feature(const_mut_refs)]
 #![feature(const_refs_to_cell)]
diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs
index 6ad72e37a..acd3d71e6 100644
--- a/compiler/rustc_query_impl/src/plumbing.rs
+++ b/compiler/rustc_query_impl/src/plumbing.rs
@@ -80,14 +80,14 @@ impl QueryContext for QueryCtxt<'_> {
         tls::with_related_context(self.tcx, |icx| icx.query)
     }
 
-    fn try_collect_active_jobs(self) -> Option<QueryMap> {
+    fn collect_active_jobs(self) -> QueryMap {
         let mut jobs = QueryMap::default();
 
         for collect in super::TRY_COLLECT_ACTIVE_JOBS.iter() {
             collect(self.tcx, &mut jobs);
         }
 
-        Some(jobs)
+        jobs
     }
 
     // Interactions with on_disk_cache
@@ -155,11 +155,11 @@ impl QueryContext for QueryCtxt<'_> {
     fn depth_limit_error(self, job: QueryJobId) {
         let mut span = None;
         let mut layout_of_depth = None;
-        if let Some(map) = self.try_collect_active_jobs() {
-            if let Some((info, depth)) = job.try_find_layout_root(map, dep_kinds::layout_of) {
-                span = Some(info.job.span);
-                layout_of_depth = Some(LayoutOfDepth { desc: info.query.description, depth });
-            }
+        if let Some((info, depth)) =
+            job.try_find_layout_root(self.collect_active_jobs(), dep_kinds::layout_of)
+        {
+            span = Some(info.job.span);
+            layout_of_depth = Some(LayoutOfDepth { desc: info.query.description, depth });
         }
 
         let suggested_limit = match self.recursion_limit() {
@@ -325,11 +325,11 @@ pub(crate) fn create_query_frame<
         Some(key.default_span(tcx))
     };
     let def_id = key.key_as_def_id();
-    let def_kind = if kind == dep_graph::dep_kinds::opt_def_kind || with_no_queries() {
+    let def_kind = if kind == dep_graph::dep_kinds::def_kind || with_no_queries() {
         // Try to avoid infinite recursion.
         None
     } else {
-        def_id.and_then(|def_id| def_id.as_local()).and_then(|def_id| tcx.opt_def_kind(def_id))
+        def_id.and_then(|def_id| def_id.as_local()).map(|def_id| tcx.def_kind(def_id))
     };
     let hash = || {
         tcx.with_stable_hashing_context(|mut hcx| {
@@ -358,7 +358,7 @@ pub(crate) fn encode_query_results<'a, 'tcx, Q>(
     assert!(query.query_state(qcx).all_inactive());
     let cache = query.query_cache(qcx);
     cache.iter(&mut |key, value, dep_node| {
-        if query.cache_on_disk(qcx.tcx, &key) {
+        if query.cache_on_disk(qcx.tcx, key) {
             let dep_node = SerializedDepNodeIndex::new(dep_node.index());
 
             // Record position of the cache entry.
diff --git a/compiler/rustc_query_system/src/dep_graph/edges.rs b/compiler/rustc_query_system/src/dep_graph/edges.rs
index 6ba3924f6..63d46f47f 100644
--- a/compiler/rustc_query_system/src/dep_graph/edges.rs
+++ b/compiler/rustc_query_system/src/dep_graph/edges.rs
@@ -1,11 +1,10 @@
 use crate::dep_graph::DepNodeIndex;
 use smallvec::SmallVec;
 use std::hash::{Hash, Hasher};
-use std::iter::Extend;
 use std::ops::Deref;
 
 #[derive(Default, Debug)]
-pub struct EdgesVec {
+pub(crate) struct EdgesVec {
     max: u32,
     edges: SmallVec<[DepNodeIndex; EdgesVec::INLINE_CAPACITY]>,
 }
@@ -18,21 +17,21 @@ impl Hash for EdgesVec {
 }
 
 impl EdgesVec {
-    pub const INLINE_CAPACITY: usize = 8;
+    pub(crate) const INLINE_CAPACITY: usize = 8;
 
     #[inline]
-    pub fn new() -> Self {
+    pub(crate) fn new() -> Self {
         Self::default()
     }
 
     #[inline]
-    pub fn push(&mut self, edge: DepNodeIndex) {
+    pub(crate) fn push(&mut self, edge: DepNodeIndex) {
         self.max = self.max.max(edge.as_u32());
         self.edges.push(edge);
     }
 
     #[inline]
-    pub fn max_index(&self) -> u32 {
+    pub(crate) fn max_index(&self) -> u32 {
         self.max
     }
 }
diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_query_system/src/dep_graph/graph.rs
index 6cace0195..3556a5ec1 100644
--- a/compiler/rustc_query_system/src/dep_graph/graph.rs
+++ b/compiler/rustc_query_system/src/dep_graph/graph.rs
@@ -18,7 +18,7 @@ use std::sync::atomic::Ordering::Relaxed;
 use super::query::DepGraphQuery;
 use super::serialized::{GraphEncoder, SerializedDepGraph, SerializedDepNodeIndex};
 use super::{DepContext, DepKind, DepNode, Deps, HasDepContext, WorkProductId};
-use crate::dep_graph::EdgesVec;
+use crate::dep_graph::edges::EdgesVec;
 use crate::ich::StableHashingContext;
 use crate::query::{QueryContext, QuerySideEffects};
 
@@ -41,8 +41,7 @@ rustc_index::newtype_index! {
 }
 
 impl DepNodeIndex {
-    pub const INVALID: DepNodeIndex = DepNodeIndex::MAX;
-    pub const SINGLETON_DEPENDENCYLESS_ANON_NODE: DepNodeIndex = DepNodeIndex::from_u32(0);
+    const SINGLETON_DEPENDENCYLESS_ANON_NODE: DepNodeIndex = DepNodeIndex::from_u32(0);
     pub const FOREVER_RED_NODE: DepNodeIndex = DepNodeIndex::from_u32(1);
 }
 
@@ -53,20 +52,20 @@ impl From<DepNodeIndex> for QueryInvocationId {
     }
 }
 
-pub struct MarkFrame<'a> {
+pub(crate) struct MarkFrame<'a> {
     index: SerializedDepNodeIndex,
     parent: Option<&'a MarkFrame<'a>>,
 }
 
 #[derive(PartialEq)]
-pub enum DepNodeColor {
+enum DepNodeColor {
     Red,
     Green(DepNodeIndex),
 }
 
 impl DepNodeColor {
     #[inline]
-    pub fn is_green(self) -> bool {
+    fn is_green(self) -> bool {
         match self {
             DepNodeColor::Red => false,
             DepNodeColor::Green(_) => true,
@@ -74,7 +73,7 @@ impl DepNodeColor {
     }
 }
 
-pub struct DepGraphData<D: Deps> {
+pub(crate) struct DepGraphData<D: Deps> {
     /// The new encoding of the dependency graph, optimized for red/green
     /// tracking. The `current` field is the dependency graph of only the
     /// current compilation session: We don't merge the previous dep-graph into
@@ -185,7 +184,7 @@ impl<D: Deps> DepGraph<D> {
     }
 
     #[inline]
-    pub fn data(&self) -> Option<&DepGraphData<D>> {
+    pub(crate) fn data(&self) -> Option<&DepGraphData<D>> {
         self.data.as_deref()
     }
 
@@ -333,7 +332,7 @@ impl<D: Deps> DepGraphData<D> {
     ///
     /// [rustc dev guide]: https://rustc-dev-guide.rust-lang.org/queries/incremental-compilation.html
     #[inline(always)]
-    pub fn with_task<Ctxt: HasDepContext<Deps = D>, A: Debug, R>(
+    pub(crate) fn with_task<Ctxt: HasDepContext<Deps = D>, A: Debug, R>(
         &self,
         key: DepNode,
         cx: Ctxt,
@@ -398,7 +397,7 @@ impl<D: Deps> DepGraphData<D> {
 
     /// Executes something within an "anonymous" task, that is, a task the
     /// `DepNode` of which is determined by the list of inputs it read from.
-    pub fn with_anon_task<Tcx: DepContext<Deps = D>, OP, R>(
+    pub(crate) fn with_anon_task<Tcx: DepContext<Deps = D>, OP, R>(
         &self,
         cx: Tcx,
         dep_kind: DepKind,
@@ -618,7 +617,7 @@ impl<D: Deps> DepGraph<D> {
 
 impl<D: Deps> DepGraphData<D> {
     #[inline]
-    pub fn dep_node_index_of_opt(&self, dep_node: &DepNode) -> Option<DepNodeIndex> {
+    fn dep_node_index_of_opt(&self, dep_node: &DepNode) -> Option<DepNodeIndex> {
         if let Some(prev_index) = self.previous.node_to_index_opt(dep_node) {
             self.current.prev_index_to_index.lock()[prev_index]
         } else {
@@ -627,7 +626,7 @@ impl<D: Deps> DepGraphData<D> {
     }
 
     #[inline]
-    pub fn dep_node_exists(&self, dep_node: &DepNode) -> bool {
+    fn dep_node_exists(&self, dep_node: &DepNode) -> bool {
         self.dep_node_index_of_opt(dep_node).is_some()
     }
 
@@ -643,21 +642,21 @@ impl<D: Deps> DepGraphData<D> {
     /// Returns true if the given node has been marked as green during the
     /// current compilation session. Used in various assertions
     #[inline]
-    pub fn is_index_green(&self, prev_index: SerializedDepNodeIndex) -> bool {
+    pub(crate) fn is_index_green(&self, prev_index: SerializedDepNodeIndex) -> bool {
         self.colors.get(prev_index).is_some_and(|c| c.is_green())
     }
 
     #[inline]
-    pub fn prev_fingerprint_of(&self, prev_index: SerializedDepNodeIndex) -> Fingerprint {
+    pub(crate) fn prev_fingerprint_of(&self, prev_index: SerializedDepNodeIndex) -> Fingerprint {
         self.previous.fingerprint_by_index(prev_index)
     }
 
     #[inline]
-    pub fn prev_node_of(&self, prev_index: SerializedDepNodeIndex) -> DepNode {
+    pub(crate) fn prev_node_of(&self, prev_index: SerializedDepNodeIndex) -> DepNode {
         self.previous.index_to_node(prev_index)
     }
 
-    pub fn mark_debug_loaded_from_disk(&self, dep_node: DepNode) {
+    pub(crate) fn mark_debug_loaded_from_disk(&self, dep_node: DepNode) {
         self.debug_loaded_from_disk.lock().insert(dep_node);
     }
 }
@@ -684,8 +683,9 @@ impl<D: Deps> DepGraph<D> {
         self.data.as_ref().unwrap().debug_loaded_from_disk.lock().contains(&dep_node)
     }
 
+    #[cfg(debug_assertions)]
     #[inline(always)]
-    pub fn register_dep_node_debug_str<F>(&self, dep_node: DepNode, debug_str_gen: F)
+    pub(crate) fn register_dep_node_debug_str<F>(&self, dep_node: DepNode, debug_str_gen: F)
     where
         F: FnOnce() -> String,
     {
@@ -725,7 +725,7 @@ impl<D: Deps> DepGraphData<D> {
     /// A node will have an index, when it's already been marked green, or when we can mark it
     /// green. This function will mark the current task as a reader of the specified node, when
     /// a node index can be found for that node.
-    pub fn try_mark_green<Qcx: QueryContext<Deps = D>>(
+    pub(crate) fn try_mark_green<Qcx: QueryContext<Deps = D>>(
         &self,
         qcx: Qcx,
         dep_node: &DepNode,
@@ -743,7 +743,7 @@ impl<D: Deps> DepGraphData<D> {
                 // in the previous compilation session too, so we can try to
                 // mark it as green by recursively marking all of its
                 // dependencies green.
-                self.try_mark_previous_green(qcx, prev_index, &dep_node, None)
+                self.try_mark_previous_green(qcx, prev_index, dep_node, None)
                     .map(|dep_node_index| (prev_index, dep_node_index))
             }
         }
@@ -818,7 +818,7 @@ impl<D: Deps> DepGraphData<D> {
             None => {}
         }
 
-        if let None = qcx.dep_context().sess().has_errors_or_delayed_span_bugs() {
+        if let None = qcx.dep_context().sess().has_errors_or_span_delayed_bugs() {
             panic!("try_mark_previous_green() - Forcing the DepNode should have set its color")
         }
 
@@ -924,10 +924,10 @@ impl<D: Deps> DepGraphData<D> {
             // Promote the previous diagnostics to the current session.
             qcx.store_side_effects(dep_node_index, side_effects.clone());
 
-            let handle = qcx.dep_context().sess().diagnostic();
+            let dcx = qcx.dep_context().sess().dcx();
 
-            for mut diagnostic in side_effects.diagnostics {
-                handle.emit_diagnostic(&mut diagnostic);
+            for diagnostic in side_effects.diagnostics {
+                dcx.emit_diagnostic(diagnostic);
             }
         }
     }
@@ -982,7 +982,7 @@ impl<D: Deps> DepGraph<D> {
         }
     }
 
-    pub fn encode(&self, profiler: &SelfProfilerRef) -> FileEncodeResult {
+    pub fn finish_encoding(&self, profiler: &SelfProfilerRef) -> FileEncodeResult {
         if let Some(data) = &self.data {
             data.current.encoder.steal().finish(profiler)
         } else {
diff --git a/compiler/rustc_query_system/src/dep_graph/mod.rs b/compiler/rustc_query_system/src/dep_graph/mod.rs
index 624ae680a..feb69ecd0 100644
--- a/compiler/rustc_query_system/src/dep_graph/mod.rs
+++ b/compiler/rustc_query_system/src/dep_graph/mod.rs
@@ -6,11 +6,8 @@ mod query;
 mod serialized;
 
 pub use dep_node::{DepKind, DepKindStruct, DepNode, DepNodeParams, WorkProductId};
-pub use edges::EdgesVec;
-pub use graph::{
-    hash_result, DepGraph, DepGraphData, DepNodeColor, DepNodeIndex, TaskDeps, TaskDepsRef,
-    WorkProduct, WorkProductMap,
-};
+pub(crate) use graph::DepGraphData;
+pub use graph::{hash_result, DepGraph, DepNodeIndex, TaskDepsRef, WorkProduct, WorkProductMap};
 pub use query::DepGraphQuery;
 pub use serialized::{SerializedDepGraph, SerializedDepNodeIndex};
 
diff --git a/compiler/rustc_query_system/src/dep_graph/serialized.rs b/compiler/rustc_query_system/src/dep_graph/serialized.rs
index fcf46be6e..504763f6c 100644
--- a/compiler/rustc_query_system/src/dep_graph/serialized.rs
+++ b/compiler/rustc_query_system/src/dep_graph/serialized.rs
@@ -37,7 +37,7 @@
 
 use super::query::DepGraphQuery;
 use super::{DepKind, DepNode, DepNodeIndex, Deps};
-use crate::dep_graph::EdgesVec;
+use crate::dep_graph::edges::EdgesVec;
 use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::fingerprint::PackedFingerprint;
 use rustc_data_structures::fx::FxHashMap;
@@ -54,6 +54,7 @@ use std::marker::PhantomData;
 // unused so that we can store multiple index types in `CompressedHybridIndex`,
 // and use those bits to encode which index type it contains.
 rustc_index::newtype_index! {
+    #[encodable]
     #[max = 0x7FFF_FFFF]
     pub struct SerializedDepNodeIndex {}
 }
@@ -69,7 +70,7 @@ const DEP_NODE_PAD: usize = DEP_NODE_SIZE - 1;
 const DEP_NODE_WIDTH_BITS: usize = DEP_NODE_SIZE / 2;
 
 /// Data for use when recompiling the **current crate**.
-#[derive(Debug)]
+#[derive(Debug, Default)]
 pub struct SerializedDepGraph {
     /// The set of all DepNodes in the graph
     nodes: IndexVec<SerializedDepNodeIndex, DepNode>,
@@ -88,18 +89,6 @@ pub struct SerializedDepGraph {
     index: Vec<UnhashMap<PackedFingerprint, SerializedDepNodeIndex>>,
 }
 
-impl Default for SerializedDepGraph {
-    fn default() -> Self {
-        SerializedDepGraph {
-            nodes: Default::default(),
-            fingerprints: Default::default(),
-            edge_list_indices: Default::default(),
-            edge_list_data: Default::default(),
-            index: Default::default(),
-        }
-    }
-}
-
 impl SerializedDepGraph {
     #[inline]
     pub fn edge_targets_from(
diff --git a/compiler/rustc_query_system/src/ich/hcx.rs b/compiler/rustc_query_system/src/ich/hcx.rs
index 5593a1541..eec0433ae 100644
--- a/compiler/rustc_query_system/src/ich/hcx.rs
+++ b/compiler/rustc_query_system/src/ich/hcx.rs
@@ -28,7 +28,7 @@ pub struct StableHashingContext<'a> {
     // `CachingSourceMapView`, so we initialize it lazily.
     raw_source_map: &'a SourceMap,
     caching_source_map: Option<CachingSourceMapView<'a>>,
-    pub(super) hashing_controls: HashingControls,
+    hashing_controls: HashingControls,
 }
 
 /// The `BodyResolver` allows mapping a `BodyId` to the corresponding `hir::Body`.
diff --git a/compiler/rustc_query_system/src/ich/impls_syntax.rs b/compiler/rustc_query_system/src/ich/impls_syntax.rs
index b2177be0e..f2387017c 100644
--- a/compiler/rustc_query_system/src/ich/impls_syntax.rs
+++ b/compiler/rustc_query_system/src/ich/impls_syntax.rs
@@ -117,11 +117,9 @@ impl<'tcx> HashStable<StableHashingContext<'tcx>> for rustc_feature::Features {
         self.declared_lang_features.hash_stable(hcx, hasher);
         self.declared_lib_features.hash_stable(hcx, hasher);
 
-        self.walk_feature_fields(|feature_name, value| {
-            feature_name.hash_stable(hcx, hasher);
-            value.hash_stable(hcx, hasher);
-        });
+        self.all_features()[..].hash_stable(hcx, hasher);
+        for feature in rustc_feature::UNSTABLE_FEATURES.iter() {
+            feature.feature.name.hash_stable(hcx, hasher);
+        }
     }
 }
-
-impl<'ctx> rustc_type_ir::HashStableContext for StableHashingContext<'ctx> {}
diff --git a/compiler/rustc_query_system/src/lib.rs b/compiler/rustc_query_system/src/lib.rs
index 1944ac443..9b66b9a48 100644
--- a/compiler/rustc_query_system/src/lib.rs
+++ b/compiler/rustc_query_system/src/lib.rs
@@ -2,10 +2,8 @@
 #![feature(core_intrinsics)]
 #![feature(hash_raw_entry)]
 #![feature(min_specialization)]
-#![feature(extern_types)]
 #![feature(let_chains)]
-#![feature(inline_const)]
-#![allow(rustc::potential_query_instability)]
+#![allow(rustc::potential_query_instability, internal_features)]
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
 
@@ -16,9 +14,6 @@ extern crate rustc_data_structures;
 #[macro_use]
 extern crate rustc_macros;
 
-use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
-use rustc_fluent_macro::fluent_messages;
-
 pub mod cache;
 pub mod dep_graph;
 mod error;
@@ -31,4 +26,4 @@ pub use error::LayoutOfDepth;
 pub use error::QueryOverflow;
 pub use values::Value;
 
-fluent_messages! { "../messages.ftl" }
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
diff --git a/compiler/rustc_query_system/src/query/job.rs b/compiler/rustc_query_system/src/query/job.rs
index f2c1f84fc..2a34ffb75 100644
--- a/compiler/rustc_query_system/src/query/job.rs
+++ b/compiler/rustc_query_system/src/query/job.rs
@@ -5,7 +5,7 @@ use crate::query::DepKind;
 use crate::query::{QueryContext, QueryStackFrame};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_errors::{
-    Diagnostic, DiagnosticBuilder, ErrorGuaranteed, Handler, IntoDiagnostic, Level,
+    DiagCtxt, Diagnostic, DiagnosticBuilder, ErrorGuaranteed, IntoDiagnostic, Level,
 };
 use rustc_hir::def::DefKind;
 use rustc_session::Session;
@@ -18,7 +18,6 @@ use std::num::NonZeroU64;
 #[cfg(parallel_compiler)]
 use {
     parking_lot::{Condvar, Mutex},
-    rayon_core,
     rustc_data_structures::fx::FxHashSet,
     rustc_data_structures::{defer, jobserver},
     rustc_span::DUMMY_SP,
@@ -38,7 +37,7 @@ pub struct QueryInfo {
 pub type QueryMap = FxHashMap<QueryJobId, QueryJobInfo>;
 
 /// A value uniquely identifying an active query job.
-#[derive(Copy, Clone, Eq, PartialEq, Hash)]
+#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)]
 pub struct QueryJobId(pub NonZeroU64);
 
 impl QueryJobId {
@@ -62,14 +61,14 @@ impl QueryJobId {
     }
 }
 
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub struct QueryJobInfo {
     pub query: QueryStackFrame,
     pub job: QueryJob,
 }
 
 /// Represents an active query job.
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub struct QueryJob {
     pub id: QueryJobId,
 
@@ -182,6 +181,7 @@ impl QueryJobId {
 }
 
 #[cfg(parallel_compiler)]
+#[derive(Debug)]
 struct QueryWaiter {
     query: Option<QueryJobId>,
     condvar: Condvar,
@@ -198,13 +198,14 @@ impl QueryWaiter {
 }
 
 #[cfg(parallel_compiler)]
+#[derive(Debug)]
 struct QueryLatchInfo {
     complete: bool,
     waiters: Vec<Arc<QueryWaiter>>,
 }
 
 #[cfg(parallel_compiler)]
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub(super) struct QueryLatch {
     info: Arc<Mutex<QueryLatchInfo>>,
 }
@@ -540,7 +541,11 @@ pub fn deadlock(query_map: QueryMap, registry: &rayon_core::Registry) {
     // X to Y due to Rayon waiting and a true dependency from Y to X. The algorithm here
     // only considers the true dependency and won't detect a cycle.
     if !found_cycle {
-        panic!("deadlock detected");
+        if query_map.len() == 0 {
+            panic!("deadlock detected without any query!")
+        } else {
+            panic!("deadlock detected! current query map:\n{:#?}", query_map);
+        }
     }
 
     // FIXME: Ensure this won't cause a deadlock before we return
@@ -599,28 +604,28 @@ pub(crate) fn report_cycle<'a>(
         note_span: (),
     };
 
-    cycle_diag.into_diagnostic(&sess.parse_sess.span_diagnostic)
+    cycle_diag.into_diagnostic(sess.dcx())
 }
 
 pub fn print_query_stack<Qcx: QueryContext>(
     qcx: Qcx,
     mut current_query: Option<QueryJobId>,
-    handler: &Handler,
+    dcx: &DiagCtxt,
     num_frames: Option<usize>,
     mut file: Option<std::fs::File>,
 ) -> usize {
     // Be careful relying on global state here: this code is called from
-    // a panic hook, which means that the global `Handler` may be in a weird
+    // a panic hook, which means that the global `DiagCtxt` may be in a weird
     // state if it was responsible for triggering the panic.
     let mut count_printed = 0;
     let mut count_total = 0;
-    let query_map = qcx.try_collect_active_jobs();
+    let query_map = qcx.collect_active_jobs();
 
     if let Some(ref mut file) = file {
         let _ = writeln!(file, "\n\nquery stack during panic:");
     }
     while let Some(query) = current_query {
-        let Some(query_info) = query_map.as_ref().and_then(|map| map.get(&query)) else {
+        let Some(query_info) = query_map.get(&query) else {
             break;
         };
         if Some(count_printed) < num_frames || num_frames.is_none() {
@@ -633,7 +638,7 @@ pub fn print_query_stack<Qcx: QueryContext>(
                 ),
             );
             diag.span = query_info.job.span.into();
-            handler.force_print_diagnostic(diag);
+            dcx.force_print_diagnostic(diag);
             count_printed += 1;
         }
 
diff --git a/compiler/rustc_query_system/src/query/mod.rs b/compiler/rustc_query_system/src/query/mod.rs
index 05dee9f12..96a0c7a03 100644
--- a/compiler/rustc_query_system/src/query/mod.rs
+++ b/compiler/rustc_query_system/src/query/mod.rs
@@ -106,7 +106,7 @@ pub trait QueryContext: HasDepContext {
     /// Get the query information from the TLS context.
     fn current_query_job(self) -> Option<QueryJobId>;
 
-    fn try_collect_active_jobs(self) -> Option<QueryMap>;
+    fn collect_active_jobs(self) -> QueryMap;
 
     /// Load side effects associated to the node in the previous session.
     fn load_side_effects(self, prev_dep_node_index: SerializedDepNodeIndex) -> QuerySideEffects;
diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs
index 1f3403d09..41638b38c 100644
--- a/compiler/rustc_query_system/src/query/plumbing.rs
+++ b/compiler/rustc_query_system/src/query/plumbing.rs
@@ -138,7 +138,7 @@ where
                 && let Some(span) = root.query.span
             {
                 error.stash(span, StashKey::Cycle);
-                qcx.dep_context().sess().delay_span_bug(span, "delayed cycle error")
+                qcx.dep_context().sess().span_delayed_bug(span, "delayed cycle error")
             } else {
                 error.emit()
             };
@@ -203,7 +203,7 @@ where
     }
 }
 
-#[derive(Clone)]
+#[derive(Clone, Debug)]
 pub(crate) struct CycleError {
     /// The query and related span that uses the cycle.
     pub usage: Option<(Span, QueryStackFrame)>,
@@ -220,7 +220,7 @@ where
     C: QueryCache,
     Tcx: DepContext,
 {
-    match cache.lookup(&key) {
+    match cache.lookup(key) {
         Some((value, index)) => {
             tcx.profiler().query_cache_hit(index.into());
             tcx.dep_graph().read_index(index);
@@ -242,11 +242,8 @@ where
     Q: QueryConfig<Qcx>,
     Qcx: QueryContext,
 {
-    let error = try_execute.find_cycle_in_stack(
-        qcx.try_collect_active_jobs().unwrap(),
-        &qcx.current_query_job(),
-        span,
-    );
+    let error =
+        try_execute.find_cycle_in_stack(qcx.collect_active_jobs(), &qcx.current_query_job(), span);
     (mk_cycle(query, qcx, error), None)
 }
 
@@ -424,7 +421,7 @@ where
                 // We have an inconsistency. This can happen if one of the two
                 // results is tainted by errors. In this case, delay a bug to
                 // ensure compilation is doomed.
-                qcx.dep_context().sess().delay_span_bug(
+                qcx.dep_context().sess().span_delayed_bug(
                     DUMMY_SP,
                     format!(
                         "Computed query value for {:?}({:?}) is inconsistent with fed value,\n\
@@ -502,7 +499,7 @@ where
         // The diagnostics for this query will be promoted to the current session during
         // `try_mark_green()`, so we can ignore them here.
         if let Some(ret) = qcx.start_query(job_id, false, None, || {
-            try_load_from_disk_and_cache_in_memory(query, dep_graph_data, qcx, &key, &dep_node)
+            try_load_from_disk_and_cache_in_memory(query, dep_graph_data, qcx, &key, dep_node)
         }) {
             return ret;
         }
@@ -563,7 +560,7 @@ where
     // Note this function can be called concurrently from the same query
     // We must ensure that this is handled correctly.
 
-    let (prev_dep_node_index, dep_node_index) = dep_graph_data.try_mark_green(qcx, &dep_node)?;
+    let (prev_dep_node_index, dep_node_index) = dep_graph_data.try_mark_green(qcx, dep_node)?;
 
     debug_assert!(dep_graph_data.is_index_green(prev_dep_node_index));
 
@@ -610,7 +607,7 @@ where
     // Sanity check for the logic in `ensure`: if the node is green and the result loadable,
     // we should actually be able to load it.
     debug_assert!(
-        !query.loadable_from_disk(qcx, &key, prev_dep_node_index),
+        !query.loadable_from_disk(qcx, key, prev_dep_node_index),
         "missing on-disk cache entry for loadable {dep_node:?}"
     );
 
@@ -667,7 +664,7 @@ pub(crate) fn incremental_verify_ich<Tcx, V>(
     let old_hash = dep_graph_data.prev_fingerprint_of(prev_index);
 
     if new_hash != old_hash {
-        incremental_verify_ich_failed(tcx, prev_index, &|| format_value(&result));
+        incremental_verify_ich_failed(tcx, prev_index, &|| format_value(result));
     }
 }
 
diff --git a/compiler/rustc_resolve/messages.ftl b/compiler/rustc_resolve/messages.ftl
index 272483d4a..3f8df16e0 100644
--- a/compiler/rustc_resolve/messages.ftl
+++ b/compiler/rustc_resolve/messages.ftl
@@ -127,8 +127,6 @@ resolve_generic_params_from_outer_item_self_ty_param = can't use `Self` here
 
 resolve_generic_params_from_outer_item_ty_param = type parameter from outer item
 
-resolve_glob_import_doesnt_reexport =
-    glob import doesn't reexport anything because no candidate is public enough
 
 resolve_ident_bound_more_than_once_in_parameter_list =
     identifier `{$identifier}` is bound more than once in this parameter list
@@ -183,6 +181,8 @@ resolve_method_not_member_of_trait =
     method `{$method}` is not a member of trait `{$trait_}`
     .label = not a member of trait `{$trait_}`
 
+resolve_missing_macro_rules_name = maybe you have forgotten to define a name for this `macro_rules!`
+
 resolve_module_only =
     visibility must resolve to a module
 
@@ -200,7 +200,7 @@ resolve_param_in_non_trivial_anon_const =
     .label = cannot perform const operation using `{$name}`
 
 resolve_param_in_non_trivial_anon_const_help =
-    use `#![feature(generic_const_exprs)]` to allow generic const expressions
+    add `#![feature(generic_const_exprs)]` to allow generic const expressions
 
 resolve_param_in_ty_of_const_param =
     the type of const parameters must not depend on other generic parameters
diff --git a/compiler/rustc_resolve/src/build_reduced_graph.rs b/compiler/rustc_resolve/src/build_reduced_graph.rs
index 0407db528..98a9d0ba4 100644
--- a/compiler/rustc_resolve/src/build_reduced_graph.rs
+++ b/compiler/rustc_resolve/src/build_reduced_graph.rs
@@ -123,7 +123,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
                     .map(|parent_id| self.get_nearest_non_block_module(parent_id));
                 // Query `expn_that_defined` is not used because
                 // hashing spans in its result is expensive.
-                let expn_id = self.cstore().expn_that_defined_untracked(def_id, &self.tcx.sess);
+                let expn_id = self.cstore().expn_that_defined_untracked(def_id, self.tcx.sess);
                 return Some(self.new_module(
                     parent,
                     ModuleKind::Def(def_kind, def_id, self.tcx.item_name(def_id)),
@@ -156,33 +156,26 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
         }
     }
 
-    pub(crate) fn get_macro(&mut self, res: Res) -> Option<MacroData> {
+    pub(crate) fn get_macro(&mut self, res: Res) -> Option<&MacroData> {
         match res {
             Res::Def(DefKind::Macro(..), def_id) => Some(self.get_macro_by_def_id(def_id)),
-            Res::NonMacroAttr(_) => {
-                Some(MacroData { ext: self.non_macro_attr.clone(), macro_rules: false })
-            }
+            Res::NonMacroAttr(_) => Some(&self.non_macro_attr),
             _ => None,
         }
     }
 
-    pub(crate) fn get_macro_by_def_id(&mut self, def_id: DefId) -> MacroData {
-        if let Some(macro_data) = self.macro_map.get(&def_id) {
-            return macro_data.clone();
+    pub(crate) fn get_macro_by_def_id(&mut self, def_id: DefId) -> &MacroData {
+        if self.macro_map.contains_key(&def_id) {
+            return &self.macro_map[&def_id];
         }
 
-        let load_macro_untracked = self.cstore().load_macro_untracked(def_id, self.tcx);
-        let (ext, macro_rules) = match load_macro_untracked {
-            LoadedMacro::MacroDef(item, edition) => (
-                Lrc::new(self.compile_macro(&item, edition).0),
-                matches!(item.kind, ItemKind::MacroDef(def) if def.macro_rules),
-            ),
-            LoadedMacro::ProcMacro(extz) => (Lrc::new(extz), false),
+        let loaded_macro = self.cstore().load_macro_untracked(def_id, self.tcx);
+        let macro_data = match loaded_macro {
+            LoadedMacro::MacroDef(item, edition) => self.compile_macro(&item, edition),
+            LoadedMacro::ProcMacro(ext) => MacroData::new(Lrc::new(ext)),
         };
 
-        let macro_data = MacroData { ext, macro_rules };
-        self.macro_map.insert(def_id, macro_data.clone());
-        macro_data
+        self.macro_map.entry(def_id).or_insert(macro_data)
     }
 
     pub(crate) fn build_reduced_graph(
@@ -217,6 +210,11 @@ impl<'a, 'tcx> AsMut<Resolver<'a, 'tcx>> for BuildReducedGraphVisitor<'a, '_, 't
 }
 
 impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
+    fn res(&self, def_id: impl Into<DefId>) -> Res {
+        let def_id = def_id.into();
+        Res::Def(self.r.tcx.def_kind(def_id), def_id)
+    }
+
     fn resolve_visibility(&mut self, vis: &ast::Visibility) -> ty::Visibility {
         self.try_resolve_visibility(vis, true).unwrap_or_else(|err| {
             self.r.report_vis_error(err);
@@ -238,7 +236,7 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
                     // (i.e. variants, fields, and trait items) inherits from the visibility
                     // of the enum or trait.
                     ModuleKind::Def(DefKind::Enum | DefKind::Trait, def_id, _) => {
-                        self.r.visibilities[&def_id.expect_local()]
+                        self.r.tcx.visibility(def_id).expect_local()
                     }
                     // Otherwise, the visibility is restricted to the nearest parent `mod` item.
                     _ => ty::Visibility::Restricted(
@@ -396,6 +394,7 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
         id: NodeId,
         parent_prefix: &[Segment],
         nested: bool,
+        list_stem: bool,
         // The whole `use` item
         item: &Item,
         vis: ty::Visibility,
@@ -406,6 +405,12 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
             parent_prefix, use_tree, nested
         );
 
+        // Top level use tree reuses the item's id and list stems reuse their parent
+        // use tree's ids, so in both cases their visibilities are already filled.
+        if nested && !list_stem {
+            self.r.feed_visibility(self.r.local_def_id(id), vis);
+        }
+
         let mut prefix_iter = parent_prefix
             .iter()
             .cloned()
@@ -444,8 +449,6 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
                 let mut source = module_path.pop().unwrap();
                 let mut type_ns_only = false;
 
-                self.r.visibilities.insert(self.r.local_def_id(id), vis);
-
                 if nested {
                     // Correctly handle `self`
                     if source.ident.name == kw::SelfLower {
@@ -559,7 +562,7 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
                     max_vis: Cell::new(None),
                     id,
                 };
-                self.r.visibilities.insert(self.r.local_def_id(id), vis);
+
                 self.add_import(prefix, kind, use_tree.span, item, root_span, item.id, vis);
             }
             ast::UseTreeKind::Nested(ref items) => {
@@ -592,7 +595,7 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
                 for &(ref tree, id) in items {
                     self.build_reduced_graph_for_use_tree(
                         // This particular use tree
-                        tree, id, &prefix, true, // The whole `use` item
+                        tree, id, &prefix, true, false, // The whole `use` item
                         item, vis, root_span,
                     );
                 }
@@ -613,6 +616,7 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
                         id,
                         &prefix,
                         true,
+                        true,
                         // The whole `use` item
                         item,
                         ty::Visibility::Restricted(
@@ -635,8 +639,10 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
         let vis = self.resolve_visibility(&item.vis);
         let local_def_id = self.r.local_def_id(item.id);
         let def_id = local_def_id.to_def_id();
+        let def_kind = self.r.tcx.def_kind(def_id);
+        let res = Res::Def(def_kind, def_id);
 
-        self.r.visibilities.insert(local_def_id, vis);
+        self.r.feed_visibility(local_def_id, vis);
 
         match item.kind {
             ItemKind::Use(ref use_tree) => {
@@ -646,6 +652,7 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
                     item.id,
                     &[],
                     false,
+                    false,
                     // The whole `use` item
                     item,
                     vis,
@@ -666,7 +673,7 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
             ItemKind::Mod(..) => {
                 let module = self.r.new_module(
                     Some(parent),
-                    ModuleKind::Def(DefKind::Mod, def_id, ident.name),
+                    ModuleKind::Def(def_kind, def_id, ident.name),
                     expansion.to_expn_id(),
                     item.span,
                     parent.no_implicit_prelude
@@ -679,16 +686,13 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
             }
 
             // These items live in the value namespace.
-            ItemKind::Static(box ast::StaticItem { mutability, .. }) => {
-                let res = Res::Def(DefKind::Static(mutability), def_id);
+            ItemKind::Static(..) => {
                 self.r.define(parent, ident, ValueNS, (res, vis, sp, expansion));
             }
             ItemKind::Const(..) => {
-                let res = Res::Def(DefKind::Const, def_id);
                 self.r.define(parent, ident, ValueNS, (res, vis, sp, expansion));
             }
             ItemKind::Fn(..) => {
-                let res = Res::Def(DefKind::Fn, def_id);
                 self.r.define(parent, ident, ValueNS, (res, vis, sp, expansion));
 
                 // Functions introducing procedural macros reserve a slot
@@ -698,14 +702,13 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
 
             // These items live in the type namespace.
             ItemKind::TyAlias(..) => {
-                let res = Res::Def(DefKind::TyAlias, def_id);
                 self.r.define(parent, ident, TypeNS, (res, vis, sp, expansion));
             }
 
             ItemKind::Enum(_, _) => {
                 let module = self.r.new_module(
                     Some(parent),
-                    ModuleKind::Def(DefKind::Enum, def_id, ident.name),
+                    ModuleKind::Def(def_kind, def_id, ident.name),
                     expansion.to_expn_id(),
                     item.span,
                     parent.no_implicit_prelude,
@@ -715,14 +718,12 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
             }
 
             ItemKind::TraitAlias(..) => {
-                let res = Res::Def(DefKind::TraitAlias, def_id);
                 self.r.define(parent, ident, TypeNS, (res, vis, sp, expansion));
             }
 
             // These items live in both the type and value namespaces.
             ItemKind::Struct(ref vdata, _) => {
                 // Define a name in the type namespace.
-                let res = Res::Def(DefKind::Struct, def_id);
                 self.r.define(parent, ident, TypeNS, (res, vis, sp, expansion));
 
                 // Record field names for error reporting.
@@ -731,7 +732,7 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
 
                 // If this is a tuple or unit struct, define a name
                 // in the value namespace as well.
-                if let Some((ctor_kind, ctor_node_id)) = CtorKind::from_ast(vdata) {
+                if let Some(ctor_node_id) = vdata.ctor_node_id() {
                     // If the structure is marked as non_exhaustive then lower the visibility
                     // to within the crate.
                     let mut ctor_vis = if vis.is_public()
@@ -757,10 +758,9 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
                         ret_fields.push(field_vis.to_def_id());
                     }
                     let ctor_def_id = self.r.local_def_id(ctor_node_id);
-                    let ctor_res =
-                        Res::Def(DefKind::Ctor(CtorOf::Struct, ctor_kind), ctor_def_id.to_def_id());
+                    let ctor_res = self.res(ctor_def_id);
                     self.r.define(parent, ident, ValueNS, (ctor_res, ctor_vis, sp, expansion));
-                    self.r.visibilities.insert(ctor_def_id, ctor_vis);
+                    self.r.feed_visibility(ctor_def_id, ctor_vis);
                     // We need the field visibility spans also for the constructor for E0603.
                     self.insert_field_visibilities_local(ctor_def_id.to_def_id(), vdata);
 
@@ -771,7 +771,6 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
             }
 
             ItemKind::Union(ref vdata, _) => {
-                let res = Res::Def(DefKind::Union, def_id);
                 self.r.define(parent, ident, TypeNS, (res, vis, sp, expansion));
 
                 // Record field names for error reporting.
@@ -783,7 +782,7 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
                 // Add all the items within to a new module.
                 let module = self.r.new_module(
                     Some(parent),
-                    ModuleKind::Def(DefKind::Trait, def_id, ident.name),
+                    ModuleKind::Def(def_kind, def_id, ident.name),
                     expansion.to_expn_id(),
                     item.span,
                     parent.no_implicit_prelude,
@@ -895,18 +894,17 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
     fn build_reduced_graph_for_foreign_item(&mut self, item: &ForeignItem) {
         let local_def_id = self.r.local_def_id(item.id);
         let def_id = local_def_id.to_def_id();
-        let (def_kind, ns) = match item.kind {
-            ForeignItemKind::Fn(..) => (DefKind::Fn, ValueNS),
-            ForeignItemKind::Static(_, mt, _) => (DefKind::Static(mt), ValueNS),
-            ForeignItemKind::TyAlias(..) => (DefKind::ForeignTy, TypeNS),
-            ForeignItemKind::MacCall(_) => unreachable!(),
+        let ns = match item.kind {
+            ForeignItemKind::Fn(..) => ValueNS,
+            ForeignItemKind::Static(..) => ValueNS,
+            ForeignItemKind::TyAlias(..) => TypeNS,
+            ForeignItemKind::MacCall(..) => unreachable!(),
         };
         let parent = self.parent_scope.module;
         let expansion = self.parent_scope.expansion;
         let vis = self.resolve_visibility(&item.vis);
-        let res = Res::Def(def_kind, def_id);
-        self.r.define(parent, item.ident, ns, (res, vis, item.span, expansion));
-        self.r.visibilities.insert(local_def_id, vis);
+        self.r.define(parent, item.ident, ns, (self.res(def_id), vis, item.span, expansion));
+        self.r.feed_visibility(local_def_id, vis);
     }
 
     fn build_reduced_graph_for_block(&mut self, block: &Block) {
@@ -980,8 +978,7 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
                 | DefKind::LifetimeParam
                 | DefKind::GlobalAsm
                 | DefKind::Closure
-                | DefKind::Impl { .. }
-                | DefKind::Coroutine,
+                | DefKind::Impl { .. },
                 _,
             )
             | Res::Local(..)
@@ -1175,16 +1172,10 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
     // Mark the given macro as unused unless its name starts with `_`.
     // Macro uses will remove items from this set, and the remaining
     // items will be reported as `unused_macros`.
-    fn insert_unused_macro(
-        &mut self,
-        ident: Ident,
-        def_id: LocalDefId,
-        node_id: NodeId,
-        rule_spans: &[(usize, Span)],
-    ) {
+    fn insert_unused_macro(&mut self, ident: Ident, def_id: LocalDefId, node_id: NodeId) {
         if !ident.as_str().starts_with('_') {
             self.r.unused_macros.insert(def_id, (node_id, ident));
-            for (rule_i, rule_span) in rule_spans.iter() {
+            for (rule_i, rule_span) in &self.r.macro_map[&def_id.to_def_id()].rule_spans {
                 self.r.unused_macro_rules.insert((def_id, *rule_i), (ident, *rule_span));
             }
         }
@@ -1194,24 +1185,21 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
         let parent_scope = self.parent_scope;
         let expansion = parent_scope.expansion;
         let def_id = self.r.local_def_id(item.id);
-        let (ext, ident, span, macro_rules, rule_spans) = match &item.kind {
-            ItemKind::MacroDef(def) => {
-                let (ext, rule_spans) = self.r.compile_macro(item, self.r.tcx.sess.edition());
-                let ext = Lrc::new(ext);
-                (ext, item.ident, item.span, def.macro_rules, rule_spans)
-            }
+        let (res, ident, span, macro_rules) = match &item.kind {
+            ItemKind::MacroDef(def) => (self.res(def_id), item.ident, item.span, def.macro_rules),
             ItemKind::Fn(..) => match self.proc_macro_stub(item) {
                 Some((macro_kind, ident, span)) => {
+                    let res = Res::Def(DefKind::Macro(macro_kind), def_id.to_def_id());
+                    let macro_data = MacroData::new(self.r.dummy_ext(macro_kind));
+                    self.r.macro_map.insert(def_id.to_def_id(), macro_data);
                     self.r.proc_macro_stubs.insert(def_id);
-                    (self.r.dummy_ext(macro_kind), ident, span, false, Vec::new())
+                    (res, ident, span, false)
                 }
                 None => return parent_scope.macro_rules,
             },
             _ => unreachable!(),
         };
 
-        let res = Res::Def(DefKind::Macro(ext.macro_kind()), def_id.to_def_id());
-        self.r.macro_map.insert(def_id.to_def_id(), MacroData { ext, macro_rules });
         self.r.local_macro_def_scopes.insert(def_id, parent_scope.module);
 
         if macro_rules {
@@ -1245,9 +1233,9 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
                 self.r.define(self.r.graph_root, ident, MacroNS, import_binding);
             } else {
                 self.r.check_reserved_macro_name(ident, res);
-                self.insert_unused_macro(ident, def_id, item.id, &rule_spans);
+                self.insert_unused_macro(ident, def_id, item.id);
             }
-            self.r.visibilities.insert(def_id, vis);
+            self.r.feed_visibility(def_id, vis);
             let scope = self.r.arenas.alloc_macro_rules_scope(MacroRulesScope::Binding(
                 self.r.arenas.alloc_macro_rules_binding(MacroRulesBinding {
                     parent_macro_rules_scope: parent_scope.macro_rules,
@@ -1268,10 +1256,10 @@ impl<'a, 'b, 'tcx> BuildReducedGraphVisitor<'a, 'b, 'tcx> {
                 _ => self.resolve_visibility(&item.vis),
             };
             if !vis.is_public() {
-                self.insert_unused_macro(ident, def_id, item.id, &rule_spans);
+                self.insert_unused_macro(ident, def_id, item.id);
             }
             self.r.define(module, ident, MacroNS, (res, vis, span, expansion));
-            self.r.visibilities.insert(def_id, vis);
+            self.r.feed_visibility(def_id, vis);
             self.parent_scope.macro_rules
         }
     }
@@ -1373,26 +1361,25 @@ impl<'a, 'b, 'tcx> Visitor<'b> for BuildReducedGraphVisitor<'a, 'b, 'tcx> {
             // Trait impl item visibility is inherited from its trait when not specified
             // explicitly. In that case we cannot determine it here in early resolve,
             // so we leave a hole in the visibility table to be filled later.
-            self.r.visibilities.insert(local_def_id, vis);
+            self.r.feed_visibility(local_def_id, vis);
         }
 
         if ctxt == AssocCtxt::Trait {
-            let (def_kind, ns) = match item.kind {
-                AssocItemKind::Const(..) => (DefKind::AssocConst, ValueNS),
+            let ns = match item.kind {
+                AssocItemKind::Const(..) => ValueNS,
                 AssocItemKind::Fn(box Fn { ref sig, .. }) => {
                     if sig.decl.has_self() {
                         self.r.has_self.insert(local_def_id);
                     }
-                    (DefKind::AssocFn, ValueNS)
+                    ValueNS
                 }
-                AssocItemKind::Type(..) => (DefKind::AssocTy, TypeNS),
+                AssocItemKind::Type(..) => TypeNS,
                 AssocItemKind::MacCall(_) => bug!(), // handled above
             };
 
             let parent = self.parent_scope.module;
             let expansion = self.parent_scope.expansion;
-            let res = Res::Def(def_kind, def_id);
-            self.r.define(parent, item.ident, ns, (res, vis, item.span, expansion));
+            self.r.define(parent, item.ident, ns, (self.res(def_id), vis, item.span, expansion));
         }
 
         visit::walk_assoc_item(self, item, ctxt);
@@ -1452,7 +1439,7 @@ impl<'a, 'b, 'tcx> Visitor<'b> for BuildReducedGraphVisitor<'a, 'b, 'tcx> {
             self.visit_invoc(sf.id);
         } else {
             let vis = self.resolve_visibility(&sf.vis);
-            self.r.visibilities.insert(self.r.local_def_id(sf.id), vis);
+            self.r.feed_visibility(self.r.local_def_id(sf.id), vis);
             visit::walk_field_def(self, sf);
         }
     }
@@ -1471,10 +1458,9 @@ impl<'a, 'b, 'tcx> Visitor<'b> for BuildReducedGraphVisitor<'a, 'b, 'tcx> {
 
         // Define a name in the type namespace.
         let def_id = self.r.local_def_id(variant.id);
-        let res = Res::Def(DefKind::Variant, def_id.to_def_id());
         let vis = self.resolve_visibility(&variant.vis);
-        self.r.define(parent, ident, TypeNS, (res, vis, variant.span, expn_id));
-        self.r.visibilities.insert(def_id, vis);
+        self.r.define(parent, ident, TypeNS, (self.res(def_id), vis, variant.span, expn_id));
+        self.r.feed_visibility(def_id, vis);
 
         // If the variant is marked as non_exhaustive then lower the visibility to within the crate.
         let ctor_vis =
@@ -1485,12 +1471,11 @@ impl<'a, 'b, 'tcx> Visitor<'b> for BuildReducedGraphVisitor<'a, 'b, 'tcx> {
             };
 
         // Define a constructor name in the value namespace.
-        if let Some((ctor_kind, ctor_node_id)) = CtorKind::from_ast(&variant.data) {
+        if let Some(ctor_node_id) = variant.data.ctor_node_id() {
             let ctor_def_id = self.r.local_def_id(ctor_node_id);
-            let ctor_res =
-                Res::Def(DefKind::Ctor(CtorOf::Variant, ctor_kind), ctor_def_id.to_def_id());
+            let ctor_res = self.res(ctor_def_id);
             self.r.define(parent, ident, ValueNS, (ctor_res, ctor_vis, variant.span, expn_id));
-            self.r.visibilities.insert(ctor_def_id, ctor_vis);
+            self.r.feed_visibility(ctor_def_id, ctor_vis);
         }
 
         // Record field names for error reporting.
diff --git a/compiler/rustc_resolve/src/def_collector.rs b/compiler/rustc_resolve/src/def_collector.rs
index 13df7efe6..02553d500 100644
--- a/compiler/rustc_resolve/src/def_collector.rs
+++ b/compiler/rustc_resolve/src/def_collector.rs
@@ -2,10 +2,10 @@ use crate::{ImplTraitContext, Resolver};
 use rustc_ast::visit::{self, FnKind};
 use rustc_ast::*;
 use rustc_expand::expand::AstFragment;
+use rustc_hir::def::{CtorKind, CtorOf, DefKind};
 use rustc_hir::def_id::LocalDefId;
-use rustc_hir::definitions::*;
 use rustc_span::hygiene::LocalExpnId;
-use rustc_span::symbol::sym;
+use rustc_span::symbol::{kw, sym, Symbol};
 use rustc_span::Span;
 
 pub(crate) fn collect_definitions(
@@ -26,13 +26,23 @@ struct DefCollector<'a, 'b, 'tcx> {
 }
 
 impl<'a, 'b, 'tcx> DefCollector<'a, 'b, 'tcx> {
-    fn create_def(&mut self, node_id: NodeId, data: DefPathData, span: Span) -> LocalDefId {
+    fn create_def(
+        &mut self,
+        node_id: NodeId,
+        name: Symbol,
+        def_kind: DefKind,
+        span: Span,
+    ) -> LocalDefId {
         let parent_def = self.parent_def;
-        debug!("create_def(node_id={:?}, data={:?}, parent_def={:?})", node_id, data, parent_def);
+        debug!(
+            "create_def(node_id={:?}, def_kind={:?}, parent_def={:?})",
+            node_id, def_kind, parent_def
+        );
         self.resolver.create_def(
             parent_def,
             node_id,
-            data,
+            name,
+            def_kind,
             self.expansion.to_expn_id(),
             span.with_parent(None),
         )
@@ -68,7 +78,7 @@ impl<'a, 'b, 'tcx> DefCollector<'a, 'b, 'tcx> {
             self.visit_macro_invoc(field.id);
         } else {
             let name = field.ident.map_or_else(|| sym::integer(index(self)), |ident| ident.name);
-            let def = self.create_def(field.id, DefPathData::ValueNs(name), field.span);
+            let def = self.create_def(field.id, name, DefKind::Field, field.span);
             self.with_parent(def, |this| visit::walk_field_def(this, field));
         }
     }
@@ -87,39 +97,54 @@ impl<'a, 'b, 'tcx> visit::Visitor<'a> for DefCollector<'a, 'b, 'tcx> {
 
         // Pick the def data. This need not be unique, but the more
         // information we encapsulate into, the better
-        let def_data = match &i.kind {
-            ItemKind::Impl { .. } => DefPathData::Impl,
-            ItemKind::ForeignMod(..) => DefPathData::ForeignMod,
-            ItemKind::Mod(..)
-            | ItemKind::Trait(..)
-            | ItemKind::TraitAlias(..)
-            | ItemKind::Enum(..)
-            | ItemKind::Struct(..)
-            | ItemKind::Union(..)
-            | ItemKind::ExternCrate(..)
-            | ItemKind::TyAlias(..) => DefPathData::TypeNs(i.ident.name),
-            ItemKind::Static(..) | ItemKind::Const(..) | ItemKind::Fn(..) => {
-                DefPathData::ValueNs(i.ident.name)
+        let mut opt_macro_data = None;
+        let def_kind = match &i.kind {
+            ItemKind::Impl(i) => DefKind::Impl { of_trait: i.of_trait.is_some() },
+            ItemKind::ForeignMod(..) => DefKind::ForeignMod,
+            ItemKind::Mod(..) => DefKind::Mod,
+            ItemKind::Trait(..) => DefKind::Trait,
+            ItemKind::TraitAlias(..) => DefKind::TraitAlias,
+            ItemKind::Enum(..) => DefKind::Enum,
+            ItemKind::Struct(..) => DefKind::Struct,
+            ItemKind::Union(..) => DefKind::Union,
+            ItemKind::ExternCrate(..) => DefKind::ExternCrate,
+            ItemKind::TyAlias(..) => DefKind::TyAlias,
+            ItemKind::Static(s) => DefKind::Static(s.mutability),
+            ItemKind::Const(..) => DefKind::Const,
+            ItemKind::Fn(..) => DefKind::Fn,
+            ItemKind::MacroDef(..) => {
+                let macro_data = self.resolver.compile_macro(i, self.resolver.tcx.sess.edition());
+                let macro_kind = macro_data.ext.macro_kind();
+                opt_macro_data = Some(macro_data);
+                DefKind::Macro(macro_kind)
             }
-            ItemKind::MacroDef(..) => DefPathData::MacroNs(i.ident.name),
             ItemKind::MacCall(..) => {
                 visit::walk_item(self, i);
                 return self.visit_macro_invoc(i.id);
             }
-            ItemKind::GlobalAsm(..) => DefPathData::GlobalAsm,
+            ItemKind::GlobalAsm(..) => DefKind::GlobalAsm,
             ItemKind::Use(..) => {
                 return visit::walk_item(self, i);
             }
         };
-        let def = self.create_def(i.id, def_data, i.span);
+        let def_id = self.create_def(i.id, i.ident.name, def_kind, i.span);
 
-        self.with_parent(def, |this| {
+        if let Some(macro_data) = opt_macro_data {
+            self.resolver.macro_map.insert(def_id.to_def_id(), macro_data);
+        }
+
+        self.with_parent(def_id, |this| {
             this.with_impl_trait(ImplTraitContext::Existential, |this| {
                 match i.kind {
                     ItemKind::Struct(ref struct_def, _) | ItemKind::Union(ref struct_def, _) => {
                         // If this is a unit or tuple-like struct, register the constructor.
-                        if let Some(ctor_node_id) = struct_def.ctor_node_id() {
-                            this.create_def(ctor_node_id, DefPathData::Ctor, i.span);
+                        if let Some((ctor_kind, ctor_node_id)) = CtorKind::from_ast(struct_def) {
+                            this.create_def(
+                                ctor_node_id,
+                                kw::Empty,
+                                DefKind::Ctor(CtorOf::Struct, ctor_kind),
+                                i.span,
+                            );
                         }
                     }
                     _ => {}
@@ -131,25 +156,33 @@ impl<'a, 'b, 'tcx> visit::Visitor<'a> for DefCollector<'a, 'b, 'tcx> {
 
     fn visit_fn(&mut self, fn_kind: FnKind<'a>, span: Span, _: NodeId) {
         if let FnKind::Fn(_, _, sig, _, generics, body) = fn_kind {
-            if let Async::Yes { closure_id, .. } = sig.header.asyncness {
-                self.visit_generics(generics);
-
-                // For async functions, we need to create their inner defs inside of a
-                // closure to match their desugared representation. Besides that,
-                // we must mirror everything that `visit::walk_fn` below does.
-                self.visit_fn_header(&sig.header);
-                for param in &sig.decl.inputs {
-                    self.visit_param(param);
-                }
-                self.visit_fn_ret_ty(&sig.decl.output);
-                // If this async fn has no body (i.e. it's an async fn signature in a trait)
-                // then the closure_def will never be used, and we should avoid generating a
-                // def-id for it.
-                if let Some(body) = body {
-                    let closure_def = self.create_def(closure_id, DefPathData::ClosureExpr, span);
-                    self.with_parent(closure_def, |this| this.visit_block(body));
+            match sig.header.coroutine_kind {
+                Some(coroutine_kind) => {
+                    self.visit_generics(generics);
+
+                    // For async functions, we need to create their inner defs inside of a
+                    // closure to match their desugared representation. Besides that,
+                    // we must mirror everything that `visit::walk_fn` below does.
+                    self.visit_fn_header(&sig.header);
+                    for param in &sig.decl.inputs {
+                        self.visit_param(param);
+                    }
+                    self.visit_fn_ret_ty(&sig.decl.output);
+                    // If this async fn has no body (i.e. it's an async fn signature in a trait)
+                    // then the closure_def will never be used, and we should avoid generating a
+                    // def-id for it.
+                    if let Some(body) = body {
+                        let closure_def = self.create_def(
+                            coroutine_kind.closure_id(),
+                            kw::Empty,
+                            DefKind::Closure,
+                            span,
+                        );
+                        self.with_parent(closure_def, |this| this.visit_block(body));
+                    }
+                    return;
                 }
-                return;
+                None => {}
             }
         }
 
@@ -157,34 +190,36 @@ impl<'a, 'b, 'tcx> visit::Visitor<'a> for DefCollector<'a, 'b, 'tcx> {
     }
 
     fn visit_use_tree(&mut self, use_tree: &'a UseTree, id: NodeId, _nested: bool) {
-        self.create_def(id, DefPathData::Use, use_tree.span);
+        self.create_def(id, kw::Empty, DefKind::Use, use_tree.span);
         visit::walk_use_tree(self, use_tree, id);
     }
 
-    fn visit_foreign_item(&mut self, foreign_item: &'a ForeignItem) {
-        if let ForeignItemKind::MacCall(_) = foreign_item.kind {
-            return self.visit_macro_invoc(foreign_item.id);
-        }
+    fn visit_foreign_item(&mut self, fi: &'a ForeignItem) {
+        let def_kind = match fi.kind {
+            ForeignItemKind::Static(_, mt, _) => DefKind::Static(mt),
+            ForeignItemKind::Fn(_) => DefKind::Fn,
+            ForeignItemKind::TyAlias(_) => DefKind::ForeignTy,
+            ForeignItemKind::MacCall(_) => return self.visit_macro_invoc(fi.id),
+        };
 
-        let def = self.create_def(
-            foreign_item.id,
-            DefPathData::ValueNs(foreign_item.ident.name),
-            foreign_item.span,
-        );
+        let def = self.create_def(fi.id, fi.ident.name, def_kind, fi.span);
 
-        self.with_parent(def, |this| {
-            visit::walk_foreign_item(this, foreign_item);
-        });
+        self.with_parent(def, |this| visit::walk_foreign_item(this, fi));
     }
 
     fn visit_variant(&mut self, v: &'a Variant) {
         if v.is_placeholder {
             return self.visit_macro_invoc(v.id);
         }
-        let def = self.create_def(v.id, DefPathData::TypeNs(v.ident.name), v.span);
+        let def = self.create_def(v.id, v.ident.name, DefKind::Variant, v.span);
         self.with_parent(def, |this| {
-            if let Some(ctor_node_id) = v.data.ctor_node_id() {
-                this.create_def(ctor_node_id, DefPathData::Ctor, v.span);
+            if let Some((ctor_kind, ctor_node_id)) = CtorKind::from_ast(&v.data) {
+                this.create_def(
+                    ctor_node_id,
+                    kw::Empty,
+                    DefKind::Ctor(CtorOf::Variant, ctor_kind),
+                    v.span,
+                );
             }
             visit::walk_variant(this, v)
         });
@@ -204,13 +239,12 @@ impl<'a, 'b, 'tcx> visit::Visitor<'a> for DefCollector<'a, 'b, 'tcx> {
             self.visit_macro_invoc(param.id);
             return;
         }
-        let name = param.ident.name;
-        let def_path_data = match param.kind {
-            GenericParamKind::Lifetime { .. } => DefPathData::LifetimeNs(name),
-            GenericParamKind::Type { .. } => DefPathData::TypeNs(name),
-            GenericParamKind::Const { .. } => DefPathData::ValueNs(name),
+        let def_kind = match param.kind {
+            GenericParamKind::Lifetime { .. } => DefKind::LifetimeParam,
+            GenericParamKind::Type { .. } => DefKind::TyParam,
+            GenericParamKind::Const { .. } => DefKind::ConstParam,
         };
-        self.create_def(param.id, def_path_data, param.ident.span);
+        self.create_def(param.id, param.ident.name, def_kind, param.ident.span);
 
         // impl-Trait can happen inside generic parameters, like
         // ```
@@ -218,19 +252,20 @@ impl<'a, 'b, 'tcx> visit::Visitor<'a> for DefCollector<'a, 'b, 'tcx> {
         // ```
         //
         // In that case, the impl-trait is lowered as an additional generic parameter.
-        self.with_impl_trait(ImplTraitContext::Universal(self.parent_def), |this| {
+        self.with_impl_trait(ImplTraitContext::Universal, |this| {
             visit::walk_generic_param(this, param)
         });
     }
 
     fn visit_assoc_item(&mut self, i: &'a AssocItem, ctxt: visit::AssocCtxt) {
-        let def_data = match &i.kind {
-            AssocItemKind::Fn(..) | AssocItemKind::Const(..) => DefPathData::ValueNs(i.ident.name),
-            AssocItemKind::Type(..) => DefPathData::TypeNs(i.ident.name),
+        let def_kind = match &i.kind {
+            AssocItemKind::Fn(..) => DefKind::AssocFn,
+            AssocItemKind::Const(..) => DefKind::AssocConst,
+            AssocItemKind::Type(..) => DefKind::AssocTy,
             AssocItemKind::MacCall(..) => return self.visit_macro_invoc(i.id),
         };
 
-        let def = self.create_def(i.id, def_data, i.span);
+        let def = self.create_def(i.id, i.ident.name, def_kind, i.span);
         self.with_parent(def, |this| visit::walk_assoc_item(this, i, ctxt));
     }
 
@@ -242,7 +277,7 @@ impl<'a, 'b, 'tcx> visit::Visitor<'a> for DefCollector<'a, 'b, 'tcx> {
     }
 
     fn visit_anon_const(&mut self, constant: &'a AnonConst) {
-        let def = self.create_def(constant.id, DefPathData::AnonConst, constant.value.span);
+        let def = self.create_def(constant.id, kw::Empty, DefKind::AnonConst, constant.value.span);
         self.with_parent(def, |this| visit::walk_anon_const(this, constant));
     }
 
@@ -252,15 +287,30 @@ impl<'a, 'b, 'tcx> visit::Visitor<'a> for DefCollector<'a, 'b, 'tcx> {
             ExprKind::Closure(ref closure) => {
                 // Async closures desugar to closures inside of closures, so
                 // we must create two defs.
-                let closure_def = self.create_def(expr.id, DefPathData::ClosureExpr, expr.span);
-                match closure.asyncness {
-                    Async::Yes { closure_id, .. } => {
-                        self.create_def(closure_id, DefPathData::ClosureExpr, expr.span)
-                    }
-                    Async::No => closure_def,
+                let closure_def = self.create_def(expr.id, kw::Empty, DefKind::Closure, expr.span);
+                match closure.coroutine_kind {
+                    Some(coroutine_kind) => self.create_def(
+                        coroutine_kind.closure_id(),
+                        kw::Empty,
+                        DefKind::Closure,
+                        expr.span,
+                    ),
+                    None => closure_def,
                 }
             }
-            ExprKind::Gen(_, _, _) => self.create_def(expr.id, DefPathData::ClosureExpr, expr.span),
+            ExprKind::Gen(_, _, _) => {
+                self.create_def(expr.id, kw::Empty, DefKind::Closure, expr.span)
+            }
+            ExprKind::ConstBlock(ref constant) => {
+                let def = self.create_def(
+                    constant.id,
+                    kw::Empty,
+                    DefKind::InlineConst,
+                    constant.value.span,
+                );
+                self.with_parent(def, |this| visit::walk_anon_const(this, constant));
+                return;
+            }
             _ => self.parent_def,
         };
 
@@ -305,9 +355,7 @@ impl<'a, 'b, 'tcx> visit::Visitor<'a> for DefCollector<'a, 'b, 'tcx> {
         if p.is_placeholder {
             self.visit_macro_invoc(p.id)
         } else {
-            self.with_impl_trait(ImplTraitContext::Universal(self.parent_def), |this| {
-                visit::walk_param(this, p)
-            })
+            self.with_impl_trait(ImplTraitContext::Universal, |this| visit::walk_param(this, p))
         }
     }
 
diff --git a/compiler/rustc_resolve/src/diagnostics.rs b/compiler/rustc_resolve/src/diagnostics.rs
index 93db6cfc4..542aff69e 100644
--- a/compiler/rustc_resolve/src/diagnostics.rs
+++ b/compiler/rustc_resolve/src/diagnostics.rs
@@ -27,10 +27,8 @@ use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::{BytePos, Span, SyntaxContext};
 use thin_vec::{thin_vec, ThinVec};
 
-use crate::errors::{
-    AddedMacroUse, ChangeImportBinding, ChangeImportBindingSuggestion, ConsiderAddingADerive,
-    ExplicitUnsafeTraits,
-};
+use crate::errors::{AddedMacroUse, ChangeImportBinding, ChangeImportBindingSuggestion};
+use crate::errors::{ConsiderAddingADerive, ExplicitUnsafeTraits, MaybeMissingMacroRulesName};
 use crate::imports::{Import, ImportKind};
 use crate::late::{PatternSource, Rib};
 use crate::path_names_to_string;
@@ -979,7 +977,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
                     path_span: path.span,
                     // intentionally converting to String, as the text would also be used as
                     // in suggestion context
-                    path_str: pprust::path_to_string(&path),
+                    path_str: pprust::path_to_string(path),
                 })
             }
             VisResolutionError::AncestorOnly(span) => {
@@ -1421,14 +1419,21 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
             "",
         );
 
+        if macro_kind == MacroKind::Bang && ident.name == sym::macro_rules {
+            err.subdiagnostic(MaybeMissingMacroRulesName { span: ident.span });
+            return;
+        }
+
         if macro_kind == MacroKind::Derive && (ident.name == sym::Send || ident.name == sym::Sync) {
             err.subdiagnostic(ExplicitUnsafeTraits { span: ident.span, ident });
             return;
         }
+
         if self.macro_names.contains(&ident.normalize_to_macros_2_0()) {
             err.subdiagnostic(AddedMacroUse);
             return;
         }
+
         if ident.name == kw::Default
             && let ModuleKind::Def(DefKind::Enum, def_id, _) = parent_scope.module.kind
         {
@@ -1444,7 +1449,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
             if let Ok(binding) = self.early_resolve_ident_in_lexical_scope(
                 ident,
                 ScopeSet::All(ns),
-                &parent_scope,
+                parent_scope,
                 None,
                 false,
                 None,
@@ -1697,6 +1702,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
             struct_span_err!(self.tcx.sess, ident.span, E0603, "{} `{}` is private", descr, ident);
         err.span_label(ident.span, format!("private {descr}"));
 
+        let mut not_publicly_reexported = false;
         if let Some((this_res, outer_ident)) = outermost_res {
             let import_suggestions = self.lookup_import_candidates(
                 outer_ident,
@@ -1717,6 +1723,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
             );
             // If we suggest importing a public re-export, don't point at the definition.
             if point_to_def && ident.span != outer_ident.span {
+                not_publicly_reexported = true;
                 err.span_label(
                     outer_ident.span,
                     format!("{} `{outer_ident}` is not publicly re-exported", this_res.descr()),
@@ -1749,10 +1756,51 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
             }
         }
 
+        let mut sugg_paths = vec![];
+        if let Some(mut def_id) = res.opt_def_id() {
+            // We can't use `def_path_str` in resolve.
+            let mut path = vec![def_id];
+            while let Some(parent) = self.tcx.opt_parent(def_id) {
+                def_id = parent;
+                if !def_id.is_top_level_module() {
+                    path.push(def_id);
+                } else {
+                    break;
+                }
+            }
+            // We will only suggest importing directly if it is accessible through that path.
+            let path_names: Option<Vec<String>> = path
+                .iter()
+                .rev()
+                .map(|def_id| {
+                    self.tcx.opt_item_name(*def_id).map(|n| {
+                        if def_id.is_top_level_module() {
+                            "crate".to_string()
+                        } else {
+                            n.to_string()
+                        }
+                    })
+                })
+                .collect();
+            if let Some(def_id) = path.get(0)
+                && let Some(path) = path_names
+            {
+                if let Some(def_id) = def_id.as_local() {
+                    if self.effective_visibilities.is_directly_public(def_id) {
+                        sugg_paths.push((path, false));
+                    }
+                } else if self.is_accessible_from(self.tcx.visibility(def_id), parent_scope.module)
+                {
+                    sugg_paths.push((path, false));
+                }
+            }
+        }
+
         // Print the whole import chain to make it easier to see what happens.
         let first_binding = binding;
         let mut next_binding = Some(binding);
         let mut next_ident = ident;
+        let mut path = vec![];
         while let Some(binding) = next_binding {
             let name = next_ident;
             next_binding = match binding.kind {
@@ -1771,6 +1819,21 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
                 _ => None,
             };
 
+            match binding.kind {
+                NameBindingKind::Import { import, .. } => {
+                    for segment in import.module_path.iter().skip(1) {
+                        path.push(segment.ident.to_string());
+                    }
+                    sugg_paths.push((
+                        path.iter()
+                            .cloned()
+                            .chain(vec![ident.to_string()].into_iter())
+                            .collect::<Vec<_>>(),
+                        true, // re-export
+                    ));
+                }
+                NameBindingKind::Res(_) | NameBindingKind::Module(_) => {}
+            }
             let first = binding == first_binding;
             let msg = format!(
                 "{and_refers_to}the {item} `{name}`{which} is defined here{dots}",
@@ -1782,7 +1845,11 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
             let def_span = self.tcx.sess.source_map().guess_head_span(binding.span);
             let mut note_span = MultiSpan::from_span(def_span);
             if !first && binding.vis.is_public() {
-                note_span.push_span_label(def_span, "consider importing it directly");
+                let desc = match binding.kind {
+                    NameBindingKind::Import { .. } => "re-export",
+                    _ => "directly",
+                };
+                note_span.push_span_label(def_span, format!("you could import this {desc}"));
             }
             // Final step in the import chain, point out if the ADT is `non_exhaustive`
             // which is probably why this privacy violation occurred.
@@ -1796,6 +1863,29 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
             }
             err.span_note(note_span, msg);
         }
+        // We prioritize shorter paths, non-core imports and direct imports over the alternatives.
+        sugg_paths.sort_by_key(|(p, reexport)| (p.len(), p[0] == "core", *reexport));
+        for (sugg, reexport) in sugg_paths {
+            if not_publicly_reexported {
+                break;
+            }
+            if sugg.len() <= 1 {
+                // A single path segment suggestion is wrong. This happens on circular imports.
+                // `tests/ui/imports/issue-55884-2.rs`
+                continue;
+            }
+            let path = sugg.join("::");
+            err.span_suggestion_verbose(
+                dedup_span,
+                format!(
+                    "import `{ident}` {}",
+                    if reexport { "through the re-export" } else { "directly" }
+                ),
+                path,
+                Applicability::MachineApplicable,
+            );
+            break;
+        }
 
         err.emit();
     }
@@ -1858,6 +1948,15 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
                         Applicability::MaybeIncorrect,
                     )),
                 )
+            } else if ident.name == sym::core {
+                (
+                    format!("maybe a missing crate `{ident}`?"),
+                    Some((
+                        vec![(ident.span, "std".to_string())],
+                        "try using `std` instead of `core`".to_string(),
+                        Applicability::MaybeIncorrect,
+                    )),
+                )
             } else if self.tcx.sess.is_rust_2015() {
                 (
                     format!("maybe a missing crate `{ident}`?"),
@@ -2028,7 +2127,19 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
                     },
                 )
             });
-            (format!("use of undeclared crate or module `{ident}`"), suggestion)
+            if let Ok(binding) = self.early_resolve_ident_in_lexical_scope(
+                ident,
+                ScopeSet::All(ValueNS),
+                parent_scope,
+                None,
+                false,
+                ignore_binding,
+            ) {
+                let descr = binding.res().descr();
+                (format!("{descr} `{ident}` is not a crate or module"), suggestion)
+            } else {
+                (format!("use of undeclared crate or module `{ident}`"), suggestion)
+            }
         }
     }
 
@@ -2596,6 +2707,7 @@ fn show_candidates(
         path_strings.extend(core_path_strings);
         path_strings.dedup_by(|a, b| a.0 == b.0);
     }
+    accessible_path_strings.sort();
 
     if !accessible_path_strings.is_empty() {
         let (determiner, kind, name, through) =
diff --git a/compiler/rustc_resolve/src/effective_visibilities.rs b/compiler/rustc_resolve/src/effective_visibilities.rs
index 4477b9672..503521692 100644
--- a/compiler/rustc_resolve/src/effective_visibilities.rs
+++ b/compiler/rustc_resolve/src/effective_visibilities.rs
@@ -115,7 +115,7 @@ impl<'r, 'a, 'tcx> EffectiveVisibilitiesVisitor<'r, 'a, 'tcx> {
     /// Update effective visibilities of bindings in the given module,
     /// including their whole reexport chains.
     fn set_bindings_effective_visibilities(&mut self, module_id: LocalDefId) {
-        assert!(self.r.module_map.contains_key(&&module_id.to_def_id()));
+        assert!(self.r.module_map.contains_key(&module_id.to_def_id()));
         let module = self.r.get_module(module_id.to_def_id()).unwrap();
         let resolutions = self.r.resolutions(module);
 
@@ -186,7 +186,7 @@ impl<'r, 'a, 'tcx> EffectiveVisibilitiesVisitor<'r, 'a, 'tcx> {
     ) -> Option<Option<Visibility>> {
         match parent_id {
             ParentId::Def(def_id) => (nominal_vis != self.current_private_vis
-                && self.r.visibilities[&def_id] != self.current_private_vis)
+                && self.r.tcx.local_visibility(def_id) != self.current_private_vis)
                 .then_some(Some(self.current_private_vis)),
             ParentId::Import(_) => Some(None),
         }
@@ -222,7 +222,7 @@ impl<'r, 'a, 'tcx> EffectiveVisibilitiesVisitor<'r, 'a, 'tcx> {
     }
 
     fn update_field(&mut self, def_id: LocalDefId, parent_id: LocalDefId) {
-        self.update_def(def_id, self.r.visibilities[&def_id], ParentId::Def(parent_id));
+        self.update_def(def_id, self.r.tcx.local_visibility(def_id), ParentId::Def(parent_id));
     }
 }
 
diff --git a/compiler/rustc_resolve/src/errors.rs b/compiler/rustc_resolve/src/errors.rs
index 72ff959bb..1fdb193e5 100644
--- a/compiler/rustc_resolve/src/errors.rs
+++ b/compiler/rustc_resolve/src/errors.rs
@@ -666,6 +666,13 @@ pub(crate) struct ExplicitUnsafeTraits {
 }
 
 #[derive(Subdiagnostic)]
+#[note(resolve_missing_macro_rules_name)]
+pub(crate) struct MaybeMissingMacroRulesName {
+    #[primary_span]
+    pub(crate) span: Span,
+}
+
+#[derive(Subdiagnostic)]
 #[help(resolve_added_macro_use)]
 pub(crate) struct AddedMacroUse;
 
diff --git a/compiler/rustc_resolve/src/ident.rs b/compiler/rustc_resolve/src/ident.rs
index 1a50bd5ec..a9f7002e5 100644
--- a/compiler/rustc_resolve/src/ident.rs
+++ b/compiler/rustc_resolve/src/ident.rs
@@ -240,7 +240,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
                 {
                     // The macro is a proc macro derive
                     if let Some(def_id) = module.expansion.expn_data().macro_def_id {
-                        let ext = self.get_macro_by_def_id(def_id).ext;
+                        let ext = &self.get_macro_by_def_id(def_id).ext;
                         if ext.builtin_name.is_none()
                             && ext.macro_kind() == MacroKind::Derive
                             && parent.expansion.outer_expn_is_descendant_of(*ctxt)
@@ -1201,7 +1201,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
                                             }
                                         };
                                         self.report_error(span, error);
-                                        self.tcx.sess.delay_span_bug(span, CG_BUG_STR);
+                                        self.tcx.sess.span_delayed_bug(span, CG_BUG_STR);
                                     }
 
                                     return Res::Err;
diff --git a/compiler/rustc_resolve/src/imports.rs b/compiler/rustc_resolve/src/imports.rs
index b34790a92..39e82da6d 100644
--- a/compiler/rustc_resolve/src/imports.rs
+++ b/compiler/rustc_resolve/src/imports.rs
@@ -8,7 +8,7 @@ use crate::errors::{
     ItemsInTraitsAreNotImportable,
 };
 use crate::Determinacy::{self, *};
-use crate::{fluent_generated as fluent, Namespace::*};
+use crate::Namespace::*;
 use crate::{module_to_string, names_to_string, ImportSuggestion};
 use crate::{AmbiguityKind, BindingKey, ModuleKind, ResolutionError, Resolver, Segment};
 use crate::{Finalize, Module, ModuleOrUniformRoot, ParentScope, PerNS, ScopeSet};
@@ -477,6 +477,9 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
             self.per_ns(|this, ns| {
                 let key = BindingKey::new(target, ns);
                 let _ = this.try_define(import.parent_scope.module, key, dummy_binding, false);
+                this.update_resolution(import.parent_scope.module, key, false, |_, resolution| {
+                    resolution.single_imports.remove(&import);
+                })
             });
             self.record_use(target, dummy_binding, false);
         } else if import.imported_module.get().is_none() {
@@ -708,7 +711,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
                         self.tcx,
                         &mut diag,
                         Some(err.span),
-                        &candidates,
+                        candidates,
                         DiagnosticMode::Import,
                         (source != target)
                             .then(|| format!(" as {target}"))
@@ -720,7 +723,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
                             self.tcx,
                             &mut diag,
                             None,
-                            &candidates,
+                            candidates,
                             DiagnosticMode::Normal,
                             (source != target)
                                 .then(|| format!(" as {target}"))
@@ -987,13 +990,23 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
                     }
                     if !is_prelude
                         && let Some(max_vis) = max_vis.get()
-                        && !max_vis.is_at_least(import.expect_vis(), self.tcx)
+                        && let import_vis = import.expect_vis()
+                        && !max_vis.is_at_least(import_vis, self.tcx)
                     {
-                        self.lint_buffer.buffer_lint(
+                        let def_id = self.local_def_id(id);
+                        let msg = format!(
+                            "glob import doesn't reexport anything with visibility `{}` because no imported item is public enough",
+                            import_vis.to_string(def_id, self.tcx)
+                        );
+                        self.lint_buffer.buffer_lint_with_diagnostic(
                             UNUSED_IMPORTS,
                             id,
                             import.span,
-                            fluent::resolve_glob_import_doesnt_reexport,
+                            msg,
+                            BuiltinLintDiagnostics::RedundantImportVisibility {
+                                max_vis: max_vis.to_string(def_id, self.tcx),
+                                span: import.span,
+                            },
                         );
                     }
                     return None;
@@ -1050,16 +1063,12 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
                             initial_binding.res()
                         });
                         let res = binding.res();
-                        let has_ambiguity_error = this
-                            .ambiguity_errors
-                            .iter()
-                            .filter(|error| !error.warning)
-                            .next()
-                            .is_some();
+                        let has_ambiguity_error =
+                            this.ambiguity_errors.iter().any(|error| !error.warning);
                         if res == Res::Err || has_ambiguity_error {
                             this.tcx
                                 .sess
-                                .delay_span_bug(import.span, "some error happened for an import");
+                                .span_delayed_bug(import.span, "some error happened for an import");
                             return;
                         }
                         if let Ok(initial_res) = initial_res {
diff --git a/compiler/rustc_resolve/src/late.rs b/compiler/rustc_resolve/src/late.rs
index 3be962dab..cf50f630b 100644
--- a/compiler/rustc_resolve/src/late.rs
+++ b/compiler/rustc_resolve/src/late.rs
@@ -603,6 +603,8 @@ struct DiagnosticMetadata<'ast> {
     /// Only used for better errors on `let <pat>: <expr, not type>;`.
     current_let_binding: Option<(Span, Option<Span>, Option<Span>)>,
 
+    current_pat: Option<&'ast Pat>,
+
     /// Used to detect possible `if let` written without `let` and to provide structured suggestion.
     in_if_condition: Option<&'ast Expr>,
 
@@ -703,6 +705,12 @@ impl<'a: 'ast, 'ast, 'tcx> Visitor<'ast> for LateResolutionVisitor<'a, '_, 'ast,
     fn visit_expr(&mut self, expr: &'ast Expr) {
         self.resolve_expr(expr, None);
     }
+    fn visit_pat(&mut self, p: &'ast Pat) {
+        let prev = self.diagnostic_metadata.current_pat;
+        self.diagnostic_metadata.current_pat = Some(p);
+        visit::walk_pat(self, p);
+        self.diagnostic_metadata.current_pat = prev;
+    }
     fn visit_local(&mut self, local: &'ast Local) {
         let local_spans = match local.pat.kind {
             // We check for this to avoid tuple struct fields.
@@ -731,7 +739,7 @@ impl<'a: 'ast, 'ast, 'tcx> Visitor<'ast> for LateResolutionVisitor<'a, '_, 'ast,
             }
             TyKind::Path(qself, path) => {
                 self.diagnostic_metadata.current_type_path = Some(ty);
-                self.smart_resolve_path(ty.id, &qself, path, PathSource::Type);
+                self.smart_resolve_path(ty.id, qself, path, PathSource::Type);
 
                 // Check whether we should interpret this as a bare trait object.
                 if qself.is_none()
@@ -751,7 +759,7 @@ impl<'a: 'ast, 'ast, 'tcx> Visitor<'ast> for LateResolutionVisitor<'a, '_, 'ast,
                             kind: LifetimeBinderKind::PolyTrait,
                             span,
                         },
-                        |this| this.visit_path(&path, ty.id),
+                        |this| this.visit_path(path, ty.id),
                     );
                 } else {
                     visit::walk_ty(self, ty)
@@ -908,8 +916,12 @@ impl<'a: 'ast, 'ast, 'tcx> Visitor<'ast> for LateResolutionVisitor<'a, '_, 'ast,
                             &sig.decl.output,
                         );
 
-                        if let Some((async_node_id, _)) = sig.header.asyncness.opt_return_id() {
-                            this.record_lifetime_params_for_impl_trait(async_node_id);
+                        if let Some((coro_node_id, _)) = sig
+                            .header
+                            .coroutine_kind
+                            .map(|coroutine_kind| coroutine_kind.return_id())
+                        {
+                            this.record_lifetime_params_for_impl_trait(coro_node_id);
                         }
                     },
                 );
@@ -932,12 +944,15 @@ impl<'a: 'ast, 'ast, 'tcx> Visitor<'ast> for LateResolutionVisitor<'a, '_, 'ast,
                         this.visit_generics(generics);
 
                         let declaration = &sig.decl;
-                        let async_node_id = sig.header.asyncness.opt_return_id();
+                        let coro_node_id = sig
+                            .header
+                            .coroutine_kind
+                            .map(|coroutine_kind| coroutine_kind.return_id());
 
                         this.with_lifetime_rib(
                             LifetimeRibKind::AnonymousCreateParameter {
                                 binder: fn_id,
-                                report_in_path: async_node_id.is_some(),
+                                report_in_path: coro_node_id.is_some(),
                             },
                             |this| {
                                 this.resolve_fn_signature(
@@ -950,7 +965,7 @@ impl<'a: 'ast, 'ast, 'tcx> Visitor<'ast> for LateResolutionVisitor<'a, '_, 'ast,
                                     &declaration.output,
                                 );
 
-                                if let Some((async_node_id, _)) = async_node_id {
+                                if let Some((async_node_id, _)) = coro_node_id {
                                     this.record_lifetime_params_for_impl_trait(async_node_id);
                                 }
                             },
@@ -1035,7 +1050,7 @@ impl<'a: 'ast, 'ast, 'tcx> Visitor<'ast> for LateResolutionVisitor<'a, '_, 'ast,
             ClosureBinder::NotPresent => {}
             ClosureBinder::For { generic_params, .. } => {
                 self.visit_generic_params(
-                    &generic_params,
+                    generic_params,
                     self.diagnostic_metadata.current_self_item.is_some(),
                 );
             }
@@ -1179,7 +1194,7 @@ impl<'a: 'ast, 'ast, 'tcx> Visitor<'ast> for LateResolutionVisitor<'a, '_, 'ast,
             {
                 let span = predicate_span.shrink_to_lo().to(bounded_ty.span.shrink_to_lo());
                 this.with_generic_param_rib(
-                    &bound_generic_params,
+                    bound_generic_params,
                     RibKind::Normal,
                     LifetimeRibKind::Generics {
                         binder: bounded_ty.id,
@@ -1187,7 +1202,7 @@ impl<'a: 'ast, 'ast, 'tcx> Visitor<'ast> for LateResolutionVisitor<'a, '_, 'ast,
                         span,
                     },
                     |this| {
-                        this.visit_generic_params(&bound_generic_params, false);
+                        this.visit_generic_params(bound_generic_params, false);
                         this.visit_ty(bounded_ty);
                         for bound in bounds {
                             this.visit_param_bound(bound, BoundKind::Bound)
@@ -1989,7 +2004,7 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
         } else {
             LifetimeRibKind::ElisionFailure
         };
-        self.with_lifetime_rib(output_rib, |this| visit::walk_fn_ret_ty(this, &output_ty));
+        self.with_lifetime_rib(output_rib, |this| visit::walk_fn_ret_ty(this, output_ty));
         let elision_failures =
             replace(&mut self.diagnostic_metadata.current_elision_failures, outer_failures);
         if !elision_failures.is_empty() {
@@ -2371,7 +2386,7 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
                     &item.attrs,
                     generics,
                     of_trait,
-                    &self_ty,
+                    self_ty,
                     item.id,
                     impl_items,
                 );
@@ -2735,7 +2750,7 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
     /// When evaluating a `trait` use its associated types' idents for suggestions in E0412.
     fn resolve_trait_items(&mut self, trait_items: &'ast [P<AssocItem>]) {
         let trait_assoc_items =
-            replace(&mut self.diagnostic_metadata.current_trait_assoc_items, Some(&trait_items));
+            replace(&mut self.diagnostic_metadata.current_trait_assoc_items, Some(trait_items));
 
         let walk_assoc_item =
             |this: &mut Self, generics: &Generics, kind, item: &'ast AssocItem| {
@@ -3063,17 +3078,25 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
             binding = self.r.resolution(module, key).try_borrow().ok().and_then(|r| r.binding);
             debug!(?binding);
         }
+
+        let feed_visibility = |this: &mut Self, def_id| {
+            let vis = this.r.tcx.visibility(def_id).expect_local();
+            this.r.feed_visibility(this.r.local_def_id(id), vis);
+        };
+
         let Some(binding) = binding else {
             // We could not find the method: report an error.
             let candidate = self.find_similarly_named_assoc_item(ident.name, kind);
             let path = &self.current_trait_ref.as_ref().unwrap().1.path;
             let path_names = path_names_to_string(path);
             self.report_error(span, err(ident, path_names, candidate));
+            feed_visibility(self, module.def_id());
             return;
         };
 
         let res = binding.res();
         let Res::Def(def_kind, id_in_trait) = res else { bug!() };
+        feed_visibility(self, id_in_trait);
 
         match seen_trait_items.entry(id_in_trait) {
             Entry::Occupied(entry) => {
@@ -3286,7 +3309,7 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
         self.with_rib(ValueNS, RibKind::Normal, |this| {
             this.resolve_pattern_top(&arm.pat, PatternSource::Match);
             walk_list!(this, visit_expr, &arm.guard);
-            this.visit_expr(&arm.body);
+            walk_list!(this, visit_expr, &arm.body);
         });
     }
 
@@ -3536,7 +3559,7 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
             Res::SelfCtor(_) => {
                 // We resolve `Self` in pattern position as an ident sometimes during recovery,
                 // so delay a bug instead of ICEing.
-                self.r.tcx.sess.delay_span_bug(
+                self.r.tcx.sess.span_delayed_bug(
                     ident.span,
                     "unexpected `SelfCtor` in pattern, expected identifier"
                 );
@@ -3937,7 +3960,7 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
             )));
         }
 
-        let result = match self.resolve_path(&path, Some(ns), Some(finalize)) {
+        let result = match self.resolve_path(path, Some(ns), Some(finalize)) {
             PathResult::NonModule(path_res) => path_res,
             PathResult::Module(ModuleOrUniformRoot::Module(module)) if !module.is_normal() => {
                 PartialRes::new(module.res().unwrap())
@@ -4200,7 +4223,7 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
             ExprKind::Break(None, Some(ref e)) => {
                 // We use this instead of `visit::walk_expr` to keep the parent expr around for
                 // better diagnostics.
-                self.resolve_expr(e, Some(&expr));
+                self.resolve_expr(e, Some(expr));
             }
 
             ExprKind::Let(ref pat, ref scrutinee, _, _) => {
@@ -4221,7 +4244,7 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
             }
 
             ExprKind::Loop(ref block, label, _) => {
-                self.resolve_labeled_block(label, expr.id, &block)
+                self.resolve_labeled_block(label, expr.id, block)
             }
 
             ExprKind::While(ref cond, ref block, label) => {
@@ -4280,8 +4303,10 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
             // `async |x| ...` gets desugared to `|x| async {...}`, so we need to
             // resolve the arguments within the proper scopes so that usages of them inside the
             // closure are detected as upvars rather than normal closure arg usages.
+            //
+            // Similarly, `gen |x| ...` gets desugared to `|x| gen {...}`, so we handle that too.
             ExprKind::Closure(box ast::Closure {
-                asyncness: Async::Yes { .. },
+                coroutine_kind: Some(_),
                 ref fn_decl,
                 ref body,
                 ..
@@ -4310,7 +4335,7 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
                 ..
             }) => {
                 self.with_generic_param_rib(
-                    &generic_params,
+                    generic_params,
                     RibKind::Normal,
                     LifetimeRibKind::Generics {
                         binder: expr.id,
diff --git a/compiler/rustc_resolve/src/late/diagnostics.rs b/compiler/rustc_resolve/src/late/diagnostics.rs
index fd5d6fabf..d767ed741 100644
--- a/compiler/rustc_resolve/src/late/diagnostics.rs
+++ b/compiler/rustc_resolve/src/late/diagnostics.rs
@@ -1,12 +1,14 @@
 use crate::diagnostics::{ImportSuggestion, LabelSuggestion, TypoSuggestion};
 use crate::late::{AliasPossibility, LateResolutionVisitor, RibKind};
 use crate::late::{LifetimeBinderKind, LifetimeRes, LifetimeRibKind, LifetimeUseSet};
+use crate::ty::fast_reject::SimplifiedType;
 use crate::{errors, path_names_to_string};
 use crate::{Module, ModuleKind, ModuleOrUniformRoot};
 use crate::{PathResult, PathSource, Segment};
 use rustc_hir::def::Namespace::{self, *};
 
-use rustc_ast::visit::{FnCtxt, FnKind, LifetimeCtxt};
+use rustc_ast::ptr::P;
+use rustc_ast::visit::{walk_ty, FnCtxt, FnKind, LifetimeCtxt, Visitor};
 use rustc_ast::{
     self as ast, AssocItemKind, Expr, ExprKind, GenericParam, GenericParamKind, Item, ItemKind,
     MethodCall, NodeId, Path, Ty, TyKind, DUMMY_NODE_ID,
@@ -15,7 +17,7 @@ use rustc_ast_pretty::pprust::where_bound_predicate_to_string;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_errors::{
     pluralize, struct_span_err, Applicability, Diagnostic, DiagnosticBuilder, ErrorGuaranteed,
-    MultiSpan,
+    MultiSpan, SuggestionStyle,
 };
 use rustc_hir as hir;
 use rustc_hir::def::{self, CtorKind, CtorOf, DefKind};
@@ -29,6 +31,8 @@ use rustc_span::hygiene::MacroKind;
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::Span;
 
+use rustc_middle::ty;
+
 use std::borrow::Cow;
 use std::iter;
 use std::ops::Deref;
@@ -431,6 +435,7 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
             code,
         );
 
+        self.suggest_at_operator_in_slice_pat_with_range(&mut err, path);
         self.suggest_swapping_misplaced_self_ty_and_trait(&mut err, source, res, base_error.span);
 
         if let Some((span, label)) = base_error.span_label {
@@ -739,6 +744,7 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
                 err,
                 span,
                 source,
+                path,
                 res,
                 &path_str,
                 &base_error.fallback_label,
@@ -901,7 +907,7 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
             }
 
             // If the trait has a single item (which wasn't matched by the algorithm), suggest it
-            let suggestion = self.get_single_associated_item(&path, &source, is_expected);
+            let suggestion = self.get_single_associated_item(path, &source, is_expected);
             if !self.r.add_typo_suggestion(err, suggestion, ident_span) {
                 fallback = !self.let_binding_suggestion(err, ident_span);
             }
@@ -1063,6 +1069,32 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
         true
     }
 
+    fn suggest_at_operator_in_slice_pat_with_range(
+        &mut self,
+        err: &mut Diagnostic,
+        path: &[Segment],
+    ) {
+        if let Some(pat) = self.diagnostic_metadata.current_pat
+            && let ast::PatKind::Range(Some(start), None, range) = &pat.kind
+            && let ExprKind::Path(None, range_path) = &start.kind
+            && let [segment] = &range_path.segments[..]
+            && let [s] = path
+            && segment.ident == s.ident
+        {
+            // We've encountered `[first, rest..]` where the user might have meant
+            // `[first, rest @ ..]` (#88404).
+            err.span_suggestion_verbose(
+                segment.ident.span.between(range.span),
+                format!(
+                    "if you meant to collect the rest of the slice in `{}`, use the at operator",
+                    segment.ident,
+                ),
+                " @ ",
+                Applicability::MaybeIncorrect,
+            );
+        }
+    }
+
     fn suggest_swapping_misplaced_self_ty_and_trait(
         &mut self,
         err: &mut Diagnostic,
@@ -1297,6 +1329,7 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
         err: &mut Diagnostic,
         span: Span,
         source: PathSource<'_>,
+        path: &[Segment],
         res: Res,
         path_str: &str,
         fallback_label: &str,
@@ -1304,7 +1337,7 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
         let ns = source.namespace();
         let is_expected = &|res| source.is_expected(res);
 
-        let path_sep = |err: &mut Diagnostic, expr: &Expr, kind: DefKind| {
+        let path_sep = |this: &mut Self, err: &mut Diagnostic, expr: &Expr, kind: DefKind| {
             const MESSAGE: &str = "use the path separator to refer to an item";
 
             let (lhs_span, rhs_span) = match &expr.kind {
@@ -1325,7 +1358,7 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
                 true
             } else if kind == DefKind::Struct
                 && let Some(lhs_source_span) = lhs_span.find_ancestor_inside(expr.span)
-                && let Ok(snippet) = self.r.tcx.sess.source_map().span_to_snippet(lhs_source_span)
+                && let Ok(snippet) = this.r.tcx.sess.source_map().span_to_snippet(lhs_source_span)
             {
                 // The LHS is a type that originates from a macro call.
                 // We have to add angle brackets around it.
@@ -1360,13 +1393,13 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
             }
         };
 
-        let mut bad_struct_syntax_suggestion = |def_id: DefId| {
-            let (followed_by_brace, closing_brace) = self.followed_by_brace(span);
+        let mut bad_struct_syntax_suggestion = |this: &mut Self, def_id: DefId| {
+            let (followed_by_brace, closing_brace) = this.followed_by_brace(span);
 
             match source {
                 PathSource::Expr(Some(
                     parent @ Expr { kind: ExprKind::Field(..) | ExprKind::MethodCall(..), .. },
-                )) if path_sep(err, &parent, DefKind::Struct) => {}
+                )) if path_sep(this, err, parent, DefKind::Struct) => {}
                 PathSource::Expr(
                     None
                     | Some(Expr {
@@ -1403,7 +1436,7 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
                 }
                 PathSource::Expr(_) | PathSource::TupleStruct(..) | PathSource::Pat => {
                     let span = find_span(&source, err);
-                    err.span_label(self.r.def_span(def_id), format!("`{path_str}` defined here"));
+                    err.span_label(this.r.def_span(def_id), format!("`{path_str}` defined here"));
 
                     let (tail, descr, applicability, old_fields) = match source {
                         PathSource::Pat => ("", "pattern", Applicability::MachineApplicable, None),
@@ -1413,50 +1446,69 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
                             Applicability::MachineApplicable,
                             Some(
                                 args.iter()
-                                    .map(|a| self.r.tcx.sess.source_map().span_to_snippet(*a).ok())
+                                    .map(|a| this.r.tcx.sess.source_map().span_to_snippet(*a).ok())
                                     .collect::<Vec<Option<String>>>(),
                             ),
                         ),
                         _ => (": val", "literal", Applicability::HasPlaceholders, None),
                     };
-                    let field_ids = self.r.field_def_ids(def_id);
-                    let (fields, applicability) = match field_ids {
-                        Some(field_ids) => {
-                            let fields = field_ids.iter().map(|&id| self.r.tcx.item_name(id));
-
-                            let fields = if let Some(old_fields) = old_fields {
-                                fields
-                                    .enumerate()
-                                    .map(|(idx, new)| (new, old_fields.get(idx)))
-                                    .map(|(new, old)| {
-                                        let new = new.to_ident_string();
-                                        if let Some(Some(old)) = old
-                                            && new != *old
-                                        {
-                                            format!("{new}: {old}")
-                                        } else {
-                                            new
-                                        }
-                                    })
-                                    .collect::<Vec<String>>()
-                            } else {
-                                fields.map(|f| format!("{f}{tail}")).collect::<Vec<String>>()
-                            };
 
-                            (fields.join(", "), applicability)
-                        }
-                        None => ("/* fields */".to_string(), Applicability::HasPlaceholders),
-                    };
-                    let pad = match field_ids {
-                        Some(field_ids) if field_ids.is_empty() => "",
-                        _ => " ",
-                    };
-                    err.span_suggestion(
-                        span,
-                        format!("use struct {descr} syntax instead"),
-                        format!("{path_str} {{{pad}{fields}{pad}}}"),
-                        applicability,
-                    );
+                    if !this.has_private_fields(def_id) {
+                        // If the fields of the type are private, we shouldn't be suggesting using
+                        // the struct literal syntax at all, as that will cause a subsequent error.
+                        let field_ids = this.r.field_def_ids(def_id);
+                        let (fields, applicability) = match field_ids {
+                            Some(field_ids) => {
+                                let fields = field_ids.iter().map(|&id| this.r.tcx.item_name(id));
+
+                                let fields = if let Some(old_fields) = old_fields {
+                                    fields
+                                        .enumerate()
+                                        .map(|(idx, new)| (new, old_fields.get(idx)))
+                                        .map(|(new, old)| {
+                                            let new = new.to_ident_string();
+                                            if let Some(Some(old)) = old
+                                                && new != *old
+                                            {
+                                                format!("{new}: {old}")
+                                            } else {
+                                                new
+                                            }
+                                        })
+                                        .collect::<Vec<String>>()
+                                } else {
+                                    fields.map(|f| format!("{f}{tail}")).collect::<Vec<String>>()
+                                };
+
+                                (fields.join(", "), applicability)
+                            }
+                            None => ("/* fields */".to_string(), Applicability::HasPlaceholders),
+                        };
+                        let pad = match field_ids {
+                            Some(field_ids) if field_ids.is_empty() => "",
+                            _ => " ",
+                        };
+                        err.span_suggestion(
+                            span,
+                            format!("use struct {descr} syntax instead"),
+                            format!("{path_str} {{{pad}{fields}{pad}}}"),
+                            applicability,
+                        );
+                    }
+                    if let PathSource::Expr(Some(Expr {
+                        kind: ExprKind::Call(path, ref args),
+                        span: call_span,
+                        ..
+                    })) = source
+                    {
+                        this.suggest_alternative_construction_methods(
+                            def_id,
+                            err,
+                            path.span,
+                            *call_span,
+                            &args[..],
+                        );
+                    }
                 }
                 _ => {
                     err.span_label(span, fallback_label.to_string());
@@ -1473,12 +1525,20 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
                 | PathSource::Struct,
             ) => {
                 err.span_label(span, fallback_label.to_string());
-                err.span_suggestion_verbose(
-                    span.shrink_to_hi(),
-                    "use `!` to invoke the macro",
-                    "!",
-                    Applicability::MaybeIncorrect,
-                );
+
+                // Don't suggest `!` for a macro invocation if there are generic args
+                if path
+                    .last()
+                    .is_some_and(|segment| !segment.has_generic_args && !segment.has_lifetime_args)
+                {
+                    err.span_suggestion_verbose(
+                        span.shrink_to_hi(),
+                        "use `!` to invoke the macro",
+                        "!",
+                        Applicability::MaybeIncorrect,
+                    );
+                }
+
                 if path_str == "try" && span.is_rust_2015() {
                     err.note("if you want the `try` keyword, you need Rust 2018 or later");
                 }
@@ -1506,7 +1566,7 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
                 Res::Def(kind @ (DefKind::Mod | DefKind::Trait), _),
                 PathSource::Expr(Some(parent)),
             ) => {
-                if !path_sep(err, &parent, kind) {
+                if !path_sep(self, err, parent, kind) {
                     return false;
                 }
             }
@@ -1540,13 +1600,13 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
                 let (ctor_def, ctor_vis, fields) = if let Some(struct_ctor) = struct_ctor {
                     if let PathSource::Expr(Some(parent)) = source {
                         if let ExprKind::Field(..) | ExprKind::MethodCall(..) = parent.kind {
-                            bad_struct_syntax_suggestion(def_id);
+                            bad_struct_syntax_suggestion(self, def_id);
                             return true;
                         }
                     }
                     struct_ctor
                 } else {
-                    bad_struct_syntax_suggestion(def_id);
+                    bad_struct_syntax_suggestion(self, def_id);
                     return true;
                 };
 
@@ -1566,30 +1626,21 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
                         Some(Vec::from(pattern_spans))
                     }
                     // e.g. `let _ = Enum::TupleVariant(field1, field2);`
-                    _ if source.is_call() => {
+                    PathSource::Expr(Some(Expr {
+                        kind: ExprKind::Call(path, ref args),
+                        span: call_span,
+                        ..
+                    })) => {
                         err.set_primary_message(
                             "cannot initialize a tuple struct which contains private fields",
                         );
-                        if !def_id.is_local()
-                            && self
-                                .r
-                                .tcx
-                                .inherent_impls(def_id)
-                                .iter()
-                                .flat_map(|impl_def_id| {
-                                    self.r.tcx.provided_trait_methods(*impl_def_id)
-                                })
-                                .any(|assoc| !assoc.fn_has_self_parameter && assoc.name == sym::new)
-                        {
-                            // FIXME: look for associated functions with Self return type,
-                            // instead of relying only on the name and lack of self receiver.
-                            err.span_suggestion_verbose(
-                                span.shrink_to_hi(),
-                                "you might have meant to use the `new` associated function",
-                                "::new".to_string(),
-                                Applicability::MaybeIncorrect,
-                            );
-                        }
+                        self.suggest_alternative_construction_methods(
+                            def_id,
+                            err,
+                            path.span,
+                            *call_span,
+                            &args[..],
+                        );
                         // Use spans of the tuple struct definition.
                         self.r.field_def_ids(def_id).map(|field_ids| {
                             field_ids
@@ -1636,7 +1687,7 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
                 err.span_label(span, "constructor is not visible here due to private fields");
             }
             (Res::Def(DefKind::Union | DefKind::Variant, def_id), _) if ns == ValueNS => {
-                bad_struct_syntax_suggestion(def_id);
+                bad_struct_syntax_suggestion(self, def_id);
             }
             (Res::Def(DefKind::Ctor(_, CtorKind::Const), def_id), _) if ns == ValueNS => {
                 match source {
@@ -1682,6 +1733,156 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
         true
     }
 
+    fn suggest_alternative_construction_methods(
+        &mut self,
+        def_id: DefId,
+        err: &mut Diagnostic,
+        path_span: Span,
+        call_span: Span,
+        args: &[P<Expr>],
+    ) {
+        if def_id.is_local() {
+            // Doing analysis on local `DefId`s would cause infinite recursion.
+            return;
+        }
+        // Look at all the associated functions without receivers in the type's
+        // inherent impls to look for builders that return `Self`
+        let mut items = self
+            .r
+            .tcx
+            .inherent_impls(def_id)
+            .iter()
+            .flat_map(|i| self.r.tcx.associated_items(i).in_definition_order())
+            // Only assoc fn with no receivers.
+            .filter(|item| matches!(item.kind, ty::AssocKind::Fn) && !item.fn_has_self_parameter)
+            .filter_map(|item| {
+                // Only assoc fns that return `Self`
+                let fn_sig = self.r.tcx.fn_sig(item.def_id).skip_binder();
+                let ret_ty = fn_sig.output();
+                let ret_ty = self
+                    .r
+                    .tcx
+                    .normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), ret_ty);
+                let ty::Adt(def, _args) = ret_ty.kind() else {
+                    return None;
+                };
+                let input_len = fn_sig.inputs().skip_binder().len();
+                if def.did() != def_id {
+                    return None;
+                }
+                let order = !item.name.as_str().starts_with("new");
+                Some((order, item.name, input_len))
+            })
+            .collect::<Vec<_>>();
+        items.sort_by_key(|(order, _, _)| *order);
+        let suggestion = |name, args| {
+            format!(
+                "::{name}({})",
+                std::iter::repeat("_").take(args).collect::<Vec<_>>().join(", ")
+            )
+        };
+        match &items[..] {
+            [] => {}
+            [(_, name, len)] if *len == args.len() => {
+                err.span_suggestion_verbose(
+                    path_span.shrink_to_hi(),
+                    format!("you might have meant to use the `{name}` associated function",),
+                    format!("::{name}"),
+                    Applicability::MaybeIncorrect,
+                );
+            }
+            [(_, name, len)] => {
+                err.span_suggestion_verbose(
+                    path_span.shrink_to_hi().with_hi(call_span.hi()),
+                    format!("you might have meant to use the `{name}` associated function",),
+                    suggestion(name, *len),
+                    Applicability::MaybeIncorrect,
+                );
+            }
+            _ => {
+                err.span_suggestions_with_style(
+                    path_span.shrink_to_hi().with_hi(call_span.hi()),
+                    "you might have meant to use an associated function to build this type",
+                    items
+                        .iter()
+                        .map(|(_, name, len)| suggestion(name, *len))
+                        .collect::<Vec<String>>(),
+                    Applicability::MaybeIncorrect,
+                    SuggestionStyle::ShowAlways,
+                );
+            }
+        }
+        // We'd ideally use `type_implements_trait` but don't have access to
+        // the trait solver here. We can't use `get_diagnostic_item` or
+        // `all_traits` in resolve either. So instead we abuse the import
+        // suggestion machinery to get `std::default::Default` and perform some
+        // checks to confirm that we got *only* that trait. We then see if the
+        // Adt we have has a direct implementation of `Default`. If so, we
+        // provide a structured suggestion.
+        let default_trait = self
+            .r
+            .lookup_import_candidates(
+                Ident::with_dummy_span(sym::Default),
+                Namespace::TypeNS,
+                &self.parent_scope,
+                &|res: Res| matches!(res, Res::Def(DefKind::Trait, _)),
+            )
+            .iter()
+            .filter_map(|candidate| candidate.did)
+            .find(|did| {
+                self.r
+                    .tcx
+                    .get_attrs(*did, sym::rustc_diagnostic_item)
+                    .any(|attr| attr.value_str() == Some(sym::Default))
+            });
+        let Some(default_trait) = default_trait else {
+            return;
+        };
+        if self
+            .r
+            .extern_crate_map
+            .iter()
+            // FIXME: This doesn't include impls like `impl Default for String`.
+            .flat_map(|(_, crate_)| self.r.tcx.implementations_of_trait((*crate_, default_trait)))
+            .filter_map(|(_, simplified_self_ty)| *simplified_self_ty)
+            .filter_map(|simplified_self_ty| match simplified_self_ty {
+                SimplifiedType::Adt(did) => Some(did),
+                _ => None,
+            })
+            .any(|did| did == def_id)
+        {
+            err.multipart_suggestion(
+                "consider using the `Default` trait",
+                vec![
+                    (path_span.shrink_to_lo(), "<".to_string()),
+                    (
+                        path_span.shrink_to_hi().with_hi(call_span.hi()),
+                        " as std::default::Default>::default()".to_string(),
+                    ),
+                ],
+                Applicability::MaybeIncorrect,
+            );
+        }
+    }
+
+    fn has_private_fields(&self, def_id: DefId) -> bool {
+        let fields = match def_id.as_local() {
+            Some(def_id) => self.r.struct_constructors.get(&def_id).cloned().map(|(_, _, f)| f),
+            None => Some(
+                self.r
+                    .tcx
+                    .associated_item_def_ids(def_id)
+                    .iter()
+                    .map(|field_id| self.r.tcx.visibility(field_id))
+                    .collect(),
+            ),
+        };
+
+        fields.is_some_and(|fields| {
+            fields.iter().any(|vis| !self.r.is_accessible_from(*vis, self.parent_scope.module))
+        })
+    }
+
     /// Given the target `ident` and `kind`, search for the similarly named associated item
     /// in `self.current_trait_ref`.
     pub(crate) fn find_similarly_named_assoc_item(
@@ -1966,13 +2167,22 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
     fn let_binding_suggestion(&mut self, err: &mut Diagnostic, ident_span: Span) -> bool {
         if let Some(Expr { kind: ExprKind::Assign(lhs, ..), .. }) =
             self.diagnostic_metadata.in_assignment
-            && let ast::ExprKind::Path(None, _) = lhs.kind
+            && let ast::ExprKind::Path(None, ref path) = lhs.kind
         {
             if !ident_span.from_expansion() {
+                let (span, text) = match path.segments.first() {
+                    Some(seg) if let Some(name) = seg.ident.as_str().strip_prefix("let") => {
+                        // a special case for #117894
+                        let name = name.strip_prefix('_').unwrap_or(name);
+                        (ident_span, format!("let {name}"))
+                    }
+                    _ => (ident_span.shrink_to_lo(), "let ".to_string()),
+                };
+
                 err.span_suggestion_verbose(
-                    ident_span.shrink_to_lo(),
+                    span,
                     "you might have meant to introduce a new binding",
-                    "let ".to_string(),
+                    text,
                     Applicability::MaybeIncorrect,
                 );
                 return true;
@@ -2062,11 +2272,12 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
         if suggest_only_tuple_variants {
             // Suggest only tuple variants regardless of whether they have fields and do not
             // suggest path with added parentheses.
-            let suggestable_variants = variants
+            let mut suggestable_variants = variants
                 .iter()
                 .filter(|(.., kind)| *kind == CtorKind::Fn)
                 .map(|(variant, ..)| path_names_to_string(variant))
                 .collect::<Vec<_>>();
+            suggestable_variants.sort();
 
             let non_suggestable_variant_count = variants.len() - suggestable_variants.len();
 
@@ -2117,7 +2328,7 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
                 }
             };
 
-            let suggestable_variants = variants
+            let mut suggestable_variants = variants
                 .iter()
                 .filter(|(_, def_id, kind)| !needs_placeholder(*def_id, *kind))
                 .map(|(variant, _, kind)| (path_names_to_string(variant), kind))
@@ -2126,6 +2337,7 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
                     CtorKind::Fn => format!("({variant}())"),
                 })
                 .collect::<Vec<_>>();
+            suggestable_variants.sort();
             let no_suggestable_variant = suggestable_variants.is_empty();
 
             if !no_suggestable_variant {
@@ -2143,7 +2355,7 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
                 );
             }
 
-            let suggestable_variants_with_placeholders = variants
+            let mut suggestable_variants_with_placeholders = variants
                 .iter()
                 .filter(|(_, def_id, kind)| needs_placeholder(*def_id, *kind))
                 .map(|(variant, _, kind)| (path_names_to_string(variant), kind))
@@ -2152,6 +2364,7 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
                     _ => None,
                 })
                 .collect::<Vec<_>>();
+            suggestable_variants_with_placeholders.sort();
 
             if !suggestable_variants_with_placeholders.is_empty() {
                 let msg =
@@ -2612,6 +2825,7 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
             .collect();
         debug!(?in_scope_lifetimes);
 
+        let mut maybe_static = false;
         debug!(?function_param_lifetimes);
         if let Some((param_lifetimes, params)) = &function_param_lifetimes {
             let elided_len = param_lifetimes.len();
@@ -2650,10 +2864,11 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
 
             if num_params == 0 {
                 err.help(
-                    "this function's return type contains a borrowed value, \
-                 but there is no value for it to be borrowed from",
+                    "this function's return type contains a borrowed value, but there is no value \
+                     for it to be borrowed from",
                 );
                 if in_scope_lifetimes.is_empty() {
+                    maybe_static = true;
                     in_scope_lifetimes = vec![(
                         Ident::with_dummy_span(kw::StaticLifetime),
                         (DUMMY_NODE_ID, LifetimeRes::Static),
@@ -2661,11 +2876,11 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
                 }
             } else if elided_len == 0 {
                 err.help(
-                    "this function's return type contains a borrowed value with \
-                 an elided lifetime, but the lifetime cannot be derived from \
-                 the arguments",
+                    "this function's return type contains a borrowed value with an elided \
+                     lifetime, but the lifetime cannot be derived from the arguments",
                 );
                 if in_scope_lifetimes.is_empty() {
+                    maybe_static = true;
                     in_scope_lifetimes = vec![(
                         Ident::with_dummy_span(kw::StaticLifetime),
                         (DUMMY_NODE_ID, LifetimeRes::Static),
@@ -2673,13 +2888,13 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
                 }
             } else if num_params == 1 {
                 err.help(format!(
-                    "this function's return type contains a borrowed value, \
-                 but the signature does not say which {m} it is borrowed from"
+                    "this function's return type contains a borrowed value, but the signature does \
+                     not say which {m} it is borrowed from",
                 ));
             } else {
                 err.help(format!(
-                    "this function's return type contains a borrowed value, \
-                 but the signature does not say whether it is borrowed from {m}"
+                    "this function's return type contains a borrowed value, but the signature does \
+                     not say whether it is borrowed from {m}",
                 ));
             }
         }
@@ -2744,11 +2959,238 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
                 );
             }
             1 => {
+                let post = if maybe_static {
+                    let owned = if let [lt] = &lifetime_refs[..]
+                        && lt.kind != MissingLifetimeKind::Ampersand
+                    {
+                        ", or if you will only have owned values"
+                    } else {
+                        ""
+                    };
+                    format!(
+                        ", but this is uncommon unless you're returning a borrowed value from a \
+                         `const` or a `static`{owned}",
+                    )
+                } else {
+                    String::new()
+                };
                 err.multipart_suggestion_verbose(
-                    format!("consider using the `{existing_name}` lifetime"),
+                    format!("consider using the `{existing_name}` lifetime{post}"),
                     spans_suggs,
                     Applicability::MaybeIncorrect,
                 );
+                if maybe_static {
+                    // FIXME: what follows are general suggestions, but we'd want to perform some
+                    // minimal flow analysis to provide more accurate suggestions. For example, if
+                    // we identified that the return expression references only one argument, we
+                    // would suggest borrowing only that argument, and we'd skip the prior
+                    // "use `'static`" suggestion entirely.
+                    if let [lt] = &lifetime_refs[..]
+                        && (lt.kind == MissingLifetimeKind::Ampersand
+                            || lt.kind == MissingLifetimeKind::Underscore)
+                    {
+                        let pre = if lt.kind == MissingLifetimeKind::Ampersand
+                            && let Some((kind, _span)) = self.diagnostic_metadata.current_function
+                            && let FnKind::Fn(_, _, sig, _, _, _) = kind
+                            && !sig.decl.inputs.is_empty()
+                            && let sugg = sig
+                                .decl
+                                .inputs
+                                .iter()
+                                .filter_map(|param| {
+                                    if param.ty.span.contains(lt.span) {
+                                        // We don't want to suggest `fn elision(_: &fn() -> &i32)`
+                                        // when we have `fn elision(_: fn() -> &i32)`
+                                        None
+                                    } else if let TyKind::CVarArgs = param.ty.kind {
+                                        // Don't suggest `&...` for ffi fn with varargs
+                                        None
+                                    } else if let TyKind::ImplTrait(..) = &param.ty.kind {
+                                        // We handle these in the next `else if` branch.
+                                        None
+                                    } else {
+                                        Some((param.ty.span.shrink_to_lo(), "&".to_string()))
+                                    }
+                                })
+                                .collect::<Vec<_>>()
+                            && !sugg.is_empty()
+                        {
+                            let (the, s) = if sig.decl.inputs.len() == 1 {
+                                ("the", "")
+                            } else {
+                                ("one of the", "s")
+                            };
+                            err.multipart_suggestion_verbose(
+                                format!(
+                                    "instead, you are more likely to want to change {the} \
+                                     argument{s} to be borrowed...",
+                                ),
+                                sugg,
+                                Applicability::MaybeIncorrect,
+                            );
+                            "...or alternatively, you might want"
+                        } else if (lt.kind == MissingLifetimeKind::Ampersand
+                            || lt.kind == MissingLifetimeKind::Underscore)
+                            && let Some((kind, _span)) = self.diagnostic_metadata.current_function
+                            && let FnKind::Fn(_, _, sig, _, _, _) = kind
+                            && let ast::FnRetTy::Ty(ret_ty) = &sig.decl.output
+                            && !sig.decl.inputs.is_empty()
+                            && let arg_refs = sig
+                                .decl
+                                .inputs
+                                .iter()
+                                .filter_map(|param| match &param.ty.kind {
+                                    TyKind::ImplTrait(_, bounds) => Some(bounds),
+                                    _ => None,
+                                })
+                                .flat_map(|bounds| bounds.into_iter())
+                                .collect::<Vec<_>>()
+                            && !arg_refs.is_empty()
+                        {
+                            // We have a situation like
+                            // fn g(mut x: impl Iterator<Item = &()>) -> Option<&()>
+                            // So we look at every ref in the trait bound. If there's any, we
+                            // suggest
+                            // fn g<'a>(mut x: impl Iterator<Item = &'a ()>) -> Option<&'a ()>
+                            let mut lt_finder =
+                                LifetimeFinder { lifetime: lt.span, found: None, seen: vec![] };
+                            for bound in arg_refs {
+                                if let ast::GenericBound::Trait(trait_ref, _) = bound {
+                                    lt_finder.visit_trait_ref(&trait_ref.trait_ref);
+                                }
+                            }
+                            lt_finder.visit_ty(ret_ty);
+                            let spans_suggs: Vec<_> = lt_finder
+                                .seen
+                                .iter()
+                                .filter_map(|ty| match &ty.kind {
+                                    TyKind::Ref(_, mut_ty) => {
+                                        let span = ty.span.with_hi(mut_ty.ty.span.lo());
+                                        Some((span, "&'a ".to_string()))
+                                    }
+                                    _ => None,
+                                })
+                                .collect();
+                            self.suggest_introducing_lifetime(
+                                err,
+                                None,
+                                |err, higher_ranked, span, message, intro_sugg| {
+                                    err.multipart_suggestion_verbose(
+                                        message,
+                                        std::iter::once((span, intro_sugg))
+                                            .chain(spans_suggs.iter().cloned())
+                                            .collect(),
+                                        Applicability::MaybeIncorrect,
+                                    );
+                                    higher_ranked
+                                },
+                            );
+                            "alternatively, you might want"
+                        } else {
+                            "instead, you are more likely to want"
+                        };
+                        let mut owned_sugg = lt.kind == MissingLifetimeKind::Ampersand;
+                        let mut sugg = vec![(lt.span, String::new())];
+                        if let Some((kind, _span)) = self.diagnostic_metadata.current_function
+                            && let FnKind::Fn(_, _, sig, _, _, _) = kind
+                            && let ast::FnRetTy::Ty(ty) = &sig.decl.output
+                        {
+                            let mut lt_finder =
+                                LifetimeFinder { lifetime: lt.span, found: None, seen: vec![] };
+                            lt_finder.visit_ty(&ty);
+
+                            if let [Ty { span, kind: TyKind::Ref(_, mut_ty), .. }] =
+                                &lt_finder.seen[..]
+                            {
+                                // We might have a situation like
+                                // fn g(mut x: impl Iterator<Item = &'_ ()>) -> Option<&'_ ()>
+                                // but `lt.span` only points at `'_`, so to suggest `-> Option<()>`
+                                // we need to find a more accurate span to end up with
+                                // fn g<'a>(mut x: impl Iterator<Item = &'_ ()>) -> Option<()>
+                                sugg = vec![(span.with_hi(mut_ty.ty.span.lo()), String::new())];
+                                owned_sugg = true;
+                            }
+                            if let Some(ty) = lt_finder.found {
+                                if let TyKind::Path(None, path) = &ty.kind {
+                                    // Check if the path being borrowed is likely to be owned.
+                                    let path: Vec<_> = Segment::from_path(path);
+                                    match self.resolve_path(&path, Some(TypeNS), None) {
+                                        PathResult::Module(ModuleOrUniformRoot::Module(module)) => {
+                                            match module.res() {
+                                                Some(Res::PrimTy(PrimTy::Str)) => {
+                                                    // Don't suggest `-> str`, suggest `-> String`.
+                                                    sugg = vec![(
+                                                        lt.span.with_hi(ty.span.hi()),
+                                                        "String".to_string(),
+                                                    )];
+                                                }
+                                                Some(Res::PrimTy(..)) => {}
+                                                Some(Res::Def(
+                                                    DefKind::Struct
+                                                    | DefKind::Union
+                                                    | DefKind::Enum
+                                                    | DefKind::ForeignTy
+                                                    | DefKind::AssocTy
+                                                    | DefKind::OpaqueTy
+                                                    | DefKind::TyParam,
+                                                    _,
+                                                )) => {}
+                                                _ => {
+                                                    // Do not suggest in all other cases.
+                                                    owned_sugg = false;
+                                                }
+                                            }
+                                        }
+                                        PathResult::NonModule(res) => {
+                                            match res.base_res() {
+                                                Res::PrimTy(PrimTy::Str) => {
+                                                    // Don't suggest `-> str`, suggest `-> String`.
+                                                    sugg = vec![(
+                                                        lt.span.with_hi(ty.span.hi()),
+                                                        "String".to_string(),
+                                                    )];
+                                                }
+                                                Res::PrimTy(..) => {}
+                                                Res::Def(
+                                                    DefKind::Struct
+                                                    | DefKind::Union
+                                                    | DefKind::Enum
+                                                    | DefKind::ForeignTy
+                                                    | DefKind::AssocTy
+                                                    | DefKind::OpaqueTy
+                                                    | DefKind::TyParam,
+                                                    _,
+                                                ) => {}
+                                                _ => {
+                                                    // Do not suggest in all other cases.
+                                                    owned_sugg = false;
+                                                }
+                                            }
+                                        }
+                                        _ => {
+                                            // Do not suggest in all other cases.
+                                            owned_sugg = false;
+                                        }
+                                    }
+                                }
+                                if let TyKind::Slice(inner_ty) = &ty.kind {
+                                    // Don't suggest `-> [T]`, suggest `-> Vec<T>`.
+                                    sugg = vec![
+                                        (lt.span.with_hi(inner_ty.span.lo()), "Vec<".to_string()),
+                                        (ty.span.with_lo(inner_ty.span.hi()), ">".to_string()),
+                                    ];
+                                }
+                            }
+                        }
+                        if owned_sugg {
+                            err.multipart_suggestion_verbose(
+                                format!("{pre} to return an owned value"),
+                                sugg,
+                                Applicability::MaybeIncorrect,
+                            );
+                        }
+                    }
+                }
 
                 // Record as using the suggested resolution.
                 let (_, (_, res)) = in_scope_lifetimes[0];
@@ -2778,7 +3220,7 @@ impl<'a: 'ast, 'ast, 'tcx> LateResolutionVisitor<'a, '_, 'ast, 'tcx> {
 fn mk_where_bound_predicate(
     path: &Path,
     poly_trait_ref: &ast::PolyTraitRef,
-    ty: &ast::Ty,
+    ty: &Ty,
 ) -> Option<ast::WhereBoundPredicate> {
     use rustc_span::DUMMY_SP;
     let modified_segments = {
@@ -2855,6 +3297,24 @@ pub(super) fn signal_lifetime_shadowing(sess: &Session, orig: Ident, shadower: I
     err.emit();
 }
 
+struct LifetimeFinder<'ast> {
+    lifetime: Span,
+    found: Option<&'ast Ty>,
+    seen: Vec<&'ast Ty>,
+}
+
+impl<'ast> Visitor<'ast> for LifetimeFinder<'ast> {
+    fn visit_ty(&mut self, t: &'ast Ty) {
+        if let TyKind::Ref(_, mut_ty) = &t.kind {
+            self.seen.push(t);
+            if t.span.lo() == self.lifetime.lo() {
+                self.found = Some(&mut_ty.ty);
+            }
+        }
+        walk_ty(self, t)
+    }
+}
+
 /// Shadowing involving a label is only a warning for historical reasons.
 //FIXME: make this a proper lint.
 pub(super) fn signal_label_shadowing(sess: &Session, orig: Span, shadower: Ident) {
diff --git a/compiler/rustc_resolve/src/lib.rs b/compiler/rustc_resolve/src/lib.rs
index 501747df5..75ec594eb 100644
--- a/compiler/rustc_resolve/src/lib.rs
+++ b/compiler/rustc_resolve/src/lib.rs
@@ -7,8 +7,8 @@
 //! Type-relative name resolution (methods, fields, associated items) happens in `rustc_hir_analysis`.
 
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
+#![doc(rust_logo)]
+#![feature(rustdoc_internals)]
 #![feature(assert_matches)]
 #![feature(box_patterns)]
 #![feature(extract_if)]
@@ -37,18 +37,14 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexMap, FxIndexSet};
 use rustc_data_structures::intern::Interned;
 use rustc_data_structures::steal::Steal;
 use rustc_data_structures::sync::{FreezeReadGuard, Lrc};
-use rustc_errors::{
-    Applicability, DiagnosticBuilder, DiagnosticMessage, ErrorGuaranteed, SubdiagnosticMessage,
-};
+use rustc_errors::{Applicability, DiagnosticBuilder, ErrorGuaranteed};
 use rustc_expand::base::{DeriveResolutions, SyntaxExtension, SyntaxExtensionKind};
 use rustc_feature::BUILTIN_ATTRIBUTES;
-use rustc_fluent_macro::fluent_messages;
 use rustc_hir::def::Namespace::{self, *};
 use rustc_hir::def::NonMacroAttrKind;
 use rustc_hir::def::{self, CtorOf, DefKind, DocLinkResMap, LifetimeRes, PartialRes, PerNS};
 use rustc_hir::def_id::{CrateNum, DefId, LocalDefId, LocalDefIdMap, LocalDefIdSet};
 use rustc_hir::def_id::{CRATE_DEF_ID, LOCAL_CRATE};
-use rustc_hir::definitions::DefPathData;
 use rustc_hir::{PrimTy, TraitCandidate};
 use rustc_index::IndexVec;
 use rustc_metadata::creader::{CStore, CrateLoader};
@@ -90,7 +86,7 @@ mod late;
 mod macros;
 pub mod rustdoc;
 
-fluent_messages! { "../messages.ftl" }
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
 
 #[derive(Debug)]
 enum Weak {
@@ -175,7 +171,7 @@ impl<'a> ParentScope<'a> {
 #[derive(Copy, Debug, Clone)]
 enum ImplTraitContext {
     Existential,
-    Universal(LocalDefId),
+    Universal,
 }
 
 #[derive(Debug)]
@@ -927,9 +923,16 @@ struct DeriveData {
 #[derive(Clone)]
 struct MacroData {
     ext: Lrc<SyntaxExtension>,
+    rule_spans: Vec<(usize, Span)>,
     macro_rules: bool,
 }
 
+impl MacroData {
+    fn new(ext: Lrc<SyntaxExtension>) -> MacroData {
+        MacroData { ext, rule_spans: Vec::new(), macro_rules: false }
+    }
+}
+
 /// The main resolver class.
 ///
 /// This is the visitor that walks the whole crate.
@@ -1004,8 +1007,7 @@ pub struct Resolver<'a, 'tcx> {
 
     /// Maps glob imports to the names of items actually imported.
     glob_map: FxHashMap<LocalDefId, FxHashSet<Symbol>>,
-    /// Visibilities in "lowered" form, for all entities that have them.
-    visibilities: FxHashMap<LocalDefId, ty::Visibility>,
+    visibilities_for_hashing: Vec<(LocalDefId, ty::Visibility)>,
     used_imports: FxHashSet<NodeId>,
     maybe_unused_trait_imports: FxIndexSet<LocalDefId>,
 
@@ -1030,15 +1032,12 @@ pub struct Resolver<'a, 'tcx> {
     used_extern_options: FxHashSet<Symbol>,
     macro_names: FxHashSet<Ident>,
     builtin_macros: FxHashMap<Symbol, BuiltinMacroState>,
-    /// A small map keeping true kinds of built-in macros that appear to be fn-like on
-    /// the surface (`macro` items in libcore), but are actually attributes or derives.
-    builtin_macro_kinds: FxHashMap<LocalDefId, MacroKind>,
     registered_tools: &'tcx RegisteredTools,
     macro_use_prelude: FxHashMap<Symbol, NameBinding<'a>>,
     macro_map: FxHashMap<DefId, MacroData>,
     dummy_ext_bang: Lrc<SyntaxExtension>,
     dummy_ext_derive: Lrc<SyntaxExtension>,
-    non_macro_attr: Lrc<SyntaxExtension>,
+    non_macro_attr: MacroData,
     local_macro_def_scopes: FxHashMap<LocalDefId, Module<'a>>,
     ast_transform_scopes: FxHashMap<LocalExpnId, Module<'a>>,
     unused_macros: FxHashMap<LocalDefId, (NodeId, Ident)>,
@@ -1085,7 +1084,7 @@ pub struct Resolver<'a, 'tcx> {
 
     next_node_id: NodeId,
 
-    node_id_to_def_id: FxHashMap<ast::NodeId, LocalDefId>,
+    node_id_to_def_id: NodeMap<LocalDefId>,
     def_id_to_node_id: IndexVec<LocalDefId, ast::NodeId>,
 
     /// Indices of unnamed struct or variant fields with unresolved attributes.
@@ -1211,10 +1210,12 @@ impl<'tcx> Resolver<'_, 'tcx> {
         &mut self,
         parent: LocalDefId,
         node_id: ast::NodeId,
-        data: DefPathData,
+        name: Symbol,
+        def_kind: DefKind,
         expn_id: ExpnId,
         span: Span,
     ) -> LocalDefId {
+        let data = def_kind.def_path_data(name);
         assert!(
             !self.node_id_to_def_id.contains_key(&node_id),
             "adding a def'n for node-id {:?} and data {:?} but a previous def'n exists: {:?}",
@@ -1224,7 +1225,7 @@ impl<'tcx> Resolver<'_, 'tcx> {
         );
 
         // FIXME: remove `def_span` body, pass in the right spans here and call `tcx.at().create_def()`
-        let def_id = self.tcx.untracked().definitions.write().create_def(parent, data);
+        let def_id = self.tcx.create_def(parent, name, def_kind);
 
         // Create the definition.
         if expn_id != ExpnId::root() {
@@ -1290,12 +1291,9 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
             &mut FxHashMap::default(),
         );
 
-        let mut visibilities = FxHashMap::default();
-        visibilities.insert(CRATE_DEF_ID, ty::Visibility::Public);
-
         let mut def_id_to_node_id = IndexVec::default();
         assert_eq!(def_id_to_node_id.push(CRATE_NODE_ID), CRATE_DEF_ID);
-        let mut node_id_to_def_id = FxHashMap::default();
+        let mut node_id_to_def_id = NodeMap::default();
         node_id_to_def_id.insert(CRATE_NODE_ID, CRATE_DEF_ID);
 
         let mut invocation_parents = FxHashMap::default();
@@ -1321,6 +1319,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
 
         let features = tcx.features();
         let pub_vis = ty::Visibility::<DefId>::Public;
+        let edition = tcx.sess.edition();
 
         let mut resolver = Resolver {
             tcx,
@@ -1357,7 +1356,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
             ast_transform_scopes: FxHashMap::default(),
 
             glob_map: Default::default(),
-            visibilities,
+            visibilities_for_hashing: Default::default(),
             used_imports: FxHashSet::default(),
             maybe_unused_trait_imports: Default::default(),
 
@@ -1398,13 +1397,12 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
             used_extern_options: Default::default(),
             macro_names: FxHashSet::default(),
             builtin_macros: Default::default(),
-            builtin_macro_kinds: Default::default(),
             registered_tools,
             macro_use_prelude: FxHashMap::default(),
             macro_map: FxHashMap::default(),
-            dummy_ext_bang: Lrc::new(SyntaxExtension::dummy_bang(tcx.sess.edition())),
-            dummy_ext_derive: Lrc::new(SyntaxExtension::dummy_derive(tcx.sess.edition())),
-            non_macro_attr: Lrc::new(SyntaxExtension::non_macro_attr(tcx.sess.edition())),
+            dummy_ext_bang: Lrc::new(SyntaxExtension::dummy_bang(edition)),
+            dummy_ext_derive: Lrc::new(SyntaxExtension::dummy_derive(edition)),
+            non_macro_attr: MacroData::new(Lrc::new(SyntaxExtension::non_macro_attr(edition))),
             invocation_parent_scopes: Default::default(),
             output_macro_rules_scopes: Default::default(),
             macro_rules_scopes: Default::default(),
@@ -1445,6 +1443,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
 
         let root_parent_scope = ParentScope::module(graph_root, &resolver);
         resolver.invocation_parent_scopes.insert(LocalExpnId::ROOT, root_parent_scope);
+        resolver.feed_visibility(CRATE_DEF_ID, ty::Visibility::Public);
 
         resolver
     }
@@ -1492,10 +1491,14 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
         Default::default()
     }
 
+    fn feed_visibility(&mut self, def_id: LocalDefId, vis: ty::Visibility) {
+        self.tcx.feed_local_def_id(def_id).visibility(vis.to_def_id());
+        self.visibilities_for_hashing.push((def_id, vis));
+    }
+
     pub fn into_outputs(self) -> ResolverOutputs {
         let proc_macros = self.proc_macros.iter().map(|id| self.local_def_id(*id)).collect();
         let expn_that_defined = self.expn_that_defined;
-        let visibilities = self.visibilities;
         let extern_crate_map = self.extern_crate_map;
         let maybe_unused_trait_imports = self.maybe_unused_trait_imports;
         let glob_map = self.glob_map;
@@ -1512,7 +1515,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
 
         let global_ctxt = ResolverGlobalCtxt {
             expn_that_defined,
-            visibilities,
+            visibilities_for_hashing: self.visibilities_for_hashing,
             effective_visibilities,
             extern_crate_map,
             module_children: self.module_children,
@@ -1537,7 +1540,6 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
             node_id_to_def_id: self.node_id_to_def_id,
             def_id_to_node_id: self.def_id_to_node_id,
             trait_map: self.trait_map,
-            builtin_macro_kinds: self.builtin_macro_kinds,
             lifetime_elision_allowed: self.lifetime_elision_allowed,
             lint_buffer: Steal::new(self.lint_buffer),
         };
@@ -1564,7 +1566,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
         match macro_kind {
             MacroKind::Bang => self.dummy_ext_bang.clone(),
             MacroKind::Derive => self.dummy_ext_derive.clone(),
-            MacroKind::Attr => self.non_macro_attr.clone(),
+            MacroKind::Attr => self.non_macro_attr.ext.clone(),
         }
     }
 
diff --git a/compiler/rustc_resolve/src/macros.rs b/compiler/rustc_resolve/src/macros.rs
index 2ff6fb424..1001286b6 100644
--- a/compiler/rustc_resolve/src/macros.rs
+++ b/compiler/rustc_resolve/src/macros.rs
@@ -6,7 +6,7 @@ use crate::errors::{
     MacroExpectedFound, RemoveSurroundingDerive,
 };
 use crate::Namespace::*;
-use crate::{BuiltinMacroState, Determinacy};
+use crate::{BuiltinMacroState, Determinacy, MacroData};
 use crate::{DeriveData, Finalize, ParentScope, ResolutionError, Resolver, ScopeSet};
 use crate::{ModuleKind, ModuleOrUniformRoot, NameBinding, PathResult, Segment, ToNameBinding};
 use rustc_ast::expand::StrippedCfgItem;
@@ -205,10 +205,7 @@ impl<'a, 'tcx> ResolverExpand for Resolver<'a, 'tcx> {
 
     fn register_builtin_macro(&mut self, name: Symbol, ext: SyntaxExtensionKind) {
         if self.builtin_macros.insert(name, BuiltinMacroState::NotYetSeen(ext)).is_some() {
-            self.tcx
-                .sess
-                .diagnostic()
-                .bug(format!("built-in macro `{name}` was already registered"));
+            self.tcx.sess.dcx().bug(format!("built-in macro `{name}` was already registered"));
         }
     }
 
@@ -371,7 +368,7 @@ impl<'a, 'tcx> ResolverExpand for Resolver<'a, 'tcx> {
             if opt_ext.is_none() {
                 *opt_ext = Some(
                     match self.resolve_macro_path(
-                        &path,
+                        path,
                         Some(MacroKind::Derive),
                         &parent_scope,
                         true,
@@ -485,7 +482,7 @@ impl<'a, 'tcx> ResolverExpand for Resolver<'a, 'tcx> {
     }
 
     fn registered_tools(&self) -> &RegisteredTools {
-        &self.registered_tools
+        self.registered_tools
     }
 }
 
@@ -695,7 +692,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
             res
         };
 
-        res.map(|res| (self.get_macro(res).map(|macro_data| macro_data.ext), res))
+        res.map(|res| (self.get_macro(res).map(|macro_data| macro_data.ext.clone()), res))
     }
 
     pub(crate) fn finalize_macro_resolutions(&mut self, krate: &Crate) {
@@ -710,7 +707,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
                     // Make sure compilation does not succeed if preferred macro resolution
                     // has changed after the macro had been expanded. In theory all such
                     // situations should be reported as errors, so this is a bug.
-                    this.tcx.sess.delay_span_bug(span, "inconsistent resolution for a macro");
+                    this.tcx.sess.span_delayed_bug(span, "inconsistent resolution for a macro");
                 }
             } else {
                 // It's possible that the macro was unresolved (indeterminate) and silently
@@ -887,7 +884,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
             }
         }
         if let Some(depr) = &ext.deprecation {
-            let path = pprust::path_to_string(&path);
+            let path = pprust::path_to_string(path);
             let (message, lint) = stability::deprecation_message_and_lint(depr, "macro", &path);
             stability::early_report_deprecation(
                 &mut self.lint_buffer,
@@ -936,28 +933,20 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
     /// Compile the macro into a `SyntaxExtension` and its rule spans.
     ///
     /// Possibly replace its expander to a pre-defined one for built-in macros.
-    pub(crate) fn compile_macro(
-        &mut self,
-        item: &ast::Item,
-        edition: Edition,
-    ) -> (SyntaxExtension, Vec<(usize, Span)>) {
-        let (mut result, mut rule_spans) =
+    pub(crate) fn compile_macro(&mut self, item: &ast::Item, edition: Edition) -> MacroData {
+        let (mut ext, mut rule_spans) =
             compile_declarative_macro(self.tcx.sess, self.tcx.features(), item, edition);
 
-        if let Some(builtin_name) = result.builtin_name {
+        if let Some(builtin_name) = ext.builtin_name {
             // The macro was marked with `#[rustc_builtin_macro]`.
             if let Some(builtin_macro) = self.builtin_macros.get_mut(&builtin_name) {
                 // The macro is a built-in, replace its expander function
                 // while still taking everything else from the source code.
                 // If we already loaded this builtin macro, give a better error message than 'no such builtin macro'.
                 match mem::replace(builtin_macro, BuiltinMacroState::AlreadySeen(item.span)) {
-                    BuiltinMacroState::NotYetSeen(ext) => {
-                        result.kind = ext;
+                    BuiltinMacroState::NotYetSeen(builtin_ext) => {
+                        ext.kind = builtin_ext;
                         rule_spans = Vec::new();
-                        if item.id != ast::DUMMY_NODE_ID {
-                            self.builtin_macro_kinds
-                                .insert(self.local_def_id(item.id), result.macro_kind());
-                        }
                     }
                     BuiltinMacroState::AlreadySeen(span) => {
                         struct_span_err!(
@@ -976,6 +965,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
             }
         }
 
-        (result, rule_spans)
+        let ItemKind::MacroDef(def) = &item.kind else { unreachable!() };
+        MacroData { ext: Lrc::new(ext), rule_spans, macro_rules: def.macro_rules }
     }
 }
diff --git a/compiler/rustc_resolve/src/rustdoc.rs b/compiler/rustc_resolve/src/rustdoc.rs
index fe4b8c7f6..4ff4ccf5e 100644
--- a/compiler/rustc_resolve/src/rustdoc.rs
+++ b/compiler/rustc_resolve/src/rustdoc.rs
@@ -404,11 +404,10 @@ pub(crate) fn attrs_to_preprocessed_links(attrs: &[ast::Attribute]) -> Vec<Box<s
 fn parse_links<'md>(doc: &'md str) -> Vec<Box<str>> {
     let mut broken_link_callback = |link: BrokenLink<'md>| Some((link.reference, "".into()));
     let mut event_iter = Parser::new_with_broken_link_callback(
-        &doc,
+        doc,
         main_body_opts(),
         Some(&mut broken_link_callback),
-    )
-    .into_iter();
+    );
     let mut links = Vec::new();
 
     while let Some(event) = event_iter.next() {
diff --git a/compiler/rustc_serialize/src/lib.rs b/compiler/rustc_serialize/src/lib.rs
index cfa54072e..352758214 100644
--- a/compiler/rustc_serialize/src/lib.rs
+++ b/compiler/rustc_serialize/src/lib.rs
@@ -5,9 +5,9 @@
     html_playground_url = "https://play.rust-lang.org/",
     test(attr(allow(unused_variables), deny(warnings)))
 )]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
-#![cfg_attr(not(bootstrap), allow(internal_features))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
+#![doc(rust_logo)]
+#![allow(internal_features)]
+#![feature(rustdoc_internals)]
 #![feature(allocator_api)]
 #![feature(associated_type_bounds)]
 #![feature(const_option)]
diff --git a/compiler/rustc_serialize/src/opaque.rs b/compiler/rustc_serialize/src/opaque.rs
index 552554390..cc8d1c250 100644
--- a/compiler/rustc_serialize/src/opaque.rs
+++ b/compiler/rustc_serialize/src/opaque.rs
@@ -5,12 +5,13 @@ use std::io::{self, Write};
 use std::marker::PhantomData;
 use std::ops::Range;
 use std::path::Path;
+use std::path::PathBuf;
 
 // -----------------------------------------------------------------------------
 // Encoder
 // -----------------------------------------------------------------------------
 
-pub type FileEncodeResult = Result<usize, io::Error>;
+pub type FileEncodeResult = Result<usize, (PathBuf, io::Error)>;
 
 /// The size of the buffer in `FileEncoder`.
 const BUF_SIZE: usize = 8192;
@@ -34,6 +35,9 @@ pub struct FileEncoder {
     // This is used to implement delayed error handling, as described in the
     // comment on `trait Encoder`.
     res: Result<(), io::Error>,
+    path: PathBuf,
+    #[cfg(debug_assertions)]
+    finished: bool,
 }
 
 impl FileEncoder {
@@ -41,14 +45,18 @@ impl FileEncoder {
         // File::create opens the file for writing only. When -Zmeta-stats is enabled, the metadata
         // encoder rewinds the file to inspect what was written. So we need to always open the file
         // for reading and writing.
-        let file = File::options().read(true).write(true).create(true).truncate(true).open(path)?;
+        let file =
+            File::options().read(true).write(true).create(true).truncate(true).open(&path)?;
 
         Ok(FileEncoder {
             buf: vec![0u8; BUF_SIZE].into_boxed_slice().try_into().unwrap(),
+            path: path.as_ref().into(),
             buffered: 0,
             flushed: 0,
             file,
             res: Ok(()),
+            #[cfg(debug_assertions)]
+            finished: false,
         })
     }
 
@@ -63,6 +71,10 @@ impl FileEncoder {
     #[cold]
     #[inline(never)]
     pub fn flush(&mut self) {
+        #[cfg(debug_assertions)]
+        {
+            self.finished = false;
+        }
         if self.res.is_ok() {
             self.res = self.file.write_all(&self.buf[..self.buffered]);
         }
@@ -74,6 +86,10 @@ impl FileEncoder {
         &self.file
     }
 
+    pub fn path(&self) -> &Path {
+        &self.path
+    }
+
     #[inline]
     fn buffer_empty(&mut self) -> &mut [u8] {
         // SAFETY: self.buffered is inbounds as an invariant of the type
@@ -97,6 +113,10 @@ impl FileEncoder {
 
     #[inline]
     fn write_all(&mut self, buf: &[u8]) {
+        #[cfg(debug_assertions)]
+        {
+            self.finished = false;
+        }
         if let Some(dest) = self.buffer_empty().get_mut(..buf.len()) {
             dest.copy_from_slice(buf);
             self.buffered += buf.len();
@@ -121,6 +141,10 @@ impl FileEncoder {
     /// with one instruction, so while this does in some sense do wasted work, we come out ahead.
     #[inline]
     pub fn write_with<const N: usize>(&mut self, visitor: impl FnOnce(&mut [u8; N]) -> usize) {
+        #[cfg(debug_assertions)]
+        {
+            self.finished = false;
+        }
         let flush_threshold = const { BUF_SIZE.checked_sub(N).unwrap() };
         if std::intrinsics::unlikely(self.buffered > flush_threshold) {
             self.flush();
@@ -152,20 +176,25 @@ impl FileEncoder {
         })
     }
 
-    pub fn finish(mut self) -> Result<usize, io::Error> {
+    pub fn finish(&mut self) -> FileEncodeResult {
         self.flush();
+        #[cfg(debug_assertions)]
+        {
+            self.finished = true;
+        }
         match std::mem::replace(&mut self.res, Ok(())) {
             Ok(()) => Ok(self.position()),
-            Err(e) => Err(e),
+            Err(e) => Err((self.path.clone(), e)),
         }
     }
 }
 
+#[cfg(debug_assertions)]
 impl Drop for FileEncoder {
     fn drop(&mut self) {
-        // Likely to be a no-op, because `finish` should have been called and
-        // it also flushes. But do it just in case.
-        self.flush();
+        if !std::thread::panicking() {
+            assert!(self.finished);
+        }
     }
 }
 
diff --git a/compiler/rustc_session/messages.ftl b/compiler/rustc_session/messages.ftl
index fa1b6f9f1..f2e646c70 100644
--- a/compiler/rustc_session/messages.ftl
+++ b/compiler/rustc_session/messages.ftl
@@ -16,6 +16,8 @@ session_crate_name_invalid = crate names cannot start with a `-`, but `{$s}` has
 
 session_expr_parentheses_needed = parentheses are required to parse this as an expression
 
+session_failed_to_create_profiler = failed to create profiler: {$err}
+
 session_feature_diagnostic_for_issue =
     see issue #{$n} <https://github.com/rust-lang/rust/issues/{$n}> for more information
 
@@ -26,6 +28,10 @@ session_file_is_not_writeable = output file {$file} is not writeable -- check it
 
 session_file_write_fail = failed to write `{$path}` due to error `{$err}`
 
+session_function_return_requires_x86_or_x86_64 = `-Zfunction-return` (except `keep`) is only supported on x86 and x86_64
+
+session_function_return_thunk_extern_requires_non_large_code_model = `-Zfunction-return=thunk-extern` is only supported on non-large code models
+
 session_hexadecimal_float_literal_not_supported = hexadecimal float literal is not supported
 
 session_incompatible_linker_flavor = linker flavor `{$flavor}` is incompatible with the current target
@@ -69,6 +75,7 @@ session_not_supported = not supported
 session_nul_in_c_str = null characters in C string literals are not supported
 
 session_octal_float_literal_not_supported = octal float literal is not supported
+
 session_optimization_fuel_exhausted = optimization-fuel-exhausted: {$msg}
 
 session_profile_sample_use_file_does_not_exist = file `{$path}` passed to `-C profile-sample-use` does not exist.
diff --git a/compiler/rustc_session/src/code_stats.rs b/compiler/rustc_session/src/code_stats.rs
index e1eb58fec..2553df33c 100644
--- a/compiler/rustc_session/src/code_stats.rs
+++ b/compiler/rustc_session/src/code_stats.rs
@@ -132,6 +132,8 @@ impl CodeStats {
 
     pub fn print_type_sizes(&self) {
         let type_sizes = self.type_sizes.borrow();
+        // We will soon sort, so the initial order does not matter.
+        #[allow(rustc::potential_query_instability)]
         let mut sorted: Vec<_> = type_sizes.iter().collect();
 
         // Primary sort: large-to-small.
@@ -227,6 +229,8 @@ impl CodeStats {
     }
 
     pub fn print_vtable_sizes(&self, crate_name: Symbol) {
+        // We will soon sort, so the initial order does not matter.
+        #[allow(rustc::potential_query_instability)]
         let mut infos =
             std::mem::take(&mut *self.vtable_sizes.lock()).into_values().collect::<Vec<_>>();
 
diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs
index f745bc390..0c21e4eb4 100644
--- a/compiler/rustc_session/src/config.rs
+++ b/compiler/rustc_session/src/config.rs
@@ -7,24 +7,20 @@ use crate::errors::FileWriteFail;
 use crate::search_paths::SearchPath;
 use crate::utils::{CanonicalizedPath, NativeLib, NativeLibKind};
 use crate::{lint, HashStableContext};
-use crate::{EarlyErrorHandler, Session};
-
-use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet};
+use crate::{EarlyDiagCtxt, Session};
+use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexMap, FxIndexSet};
 use rustc_data_structures::stable_hasher::{StableOrd, ToStableHashKey};
-use rustc_target::abi::Align;
-use rustc_target::spec::LinkSelfContainedComponents;
-use rustc_target::spec::{PanicStrategy, RelocModel, SanitizerSet, SplitDebuginfo};
-use rustc_target::spec::{Target, TargetTriple, TargetWarnings, TARGETS};
-
+use rustc_errors::emitter::HumanReadableErrorType;
+use rustc_errors::{ColorConfig, DiagCtxtFlags, DiagnosticArgValue, IntoDiagnosticArg};
 use rustc_feature::UnstableFeatures;
 use rustc_span::edition::{Edition, DEFAULT_EDITION, EDITION_NAME_LIST, LATEST_STABLE_EDITION};
 use rustc_span::source_map::FilePathMapping;
 use rustc_span::symbol::{sym, Symbol};
 use rustc_span::{FileName, FileNameDisplayPreference, RealFileName, SourceFileHashAlgorithm};
-
-use rustc_errors::emitter::HumanReadableErrorType;
-use rustc_errors::{ColorConfig, DiagnosticArgValue, HandlerFlags, IntoDiagnosticArg};
-
+use rustc_target::abi::Align;
+use rustc_target::spec::LinkSelfContainedComponents;
+use rustc_target::spec::{PanicStrategy, RelocModel, SanitizerSet, SplitDebuginfo};
+use rustc_target::spec::{Target, TargetTriple, TargetWarnings, TARGETS};
 use std::collections::btree_map::{
     Iter as BTreeMapIter, Keys as BTreeMapKeysIter, Values as BTreeMapValuesIter,
 };
@@ -555,12 +551,15 @@ impl Default for ErrorOutputType {
 /// Parameter to control path trimming.
 #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)]
 pub enum TrimmedDefPaths {
-    /// `try_print_trimmed_def_path` never prints a trimmed path and never calls the expensive query
+    /// `try_print_trimmed_def_path` never prints a trimmed path and never calls the expensive
+    /// query.
     #[default]
     Never,
-    /// `try_print_trimmed_def_path` calls the expensive query, the query doesn't call `delay_good_path_bug`
+    /// `try_print_trimmed_def_path` calls the expensive query, the query doesn't call
+    /// `good_path_delayed_bug`.
     Always,
-    /// `try_print_trimmed_def_path` calls the expensive query, the query calls `delay_good_path_bug`
+    /// `try_print_trimmed_def_path` calls the expensive query, the query calls
+    /// `good_path_delayed_bug`.
     GoodPath,
 }
 
@@ -580,7 +579,7 @@ pub enum ResolveDocLinks {
 /// *Do not* switch `BTreeMap` out for an unsorted container type! That would break
 /// dependency tracking for command-line arguments. Also only hash keys, since tracking
 /// should only depend on the output types, not the paths they're written to.
-#[derive(Clone, Debug, Hash, HashStable_Generic)]
+#[derive(Clone, Debug, Hash, HashStable_Generic, Encodable, Decodable)]
 pub struct OutputTypes(BTreeMap<OutputType, Option<OutFileName>>);
 
 impl OutputTypes {
@@ -756,13 +755,14 @@ pub enum PrintKind {
 }
 
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
-pub enum TraitSolver {
-    /// Classic trait solver in `rustc_trait_selection::traits::select`
-    Classic,
-    /// Experimental trait solver in `rustc_trait_selection::solve`
-    Next,
-    /// Use the new trait solver during coherence
-    NextCoherence,
+pub struct NextSolverConfig {
+    /// Whether the new trait solver should be enabled in coherence.
+    pub coherence: bool,
+    /// Whether the new trait solver should be enabled everywhere.
+    /// This is only `true` if `coherence` is also enabled.
+    pub globally: bool,
+    /// Whether to dump proof trees after computing a proof tree.
+    pub dump_tree: DumpSolverProofTree,
 }
 
 #[derive(Default, Debug, Copy, Clone, Hash, PartialEq, Eq)]
@@ -818,7 +818,7 @@ impl Input {
     }
 }
 
-#[derive(Clone, Hash, Debug, HashStable_Generic, PartialEq)]
+#[derive(Clone, Hash, Debug, HashStable_Generic, PartialEq, Encodable, Decodable)]
 pub enum OutFileName {
     Real(PathBuf),
     Stdout,
@@ -857,7 +857,7 @@ impl OutFileName {
     pub fn as_path(&self) -> &Path {
         match *self {
             OutFileName::Real(ref path) => path.as_ref(),
-            OutFileName::Stdout => &Path::new("stdout"),
+            OutFileName::Stdout => Path::new("stdout"),
         }
     }
 
@@ -890,7 +890,7 @@ impl OutFileName {
     }
 }
 
-#[derive(Clone, Hash, Debug, HashStable_Generic)]
+#[derive(Clone, Hash, Debug, HashStable_Generic, Encodable, Decodable)]
 pub struct OutputFilenames {
     pub out_directory: PathBuf,
     /// Crate name. Never contains '-'.
@@ -1115,6 +1115,7 @@ impl Default for Options {
             pretty: None,
             working_dir: RealFileName::LocalPath(std::env::current_dir().unwrap()),
             color: ColorConfig::Auto,
+            logical_env: FxIndexMap::default(),
         }
     }
 }
@@ -1154,8 +1155,8 @@ impl Options {
 }
 
 impl UnstableOptions {
-    pub fn diagnostic_handler_flags(&self, can_emit_warnings: bool) -> HandlerFlags {
-        HandlerFlags {
+    pub fn dcx_flags(&self, can_emit_warnings: bool) -> DiagCtxtFlags {
+        DiagCtxtFlags {
             can_emit_warnings,
             treat_err_as_bug: self.treat_err_as_bug,
             dont_buffer_diagnostics: self.dont_buffer_diagnostics,
@@ -1247,46 +1248,85 @@ pub const fn default_lib_output() -> CrateType {
 }
 
 fn default_configuration(sess: &Session) -> Cfg {
-    // NOTE: This should be kept in sync with `CheckCfg::fill_well_known` below.
-    let end = &sess.target.endian;
-    let arch = &sess.target.arch;
-    let wordsz = sess.target.pointer_width.to_string();
-    let os = &sess.target.os;
-    let env = &sess.target.env;
-    let abi = &sess.target.abi;
-    let relocation_model = sess.target.relocation_model.desc_symbol();
-    let vendor = &sess.target.vendor;
-    let min_atomic_width = sess.target.min_atomic_width();
-    let max_atomic_width = sess.target.max_atomic_width();
-    let atomic_cas = sess.target.atomic_cas;
-    let layout = sess.target.parse_data_layout().unwrap_or_else(|err| {
-        sess.emit_fatal(err);
-    });
-
     let mut ret = Cfg::default();
-    ret.reserve(7); // the minimum number of insertions
-    // Target bindings.
-    ret.insert((sym::target_os, Some(Symbol::intern(os))));
-    for fam in sess.target.families.as_ref() {
-        ret.insert((sym::target_family, Some(Symbol::intern(fam))));
-        if fam == "windows" {
-            ret.insert((sym::windows, None));
-        } else if fam == "unix" {
-            ret.insert((sym::unix, None));
-        }
-    }
-    ret.insert((sym::target_arch, Some(Symbol::intern(arch))));
-    ret.insert((sym::target_endian, Some(Symbol::intern(end.as_str()))));
-    ret.insert((sym::target_pointer_width, Some(Symbol::intern(&wordsz))));
-    ret.insert((sym::target_env, Some(Symbol::intern(env))));
-    ret.insert((sym::target_abi, Some(Symbol::intern(abi))));
+
+    macro_rules! ins_none {
+        ($key:expr) => {
+            ret.insert(($key, None));
+        };
+    }
+    macro_rules! ins_str {
+        ($key:expr, $val_str:expr) => {
+            ret.insert(($key, Some(Symbol::intern($val_str))));
+        };
+    }
+    macro_rules! ins_sym {
+        ($key:expr, $val_sym:expr) => {
+            ret.insert(($key, Some($val_sym)));
+        };
+    }
+
+    // Symbols are inserted in alphabetical order as much as possible.
+    // The exceptions are where control flow forces things out of order.
+    //
+    // Run `rustc --print cfg` to see the configuration in practice.
+    //
+    // NOTE: These insertions should be kept in sync with
+    // `CheckCfg::fill_well_known` below.
+
+    if sess.opts.debug_assertions {
+        ins_none!(sym::debug_assertions);
+    }
+
+    if sess.overflow_checks() {
+        ins_none!(sym::overflow_checks);
+    }
+
+    ins_sym!(sym::panic, sess.panic_strategy().desc_symbol());
+
+    // JUSTIFICATION: before wrapper fn is available
+    #[allow(rustc::bad_opt_access)]
+    if sess.opts.crate_types.contains(&CrateType::ProcMacro) {
+        ins_none!(sym::proc_macro);
+    }
+
     if sess.is_nightly_build() {
-        ret.insert((sym::relocation_model, Some(relocation_model)));
+        ins_sym!(sym::relocation_model, sess.target.relocation_model.desc_symbol());
+    }
+
+    for mut s in sess.opts.unstable_opts.sanitizer {
+        // KASAN is still ASAN under the hood, so it uses the same attribute.
+        if s == SanitizerSet::KERNELADDRESS {
+            s = SanitizerSet::ADDRESS;
+        }
+        ins_str!(sym::sanitize, &s.to_string());
+    }
+
+    if sess.is_sanitizer_cfi_generalize_pointers_enabled() {
+        ins_none!(sym::sanitizer_cfi_generalize_pointers);
     }
-    ret.insert((sym::target_vendor, Some(Symbol::intern(vendor))));
-    if sess.target.has_thread_local {
-        ret.insert((sym::target_thread_local, None));
+    if sess.is_sanitizer_cfi_normalize_integers_enabled() {
+        ins_none!(sym::sanitizer_cfi_normalize_integers);
     }
+
+    ins_str!(sym::target_abi, &sess.target.abi);
+    ins_str!(sym::target_arch, &sess.target.arch);
+    ins_str!(sym::target_endian, sess.target.endian.as_str());
+    ins_str!(sym::target_env, &sess.target.env);
+
+    for family in sess.target.families.as_ref() {
+        ins_str!(sym::target_family, family);
+        if family == "windows" {
+            ins_none!(sym::windows);
+        } else if family == "unix" {
+            ins_none!(sym::unix);
+        }
+    }
+
+    // `target_has_atomic*`
+    let layout = sess.target.parse_data_layout().unwrap_or_else(|err| {
+        sess.emit_fatal(err);
+    });
     let mut has_atomic = false;
     for (i, align) in [
         (8, layout.i8_align.abi),
@@ -1295,64 +1335,46 @@ fn default_configuration(sess: &Session) -> Cfg {
         (64, layout.i64_align.abi),
         (128, layout.i128_align.abi),
     ] {
-        if i >= min_atomic_width && i <= max_atomic_width {
-            has_atomic = true;
-            let mut insert_atomic = |s, align: Align| {
-                ret.insert((sym::target_has_atomic_load_store, Some(Symbol::intern(s))));
-                if atomic_cas {
-                    ret.insert((sym::target_has_atomic, Some(Symbol::intern(s))));
+        if i >= sess.target.min_atomic_width() && i <= sess.target.max_atomic_width() {
+            if !has_atomic {
+                has_atomic = true;
+                if sess.is_nightly_build() {
+                    if sess.target.atomic_cas {
+                        ins_none!(sym::target_has_atomic);
+                    }
+                    ins_none!(sym::target_has_atomic_load_store);
+                }
+            }
+            let mut insert_atomic = |sym, align: Align| {
+                if sess.target.atomic_cas {
+                    ins_sym!(sym::target_has_atomic, sym);
                 }
                 if align.bits() == i {
-                    ret.insert((sym::target_has_atomic_equal_alignment, Some(Symbol::intern(s))));
+                    ins_sym!(sym::target_has_atomic_equal_alignment, sym);
                 }
+                ins_sym!(sym::target_has_atomic_load_store, sym);
             };
-            let s = i.to_string();
-            insert_atomic(&s, align);
-            if s == wordsz {
-                insert_atomic("ptr", layout.pointer_align.abi);
+            insert_atomic(sym::integer(i), align);
+            if sess.target.pointer_width as u64 == i {
+                insert_atomic(sym::ptr, layout.pointer_align.abi);
             }
         }
     }
-    if sess.is_nightly_build() && has_atomic {
-        ret.insert((sym::target_has_atomic_load_store, None));
-        if atomic_cas {
-            ret.insert((sym::target_has_atomic, None));
-        }
-    }
 
-    let panic_strategy = sess.panic_strategy();
-    ret.insert((sym::panic, Some(panic_strategy.desc_symbol())));
+    ins_str!(sym::target_os, &sess.target.os);
+    ins_sym!(sym::target_pointer_width, sym::integer(sess.target.pointer_width));
 
-    for mut s in sess.opts.unstable_opts.sanitizer {
-        // KASAN should use the same attribute name as ASAN, as it's still ASAN
-        // under the hood
-        if s == SanitizerSet::KERNELADDRESS {
-            s = SanitizerSet::ADDRESS;
-        }
-
-        let symbol = Symbol::intern(&s.to_string());
-        ret.insert((sym::sanitize, Some(symbol)));
+    if sess.opts.unstable_opts.has_thread_local.unwrap_or(sess.target.has_thread_local) {
+        ins_none!(sym::target_thread_local);
     }
 
-    if sess.is_sanitizer_cfi_generalize_pointers_enabled() {
-        ret.insert((sym::sanitizer_cfi_generalize_pointers, None));
-    }
+    ins_str!(sym::target_vendor, &sess.target.vendor);
 
-    if sess.is_sanitizer_cfi_normalize_integers_enabled() {
-        ret.insert((sym::sanitizer_cfi_normalize_integers, None));
+    // If the user wants a test runner, then add the test cfg.
+    if sess.is_test_crate() {
+        ins_none!(sym::test);
     }
 
-    if sess.opts.debug_assertions {
-        ret.insert((sym::debug_assertions, None));
-    }
-    if sess.overflow_checks() {
-        ret.insert((sym::overflow_checks, None));
-    }
-    // JUSTIFICATION: before wrapper fn is available
-    #[allow(rustc::bad_opt_access)]
-    if sess.opts.crate_types.contains(&CrateType::ProcMacro) {
-        ret.insert((sym::proc_macro, None));
-    }
     ret
 }
 
@@ -1423,88 +1445,73 @@ impl CheckCfg {
             ExpectedValues::Some(values)
         };
 
-        // NOTE: This should be kept in sync with `default_configuration`
+        macro_rules! ins {
+            ($name:expr, $values:expr) => {
+                self.expecteds.entry($name).or_insert_with($values)
+            };
+        }
+
+        // Symbols are inserted in alphabetical order as much as possible.
+        // The exceptions are where control flow forces things out of order.
+        //
+        // NOTE: This should be kept in sync with `default_configuration`.
+        // Note that symbols inserted conditionally in `default_configuration`
+        // are inserted unconditionally here.
+        //
+        // When adding a new config here you should also update
+        // `tests/ui/check-cfg/well-known-values.rs`.
 
-        let panic_values = &PanicStrategy::all();
+        ins!(sym::debug_assertions, no_values);
 
-        let atomic_values = &[
-            sym::ptr,
-            sym::integer(8usize),
-            sym::integer(16usize),
-            sym::integer(32usize),
-            sym::integer(64usize),
-            sym::integer(128usize),
-        ];
+        // These three are never set by rustc, but we set them anyway: they
+        // should not trigger a lint because `cargo doc`, `cargo test`, and
+        // `cargo miri run` (respectively) can set them.
+        ins!(sym::doc, no_values);
+        ins!(sym::doctest, no_values);
+        ins!(sym::miri, no_values);
+
+        ins!(sym::overflow_checks, no_values);
+
+        ins!(sym::panic, empty_values).extend(&PanicStrategy::all());
+
+        ins!(sym::proc_macro, no_values);
+
+        ins!(sym::relocation_model, empty_values).extend(RelocModel::all());
 
         let sanitize_values = SanitizerSet::all()
             .into_iter()
             .map(|sanitizer| Symbol::intern(sanitizer.as_str().unwrap()));
+        ins!(sym::sanitize, empty_values).extend(sanitize_values);
 
-        let relocation_model_values = RelocModel::all();
-
-        // Unknown possible values:
-        //  - `feature`
-        //  - `target_feature`
-        for name in [sym::feature, sym::target_feature] {
-            self.expecteds.entry(name).or_insert(ExpectedValues::Any);
-        }
-
-        // No-values
-        for name in [
-            sym::doc,
-            sym::miri,
-            sym::unix,
-            sym::test,
-            sym::doctest,
-            sym::windows,
-            sym::proc_macro,
-            sym::debug_assertions,
-            sym::overflow_checks,
-            sym::target_thread_local,
-        ] {
-            self.expecteds.entry(name).or_insert_with(no_values);
-        }
-
-        // Pre-defined values
-        self.expecteds.entry(sym::panic).or_insert_with(empty_values).extend(panic_values);
-        self.expecteds.entry(sym::sanitize).or_insert_with(empty_values).extend(sanitize_values);
-        self.expecteds
-            .entry(sym::target_has_atomic)
-            .or_insert_with(no_values)
-            .extend(atomic_values);
-        self.expecteds
-            .entry(sym::target_has_atomic_load_store)
-            .or_insert_with(no_values)
-            .extend(atomic_values);
-        self.expecteds
-            .entry(sym::target_has_atomic_equal_alignment)
-            .or_insert_with(no_values)
-            .extend(atomic_values);
-        self.expecteds
-            .entry(sym::relocation_model)
-            .or_insert_with(empty_values)
-            .extend(relocation_model_values);
-
-        // Target specific values
+        ins!(sym::sanitizer_cfi_generalize_pointers, no_values);
+        ins!(sym::sanitizer_cfi_normalize_integers, no_values);
+
+        ins!(sym::target_feature, empty_values).extend(
+            rustc_target::target_features::all_known_features()
+                .map(|(f, _sb)| f)
+                .chain(rustc_target::target_features::RUSTC_SPECIFIC_FEATURES.iter().cloned())
+                .map(Symbol::intern),
+        );
+
+        // sym::target_*
         {
             const VALUES: [&Symbol; 8] = [
-                &sym::target_os,
-                &sym::target_family,
+                &sym::target_abi,
                 &sym::target_arch,
                 &sym::target_endian,
                 &sym::target_env,
-                &sym::target_abi,
-                &sym::target_vendor,
+                &sym::target_family,
+                &sym::target_os,
                 &sym::target_pointer_width,
+                &sym::target_vendor,
             ];
 
             // Initialize (if not already initialized)
             for &e in VALUES {
-                let entry = self.expecteds.entry(e);
                 if !self.exhaustive_values {
-                    entry.or_insert(ExpectedValues::Any);
+                    ins!(e, || ExpectedValues::Any);
                 } else {
-                    entry.or_insert_with(empty_values);
+                    ins!(e, empty_values);
                 }
             }
 
@@ -1512,14 +1519,14 @@ impl CheckCfg {
                 // Get all values map at once otherwise it would be costly.
                 // (8 values * 220 targets ~= 1760 times, at the time of writing this comment).
                 let [
-                    values_target_os,
-                    values_target_family,
+                    values_target_abi,
                     values_target_arch,
                     values_target_endian,
                     values_target_env,
-                    values_target_abi,
-                    values_target_vendor,
+                    values_target_family,
+                    values_target_os,
                     values_target_pointer_width,
+                    values_target_vendor,
                 ] = self
                     .expecteds
                     .get_many_mut(VALUES)
@@ -1530,36 +1537,54 @@ impl CheckCfg {
                     .map(|target| Target::expect_builtin(&TargetTriple::from_triple(target)))
                     .chain(iter::once(current_target.clone()))
                 {
-                    values_target_os.insert(Symbol::intern(&target.options.os));
-                    values_target_family.extend(
-                        target.options.families.iter().map(|family| Symbol::intern(family)),
-                    );
+                    values_target_abi.insert(Symbol::intern(&target.options.abi));
                     values_target_arch.insert(Symbol::intern(&target.arch));
                     values_target_endian.insert(Symbol::intern(target.options.endian.as_str()));
                     values_target_env.insert(Symbol::intern(&target.options.env));
-                    values_target_abi.insert(Symbol::intern(&target.options.abi));
-                    values_target_vendor.insert(Symbol::intern(&target.options.vendor));
+                    values_target_family.extend(
+                        target.options.families.iter().map(|family| Symbol::intern(family)),
+                    );
+                    values_target_os.insert(Symbol::intern(&target.options.os));
                     values_target_pointer_width.insert(sym::integer(target.pointer_width));
+                    values_target_vendor.insert(Symbol::intern(&target.options.vendor));
                 }
             }
         }
+
+        let atomic_values = &[
+            sym::ptr,
+            sym::integer(8usize),
+            sym::integer(16usize),
+            sym::integer(32usize),
+            sym::integer(64usize),
+            sym::integer(128usize),
+        ];
+        for sym in [
+            sym::target_has_atomic,
+            sym::target_has_atomic_equal_alignment,
+            sym::target_has_atomic_load_store,
+        ] {
+            ins!(sym, no_values).extend(atomic_values);
+        }
+
+        ins!(sym::target_thread_local, no_values);
+
+        ins!(sym::test, no_values);
+
+        ins!(sym::unix, no_values);
+        ins!(sym::windows, no_values);
     }
 }
 
 pub fn build_configuration(sess: &Session, mut user_cfg: Cfg) -> Cfg {
     // Combine the configuration requested by the session (command line) with
     // some default and generated configuration items.
-    let default_cfg = default_configuration(sess);
-    // If the user wants a test runner, then add the test cfg.
-    if sess.is_test_crate() {
-        user_cfg.insert((sym::test, None));
-    }
-    user_cfg.extend(default_cfg.iter().cloned());
+    user_cfg.extend(default_configuration(sess));
     user_cfg
 }
 
 pub(super) fn build_target_config(
-    handler: &EarlyErrorHandler,
+    early_dcx: &EarlyDiagCtxt,
     opts: &Options,
     target_override: Option<Target>,
     sysroot: &Path,
@@ -1569,17 +1594,17 @@ pub(super) fn build_target_config(
         |t| Ok((t, TargetWarnings::empty())),
     );
     let (target, target_warnings) = target_result.unwrap_or_else(|e| {
-        handler.early_error(format!(
+        early_dcx.early_error(format!(
             "Error loading target specification: {e}. \
                  Run `rustc --print target-list` for a list of built-in targets"
         ))
     });
     for warning in target_warnings.warning_messages() {
-        handler.early_warn(warning)
+        early_dcx.early_warn(warning)
     }
 
     if !matches!(target.pointer_width, 16 | 32 | 64) {
-        handler.early_error(format!(
+        early_dcx.early_error(format!(
             "target specification was invalid: unrecognized target-pointer-width {}",
             target.pointer_width
         ))
@@ -1813,12 +1838,13 @@ pub fn rustc_optgroups() -> Vec<RustcOptGroup> {
             "Remap source names in all output (compiler messages and output files)",
             "FROM=TO",
         ),
+        opt::multi("", "env", "Inject an environment variable", "VAR=VALUE"),
     ]);
     opts
 }
 
 pub fn get_cmd_lint_options(
-    handler: &EarlyErrorHandler,
+    early_dcx: &EarlyDiagCtxt,
     matches: &getopts::Matches,
 ) -> (Vec<(String, lint::Level)>, bool, Option<lint::Level>) {
     let mut lint_opts_with_position = vec![];
@@ -1843,14 +1869,14 @@ pub fn get_cmd_lint_options(
 
     let lint_cap = matches.opt_str("cap-lints").map(|cap| {
         lint::Level::from_str(&cap)
-            .unwrap_or_else(|| handler.early_error(format!("unknown lint level: `{cap}`")))
+            .unwrap_or_else(|| early_dcx.early_error(format!("unknown lint level: `{cap}`")))
     });
 
     (lint_opts, describe_lints, lint_cap)
 }
 
 /// Parses the `--color` flag.
-pub fn parse_color(handler: &EarlyErrorHandler, matches: &getopts::Matches) -> ColorConfig {
+pub fn parse_color(early_dcx: &EarlyDiagCtxt, matches: &getopts::Matches) -> ColorConfig {
     match matches.opt_str("color").as_deref() {
         Some("auto") => ColorConfig::Auto,
         Some("always") => ColorConfig::Always,
@@ -1858,7 +1884,7 @@ pub fn parse_color(handler: &EarlyErrorHandler, matches: &getopts::Matches) -> C
 
         None => ColorConfig::Auto,
 
-        Some(arg) => handler.early_error(format!(
+        Some(arg) => early_dcx.early_error(format!(
             "argument for `--color` must be auto, \
                  always or never (instead was `{arg}`)"
         )),
@@ -1904,7 +1930,7 @@ impl JsonUnusedExterns {
 ///
 /// The first value returned is how to render JSON diagnostics, and the second
 /// is whether or not artifact notifications are enabled.
-pub fn parse_json(handler: &EarlyErrorHandler, matches: &getopts::Matches) -> JsonConfig {
+pub fn parse_json(early_dcx: &EarlyDiagCtxt, matches: &getopts::Matches) -> JsonConfig {
     let mut json_rendered: fn(ColorConfig) -> HumanReadableErrorType =
         HumanReadableErrorType::Default;
     let mut json_color = ColorConfig::Never;
@@ -1916,7 +1942,7 @@ pub fn parse_json(handler: &EarlyErrorHandler, matches: &getopts::Matches) -> Js
         // won't actually be emitting any colors and anything colorized is
         // embedded in a diagnostic message anyway.
         if matches.opt_str("color").is_some() {
-            handler.early_error("cannot specify the `--color` option with `--json`");
+            early_dcx.early_error("cannot specify the `--color` option with `--json`");
         }
 
         for sub_option in option.split(',') {
@@ -1927,7 +1953,7 @@ pub fn parse_json(handler: &EarlyErrorHandler, matches: &getopts::Matches) -> Js
                 "unused-externs" => json_unused_externs = JsonUnusedExterns::Loud,
                 "unused-externs-silent" => json_unused_externs = JsonUnusedExterns::Silent,
                 "future-incompat" => json_future_incompat = true,
-                s => handler.early_error(format!("unknown `--json` option `{s}`")),
+                s => early_dcx.early_error(format!("unknown `--json` option `{s}`")),
             }
         }
     }
@@ -1942,7 +1968,7 @@ pub fn parse_json(handler: &EarlyErrorHandler, matches: &getopts::Matches) -> Js
 
 /// Parses the `--error-format` flag.
 pub fn parse_error_format(
-    handler: &mut EarlyErrorHandler,
+    early_dcx: &mut EarlyDiagCtxt,
     matches: &getopts::Matches,
     color: ColorConfig,
     json_rendered: HumanReadableErrorType,
@@ -1964,10 +1990,10 @@ pub fn parse_error_format(
             Some("short") => ErrorOutputType::HumanReadable(HumanReadableErrorType::Short(color)),
 
             Some(arg) => {
-                handler.abort_if_error_and_set_error_format(ErrorOutputType::HumanReadable(
+                early_dcx.abort_if_error_and_set_error_format(ErrorOutputType::HumanReadable(
                     HumanReadableErrorType::Default(color),
                 ));
-                handler.early_error(format!(
+                early_dcx.early_error(format!(
                     "argument for `--error-format` must be `human`, `json` or \
                      `short` (instead was `{arg}`)"
                 ))
@@ -1984,7 +2010,7 @@ pub fn parse_error_format(
         // `--error-format=json`. This means that `--json` is specified we
         // should actually be emitting JSON blobs.
         _ if !matches.opt_strs("json").is_empty() => {
-            handler.early_error("using `--json` requires also using `--error-format=json`");
+            early_dcx.early_error("using `--json` requires also using `--error-format=json`");
         }
 
         _ => {}
@@ -1993,10 +2019,10 @@ pub fn parse_error_format(
     error_format
 }
 
-pub fn parse_crate_edition(handler: &EarlyErrorHandler, matches: &getopts::Matches) -> Edition {
+pub fn parse_crate_edition(early_dcx: &EarlyDiagCtxt, matches: &getopts::Matches) -> Edition {
     let edition = match matches.opt_str("edition") {
         Some(arg) => Edition::from_str(&arg).unwrap_or_else(|_| {
-            handler.early_error(format!(
+            early_dcx.early_error(format!(
                 "argument for `--edition` must be one of: \
                      {EDITION_NAME_LIST}. (instead was `{arg}`)"
             ))
@@ -2013,40 +2039,40 @@ pub fn parse_crate_edition(handler: &EarlyErrorHandler, matches: &getopts::Match
         } else {
             format!("edition {edition} is unstable and only available with -Z unstable-options")
         };
-        handler.early_error(msg)
+        early_dcx.early_error(msg)
     }
 
     edition
 }
 
 fn check_error_format_stability(
-    handler: &mut EarlyErrorHandler,
+    early_dcx: &mut EarlyDiagCtxt,
     unstable_opts: &UnstableOptions,
     error_format: ErrorOutputType,
     json_rendered: HumanReadableErrorType,
 ) {
     if !unstable_opts.unstable_options {
         if let ErrorOutputType::Json { pretty: true, json_rendered } = error_format {
-            handler.abort_if_error_and_set_error_format(ErrorOutputType::Json {
+            early_dcx.abort_if_error_and_set_error_format(ErrorOutputType::Json {
                 pretty: false,
                 json_rendered,
             });
-            handler.early_error("`--error-format=pretty-json` is unstable");
+            early_dcx.early_error("`--error-format=pretty-json` is unstable");
         }
         if let ErrorOutputType::HumanReadable(HumanReadableErrorType::AnnotateSnippet(_)) =
             error_format
         {
-            handler.abort_if_error_and_set_error_format(ErrorOutputType::Json {
+            early_dcx.abort_if_error_and_set_error_format(ErrorOutputType::Json {
                 pretty: false,
                 json_rendered,
             });
-            handler.early_error("`--error-format=human-annotate-rs` is unstable");
+            early_dcx.early_error("`--error-format=human-annotate-rs` is unstable");
         }
     }
 }
 
 fn parse_output_types(
-    handler: &EarlyErrorHandler,
+    early_dcx: &EarlyDiagCtxt,
     unstable_opts: &UnstableOptions,
     matches: &getopts::Matches,
 ) -> OutputTypes {
@@ -2056,7 +2082,7 @@ fn parse_output_types(
             for output_type in list.split(',') {
                 let (shorthand, path) = split_out_file_name(output_type);
                 let output_type = OutputType::from_shorthand(shorthand).unwrap_or_else(|| {
-                    handler.early_error(format!(
+                    early_dcx.early_error(format!(
                         "unknown emission type: `{shorthand}` - expected one of: {display}",
                         display = OutputType::shorthands_display(),
                     ))
@@ -2080,7 +2106,7 @@ fn split_out_file_name(arg: &str) -> (&str, Option<OutFileName>) {
 }
 
 fn should_override_cgus_and_disable_thinlto(
-    handler: &EarlyErrorHandler,
+    early_dcx: &EarlyDiagCtxt,
     output_types: &OutputTypes,
     matches: &getopts::Matches,
     mut codegen_units: Option<usize>,
@@ -2100,12 +2126,12 @@ fn should_override_cgus_and_disable_thinlto(
             Some(n) if n > 1 => {
                 if matches.opt_present("o") {
                     for ot in &incompatible {
-                        handler.early_warn(format!(
+                        early_dcx.early_warn(format!(
                             "`--emit={ot}` with `-o` incompatible with \
                                  `-C codegen-units=N` for N > 1",
                         ));
                     }
-                    handler.early_warn("resetting to default -C codegen-units=1");
+                    early_dcx.early_warn("resetting to default -C codegen-units=1");
                     codegen_units = Some(1);
                     disable_local_thinlto = true;
                 }
@@ -2118,24 +2144,14 @@ fn should_override_cgus_and_disable_thinlto(
     }
 
     if codegen_units == Some(0) {
-        handler.early_error("value for codegen units must be a positive non-zero integer");
+        early_dcx.early_error("value for codegen units must be a positive non-zero integer");
     }
 
     (disable_local_thinlto, codegen_units)
 }
 
-fn check_thread_count(handler: &EarlyErrorHandler, unstable_opts: &UnstableOptions) {
-    if unstable_opts.threads == 0 {
-        handler.early_error("value for threads must be a positive non-zero integer");
-    }
-
-    if unstable_opts.threads > 1 && unstable_opts.fuel.is_some() {
-        handler.early_error("optimization fuel is incompatible with multiple threads");
-    }
-}
-
 fn collect_print_requests(
-    handler: &EarlyErrorHandler,
+    early_dcx: &EarlyDiagCtxt,
     cg: &mut CodegenOptions,
     unstable_opts: &mut UnstableOptions,
     matches: &getopts::Matches,
@@ -2151,25 +2167,27 @@ fn collect_print_requests(
     }
 
     const PRINT_KINDS: &[(&str, PrintKind)] = &[
+        // tidy-alphabetical-start
+        ("all-target-specs-json", PrintKind::AllTargetSpecs),
+        ("calling-conventions", PrintKind::CallingConventions),
+        ("cfg", PrintKind::Cfg),
+        ("code-models", PrintKind::CodeModels),
         ("crate-name", PrintKind::CrateName),
+        ("deployment-target", PrintKind::DeploymentTarget),
         ("file-names", PrintKind::FileNames),
+        ("link-args", PrintKind::LinkArgs),
+        ("native-static-libs", PrintKind::NativeStaticLibs),
+        ("relocation-models", PrintKind::RelocationModels),
+        ("split-debuginfo", PrintKind::SplitDebuginfo),
+        ("stack-protector-strategies", PrintKind::StackProtectorStrategies),
         ("sysroot", PrintKind::Sysroot),
-        ("target-libdir", PrintKind::TargetLibdir),
-        ("cfg", PrintKind::Cfg),
-        ("calling-conventions", PrintKind::CallingConventions),
-        ("target-list", PrintKind::TargetList),
         ("target-cpus", PrintKind::TargetCPUs),
         ("target-features", PrintKind::TargetFeatures),
-        ("relocation-models", PrintKind::RelocationModels),
-        ("code-models", PrintKind::CodeModels),
-        ("tls-models", PrintKind::TlsModels),
-        ("native-static-libs", PrintKind::NativeStaticLibs),
-        ("stack-protector-strategies", PrintKind::StackProtectorStrategies),
+        ("target-libdir", PrintKind::TargetLibdir),
+        ("target-list", PrintKind::TargetList),
         ("target-spec-json", PrintKind::TargetSpec),
-        ("all-target-specs-json", PrintKind::AllTargetSpecs),
-        ("link-args", PrintKind::LinkArgs),
-        ("split-debuginfo", PrintKind::SplitDebuginfo),
-        ("deployment-target", PrintKind::DeploymentTarget),
+        ("tls-models", PrintKind::TlsModels),
+        // tidy-alphabetical-end
     ];
 
     // We disallow reusing the same path in multiple prints, such as `--print
@@ -2186,7 +2204,7 @@ fn collect_print_requests(
                 if unstable_opts.unstable_options {
                     PrintKind::TargetSpec
                 } else {
-                    handler.early_error(
+                    early_dcx.early_error(
                         "the `-Z unstable-options` flag must also be passed to \
                          enable the target-spec-json print option",
                     );
@@ -2196,7 +2214,7 @@ fn collect_print_requests(
                 if unstable_opts.unstable_options {
                     PrintKind::AllTargetSpecs
                 } else {
-                    handler.early_error(
+                    early_dcx.early_error(
                         "the `-Z unstable-options` flag must also be passed to \
                          enable the all-target-specs-json print option",
                     );
@@ -2207,7 +2225,7 @@ fn collect_print_requests(
                 let prints =
                     PRINT_KINDS.iter().map(|(name, _)| format!("`{name}`")).collect::<Vec<_>>();
                 let prints = prints.join(", ");
-                handler.early_error(format!(
+                early_dcx.early_error(format!(
                     "unknown print request `{req}`. Valid print requests are: {prints}"
                 ));
             }
@@ -2216,7 +2234,7 @@ fn collect_print_requests(
         let out = out.unwrap_or(OutFileName::Stdout);
         if let OutFileName::Real(path) = &out {
             if !printed_paths.insert(path.clone()) {
-                handler.early_error(format!(
+                early_dcx.early_error(format!(
                     "cannot print multiple outputs to the same path: {}",
                     path.display(),
                 ));
@@ -2229,15 +2247,12 @@ fn collect_print_requests(
     prints
 }
 
-pub fn parse_target_triple(
-    handler: &EarlyErrorHandler,
-    matches: &getopts::Matches,
-) -> TargetTriple {
+pub fn parse_target_triple(early_dcx: &EarlyDiagCtxt, matches: &getopts::Matches) -> TargetTriple {
     match matches.opt_str("target") {
         Some(target) if target.ends_with(".json") => {
             let path = Path::new(&target);
             TargetTriple::from_path(path).unwrap_or_else(|_| {
-                handler.early_error(format!("target file {path:?} does not exist"))
+                early_dcx.early_error(format!("target file {path:?} does not exist"))
             })
         }
         Some(target) => TargetTriple::TargetTriple(target),
@@ -2246,7 +2261,7 @@ pub fn parse_target_triple(
 }
 
 fn parse_opt_level(
-    handler: &EarlyErrorHandler,
+    early_dcx: &EarlyDiagCtxt,
     matches: &getopts::Matches,
     cg: &CodegenOptions,
 ) -> OptLevel {
@@ -2276,7 +2291,7 @@ fn parse_opt_level(
             "s" => OptLevel::Size,
             "z" => OptLevel::SizeMin,
             arg => {
-                handler.early_error(format!(
+                early_dcx.early_error(format!(
                     "optimization level needs to be \
                             between 0-3, s or z (instead was `{arg}`)"
                 ));
@@ -2298,29 +2313,22 @@ fn select_debuginfo(matches: &getopts::Matches, cg: &CodegenOptions) -> DebugInf
     if max_g > max_c { DebugInfo::Full } else { cg.debuginfo }
 }
 
-fn select_debuginfo_compression(
-    _handler: &EarlyErrorHandler,
-    unstable_opts: &UnstableOptions,
-) -> DebugInfoCompression {
-    unstable_opts.debuginfo_compression
-}
-
-pub(crate) fn parse_assert_incr_state(
-    handler: &EarlyErrorHandler,
+fn parse_assert_incr_state(
+    early_dcx: &EarlyDiagCtxt,
     opt_assertion: &Option<String>,
 ) -> Option<IncrementalStateAssertion> {
     match opt_assertion {
         Some(s) if s.as_str() == "loaded" => Some(IncrementalStateAssertion::Loaded),
         Some(s) if s.as_str() == "not-loaded" => Some(IncrementalStateAssertion::NotLoaded),
         Some(s) => {
-            handler.early_error(format!("unexpected incremental state assertion value: {s}"))
+            early_dcx.early_error(format!("unexpected incremental state assertion value: {s}"))
         }
         None => None,
     }
 }
 
 fn parse_native_lib_kind(
-    handler: &EarlyErrorHandler,
+    early_dcx: &EarlyDiagCtxt,
     matches: &getopts::Matches,
     kind: &str,
 ) -> (NativeLibKind, Option<bool>) {
@@ -2340,22 +2348,22 @@ fn parse_native_lib_kind(
                 } else {
                     ", the `-Z unstable-options` flag must also be passed to use it"
                 };
-                handler.early_error(format!("library kind `link-arg` is unstable{why}"))
+                early_dcx.early_error(format!("library kind `link-arg` is unstable{why}"))
             }
             NativeLibKind::LinkArg
         }
-        _ => handler.early_error(format!(
+        _ => early_dcx.early_error(format!(
             "unknown library kind `{kind}`, expected one of: static, dylib, framework, link-arg"
         )),
     };
     match modifiers {
         None => (kind, None),
-        Some(modifiers) => parse_native_lib_modifiers(handler, kind, modifiers, matches),
+        Some(modifiers) => parse_native_lib_modifiers(early_dcx, kind, modifiers, matches),
     }
 }
 
 fn parse_native_lib_modifiers(
-    handler: &EarlyErrorHandler,
+    early_dcx: &EarlyDiagCtxt,
     mut kind: NativeLibKind,
     modifiers: &str,
     matches: &getopts::Matches,
@@ -2364,7 +2372,7 @@ fn parse_native_lib_modifiers(
     for modifier in modifiers.split(',') {
         let (modifier, value) = match modifier.strip_prefix(['+', '-']) {
             Some(m) => (m, modifier.starts_with('+')),
-            None => handler.early_error(
+            None => early_dcx.early_error(
                 "invalid linking modifier syntax, expected '+' or '-' prefix \
                  before one of: bundle, verbatim, whole-archive, as-needed",
             ),
@@ -2377,20 +2385,20 @@ fn parse_native_lib_modifiers(
                 } else {
                     ", the `-Z unstable-options` flag must also be passed to use it"
                 };
-                handler.early_error(format!("linking modifier `{modifier}` is unstable{why}"))
+                early_dcx.early_error(format!("linking modifier `{modifier}` is unstable{why}"))
             }
         };
         let assign_modifier = |dst: &mut Option<bool>| {
             if dst.is_some() {
                 let msg = format!("multiple `{modifier}` modifiers in a single `-l` option");
-                handler.early_error(msg)
+                early_dcx.early_error(msg)
             } else {
                 *dst = Some(value);
             }
         };
         match (modifier, &mut kind) {
             ("bundle", NativeLibKind::Static { bundle, .. }) => assign_modifier(bundle),
-            ("bundle", _) => handler.early_error(
+            ("bundle", _) => early_dcx.early_error(
                 "linking modifier `bundle` is only compatible with `static` linking kind",
             ),
 
@@ -2399,7 +2407,7 @@ fn parse_native_lib_modifiers(
             ("whole-archive", NativeLibKind::Static { whole_archive, .. }) => {
                 assign_modifier(whole_archive)
             }
-            ("whole-archive", _) => handler.early_error(
+            ("whole-archive", _) => early_dcx.early_error(
                 "linking modifier `whole-archive` is only compatible with `static` linking kind",
             ),
 
@@ -2408,14 +2416,14 @@ fn parse_native_lib_modifiers(
                 report_unstable_modifier();
                 assign_modifier(as_needed)
             }
-            ("as-needed", _) => handler.early_error(
+            ("as-needed", _) => early_dcx.early_error(
                 "linking modifier `as-needed` is only compatible with \
                  `dylib` and `framework` linking kinds",
             ),
 
             // Note: this error also excludes the case with empty modifier
             // string, like `modifiers = ""`.
-            _ => handler.early_error(format!(
+            _ => early_dcx.early_error(format!(
                 "unknown linking modifier `{modifier}`, expected one \
                      of: bundle, verbatim, whole-archive, as-needed"
             )),
@@ -2425,7 +2433,7 @@ fn parse_native_lib_modifiers(
     (kind, verbatim)
 }
 
-fn parse_libs(handler: &EarlyErrorHandler, matches: &getopts::Matches) -> Vec<NativeLib> {
+fn parse_libs(early_dcx: &EarlyDiagCtxt, matches: &getopts::Matches) -> Vec<NativeLib> {
     matches
         .opt_strs("l")
         .into_iter()
@@ -2439,7 +2447,7 @@ fn parse_libs(handler: &EarlyErrorHandler, matches: &getopts::Matches) -> Vec<Na
             let (name, kind, verbatim) = match s.split_once('=') {
                 None => (s, NativeLibKind::Unspecified, None),
                 Some((kind, name)) => {
-                    let (kind, verbatim) = parse_native_lib_kind(handler, matches, kind);
+                    let (kind, verbatim) = parse_native_lib_kind(early_dcx, matches, kind);
                     (name.to_string(), kind, verbatim)
                 }
             };
@@ -2449,7 +2457,7 @@ fn parse_libs(handler: &EarlyErrorHandler, matches: &getopts::Matches) -> Vec<Na
                 Some((name, new_name)) => (name.to_string(), Some(new_name.to_owned())),
             };
             if name.is_empty() {
-                handler.early_error("library name must not be empty");
+                early_dcx.early_error("library name must not be empty");
             }
             NativeLib { name, new_name, kind, verbatim }
         })
@@ -2457,10 +2465,21 @@ fn parse_libs(handler: &EarlyErrorHandler, matches: &getopts::Matches) -> Vec<Na
 }
 
 pub fn parse_externs(
-    handler: &EarlyErrorHandler,
+    early_dcx: &EarlyDiagCtxt,
     matches: &getopts::Matches,
     unstable_opts: &UnstableOptions,
 ) -> Externs {
+    fn is_ascii_ident(string: &str) -> bool {
+        let mut chars = string.chars();
+        if let Some(start) = chars.next()
+            && (start.is_ascii_alphabetic() || start == '_')
+        {
+            chars.all(|char| char.is_ascii_alphanumeric() || char == '_')
+        } else {
+            false
+        }
+    }
+
     let is_unstable_enabled = unstable_opts.unstable_options;
     let mut externs: BTreeMap<String, ExternEntry> = BTreeMap::new();
     for arg in matches.opt_strs("extern") {
@@ -2473,12 +2492,12 @@ pub fn parse_externs(
             Some((opts, name)) => (Some(opts), name.to_string()),
         };
 
-        if !crate::utils::is_ascii_ident(&name) {
-            let mut error = handler.early_struct_error(format!(
+        if !is_ascii_ident(&name) {
+            let mut error = early_dcx.early_struct_error(format!(
                 "crate name `{name}` passed to `--extern` is not a valid ASCII identifier"
             ));
             let adjusted_name = name.replace('-', "_");
-            if crate::utils::is_ascii_ident(&adjusted_name) {
+            if is_ascii_ident(&adjusted_name) {
                 error.help(format!(
                     "consider replacing the dashes with underscores: `{adjusted_name}`"
                 ));
@@ -2536,7 +2555,7 @@ pub fn parse_externs(
         let mut force = false;
         if let Some(opts) = options {
             if !is_unstable_enabled {
-                handler.early_error(
+                early_dcx.early_error(
                     "the `-Z unstable-options` flag must also be passed to \
                      enable `--extern` options",
                 );
@@ -2548,14 +2567,14 @@ pub fn parse_externs(
                         if let ExternLocation::ExactPaths(_) = &entry.location {
                             add_prelude = false;
                         } else {
-                            handler.early_error(
+                            early_dcx.early_error(
                                 "the `noprelude` --extern option requires a file path",
                             );
                         }
                     }
                     "nounused" => nounused_dep = true,
                     "force" => force = true,
-                    _ => handler.early_error(format!("unknown --extern option `{opt}`")),
+                    _ => early_dcx.early_error(format!("unknown --extern option `{opt}`")),
                 }
             }
         }
@@ -2574,7 +2593,7 @@ pub fn parse_externs(
 }
 
 fn parse_remap_path_prefix(
-    handler: &EarlyErrorHandler,
+    early_dcx: &EarlyDiagCtxt,
     matches: &getopts::Matches,
     unstable_opts: &UnstableOptions,
 ) -> Vec<(PathBuf, PathBuf)> {
@@ -2582,7 +2601,9 @@ fn parse_remap_path_prefix(
         .opt_strs("remap-path-prefix")
         .into_iter()
         .map(|remap| match remap.rsplit_once('=') {
-            None => handler.early_error("--remap-path-prefix must contain '=' between FROM and TO"),
+            None => {
+                early_dcx.early_error("--remap-path-prefix must contain '=' between FROM and TO")
+            }
             Some((from, to)) => (PathBuf::from(from), PathBuf::from(to)),
         })
         .collect();
@@ -2596,79 +2617,107 @@ fn parse_remap_path_prefix(
     mapping
 }
 
+fn parse_logical_env(
+    early_dcx: &mut EarlyDiagCtxt,
+    matches: &getopts::Matches,
+) -> FxIndexMap<String, String> {
+    let mut vars = FxIndexMap::default();
+
+    for arg in matches.opt_strs("env") {
+        if let Some((name, val)) = arg.split_once('=') {
+            vars.insert(name.to_string(), val.to_string());
+        } else {
+            early_dcx.early_error(format!("`--env`: specify value for variable `{arg}`"));
+        }
+    }
+
+    vars
+}
+
 // JUSTIFICATION: before wrapper fn is available
 #[allow(rustc::bad_opt_access)]
-pub fn build_session_options(
-    handler: &mut EarlyErrorHandler,
-    matches: &getopts::Matches,
-) -> Options {
-    let color = parse_color(handler, matches);
+pub fn build_session_options(early_dcx: &mut EarlyDiagCtxt, matches: &getopts::Matches) -> Options {
+    let color = parse_color(early_dcx, matches);
 
-    let edition = parse_crate_edition(handler, matches);
+    let edition = parse_crate_edition(early_dcx, matches);
 
     let JsonConfig {
         json_rendered,
         json_artifact_notifications,
         json_unused_externs,
         json_future_incompat,
-    } = parse_json(handler, matches);
+    } = parse_json(early_dcx, matches);
 
-    let error_format = parse_error_format(handler, matches, color, json_rendered);
+    let error_format = parse_error_format(early_dcx, matches, color, json_rendered);
 
-    handler.abort_if_error_and_set_error_format(error_format);
+    early_dcx.abort_if_error_and_set_error_format(error_format);
 
     let diagnostic_width = matches.opt_get("diagnostic-width").unwrap_or_else(|_| {
-        handler.early_error("`--diagnostic-width` must be an positive integer");
+        early_dcx.early_error("`--diagnostic-width` must be an positive integer");
     });
 
     let unparsed_crate_types = matches.opt_strs("crate-type");
     let crate_types = parse_crate_types_from_list(unparsed_crate_types)
-        .unwrap_or_else(|e| handler.early_error(e));
+        .unwrap_or_else(|e| early_dcx.early_error(e));
 
-    let mut unstable_opts = UnstableOptions::build(handler, matches);
-    let (lint_opts, describe_lints, lint_cap) = get_cmd_lint_options(handler, matches);
+    let mut unstable_opts = UnstableOptions::build(early_dcx, matches);
+    let (lint_opts, describe_lints, lint_cap) = get_cmd_lint_options(early_dcx, matches);
 
-    check_error_format_stability(handler, &unstable_opts, error_format, json_rendered);
+    check_error_format_stability(early_dcx, &unstable_opts, error_format, json_rendered);
 
     if !unstable_opts.unstable_options && json_unused_externs.is_enabled() {
-        handler.early_error(
+        early_dcx.early_error(
             "the `-Z unstable-options` flag must also be passed to enable \
             the flag `--json=unused-externs`",
         );
     }
 
-    let output_types = parse_output_types(handler, &unstable_opts, matches);
+    let output_types = parse_output_types(early_dcx, &unstable_opts, matches);
 
-    let mut cg = CodegenOptions::build(handler, matches);
-    let (disable_local_thinlto, mut codegen_units) =
-        should_override_cgus_and_disable_thinlto(handler, &output_types, matches, cg.codegen_units);
+    let mut cg = CodegenOptions::build(early_dcx, matches);
+    let (disable_local_thinlto, mut codegen_units) = should_override_cgus_and_disable_thinlto(
+        early_dcx,
+        &output_types,
+        matches,
+        cg.codegen_units,
+    );
 
-    check_thread_count(handler, &unstable_opts);
+    if unstable_opts.threads == 0 {
+        early_dcx.early_error("value for threads must be a positive non-zero integer");
+    }
+
+    let fuel = unstable_opts.fuel.is_some() || unstable_opts.print_fuel.is_some();
+    if fuel && unstable_opts.threads > 1 {
+        early_dcx.early_error("optimization fuel is incompatible with multiple threads");
+    }
+    if fuel && cg.incremental.is_some() {
+        early_dcx.early_error("optimization fuel is incompatible with incremental compilation");
+    }
 
     let incremental = cg.incremental.as_ref().map(PathBuf::from);
 
-    let assert_incr_state = parse_assert_incr_state(handler, &unstable_opts.assert_incr_state);
+    let assert_incr_state = parse_assert_incr_state(early_dcx, &unstable_opts.assert_incr_state);
 
     if unstable_opts.profile && incremental.is_some() {
-        handler.early_error("can't instrument with gcov profiling when compiling incrementally");
+        early_dcx.early_error("can't instrument with gcov profiling when compiling incrementally");
     }
     if unstable_opts.profile {
         match codegen_units {
             Some(1) => {}
             None => codegen_units = Some(1),
-            Some(_) => handler
+            Some(_) => early_dcx
                 .early_error("can't instrument with gcov profiling with multiple codegen units"),
         }
     }
 
     if cg.profile_generate.enabled() && cg.profile_use.is_some() {
-        handler.early_error("options `-C profile-generate` and `-C profile-use` are exclusive");
+        early_dcx.early_error("options `-C profile-generate` and `-C profile-use` are exclusive");
     }
 
     if unstable_opts.profile_sample_use.is_some()
         && (cg.profile_generate.enabled() || cg.profile_use.is_some())
     {
-        handler.early_error(
+        early_dcx.early_error(
             "option `-Z profile-sample-use` cannot be used with `-C profile-generate` or `-C profile-use`",
         );
     }
@@ -2681,7 +2730,7 @@ pub fn build_session_options(
         // Unstable values:
         Some(SymbolManglingVersion::Legacy) => {
             if !unstable_opts.unstable_options {
-                handler.early_error(
+                early_dcx.early_error(
                     "`-C symbol-mangling-version=legacy` requires `-Z unstable-options`",
                 );
             }
@@ -2698,7 +2747,7 @@ pub fn build_session_options(
         | InstrumentCoverage::ExceptUnusedFunctions
         | InstrumentCoverage::ExceptUnusedGenerics => {
             if !unstable_opts.unstable_options {
-                handler.early_error(
+                early_dcx.early_error(
                     "`-C instrument-coverage=branch` and `-C instrument-coverage=except-*` \
                     require `-Z unstable-options`",
                 );
@@ -2708,7 +2757,7 @@ pub fn build_session_options(
 
     if cg.instrument_coverage != InstrumentCoverage::Off {
         if cg.profile_generate.enabled() || cg.profile_use.is_some() {
-            handler.early_error(
+            early_dcx.early_error(
                 "option `-C instrument-coverage` is not compatible with either `-C profile-use` \
                 or `-C profile-generate`",
             );
@@ -2721,7 +2770,7 @@ pub fn build_session_options(
         match cg.symbol_mangling_version {
             None => cg.symbol_mangling_version = Some(SymbolManglingVersion::V0),
             Some(SymbolManglingVersion::Legacy) => {
-                handler.early_warn(
+                early_dcx.early_warn(
                     "-C instrument-coverage requires symbol mangling version `v0`, \
                     but `-C symbol-mangling-version=legacy` was specified",
                 );
@@ -2738,7 +2787,7 @@ pub fn build_session_options(
         match cg.lto {
             LtoCli::No | LtoCli::Unspecified => {}
             LtoCli::Yes | LtoCli::NoParam | LtoCli::Thin | LtoCli::Fat => {
-                handler.early_error("options `-C embed-bitcode=no` and `-C lto` are incompatible")
+                early_dcx.early_error("options `-C embed-bitcode=no` and `-C lto` are incompatible")
             }
         }
     }
@@ -2750,7 +2799,7 @@ pub fn build_session_options(
         let uses_unstable_self_contained_option =
             cg.link_self_contained.are_unstable_variants_set();
         if uses_unstable_self_contained_option {
-            handler.early_error(
+            early_dcx.early_error(
                 "only `-C link-self-contained` values `y`/`yes`/`on`/`n`/`no`/`off` are stable, \
                 the `-Z unstable-options` flag must also be passed to use the unstable values",
             );
@@ -2758,7 +2807,7 @@ pub fn build_session_options(
 
         if let Some(flavor) = cg.linker_flavor {
             if flavor.is_unstable() {
-                handler.early_error(format!(
+                early_dcx.early_error(format!(
                     "the linker flavor `{}` is unstable, the `-Z unstable-options` \
                         flag must also be passed to use the unstable values",
                     flavor.desc()
@@ -2775,56 +2824,58 @@ pub fn build_session_options(
             .map(|c| c.as_str().unwrap())
             .intersperse(", ")
             .collect();
-        handler.early_error(format!(
+        early_dcx.early_error(format!(
             "some `-C link-self-contained` components were both enabled and disabled: {names}"
         ));
     }
 
-    let prints = collect_print_requests(handler, &mut cg, &mut unstable_opts, matches);
+    let prints = collect_print_requests(early_dcx, &mut cg, &mut unstable_opts, matches);
 
     let cg = cg;
 
     let sysroot_opt = matches.opt_str("sysroot").map(|m| PathBuf::from(&m));
-    let target_triple = parse_target_triple(handler, matches);
-    let opt_level = parse_opt_level(handler, matches, &cg);
+    let target_triple = parse_target_triple(early_dcx, matches);
+    let opt_level = parse_opt_level(early_dcx, matches, &cg);
     // The `-g` and `-C debuginfo` flags specify the same setting, so we want to be able
     // to use them interchangeably. See the note above (regarding `-O` and `-C opt-level`)
     // for more details.
     let debug_assertions = cg.debug_assertions.unwrap_or(opt_level == OptLevel::No);
     let debuginfo = select_debuginfo(matches, &cg);
-    let debuginfo_compression: DebugInfoCompression =
-        select_debuginfo_compression(handler, &unstable_opts);
+    let debuginfo_compression = unstable_opts.debuginfo_compression;
 
     let mut search_paths = vec![];
     for s in &matches.opt_strs("L") {
-        search_paths.push(SearchPath::from_cli_opt(handler, s));
+        search_paths.push(SearchPath::from_cli_opt(early_dcx, s));
     }
 
-    let libs = parse_libs(handler, matches);
+    let libs = parse_libs(early_dcx, matches);
 
     let test = matches.opt_present("test");
 
     if !cg.remark.is_empty() && debuginfo == DebugInfo::None {
-        handler.early_warn("-C remark requires \"-C debuginfo=n\" to show source locations");
+        early_dcx.early_warn("-C remark requires \"-C debuginfo=n\" to show source locations");
     }
 
     if cg.remark.is_empty() && unstable_opts.remark_dir.is_some() {
-        handler.early_warn("using -Z remark-dir without enabling remarks using e.g. -C remark=all");
+        early_dcx
+            .early_warn("using -Z remark-dir without enabling remarks using e.g. -C remark=all");
     }
 
-    let externs = parse_externs(handler, matches, &unstable_opts);
+    let externs = parse_externs(early_dcx, matches, &unstable_opts);
 
     let crate_name = matches.opt_str("crate-name");
 
-    let remap_path_prefix = parse_remap_path_prefix(handler, matches, &unstable_opts);
+    let remap_path_prefix = parse_remap_path_prefix(early_dcx, matches, &unstable_opts);
 
-    let pretty = parse_pretty(handler, &unstable_opts);
+    let pretty = parse_pretty(early_dcx, &unstable_opts);
 
     // query-dep-graph is required if dump-dep-graph is given #106736
     if unstable_opts.dump_dep_graph && !unstable_opts.query_dep_graph {
-        handler.early_error("can't dump dependency graph without `-Z query-dep-graph`");
+        early_dcx.early_error("can't dump dependency graph without `-Z query-dep-graph`");
     }
 
+    let logical_env = parse_logical_env(early_dcx, matches);
+
     // Try to find a directory containing the Rust `src`, for more details see
     // the doc comment on the `real_rust_source_base_dir` field.
     let tmp_buf;
@@ -2854,7 +2905,7 @@ pub fn build_session_options(
     };
 
     let working_dir = std::env::current_dir().unwrap_or_else(|e| {
-        handler.early_error(format!("Current directory is invalid: {e}"));
+        early_dcx.early_error(format!("Current directory is invalid: {e}"));
     });
 
     let remap = file_path_mapping(remap_path_prefix.clone(), &unstable_opts);
@@ -2905,10 +2956,11 @@ pub fn build_session_options(
         pretty,
         working_dir,
         color,
+        logical_env,
     }
 }
 
-fn parse_pretty(handler: &EarlyErrorHandler, unstable_opts: &UnstableOptions) -> Option<PpMode> {
+fn parse_pretty(early_dcx: &EarlyDiagCtxt, unstable_opts: &UnstableOptions) -> Option<PpMode> {
     use PpMode::*;
 
     let first = match unstable_opts.unpretty.as_deref()? {
@@ -2926,12 +2978,13 @@ fn parse_pretty(handler: &EarlyErrorHandler, unstable_opts: &UnstableOptions) ->
         "thir-tree" => ThirTree,
         "thir-flat" => ThirFlat,
         "mir" => Mir,
+        "stable-mir" => StableMir,
         "mir-cfg" => MirCFG,
-        name => handler.early_error(format!(
+        name => early_dcx.early_error(format!(
             "argument to `unpretty` must be one of `normal`, `identified`, \
                             `expanded`, `expanded,identified`, `expanded,hygiene`, \
                             `ast-tree`, `ast-tree,expanded`, `hir`, `hir,identified`, \
-                            `hir,typed`, `hir-tree`, `thir-tree`, `thir-flat`, `mir` or \
+                            `hir,typed`, `hir-tree`, `thir-tree`, `thir-flat`, `mir`, `stable-mir`, or \
                             `mir-cfg`; got {name}"
         )),
     };
@@ -2974,7 +3027,7 @@ pub fn parse_crate_types_from_list(list_list: Vec<String>) -> Result<Vec<CrateTy
 
 pub mod nightly_options {
     use super::{OptionStability, RustcOptGroup};
-    use crate::EarlyErrorHandler;
+    use crate::EarlyDiagCtxt;
     use rustc_feature::UnstableFeatures;
 
     pub fn is_unstable_enabled(matches: &getopts::Matches) -> bool {
@@ -2991,7 +3044,7 @@ pub mod nightly_options {
     }
 
     pub fn check_nightly_options(
-        handler: &EarlyErrorHandler,
+        early_dcx: &EarlyDiagCtxt,
         matches: &getopts::Matches,
         flags: &[RustcOptGroup],
     ) {
@@ -3007,7 +3060,7 @@ pub mod nightly_options {
                 continue;
             }
             if opt.name != "Z" && !has_z_unstable_option {
-                handler.early_error(format!(
+                early_dcx.early_error(format!(
                     "the `-Z unstable-options` flag must also be passed to enable \
                          the flag `{}`",
                     opt.name
@@ -3023,17 +3076,17 @@ pub mod nightly_options {
                         "the option `{}` is only accepted on the nightly compiler",
                         opt.name
                     );
-                    let _ = handler.early_error_no_abort(msg);
+                    let _ = early_dcx.early_error_no_abort(msg);
                 }
                 OptionStability::Stable => {}
             }
         }
         if nightly_options_on_stable > 0 {
-            handler
+            early_dcx
                 .early_help("consider switching to a nightly toolchain: `rustup default nightly`");
-            handler.early_note("selecting a toolchain with `+toolchain` arguments require a rustup proxy; see <https://rust-lang.github.io/rustup/concepts/index.html>");
-            handler.early_note("for more information about Rust's stability policy, see <https://doc.rust-lang.org/book/appendix-07-nightly-rust.html#unstable-features>");
-            handler.early_error(format!(
+            early_dcx.early_note("selecting a toolchain with `+toolchain` arguments require a rustup proxy; see <https://rust-lang.github.io/rustup/concepts/index.html>");
+            early_dcx.early_note("for more information about Rust's stability policy, see <https://doc.rust-lang.org/book/appendix-07-nightly-rust.html#unstable-features>");
+            early_dcx.early_error(format!(
                 "{} nightly option{} were parsed",
                 nightly_options_on_stable,
                 if nightly_options_on_stable > 1 { "s" } else { "" }
@@ -3106,6 +3159,8 @@ pub enum PpMode {
     Mir,
     /// `-Zunpretty=mir-cfg`
     MirCFG,
+    /// `-Zunpretty=stable-mir`
+    StableMir,
 }
 
 impl PpMode {
@@ -3122,7 +3177,8 @@ impl PpMode {
             | ThirTree
             | ThirFlat
             | Mir
-            | MirCFG => true,
+            | MirCFG
+            | StableMir => true,
         }
     }
     pub fn needs_hir(&self) -> bool {
@@ -3130,16 +3186,22 @@ impl PpMode {
         match *self {
             Source(_) | AstTree | AstTreeExpanded => false,
 
-            Hir(_) | HirTree | ThirTree | ThirFlat | Mir | MirCFG => true,
+            Hir(_) | HirTree | ThirTree | ThirFlat | Mir | MirCFG | StableMir => true,
         }
     }
 
     pub fn needs_analysis(&self) -> bool {
         use PpMode::*;
-        matches!(*self, Hir(PpHirMode::Typed) | Mir | MirCFG | ThirTree | ThirFlat)
+        matches!(*self, Hir(PpHirMode::Typed) | Mir | StableMir | MirCFG | ThirTree | ThirFlat)
     }
 }
 
+#[derive(Clone, Hash, PartialEq, Eq, Debug)]
+pub enum WasiExecModel {
+    Command,
+    Reactor,
+}
+
 /// Command-line arguments passed to the compiler have to be incorporated with
 /// the dependency tracking system for incremental compilation. This module
 /// provides some utilities to make this more convenient.
@@ -3161,14 +3223,15 @@ impl PpMode {
 pub(crate) mod dep_tracking {
     use super::{
         BranchProtection, CFGuard, CFProtection, CrateType, DebugInfo, DebugInfoCompression,
-        ErrorOutputType, InliningThreshold, InstrumentCoverage, InstrumentXRay, LinkerPluginLto,
-        LocationDetail, LtoCli, OomStrategy, OptLevel, OutFileName, OutputType, OutputTypes,
-        Polonius, RemapPathScopeComponents, ResolveDocLinks, SourceFileHashAlgorithm,
-        SplitDwarfKind, SwitchWithOptPath, SymbolManglingVersion, TraitSolver, TrimmedDefPaths,
+        ErrorOutputType, FunctionReturn, InliningThreshold, InstrumentCoverage, InstrumentXRay,
+        LinkerPluginLto, LocationDetail, LtoCli, NextSolverConfig, OomStrategy, OptLevel,
+        OutFileName, OutputType, OutputTypes, Polonius, RemapPathScopeComponents, ResolveDocLinks,
+        SourceFileHashAlgorithm, SplitDwarfKind, SwitchWithOptPath, SymbolManglingVersion,
+        TrimmedDefPaths, WasiExecModel,
     };
     use crate::lint;
-    use crate::options::WasiExecModel;
     use crate::utils::NativeLib;
+    use rustc_data_structures::fx::FxIndexMap;
     use rustc_data_structures::stable_hasher::Hash64;
     use rustc_errors::LanguageIdentifier;
     use rustc_feature::UnstableFeatures;
@@ -3178,9 +3241,8 @@ pub(crate) mod dep_tracking {
     use rustc_target::spec::{
         RelroLevel, SanitizerSet, SplitDebuginfo, StackProtector, TargetTriple, TlsModel,
     };
-    use std::collections::hash_map::DefaultHasher;
     use std::collections::BTreeMap;
-    use std::hash::Hash;
+    use std::hash::{DefaultHasher, Hash};
     use std::num::NonZeroUsize;
     use std::path::PathBuf;
 
@@ -3268,9 +3330,10 @@ pub(crate) mod dep_tracking {
         BranchProtection,
         OomStrategy,
         LanguageIdentifier,
-        TraitSolver,
+        NextSolverConfig,
         Polonius,
         InliningThreshold,
+        FunctionReturn,
     );
 
     impl<T1, T2> DepTrackingHash for (T1, T2)
@@ -3327,6 +3390,21 @@ pub(crate) mod dep_tracking {
         }
     }
 
+    impl<T: DepTrackingHash, V: DepTrackingHash> DepTrackingHash for FxIndexMap<T, V> {
+        fn hash(
+            &self,
+            hasher: &mut DefaultHasher,
+            error_format: ErrorOutputType,
+            for_crate_hash: bool,
+        ) {
+            Hash::hash(&self.len(), hasher);
+            for (key, value) in self.iter() {
+                DepTrackingHash::hash(key, hasher, error_format, for_crate_hash);
+                DepTrackingHash::hash(value, hasher, error_format, for_crate_hash);
+            }
+        }
+    }
+
     impl DepTrackingHash for OutputTypes {
         fn hash(
             &self,
@@ -3449,3 +3527,14 @@ impl Default for InliningThreshold {
         Self::Sometimes(100)
     }
 }
+
+/// The different settings that the `-Zfunction-return` flag can have.
+#[derive(Clone, Copy, PartialEq, Hash, Debug, Default)]
+pub enum FunctionReturn {
+    /// Keep the function return unmodified.
+    #[default]
+    Keep,
+
+    /// Replace returns with jumps to thunk, without emitting the thunk.
+    ThunkExtern,
+}
diff --git a/compiler/rustc_session/src/cstore.rs b/compiler/rustc_session/src/cstore.rs
index d816842b0..a0f5eb59b 100644
--- a/compiler/rustc_session/src/cstore.rs
+++ b/compiler/rustc_session/src/cstore.rs
@@ -6,7 +6,6 @@ use crate::search_paths::PathKind;
 use crate::utils::NativeLibKind;
 use crate::Session;
 use rustc_ast as ast;
-use rustc_data_structures::owned_slice::OwnedSlice;
 use rustc_data_structures::sync::{self, AppendOnlyIndexVec, FreezeLock};
 use rustc_hir::def_id::{CrateNum, DefId, LocalDefId, StableCrateId, LOCAL_CRATE};
 use rustc_hir::definitions::{DefKey, DefPath, DefPathHash, Definitions};
@@ -14,10 +13,9 @@ use rustc_span::hygiene::{ExpnHash, ExpnId};
 use rustc_span::symbol::Symbol;
 use rustc_span::Span;
 use rustc_target::spec::abi::Abi;
-use rustc_target::spec::Target;
 
 use std::any::Any;
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
 
 // lonely orphan structs and enums looking for a better home
 
@@ -197,21 +195,6 @@ pub enum ExternCrateSource {
     Path,
 }
 
-/// The backend's way to give the crate store access to the metadata in a library.
-/// Note that it returns the raw metadata bytes stored in the library file, whether
-/// it is compressed, uncompressed, some weird mix, etc.
-/// rmeta files are backend independent and not handled here.
-///
-/// At the time of this writing, there is only one backend and one way to store
-/// metadata in library -- this trait just serves to decouple rustc_metadata from
-/// the archive reader, which depends on LLVM.
-pub trait MetadataLoader: std::fmt::Debug {
-    fn get_rlib_metadata(&self, target: &Target, filename: &Path) -> Result<OwnedSlice, String>;
-    fn get_dylib_metadata(&self, target: &Target, filename: &Path) -> Result<OwnedSlice, String>;
-}
-
-pub type MetadataLoaderDyn = dyn MetadataLoader + Send + Sync + sync::DynSend + sync::DynSync;
-
 /// A store of Rust crates, through which their metadata can be accessed.
 ///
 /// Note that this trait should probably not be expanding today. All new
diff --git a/compiler/rustc_session/src/errors.rs b/compiler/rustc_session/src/errors.rs
index 31094e0d2..c3360815a 100644
--- a/compiler/rustc_session/src/errors.rs
+++ b/compiler/rustc_session/src/errors.rs
@@ -3,7 +3,7 @@ use std::num::NonZeroU32;
 use crate::parse::ParseSess;
 use rustc_ast::token;
 use rustc_ast::util::literal::LitError;
-use rustc_errors::{error_code, DiagnosticMessage, EmissionGuarantee, IntoDiagnostic, MultiSpan};
+use rustc_errors::{error_code, DiagnosticMessage, ErrorGuaranteed, IntoDiagnostic, MultiSpan};
 use rustc_macros::Diagnostic;
 use rustc_span::{BytePos, Span, Symbol};
 use rustc_target::spec::{SplitDebuginfo, StackProtector, TargetTriple};
@@ -13,13 +13,13 @@ pub struct FeatureGateError {
     pub explain: DiagnosticMessage,
 }
 
-impl<'a, T: EmissionGuarantee> IntoDiagnostic<'a, T> for FeatureGateError {
+impl<'a> IntoDiagnostic<'a> for FeatureGateError {
     #[track_caller]
     fn into_diagnostic(
         self,
-        handler: &'a rustc_errors::Handler,
-    ) -> rustc_errors::DiagnosticBuilder<'a, T> {
-        let mut diag = handler.struct_diagnostic(self.explain);
+        dcx: &'a rustc_errors::DiagCtxt,
+    ) -> rustc_errors::DiagnosticBuilder<'a, ErrorGuaranteed> {
+        let mut diag = dcx.struct_err(self.explain);
         diag.set_span(self.span);
         diag.code(error_code!(E0658));
         diag
@@ -436,3 +436,17 @@ pub struct IncompatibleLinkerFlavor {
     pub flavor: &'static str,
     pub compatible_list: String,
 }
+
+#[derive(Diagnostic)]
+#[diag(session_function_return_requires_x86_or_x86_64)]
+pub(crate) struct FunctionReturnRequiresX86OrX8664;
+
+#[derive(Diagnostic)]
+#[diag(session_function_return_thunk_extern_requires_non_large_code_model)]
+pub(crate) struct FunctionReturnThunkExternRequiresNonLargeCodeModel;
+
+#[derive(Diagnostic)]
+#[diag(session_failed_to_create_profiler)]
+pub struct FailedToCreateProfiler {
+    pub err: String,
+}
diff --git a/compiler/rustc_session/src/filesearch.rs b/compiler/rustc_session/src/filesearch.rs
index 3988416d0..6e459ac45 100644
--- a/compiler/rustc_session/src/filesearch.rs
+++ b/compiler/rustc_session/src/filesearch.rs
@@ -9,12 +9,6 @@ use std::path::{Path, PathBuf};
 use crate::search_paths::{PathKind, SearchPath};
 use rustc_fs_util::fix_windows_verbatim_for_gcc;
 
-#[derive(Copy, Clone)]
-pub enum FileMatch {
-    FileMatches,
-    FileDoesntMatch,
-}
-
 #[derive(Clone)]
 pub struct FileSearch<'a> {
     sysroot: &'a Path,
diff --git a/compiler/rustc_session/src/lib.rs b/compiler/rustc_session/src/lib.rs
index 17ac3e991..805854bd5 100644
--- a/compiler/rustc_session/src/lib.rs
+++ b/compiler/rustc_session/src/lib.rs
@@ -1,6 +1,4 @@
-#![feature(if_let_guard)]
 #![feature(let_chains)]
-#![feature(min_specialization)]
 #![feature(never_type)]
 #![feature(lazy_cell)]
 #![feature(option_get_or_insert_default)]
@@ -8,7 +6,6 @@
 #![feature(map_many_mut)]
 #![feature(iter_intersperse)]
 #![recursion_limit = "256"]
-#![allow(rustc::potential_query_instability)]
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
 #![allow(internal_features)]
@@ -20,9 +17,6 @@ pub mod errors;
 #[macro_use]
 extern crate tracing;
 
-use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
-use rustc_fluent_macro::fluent_messages;
-
 pub mod utils;
 pub use lint::{declare_lint, declare_lint_pass, declare_tool_lint, impl_lint_pass};
 pub use rustc_lint_defs as lint;
@@ -46,7 +40,7 @@ pub use getopts;
 mod version;
 pub use version::RustcVersion;
 
-fluent_messages! { "../messages.ftl" }
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
 
 /// Requirements for a `StableHashingContext` to be used in this crate.
 /// This is a hack to allow using the `HashStable_Generic` derive macro
diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs
index ed00851b4..06b554e8e 100644
--- a/compiler/rustc_session/src/options.rs
+++ b/compiler/rustc_session/src/options.rs
@@ -2,7 +2,8 @@ use crate::config::*;
 
 use crate::search_paths::SearchPath;
 use crate::utils::NativeLib;
-use crate::{lint, EarlyErrorHandler};
+use crate::{lint, EarlyDiagCtxt};
+use rustc_data_structures::fx::FxIndexMap;
 use rustc_data_structures::profiling::TimePassesFormat;
 use rustc_data_structures::stable_hasher::Hash64;
 use rustc_errors::ColorConfig;
@@ -19,8 +20,7 @@ use rustc_span::SourceFileHashAlgorithm;
 
 use std::collections::BTreeMap;
 
-use std::collections::hash_map::DefaultHasher;
-use std::hash::Hasher;
+use std::hash::{DefaultHasher, Hasher};
 use std::num::{IntErrorKind, NonZeroUsize};
 use std::path::PathBuf;
 use std::str;
@@ -151,6 +151,9 @@ top_level_options!(
 
         target_triple: TargetTriple [TRACKED],
 
+        /// Effective logical environment used by `env!`/`option_env!` macros
+        logical_env: FxIndexMap<String, String> [TRACKED],
+
         test: bool [TRACKED],
         error_format: ErrorOutputType [UNTRACKED],
         diagnostic_width: Option<usize> [UNTRACKED],
@@ -252,10 +255,10 @@ macro_rules! options {
 
     impl $struct_name {
         pub fn build(
-            handler: &EarlyErrorHandler,
+            early_dcx: &EarlyDiagCtxt,
             matches: &getopts::Matches,
         ) -> $struct_name {
-            build_options(handler, matches, $stat, $prefix, $outputname)
+            build_options(early_dcx, matches, $stat, $prefix, $outputname)
         }
 
         fn dep_tracking_hash(&self, for_crate_hash: bool, error_format: ErrorOutputType) -> u64 {
@@ -316,7 +319,7 @@ type OptionSetter<O> = fn(&mut O, v: Option<&str>) -> bool;
 type OptionDescrs<O> = &'static [(&'static str, OptionSetter<O>, &'static str, &'static str)];
 
 fn build_options<O: Default>(
-    handler: &EarlyErrorHandler,
+    early_dcx: &EarlyDiagCtxt,
     matches: &getopts::Matches,
     descrs: OptionDescrs<O>,
     prefix: &str,
@@ -334,12 +337,12 @@ fn build_options<O: Default>(
             Some((_, setter, type_desc, _)) => {
                 if !setter(&mut op, value) {
                     match value {
-                        None => handler.early_error(
+                        None => early_dcx.early_error(
                             format!(
                                 "{outputname} option `{key}` requires {type_desc} ({prefix} {key}=<value>)"
                             ),
                         ),
-                        Some(value) => handler.early_error(
+                        Some(value) => early_dcx.early_error(
                             format!(
                                 "incorrect value `{value}` for {outputname} option `{key}` - {type_desc} was expected"
                             ),
@@ -347,7 +350,7 @@ fn build_options<O: Default>(
                     }
                 }
             }
-            None => handler.early_error(format!("unknown {outputname} option: `{key}`")),
+            None => early_dcx.early_error(format!("unknown {outputname} option: `{key}`")),
         }
     }
     return op;
@@ -393,8 +396,7 @@ mod desc {
     pub const parse_instrument_xray: &str = "either a boolean (`yes`, `no`, `on`, `off`, etc), or a comma separated list of settings: `always` or `never` (mutually exclusive), `ignore-loops`, `instruction-threshold=N`, `skip-entry`, `skip-exit`";
     pub const parse_unpretty: &str = "`string` or `string=string`";
     pub const parse_treat_err_as_bug: &str = "either no value or a non-negative number";
-    pub const parse_trait_solver: &str =
-        "one of the supported solver modes (`classic`, `next`, or `next-coherence`)";
+    pub const parse_next_solver_config: &str = "a comma separated list of solver configurations: `globally` (default), `coherence`, `dump-tree`, `dump-tree-on-error";
     pub const parse_lto: &str =
         "either a boolean (`yes`, `no`, `on`, `off`, etc), `thin`, `fat`, or omitted";
     pub const parse_linker_plugin_lto: &str =
@@ -426,10 +428,11 @@ mod desc {
         "a `,` separated combination of `bti`, `b-key`, `pac-ret`, or `leaf`";
     pub const parse_proc_macro_execution_strategy: &str =
         "one of supported execution strategies (`same-thread`, or `cross-thread`)";
-    pub const parse_dump_solver_proof_tree: &str = "one of: `always`, `on-request`, `on-error`";
     pub const parse_remap_path_scope: &str = "comma separated list of scopes: `macro`, `diagnostics`, `unsplit-debuginfo`, `split-debuginfo`, `split-debuginfo-path`, `object`, `all`";
     pub const parse_inlining_threshold: &str =
         "either a boolean (`yes`, `no`, `on`, `off`, etc), or a non-negative number";
+    pub const parse_llvm_module_flag: &str = "<key>:<type>:<value>:<behavior>. Type must currently be `u32`. Behavior should be one of (`error`, `warning`, `require`, `override`, `append`, `appendunique`, `max`, `min`)";
+    pub const parse_function_return: &str = "`keep` or `thunk-extern`";
 }
 
 mod parse {
@@ -1027,15 +1030,48 @@ mod parse {
         }
     }
 
-    pub(crate) fn parse_trait_solver(slot: &mut TraitSolver, v: Option<&str>) -> bool {
-        match v {
-            Some("classic") => *slot = TraitSolver::Classic,
-            Some("next") => *slot = TraitSolver::Next,
-            Some("next-coherence") => *slot = TraitSolver::NextCoherence,
-            // default trait solver is subject to change..
-            Some("default") => *slot = TraitSolver::Classic,
-            _ => return false,
+    pub(crate) fn parse_next_solver_config(
+        slot: &mut Option<NextSolverConfig>,
+        v: Option<&str>,
+    ) -> bool {
+        if let Some(config) = v {
+            let mut coherence = false;
+            let mut globally = true;
+            let mut dump_tree = None;
+            for c in config.split(',') {
+                match c {
+                    "globally" => globally = true,
+                    "coherence" => {
+                        globally = false;
+                        coherence = true;
+                    }
+                    "dump-tree" => {
+                        if dump_tree.replace(DumpSolverProofTree::Always).is_some() {
+                            return false;
+                        }
+                    }
+                    "dump-tree-on-error" => {
+                        if dump_tree.replace(DumpSolverProofTree::OnError).is_some() {
+                            return false;
+                        }
+                    }
+                    _ => return false,
+                }
+            }
+
+            *slot = Some(NextSolverConfig {
+                coherence: coherence || globally,
+                globally,
+                dump_tree: dump_tree.unwrap_or_default(),
+            });
+        } else {
+            *slot = Some(NextSolverConfig {
+                coherence: true,
+                globally: true,
+                dump_tree: Default::default(),
+            });
         }
+
         true
     }
 
@@ -1300,19 +1336,6 @@ mod parse {
         true
     }
 
-    pub(crate) fn parse_dump_solver_proof_tree(
-        slot: &mut DumpSolverProofTree,
-        v: Option<&str>,
-    ) -> bool {
-        match v {
-            None | Some("always") => *slot = DumpSolverProofTree::Always,
-            Some("never") => *slot = DumpSolverProofTree::Never,
-            Some("on-error") => *slot = DumpSolverProofTree::OnError,
-            _ => return false,
-        };
-        true
-    }
-
     pub(crate) fn parse_inlining_threshold(slot: &mut InliningThreshold, v: Option<&str>) -> bool {
         match v {
             Some("always" | "yes") => {
@@ -1332,6 +1355,42 @@ mod parse {
         }
         true
     }
+
+    pub(crate) fn parse_llvm_module_flag(
+        slot: &mut Vec<(String, u32, String)>,
+        v: Option<&str>,
+    ) -> bool {
+        let elements = v.unwrap_or_default().split(':').collect::<Vec<_>>();
+        let [key, md_type, value, behavior] = elements.as_slice() else {
+            return false;
+        };
+        if *md_type != "u32" {
+            // Currently we only support u32 metadata flags, but require the
+            // type for forward-compatibility.
+            return false;
+        }
+        let Ok(value) = value.parse::<u32>() else {
+            return false;
+        };
+        let behavior = behavior.to_lowercase();
+        let all_behaviors =
+            ["error", "warning", "require", "override", "append", "appendunique", "max", "min"];
+        if !all_behaviors.contains(&behavior.as_str()) {
+            return false;
+        }
+
+        slot.push((key.to_string(), value, behavior));
+        true
+    }
+
+    pub(crate) fn parse_function_return(slot: &mut FunctionReturn, v: Option<&str>) -> bool {
+        match v {
+            Some("keep") => *slot = FunctionReturn::Keep,
+            Some("thunk-extern") => *slot = FunctionReturn::ThunkExtern,
+            _ => return false,
+        }
+        true
+    }
 }
 
 options! {
@@ -1511,6 +1570,8 @@ options! {
         "compress debug info sections (none, zlib, zstd, default: none)"),
     deduplicate_diagnostics: bool = (true, parse_bool, [UNTRACKED],
         "deduplicate identical diagnostics (default: yes)"),
+    default_hidden_visibility: Option<bool> = (None, parse_opt_bool, [TRACKED],
+        "overrides the `default_hidden_visibility` setting of the target"),
     dep_info_omit_d_target: bool = (false, parse_bool, [TRACKED],
         "in dep-info output, omit targets for tracking dependencies of the dep-info files \
         themselves (default: no)"),
@@ -1548,13 +1609,12 @@ options! {
         "output statistics about monomorphization collection"),
     dump_mono_stats_format: DumpMonoStatsFormat = (DumpMonoStatsFormat::Markdown, parse_dump_mono_stats, [UNTRACKED],
         "the format to use for -Z dump-mono-stats (`markdown` (default) or `json`)"),
-    dump_solver_proof_tree: DumpSolverProofTree = (DumpSolverProofTree::Never, parse_dump_solver_proof_tree, [UNTRACKED],
-        "dump a proof tree for every goal evaluated by the new trait solver. If the flag is specified without any options after it
-        then it defaults to `always`. If the flag is not specified at all it defaults to `on-request`."),
     dwarf_version: Option<u32> = (None, parse_opt_number, [TRACKED],
         "version of DWARF debug information to emit (default: 2 or 4, depending on platform)"),
     dylib_lto: bool = (false, parse_bool, [UNTRACKED],
         "enables LTO for dylib crate type"),
+    ehcont_guard: bool = (false, parse_bool, [TRACKED],
+        "generate Windows EHCont Guard tables"),
     emit_stack_sizes: bool = (false, parse_bool, [UNTRACKED],
         "emit a section containing stack size metadata (default: no)"),
     emit_thin_lto: bool = (true, parse_bool, [TRACKED],
@@ -1574,6 +1634,8 @@ options! {
         "force all crates to be `rustc_private` unstable (default: no)"),
     fuel: Option<(String, u64)> = (None, parse_optimization_fuel, [TRACKED],
         "set the optimization fuel quota for a crate"),
+    function_return: FunctionReturn = (FunctionReturn::default(), parse_function_return, [TRACKED],
+        "replace returns with jumps to `__x86_return_thunk` (default: `keep`)"),
     function_sections: Option<bool> = (None, parse_opt_bool, [TRACKED],
         "whether each function should go in its own section"),
     future_incompat_test: bool = (false, parse_bool, [UNTRACKED],
@@ -1583,6 +1645,8 @@ options! {
     graphviz_font: String = ("Courier, monospace".to_string(), parse_string, [UNTRACKED],
         "use the given `fontname` in graphviz output; can be overridden by setting \
         environment variable `RUSTC_GRAPHVIZ_FONT` (default: `Courier, monospace`)"),
+    has_thread_local: Option<bool> = (None, parse_opt_bool, [TRACKED],
+        "explicitly enable the `cfg(target_thread_local)` directive"),
     hir_stats: bool = (false, parse_bool, [UNTRACKED],
         "print some statistics about AST and HIR (default: no)"),
     human_readable_cgu_names: bool = (false, parse_bool, [TRACKED],
@@ -1622,8 +1686,6 @@ options! {
          `=skip-entry`
          `=skip-exit`
          Multiple options can be combined with commas."),
-    keep_hygiene_data: bool = (false, parse_bool, [UNTRACKED],
-        "keep hygiene data after analysis (default: no)"),
     layout_seed: Option<u64> = (None, parse_opt_number, [TRACKED],
         "seed layout randomization"),
     link_directives: bool = (true, parse_bool, [TRACKED],
@@ -1632,6 +1694,8 @@ options! {
         "link native libraries in the linker invocation (default: yes)"),
     link_only: bool = (false, parse_bool, [TRACKED],
         "link the `.rlink` file generated by `-Z no-link` (default: no)"),
+    llvm_module_flag: Vec<(String, u32, String)> = (Vec::new(), parse_llvm_module_flag, [TRACKED],
+        "a list of module flags to pass to LLVM (space separated)"),
     llvm_plugins: Vec<String> = (Vec::new(), parse_list, [TRACKED],
         "a list LLVM plugins to enable (space separated)"),
     llvm_time_trace: bool = (false, parse_bool, [UNTRACKED],
@@ -1673,6 +1737,8 @@ options! {
         "the size at which the `large_assignments` lint starts to be emitted"),
     mutable_noalias: bool = (true, parse_bool, [TRACKED],
         "emit noalias metadata for mutable references (default: yes)"),
+    next_solver: Option<NextSolverConfig> = (None, parse_next_solver_config, [TRACKED],
+        "enable and configure the next generation trait solver used by rustc"),
     nll_facts: bool = (false, parse_bool, [UNTRACKED],
         "dump facts from NLL analysis into side files (default: no)"),
     nll_facts_dir: String = ("nll-facts".to_string(), parse_string, [UNTRACKED],
@@ -1711,8 +1777,6 @@ options! {
         "panic strategy for panics in drops"),
     parse_only: bool = (false, parse_bool, [UNTRACKED],
         "parse only; do not compile, assemble, or link (default: no)"),
-    perf_stats: bool = (false, parse_bool, [UNTRACKED],
-        "print some performance-related statistics (default: no)"),
     plt: Option<bool> = (None, parse_opt_bool, [TRACKED],
         "whether to use the PLT when calling into shared libraries;
         only has effect for PIC code on systems with ELF binaries
@@ -1774,7 +1838,7 @@ options! {
         "directory into which to write optimization remarks (if not specified, they will be \
 written to standard error output)"),
     report_delayed_bugs: bool = (false, parse_bool, [TRACKED],
-        "immediately print bugs registered with `delay_span_bug` (default: no)"),
+        "immediately print bugs registered with `span_delayed_bug` (default: no)"),
     sanitizer: SanitizerSet = (SanitizerSet::empty(), parse_sanitizers, [TRACKED],
         "use a sanitizer"),
     sanitizer_cfi_canonical_jump_tables: Option<bool> = (Some(true), parse_opt_bool, [TRACKED],
@@ -1875,8 +1939,6 @@ written to standard error output)"),
         "for every macro invocation, print its name and arguments (default: no)"),
     track_diagnostics: bool = (false, parse_bool, [UNTRACKED],
         "tracks where in rustc a diagnostic was emitted"),
-    trait_solver: TraitSolver = (TraitSolver::Classic, parse_trait_solver, [TRACKED],
-        "specify the trait solver mode used by rustc (default: classic)"),
     // Diagnostics are considered side-effects of a query (see `QuerySideEffects`) and are saved
     // alongside query results and changes to translation options can affect diagnostics - so
     // translation options should be tracked.
@@ -1946,9 +2008,3 @@ written to standard error output)"),
     // - compiler/rustc_interface/src/tests.rs
     // - src/doc/unstable-book/src/compiler-flags
 }
-
-#[derive(Clone, Hash, PartialEq, Eq, Debug)]
-pub enum WasiExecModel {
-    Command,
-    Reactor,
-}
diff --git a/compiler/rustc_session/src/output.rs b/compiler/rustc_session/src/output.rs
index 9cd96895a..2ff32b5bb 100644
--- a/compiler/rustc_session/src/output.rs
+++ b/compiler/rustc_session/src/output.rs
@@ -1,4 +1,4 @@
-//! Related to out filenames of compilation (e.g. save analysis, binaries).
+//! Related to out filenames of compilation (e.g. binaries).
 use crate::config::{CrateType, Input, OutFileName, OutputFilenames, OutputType};
 use crate::errors::{
     CrateNameDoesNotMatch, CrateNameEmpty, CrateNameInvalid, FileIsNotWriteable,
diff --git a/compiler/rustc_session/src/parse.rs b/compiler/rustc_session/src/parse.rs
index 4d20d6d41..2cb47e3a9 100644
--- a/compiler/rustc_session/src/parse.rs
+++ b/compiler/rustc_session/src/parse.rs
@@ -11,10 +11,10 @@ use crate::lint::{
 use rustc_ast::node_id::NodeId;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::sync::{AppendOnlyVec, Lock, Lrc};
-use rustc_errors::{emitter::SilentEmitter, Handler};
+use rustc_errors::{emitter::SilentEmitter, DiagCtxt};
 use rustc_errors::{
     fallback_fluent_bundle, Diagnostic, DiagnosticBuilder, DiagnosticId, DiagnosticMessage,
-    EmissionGuarantee, ErrorGuaranteed, IntoDiagnostic, MultiSpan, Noted, StashKey,
+    ErrorGuaranteed, IntoDiagnostic, MultiSpan, Noted, StashKey,
 };
 use rustc_feature::{find_feature_issue, GateIssue, UnstableFeatures};
 use rustc_span::edition::Edition;
@@ -51,6 +51,9 @@ impl GatedSpans {
     /// Prepend the given set of `spans` onto the set in `self`.
     pub fn merge(&self, mut spans: FxHashMap<Symbol, Vec<Span>>) {
         let mut inner = self.spans.borrow_mut();
+        // The entries will be moved to another map so the drain order does not
+        // matter.
+        #[allow(rustc::potential_query_instability)]
         for (gate, mut gate_spans) in inner.drain() {
             spans.entry(gate).or_default().append(&mut gate_spans);
         }
@@ -100,8 +103,7 @@ pub fn feature_err_issue(
 
     // Cancel an earlier warning for this same error, if it exists.
     if let Some(span) = span.primary_span() {
-        if let Some(err) = sess.span_diagnostic.steal_diagnostic(span, StashKey::EarlySyntaxWarning)
-        {
+        if let Some(err) = sess.dcx.steal_diagnostic(span, StashKey::EarlySyntaxWarning) {
             err.cancel()
         }
     }
@@ -135,7 +137,7 @@ pub fn feature_warn_issue(
     issue: GateIssue,
     explain: &'static str,
 ) {
-    let mut err = sess.span_diagnostic.struct_span_warn(span, explain);
+    let mut err = sess.dcx.struct_span_warn(span, explain);
     add_feature_diagnostics_for_issue(&mut err, sess, feature, issue, false);
 
     // Decorate this as a future-incompatibility lint as in rustc_middle::lint::struct_lint_level
@@ -186,7 +188,7 @@ pub fn add_feature_diagnostics_for_issue(
 
 /// Info about a parsing session.
 pub struct ParseSess {
-    pub span_diagnostic: Handler,
+    pub dcx: DiagCtxt,
     pub unstable_features: UnstableFeatures,
     pub config: Cfg,
     pub check_config: CheckCfg,
@@ -214,7 +216,7 @@ pub struct ParseSess {
     pub assume_incomplete_release: bool,
     /// Spans passed to `proc_macro::quote_span`. Each span has a numerical
     /// identifier represented by its position in the vector.
-    pub proc_macro_quoted_spans: AppendOnlyVec<Span>,
+    proc_macro_quoted_spans: AppendOnlyVec<Span>,
     /// Used to generate new `AttrId`s. Every `AttrId` is unique.
     pub attr_id_generator: AttrIdGenerator,
 }
@@ -224,13 +226,13 @@ impl ParseSess {
     pub fn new(locale_resources: Vec<&'static str>, file_path_mapping: FilePathMapping) -> Self {
         let fallback_bundle = fallback_fluent_bundle(locale_resources, false);
         let sm = Lrc::new(SourceMap::new(file_path_mapping));
-        let handler = Handler::with_tty_emitter(Some(sm.clone()), fallback_bundle);
-        ParseSess::with_span_handler(handler, sm)
+        let dcx = DiagCtxt::with_tty_emitter(Some(sm.clone()), fallback_bundle);
+        ParseSess::with_dcx(dcx, sm)
     }
 
-    pub fn with_span_handler(handler: Handler, source_map: Lrc<SourceMap>) -> Self {
+    pub fn with_dcx(dcx: DiagCtxt, source_map: Lrc<SourceMap>) -> Self {
         Self {
-            span_diagnostic: handler,
+            dcx,
             unstable_features: UnstableFeatures::from_environment(None),
             config: Cfg::default(),
             check_config: CheckCfg::default(),
@@ -253,10 +255,10 @@ impl ParseSess {
     pub fn with_silent_emitter(fatal_note: Option<String>) -> Self {
         let fallback_bundle = fallback_fluent_bundle(Vec::new(), false);
         let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
-        let fatal_handler = Handler::with_tty_emitter(None, fallback_bundle).disable_warnings();
-        let handler = Handler::with_emitter(Box::new(SilentEmitter { fatal_handler, fatal_note }))
+        let fatal_dcx = DiagCtxt::with_tty_emitter(None, fallback_bundle).disable_warnings();
+        let dcx = DiagCtxt::with_emitter(Box::new(SilentEmitter { fatal_dcx, fatal_note }))
             .disable_warnings();
-        ParseSess::with_span_handler(handler, sm)
+        ParseSess::with_dcx(dcx, sm)
     }
 
     #[inline]
@@ -320,7 +322,7 @@ impl ParseSess {
         &'a self,
         err: impl IntoDiagnostic<'a>,
     ) -> DiagnosticBuilder<'a, ErrorGuaranteed> {
-        err.into_diagnostic(&self.span_diagnostic)
+        err.into_diagnostic(&self.dcx)
     }
 
     #[track_caller]
@@ -333,7 +335,7 @@ impl ParseSess {
         &'a self,
         warning: impl IntoDiagnostic<'a, ()>,
     ) -> DiagnosticBuilder<'a, ()> {
-        warning.into_diagnostic(&self.span_diagnostic)
+        warning.into_diagnostic(&self.dcx)
     }
 
     #[track_caller]
@@ -346,7 +348,7 @@ impl ParseSess {
         &'a self,
         note: impl IntoDiagnostic<'a, Noted>,
     ) -> DiagnosticBuilder<'a, Noted> {
-        note.into_diagnostic(&self.span_diagnostic)
+        note.into_diagnostic(&self.dcx)
     }
 
     #[track_caller]
@@ -359,7 +361,7 @@ impl ParseSess {
         &'a self,
         fatal: impl IntoDiagnostic<'a, !>,
     ) -> DiagnosticBuilder<'a, !> {
-        fatal.into_diagnostic(&self.span_diagnostic)
+        fatal.into_diagnostic(&self.dcx)
     }
 
     #[track_caller]
@@ -373,27 +375,18 @@ impl ParseSess {
         &self,
         msg: impl Into<DiagnosticMessage>,
     ) -> DiagnosticBuilder<'_, ErrorGuaranteed> {
-        self.span_diagnostic.struct_err(msg)
+        self.dcx.struct_err(msg)
     }
 
     #[rustc_lint_diagnostics]
     #[track_caller]
     pub fn struct_warn(&self, msg: impl Into<DiagnosticMessage>) -> DiagnosticBuilder<'_, ()> {
-        self.span_diagnostic.struct_warn(msg)
+        self.dcx.struct_warn(msg)
     }
 
     #[rustc_lint_diagnostics]
     #[track_caller]
     pub fn struct_fatal(&self, msg: impl Into<DiagnosticMessage>) -> DiagnosticBuilder<'_, !> {
-        self.span_diagnostic.struct_fatal(msg)
-    }
-
-    #[rustc_lint_diagnostics]
-    #[track_caller]
-    pub fn struct_diagnostic<G: EmissionGuarantee>(
-        &self,
-        msg: impl Into<DiagnosticMessage>,
-    ) -> DiagnosticBuilder<'_, G> {
-        self.span_diagnostic.struct_diagnostic(msg)
+        self.dcx.struct_fatal(msg)
     }
 }
diff --git a/compiler/rustc_session/src/search_paths.rs b/compiler/rustc_session/src/search_paths.rs
index 07e78d176..8ed50f6a1 100644
--- a/compiler/rustc_session/src/search_paths.rs
+++ b/compiler/rustc_session/src/search_paths.rs
@@ -1,5 +1,5 @@
 use crate::filesearch::make_target_lib_path;
-use crate::EarlyErrorHandler;
+use crate::EarlyDiagCtxt;
 use std::path::{Path, PathBuf};
 
 #[derive(Clone, Debug)]
@@ -46,7 +46,7 @@ impl PathKind {
 }
 
 impl SearchPath {
-    pub fn from_cli_opt(handler: &EarlyErrorHandler, path: &str) -> Self {
+    pub fn from_cli_opt(early_dcx: &EarlyDiagCtxt, path: &str) -> Self {
         let (kind, path) = if let Some(stripped) = path.strip_prefix("native=") {
             (PathKind::Native, stripped)
         } else if let Some(stripped) = path.strip_prefix("crate=") {
@@ -61,7 +61,7 @@ impl SearchPath {
             (PathKind::All, path)
         };
         if path.is_empty() {
-            handler.early_error("empty search path given via `-L`");
+            early_dcx.early_error("empty search path given via `-L`");
         }
 
         let dir = PathBuf::from(path);
diff --git a/compiler/rustc_session/src/session.rs b/compiler/rustc_session/src/session.rs
index 80a549b30..7f168572f 100644
--- a/compiler/rustc_session/src/session.rs
+++ b/compiler/rustc_session/src/session.rs
@@ -1,7 +1,7 @@
 use crate::code_stats::CodeStats;
 pub use crate::code_stats::{DataTypeKind, FieldInfo, FieldKind, SizeKind, VariantInfo};
 use crate::config::{
-    self, CrateType, InstrumentCoverage, OptLevel, OutFileName, OutputType,
+    self, CrateType, FunctionReturn, InstrumentCoverage, OptLevel, OutFileName, OutputType,
     RemapPathScopeComponents, SwitchWithOptPath,
 };
 use crate::config::{ErrorOutputType, Input};
@@ -10,23 +10,21 @@ use crate::parse::{add_feature_diagnostics, ParseSess};
 use crate::search_paths::{PathKind, SearchPath};
 use crate::{filesearch, lint};
 
-pub use rustc_ast::attr::MarkedAttrs;
-pub use rustc_ast::Attribute;
 use rustc_data_structures::flock;
 use rustc_data_structures::fx::{FxHashMap, FxIndexSet};
 use rustc_data_structures::jobserver::{self, Client};
-use rustc_data_structures::profiling::{duration_to_secs_str, SelfProfiler, SelfProfilerRef};
+use rustc_data_structures::profiling::{SelfProfiler, SelfProfilerRef};
 use rustc_data_structures::sync::{
-    AtomicU64, AtomicUsize, Lock, Lrc, OneThread, Ordering, Ordering::SeqCst,
+    AtomicU64, DynSend, DynSync, Lock, Lrc, OneThread, Ordering::SeqCst,
 };
 use rustc_errors::annotate_snippet_emitter_writer::AnnotateSnippetEmitterWriter;
 use rustc_errors::emitter::{DynEmitter, EmitterWriter, HumanReadableErrorType};
 use rustc_errors::json::JsonEmitter;
 use rustc_errors::registry::Registry;
 use rustc_errors::{
-    error_code, fallback_fluent_bundle, DiagnosticBuilder, DiagnosticId, DiagnosticMessage,
-    ErrorGuaranteed, FluentBundle, Handler, IntoDiagnostic, LazyFallbackBundle, MultiSpan, Noted,
-    TerminalUrl,
+    error_code, fallback_fluent_bundle, DiagCtxt, DiagnosticBuilder, DiagnosticId,
+    DiagnosticMessage, ErrorGuaranteed, FluentBundle, IntoDiagnostic, LazyFallbackBundle,
+    MultiSpan, Noted, TerminalUrl,
 };
 use rustc_macros::HashStable_Generic;
 pub use rustc_span::def_id::StableCrateId;
@@ -39,6 +37,7 @@ use rustc_target::spec::{
     DebuginfoKind, SanitizerSet, SplitDebuginfo, StackProtector, Target, TargetTriple, TlsModel,
 };
 
+use std::any::Any;
 use std::cell::{self, RefCell};
 use std::env;
 use std::fmt;
@@ -46,9 +45,8 @@ use std::ops::{Div, Mul};
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
 use std::sync::{atomic::AtomicBool, Arc};
-use std::time::Duration;
 
-pub struct OptimizationFuel {
+struct OptimizationFuel {
     /// If `-zfuel=crate=n` is specified, initially set to `n`, otherwise `0`.
     remaining: u64,
     /// We're rejecting all further optimizations.
@@ -139,6 +137,8 @@ pub struct CompilerIO {
     pub temps_dir: Option<PathBuf>,
 }
 
+pub trait LintStoreMarker: Any + DynSync + DynSend {}
+
 /// Represents the data associated with a compilation
 /// session for a single crate.
 pub struct Session {
@@ -157,9 +157,6 @@ pub struct Session {
     /// Used by `-Z self-profile`.
     pub prof: SelfProfilerRef,
 
-    /// Some measurements that are being gathered during compilation.
-    pub perf_stats: PerfStats,
-
     /// Data about code being compiled, gathered during compilation.
     pub code_stats: CodeStats,
 
@@ -173,6 +170,12 @@ pub struct Session {
     /// false positives about a job server in our environment.
     pub jobserver: Client,
 
+    /// This only ever stores a `LintStore` but we don't want a dependency on that type here.
+    pub lint_store: Option<Lrc<dyn LintStoreMarker>>,
+
+    /// Should be set if any lints are registered in `lint_store`.
+    pub registered_lints: bool,
+
     /// Cap lint level specified by a driver specifically.
     pub driver_lint_caps: FxHashMap<lint::LintId, lint::Level>,
 
@@ -215,17 +218,6 @@ pub struct Session {
     pub expanded_args: Vec<String>,
 }
 
-pub struct PerfStats {
-    /// The accumulated time spent on computing symbol hashes.
-    pub symbol_hash_time: Lock<Duration>,
-    /// Total number of values canonicalized queries constructed.
-    pub queries_canonicalized: AtomicUsize,
-    /// Number of times this query is invoked.
-    pub normalize_generic_arg_after_erasing_regions: AtomicUsize,
-    /// Number of times this query is invoked.
-    pub normalize_projection_ty: AtomicUsize,
-}
-
 #[derive(PartialEq, Eq, PartialOrd, Ord)]
 pub enum MetadataKind {
     None,
@@ -297,7 +289,7 @@ impl Session {
     /// Invoked all the way at the end to finish off diagnostics printing.
     pub fn finish_diagnostics(&self, registry: &Registry) {
         self.check_miri_unleashed_features();
-        self.diagnostic().print_error_count(registry);
+        self.dcx().print_error_count(registry);
         self.emit_future_breakage();
     }
 
@@ -306,11 +298,11 @@ impl Session {
             return;
         }
 
-        let diags = self.diagnostic().take_future_breakage_diagnostics();
+        let diags = self.dcx().take_future_breakage_diagnostics();
         if diags.is_empty() {
             return;
         }
-        self.parse_sess.span_diagnostic.emit_future_breakage_report(diags);
+        self.dcx().emit_future_breakage_report(diags);
     }
 
     /// Returns true if the crate is a testing one.
@@ -325,7 +317,7 @@ impl Session {
         sp: S,
         msg: impl Into<DiagnosticMessage>,
     ) -> DiagnosticBuilder<'_, ()> {
-        self.diagnostic().struct_span_warn(sp, msg)
+        self.dcx().struct_span_warn(sp, msg)
     }
     #[rustc_lint_diagnostics]
     #[track_caller]
@@ -335,7 +327,7 @@ impl Session {
         msg: impl Into<DiagnosticMessage>,
         id: lint::LintExpectationId,
     ) -> DiagnosticBuilder<'_, ()> {
-        self.diagnostic().struct_span_warn_with_expectation(sp, msg, id)
+        self.dcx().struct_span_warn_with_expectation(sp, msg, id)
     }
     #[rustc_lint_diagnostics]
     #[track_caller]
@@ -345,12 +337,12 @@ impl Session {
         msg: impl Into<DiagnosticMessage>,
         code: DiagnosticId,
     ) -> DiagnosticBuilder<'_, ()> {
-        self.diagnostic().struct_span_warn_with_code(sp, msg, code)
+        self.dcx().struct_span_warn_with_code(sp, msg, code)
     }
     #[rustc_lint_diagnostics]
     #[track_caller]
     pub fn struct_warn(&self, msg: impl Into<DiagnosticMessage>) -> DiagnosticBuilder<'_, ()> {
-        self.diagnostic().struct_warn(msg)
+        self.dcx().struct_warn(msg)
     }
     #[rustc_lint_diagnostics]
     #[track_caller]
@@ -359,7 +351,7 @@ impl Session {
         msg: impl Into<DiagnosticMessage>,
         id: lint::LintExpectationId,
     ) -> DiagnosticBuilder<'_, ()> {
-        self.diagnostic().struct_warn_with_expectation(msg, id)
+        self.dcx().struct_warn_with_expectation(msg, id)
     }
     #[rustc_lint_diagnostics]
     #[track_caller]
@@ -368,12 +360,12 @@ impl Session {
         sp: S,
         msg: impl Into<DiagnosticMessage>,
     ) -> DiagnosticBuilder<'_, ()> {
-        self.diagnostic().struct_span_allow(sp, msg)
+        self.dcx().struct_span_allow(sp, msg)
     }
     #[rustc_lint_diagnostics]
     #[track_caller]
     pub fn struct_allow(&self, msg: impl Into<DiagnosticMessage>) -> DiagnosticBuilder<'_, ()> {
-        self.diagnostic().struct_allow(msg)
+        self.dcx().struct_allow(msg)
     }
     #[rustc_lint_diagnostics]
     #[track_caller]
@@ -382,7 +374,7 @@ impl Session {
         msg: impl Into<DiagnosticMessage>,
         id: lint::LintExpectationId,
     ) -> DiagnosticBuilder<'_, ()> {
-        self.diagnostic().struct_expect(msg, id)
+        self.dcx().struct_expect(msg, id)
     }
     #[rustc_lint_diagnostics]
     #[track_caller]
@@ -391,7 +383,7 @@ impl Session {
         sp: S,
         msg: impl Into<DiagnosticMessage>,
     ) -> DiagnosticBuilder<'_, ErrorGuaranteed> {
-        self.diagnostic().struct_span_err(sp, msg)
+        self.dcx().struct_span_err(sp, msg)
     }
     #[rustc_lint_diagnostics]
     #[track_caller]
@@ -401,7 +393,7 @@ impl Session {
         msg: impl Into<DiagnosticMessage>,
         code: DiagnosticId,
     ) -> DiagnosticBuilder<'_, ErrorGuaranteed> {
-        self.diagnostic().struct_span_err_with_code(sp, msg, code)
+        self.dcx().struct_span_err_with_code(sp, msg, code)
     }
     // FIXME: This method should be removed (every error should have an associated error code).
     #[rustc_lint_diagnostics]
@@ -419,7 +411,7 @@ impl Session {
         msg: impl Into<DiagnosticMessage>,
         code: DiagnosticId,
     ) -> DiagnosticBuilder<'_, ErrorGuaranteed> {
-        self.diagnostic().struct_err_with_code(msg, code)
+        self.dcx().struct_err_with_code(msg, code)
     }
     #[rustc_lint_diagnostics]
     #[track_caller]
@@ -428,7 +420,7 @@ impl Session {
         msg: impl Into<DiagnosticMessage>,
         code: DiagnosticId,
     ) -> DiagnosticBuilder<'_, ()> {
-        self.diagnostic().struct_warn_with_code(msg, code)
+        self.dcx().struct_warn_with_code(msg, code)
     }
     #[rustc_lint_diagnostics]
     #[track_caller]
@@ -437,7 +429,7 @@ impl Session {
         sp: S,
         msg: impl Into<DiagnosticMessage>,
     ) -> DiagnosticBuilder<'_, !> {
-        self.diagnostic().struct_span_fatal(sp, msg)
+        self.dcx().struct_span_fatal(sp, msg)
     }
     #[rustc_lint_diagnostics]
     pub fn struct_span_fatal_with_code<S: Into<MultiSpan>>(
@@ -446,17 +438,17 @@ impl Session {
         msg: impl Into<DiagnosticMessage>,
         code: DiagnosticId,
     ) -> DiagnosticBuilder<'_, !> {
-        self.diagnostic().struct_span_fatal_with_code(sp, msg, code)
+        self.dcx().struct_span_fatal_with_code(sp, msg, code)
     }
     #[rustc_lint_diagnostics]
     pub fn struct_fatal(&self, msg: impl Into<DiagnosticMessage>) -> DiagnosticBuilder<'_, !> {
-        self.diagnostic().struct_fatal(msg)
+        self.dcx().struct_fatal(msg)
     }
 
     #[rustc_lint_diagnostics]
     #[track_caller]
     pub fn span_fatal<S: Into<MultiSpan>>(&self, sp: S, msg: impl Into<DiagnosticMessage>) -> ! {
-        self.diagnostic().span_fatal(sp, msg)
+        self.dcx().span_fatal(sp, msg)
     }
     #[rustc_lint_diagnostics]
     pub fn span_fatal_with_code<S: Into<MultiSpan>>(
@@ -465,11 +457,11 @@ impl Session {
         msg: impl Into<DiagnosticMessage>,
         code: DiagnosticId,
     ) -> ! {
-        self.diagnostic().span_fatal_with_code(sp, msg, code)
+        self.dcx().span_fatal_with_code(sp, msg, code)
     }
     #[rustc_lint_diagnostics]
     pub fn fatal(&self, msg: impl Into<DiagnosticMessage>) -> ! {
-        self.diagnostic().fatal(msg).raise()
+        self.dcx().fatal(msg)
     }
     #[rustc_lint_diagnostics]
     #[track_caller]
@@ -478,7 +470,7 @@ impl Session {
         sp: S,
         msg: impl Into<DiagnosticMessage>,
     ) -> ErrorGuaranteed {
-        self.diagnostic().span_err(sp, msg)
+        self.dcx().span_err(sp, msg)
     }
     #[rustc_lint_diagnostics]
     pub fn span_err_with_code<S: Into<MultiSpan>>(
@@ -486,14 +478,14 @@ impl Session {
         sp: S,
         msg: impl Into<DiagnosticMessage>,
         code: DiagnosticId,
-    ) {
-        self.diagnostic().span_err_with_code(sp, msg, code)
+    ) -> ErrorGuaranteed {
+        self.dcx().span_err_with_code(sp, msg, code)
     }
     #[rustc_lint_diagnostics]
     #[allow(rustc::untranslatable_diagnostic)]
     #[allow(rustc::diagnostic_outside_of_impl)]
     pub fn err(&self, msg: impl Into<DiagnosticMessage>) -> ErrorGuaranteed {
-        self.diagnostic().err(msg)
+        self.dcx().err(msg)
     }
     #[track_caller]
     pub fn create_err<'a>(
@@ -554,23 +546,23 @@ impl Session {
     }
     #[inline]
     pub fn err_count(&self) -> usize {
-        self.diagnostic().err_count()
+        self.dcx().err_count()
     }
     pub fn has_errors(&self) -> Option<ErrorGuaranteed> {
-        self.diagnostic().has_errors()
+        self.dcx().has_errors()
     }
-    pub fn has_errors_or_delayed_span_bugs(&self) -> Option<ErrorGuaranteed> {
-        self.diagnostic().has_errors_or_delayed_span_bugs()
+    pub fn has_errors_or_span_delayed_bugs(&self) -> Option<ErrorGuaranteed> {
+        self.dcx().has_errors_or_span_delayed_bugs()
     }
     pub fn is_compilation_going_to_fail(&self) -> Option<ErrorGuaranteed> {
-        self.diagnostic().is_compilation_going_to_fail()
+        self.dcx().is_compilation_going_to_fail()
     }
     pub fn abort_if_errors(&self) {
-        self.diagnostic().abort_if_errors();
+        self.dcx().abort_if_errors();
     }
     pub fn compile_status(&self) -> Result<(), ErrorGuaranteed> {
-        if let Some(reported) = self.diagnostic().has_errors_or_lint_errors() {
-            let _ = self.diagnostic().emit_stashed_diagnostics();
+        if let Some(reported) = self.dcx().has_errors_or_lint_errors() {
+            let _ = self.dcx().emit_stashed_diagnostics();
             Err(reported)
         } else {
             Ok(())
@@ -586,7 +578,7 @@ impl Session {
         if self.err_count() == old_count {
             Ok(result)
         } else {
-            Err(self.delay_span_bug(
+            Err(self.span_delayed_bug(
                 rustc_span::DUMMY_SP,
                 "`self.err_count()` changed but an error was not emitted",
             ))
@@ -598,7 +590,7 @@ impl Session {
     #[allow(rustc::diagnostic_outside_of_impl)]
     #[track_caller]
     pub fn span_warn<S: Into<MultiSpan>>(&self, sp: S, msg: impl Into<DiagnosticMessage>) {
-        self.diagnostic().span_warn(sp, msg)
+        self.dcx().span_warn(sp, msg)
     }
 
     #[rustc_lint_diagnostics]
@@ -610,14 +602,14 @@ impl Session {
         msg: impl Into<DiagnosticMessage>,
         code: DiagnosticId,
     ) {
-        self.diagnostic().span_warn_with_code(sp, msg, code)
+        self.dcx().span_warn_with_code(sp, msg, code)
     }
 
     #[rustc_lint_diagnostics]
     #[allow(rustc::untranslatable_diagnostic)]
     #[allow(rustc::diagnostic_outside_of_impl)]
     pub fn warn(&self, msg: impl Into<DiagnosticMessage>) {
-        self.diagnostic().warn(msg)
+        self.dcx().warn(msg)
     }
 
     /// Ensures that compilation cannot succeed.
@@ -627,24 +619,28 @@ impl Session {
     ///
     /// This can be used in code paths that should never run on successful compilations.
     /// For example, it can be used to create an [`ErrorGuaranteed`]
-    /// (but you should prefer threading through the [`ErrorGuaranteed`] from an error emission directly).
+    /// (but you should prefer threading through the [`ErrorGuaranteed`] from an error emission
+    /// directly).
     ///
     /// If no span is available, use [`DUMMY_SP`].
     ///
     /// [`DUMMY_SP`]: rustc_span::DUMMY_SP
+    ///
+    /// Note: this function used to be called `delay_span_bug`. It was renamed
+    /// to match similar functions like `span_err`, `span_warn`, etc.
     #[track_caller]
-    pub fn delay_span_bug<S: Into<MultiSpan>>(
+    pub fn span_delayed_bug<S: Into<MultiSpan>>(
         &self,
         sp: S,
-        msg: impl Into<String>,
+        msg: impl Into<DiagnosticMessage>,
     ) -> ErrorGuaranteed {
-        self.diagnostic().delay_span_bug(sp, msg)
+        self.dcx().span_delayed_bug(sp, msg)
     }
 
     /// Used for code paths of expensive computations that should only take place when
     /// warnings or errors are emitted. If no messages are emitted ("good path"), then
     /// it's likely a bug.
-    pub fn delay_good_path_bug(&self, msg: impl Into<DiagnosticMessage>) {
+    pub fn good_path_delayed_bug(&self, msg: impl Into<DiagnosticMessage>) {
         if self.opts.unstable_opts.print_type_sizes
             || self.opts.unstable_opts.query_dep_graph
             || self.opts.unstable_opts.dump_mir.is_some()
@@ -655,41 +651,34 @@ impl Session {
             return;
         }
 
-        self.diagnostic().delay_good_path_bug(msg)
+        self.dcx().good_path_delayed_bug(msg)
     }
 
     #[rustc_lint_diagnostics]
     #[allow(rustc::untranslatable_diagnostic)]
     #[allow(rustc::diagnostic_outside_of_impl)]
-    pub fn note_without_error(&self, msg: impl Into<DiagnosticMessage>) {
-        self.diagnostic().note_without_error(msg)
+    pub fn note(&self, msg: impl Into<DiagnosticMessage>) {
+        self.dcx().note(msg)
     }
 
     #[track_caller]
     #[rustc_lint_diagnostics]
     #[allow(rustc::untranslatable_diagnostic)]
     #[allow(rustc::diagnostic_outside_of_impl)]
-    pub fn span_note_without_error<S: Into<MultiSpan>>(
-        &self,
-        sp: S,
-        msg: impl Into<DiagnosticMessage>,
-    ) {
-        self.diagnostic().span_note_without_error(sp, msg)
+    pub fn span_note<S: Into<MultiSpan>>(&self, sp: S, msg: impl Into<DiagnosticMessage>) {
+        self.dcx().span_note(sp, msg)
     }
 
     #[rustc_lint_diagnostics]
     #[allow(rustc::untranslatable_diagnostic)]
     #[allow(rustc::diagnostic_outside_of_impl)]
-    pub fn struct_note_without_error(
-        &self,
-        msg: impl Into<DiagnosticMessage>,
-    ) -> DiagnosticBuilder<'_, ()> {
-        self.diagnostic().struct_note_without_error(msg)
+    pub fn struct_note(&self, msg: impl Into<DiagnosticMessage>) -> DiagnosticBuilder<'_, ()> {
+        self.dcx().struct_note(msg)
     }
 
     #[inline]
-    pub fn diagnostic(&self) -> &rustc_errors::Handler {
-        &self.parse_sess.span_diagnostic
+    pub fn dcx(&self) -> &DiagCtxt {
+        &self.parse_sess.dcx
     }
 
     #[inline]
@@ -813,7 +802,7 @@ impl Session {
 
     /// Returns a list of directories where target-specific tool binaries are located.
     pub fn get_tools_search_paths(&self, self_contained: bool) -> Vec<PathBuf> {
-        let rustlib_path = rustc_target::target_rustlib_path(&self.sysroot, &config::host_triple());
+        let rustlib_path = rustc_target::target_rustlib_path(&self.sysroot, config::host_triple());
         let p = PathBuf::from_iter([
             Path::new(&self.sysroot),
             Path::new(&rustlib_path),
@@ -822,12 +811,7 @@ impl Session {
         if self_contained { vec![p.clone(), p.join("self-contained")] } else { vec![p] }
     }
 
-    pub fn init_incr_comp_session(
-        &self,
-        session_dir: PathBuf,
-        lock_file: flock::Lock,
-        load_dep_graph: bool,
-    ) {
+    pub fn init_incr_comp_session(&self, session_dir: PathBuf, lock_file: flock::Lock) {
         let mut incr_comp_session = self.incr_comp_session.borrow_mut();
 
         if let IncrCompSession::NotInitialized = *incr_comp_session {
@@ -836,7 +820,7 @@ impl Session {
         }
 
         *incr_comp_session =
-            IncrCompSession::Active { session_directory: session_dir, lock_file, load_dep_graph };
+            IncrCompSession::Active { session_directory: session_dir, _lock_file: lock_file };
     }
 
     pub fn finalize_incr_comp_session(&self, new_directory_path: PathBuf) {
@@ -883,25 +867,6 @@ impl Session {
         self.opts.incremental.as_ref().map(|_| self.incr_comp_session_dir())
     }
 
-    pub fn print_perf_stats(&self) {
-        eprintln!(
-            "Total time spent computing symbol hashes:      {}",
-            duration_to_secs_str(*self.perf_stats.symbol_hash_time.lock())
-        );
-        eprintln!(
-            "Total queries canonicalized:                   {}",
-            self.perf_stats.queries_canonicalized.load(Ordering::Relaxed)
-        );
-        eprintln!(
-            "normalize_generic_arg_after_erasing_regions:   {}",
-            self.perf_stats.normalize_generic_arg_after_erasing_regions.load(Ordering::Relaxed)
-        );
-        eprintln!(
-            "normalize_projection_ty:                       {}",
-            self.perf_stats.normalize_projection_ty.load(Ordering::Relaxed)
-        );
-    }
-
     /// We want to know if we're allowed to do an optimization for crate foo from -z fuel=foo=n.
     /// This expends fuel if applicable, and records fuel if applicable.
     pub fn consider_optimizing(
@@ -916,9 +881,9 @@ impl Session {
                 let mut fuel = self.optimization_fuel.lock();
                 ret = fuel.remaining != 0;
                 if fuel.remaining == 0 && !fuel.out_of_fuel {
-                    if self.diagnostic().can_emit_warnings() {
+                    if self.dcx().can_emit_warnings() {
                         // We only call `msg` in case we can actually emit warnings.
-                        // Otherwise, this could cause a `delay_good_path_bug` to
+                        // Otherwise, this could cause a `good_path_delayed_bug` to
                         // trigger (issue #79546).
                         self.emit_warning(errors::OptimisationFuelExhausted { msg: msg() });
                     }
@@ -996,6 +961,14 @@ impl Session {
             termize::dimensions().map_or(default_column_width, |(w, _)| w)
         }
     }
+
+    /// Whether the default visibility of symbols should be "hidden" rather than "default".
+    pub fn default_hidden_visibility(&self) -> bool {
+        self.opts
+            .unstable_opts
+            .default_hidden_visibility
+            .unwrap_or(self.target.options.default_hidden_visibility)
+    }
 }
 
 // JUSTIFICATION: defn of the suggested wrapper fns
@@ -1248,7 +1221,7 @@ impl Session {
     }
 
     pub fn teach(&self, code: &DiagnosticId) -> bool {
-        self.opts.unstable_opts.teach && self.diagnostic().must_teach(code)
+        self.opts.unstable_opts.teach && self.dcx().must_teach(code)
     }
 
     pub fn edition(&self) -> Edition {
@@ -1384,7 +1357,7 @@ fn default_emitter(
 // JUSTIFICATION: literally session construction
 #[allow(rustc::bad_opt_access)]
 pub fn build_session(
-    handler: &EarlyErrorHandler,
+    early_dcx: EarlyDiagCtxt,
     sopts: config::Options,
     io: CompilerIO,
     bundle: Option<Lrc<rustc_errors::FluentBundle>>,
@@ -1414,12 +1387,13 @@ pub fn build_session(
         None => filesearch::get_or_default_sysroot().expect("Failed finding sysroot"),
     };
 
-    let target_cfg = config::build_target_config(handler, &sopts, target_override, &sysroot);
+    let target_cfg = config::build_target_config(&early_dcx, &sopts, target_override, &sysroot);
     let host_triple = TargetTriple::from_triple(config::host_triple());
-    let (host, target_warnings) = Target::search(&host_triple, &sysroot)
-        .unwrap_or_else(|e| handler.early_error(format!("Error loading host specification: {e}")));
+    let (host, target_warnings) = Target::search(&host_triple, &sysroot).unwrap_or_else(|e| {
+        early_dcx.early_error(format!("Error loading host specification: {e}"))
+    });
     for warning in target_warnings.warning_messages() {
-        handler.early_warn(warning)
+        early_dcx.early_warn(warning)
     }
 
     let loader = file_loader.unwrap_or_else(|| Box::new(RealFileLoader));
@@ -1442,12 +1416,16 @@ pub fn build_session(
     );
     let emitter = default_emitter(&sopts, registry, source_map.clone(), bundle, fallback_bundle);
 
-    let mut span_diagnostic = rustc_errors::Handler::with_emitter(emitter)
-        .with_flags(sopts.unstable_opts.diagnostic_handler_flags(can_emit_warnings));
+    let mut dcx = DiagCtxt::with_emitter(emitter)
+        .with_flags(sopts.unstable_opts.dcx_flags(can_emit_warnings));
     if let Some(ice_file) = ice_file {
-        span_diagnostic = span_diagnostic.with_ice_file(ice_file);
+        dcx = dcx.with_ice_file(ice_file);
     }
 
+    // Now that the proper handler has been constructed, drop early_dcx to
+    // prevent accidental use.
+    drop(early_dcx);
+
     let self_profiler = if let SwitchWithOptPath::Enabled(ref d) = sopts.unstable_opts.self_profile
     {
         let directory =
@@ -1462,7 +1440,7 @@ pub fn build_session(
         match profiler {
             Ok(profiler) => Some(Arc::new(profiler)),
             Err(e) => {
-                handler.early_warn(format!("failed to create profiler: {e}"));
+                dcx.emit_warning(errors::FailedToCreateProfiler { err: e.to_string() });
                 None
             }
         }
@@ -1470,7 +1448,7 @@ pub fn build_session(
         None
     };
 
-    let mut parse_sess = ParseSess::with_span_handler(span_diagnostic, source_map);
+    let mut parse_sess = ParseSess::with_dcx(dcx, source_map);
     parse_sess.assume_incomplete_release = sopts.unstable_opts.assume_incomplete_release;
 
     let host_triple = config::host_triple();
@@ -1515,16 +1493,12 @@ pub fn build_session(
         io,
         incr_comp_session: OneThread::new(RefCell::new(IncrCompSession::NotInitialized)),
         prof,
-        perf_stats: PerfStats {
-            symbol_hash_time: Lock::new(Duration::from_secs(0)),
-            queries_canonicalized: AtomicUsize::new(0),
-            normalize_generic_arg_after_erasing_regions: AtomicUsize::new(0),
-            normalize_projection_ty: AtomicUsize::new(0),
-        },
         code_stats: Default::default(),
         optimization_fuel,
         print_fuel,
         jobserver: jobserver::client(),
+        lint_store: None,
+        registered_lints: false,
         driver_lint_caps,
         ctfe_backtrace,
         miri_unleashed_features: Lock::new(Default::default()),
@@ -1707,17 +1681,41 @@ fn validate_commandline_args_with_session_available(sess: &Session) {
             sess.emit_err(errors::IncompatibleLinkerFlavor { flavor, compatible_list });
         }
     }
+
+    if sess.opts.unstable_opts.function_return != FunctionReturn::default() {
+        if sess.target.arch != "x86" && sess.target.arch != "x86_64" {
+            sess.emit_err(errors::FunctionReturnRequiresX86OrX8664);
+        }
+    }
+
+    // The code model check applies to `thunk` and `thunk-extern`, but not `thunk-inline`, so it is
+    // kept as a `match` to force a change if new ones are added, even if we currently only support
+    // `thunk-extern` like Clang.
+    match sess.opts.unstable_opts.function_return {
+        FunctionReturn::Keep => (),
+        FunctionReturn::ThunkExtern => {
+            // FIXME: In principle, the inherited base LLVM target code model could be large,
+            // but this only checks whether we were passed one explicitly (like Clang does).
+            if let Some(code_model) = sess.code_model()
+                && code_model == CodeModel::Large
+            {
+                sess.emit_err(errors::FunctionReturnThunkExternRequiresNonLargeCodeModel);
+            }
+        }
+    }
 }
 
 /// Holds data on the current incremental compilation session, if there is one.
 #[derive(Debug)]
-pub enum IncrCompSession {
+enum IncrCompSession {
     /// This is the state the session will be in until the incr. comp. dir is
     /// needed.
     NotInitialized,
     /// This is the state during which the session directory is private and can
-    /// be modified.
-    Active { session_directory: PathBuf, lock_file: flock::Lock, load_dep_graph: bool },
+    /// be modified. `_lock_file` is never directly used, but its presence
+    /// alone has an effect, because the file will unlock when the session is
+    /// dropped.
+    Active { session_directory: PathBuf, _lock_file: flock::Lock },
     /// This is the state after the session directory has been finalized. In this
     /// state, the contents of the directory must not be modified any more.
     Finalized { session_directory: PathBuf },
@@ -1727,69 +1725,78 @@ pub enum IncrCompSession {
     InvalidBecauseOfErrors { session_directory: PathBuf },
 }
 
-/// A wrapper around an [`Handler`] that is used for early error emissions.
-pub struct EarlyErrorHandler {
-    handler: Handler,
+/// A wrapper around an [`DiagCtxt`] that is used for early error emissions.
+pub struct EarlyDiagCtxt {
+    dcx: DiagCtxt,
 }
 
-impl EarlyErrorHandler {
+impl EarlyDiagCtxt {
     pub fn new(output: ErrorOutputType) -> Self {
         let emitter = mk_emitter(output);
-        Self { handler: rustc_errors::Handler::with_emitter(emitter) }
+        Self { dcx: DiagCtxt::with_emitter(emitter) }
     }
 
     pub fn abort_if_errors(&self) {
-        self.handler.abort_if_errors()
+        self.dcx.abort_if_errors()
     }
 
-    /// Swap out the underlying handler once we acquire the user's preference on error emission
+    /// Swap out the underlying dcx once we acquire the user's preference on error emission
     /// format. Any errors prior to that will cause an abort and all stashed diagnostics of the
-    /// previous handler will be emitted.
+    /// previous dcx will be emitted.
     pub fn abort_if_error_and_set_error_format(&mut self, output: ErrorOutputType) {
-        self.handler.abort_if_errors();
+        self.dcx.abort_if_errors();
 
         let emitter = mk_emitter(output);
-        self.handler = Handler::with_emitter(emitter);
+        self.dcx = DiagCtxt::with_emitter(emitter);
     }
 
     #[allow(rustc::untranslatable_diagnostic)]
     #[allow(rustc::diagnostic_outside_of_impl)]
     pub fn early_note(&self, msg: impl Into<DiagnosticMessage>) {
-        self.handler.struct_note_without_error(msg).emit()
+        self.dcx.struct_note(msg).emit()
     }
 
     #[allow(rustc::untranslatable_diagnostic)]
     #[allow(rustc::diagnostic_outside_of_impl)]
     pub fn early_help(&self, msg: impl Into<DiagnosticMessage>) {
-        self.handler.struct_help(msg).emit()
+        self.dcx.struct_help(msg).emit()
     }
 
     #[allow(rustc::untranslatable_diagnostic)]
     #[allow(rustc::diagnostic_outside_of_impl)]
     #[must_use = "ErrorGuaranteed must be returned from `run_compiler` in order to exit with a non-zero status code"]
     pub fn early_error_no_abort(&self, msg: impl Into<DiagnosticMessage>) -> ErrorGuaranteed {
-        self.handler.struct_err(msg).emit()
+        self.dcx.struct_err(msg).emit()
     }
 
     #[allow(rustc::untranslatable_diagnostic)]
     #[allow(rustc::diagnostic_outside_of_impl)]
     pub fn early_error(&self, msg: impl Into<DiagnosticMessage>) -> ! {
-        self.handler.struct_fatal(msg).emit()
+        self.dcx.struct_fatal(msg).emit()
     }
 
     #[allow(rustc::untranslatable_diagnostic)]
     #[allow(rustc::diagnostic_outside_of_impl)]
-    pub(crate) fn early_struct_error(
+    pub fn early_struct_error(
         &self,
         msg: impl Into<DiagnosticMessage>,
     ) -> DiagnosticBuilder<'_, !> {
-        self.handler.struct_fatal(msg)
+        self.dcx.struct_fatal(msg)
     }
 
     #[allow(rustc::untranslatable_diagnostic)]
     #[allow(rustc::diagnostic_outside_of_impl)]
     pub fn early_warn(&self, msg: impl Into<DiagnosticMessage>) {
-        self.handler.struct_warn(msg).emit()
+        self.dcx.struct_warn(msg).emit()
+    }
+
+    pub fn initialize_checked_jobserver(&self) {
+        // initialize jobserver before getting `jobserver::client` and `build_session`.
+        jobserver::initialize_checked(|err| {
+            #[allow(rustc::untranslatable_diagnostic)]
+            #[allow(rustc::diagnostic_outside_of_impl)]
+            self.dcx.struct_warn(err).note("the build environment is likely misconfigured").emit()
+        });
     }
 }
 
diff --git a/compiler/rustc_session/src/utils.rs b/compiler/rustc_session/src/utils.rs
index 3ed044ad7..f76c69af5 100644
--- a/compiler/rustc_session/src/utils.rs
+++ b/compiler/rustc_session/src/utils.rs
@@ -158,14 +158,3 @@ pub fn extra_compiler_flags() -> Option<(Vec<String>, bool)> {
 
     if !result.is_empty() { Some((result, excluded_cargo_defaults)) } else { None }
 }
-
-pub(crate) fn is_ascii_ident(string: &str) -> bool {
-    let mut chars = string.chars();
-    if let Some(start) = chars.next()
-        && (start.is_ascii_alphabetic() || start == '_')
-    {
-        chars.all(|char| char.is_ascii_alphanumeric() || char == '_')
-    } else {
-        false
-    }
-}
diff --git a/compiler/rustc_session/src/version.rs b/compiler/rustc_session/src/version.rs
index 1ad8620bf..c0c088bce 100644
--- a/compiler/rustc_session/src/version.rs
+++ b/compiler/rustc_session/src/version.rs
@@ -9,7 +9,7 @@ pub struct RustcVersion {
 }
 
 impl RustcVersion {
-    pub const CURRENT: Self = current_rustc_version!(env!("CFG_RELEASE"));
+    pub const CURRENT: Self = current_rustc_version!();
 }
 
 impl Display for RustcVersion {
diff --git a/compiler/rustc_smir/Cargo.toml b/compiler/rustc_smir/Cargo.toml
index 836ea046f..c9e23efcb 100644
--- a/compiler/rustc_smir/Cargo.toml
+++ b/compiler/rustc_smir/Cargo.toml
@@ -5,6 +5,7 @@ edition = "2021"
 
 [dependencies]
 # tidy-alphabetical-start
+rustc_abi = { path = "../rustc_abi" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_hir = { path = "../rustc_hir" }
 rustc_middle = { path = "../rustc_middle" }
diff --git a/compiler/rustc_smir/src/lib.rs b/compiler/rustc_smir/src/lib.rs
index dcf6b9040..ddd5ea551 100644
--- a/compiler/rustc_smir/src/lib.rs
+++ b/compiler/rustc_smir/src/lib.rs
@@ -10,9 +10,9 @@
     html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/",
     test(attr(allow(unused_variables), deny(warnings)))
 )]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
-#![cfg_attr(not(bootstrap), allow(internal_features))]
+#![doc(rust_logo)]
+#![feature(rustdoc_internals)]
+#![allow(internal_features)]
 #![allow(rustc::usage_of_ty_tykind)]
 
 pub mod rustc_internal;
diff --git a/compiler/rustc_smir/src/rustc_internal/internal.rs b/compiler/rustc_smir/src/rustc_internal/internal.rs
index 7cfdbbbf7..bbc98af45 100644
--- a/compiler/rustc_smir/src/rustc_internal/internal.rs
+++ b/compiler/rustc_smir/src/rustc_internal/internal.rs
@@ -6,11 +6,28 @@
 // Prefer importing stable_mir over internal rustc constructs to make this file more readable.
 use crate::rustc_smir::Tables;
 use rustc_middle::ty::{self as rustc_ty, Ty as InternalTy};
-use stable_mir::ty::{Const, GenericArgKind, GenericArgs, Region, Ty};
-use stable_mir::DefId;
+use rustc_span::Symbol;
+use stable_mir::abi::Layout;
+use stable_mir::mir::alloc::AllocId;
+use stable_mir::mir::mono::{Instance, MonoItem, StaticDef};
+use stable_mir::mir::{Mutability, Safety};
+use stable_mir::ty::{
+    Abi, AdtDef, Binder, BoundRegionKind, BoundTyKind, BoundVariableKind, ClosureKind, Const,
+    DynKind, ExistentialPredicate, ExistentialProjection, ExistentialTraitRef, FloatTy, FnSig,
+    GenericArgKind, GenericArgs, IndexedVal, IntTy, Movability, Region, RigidTy, Span, TermKind,
+    TraitRef, Ty, UintTy, VariantDef, VariantIdx,
+};
+use stable_mir::{CrateItem, DefId};
 
 use super::RustcInternal;
 
+impl<'tcx> RustcInternal<'tcx> for CrateItem {
+    type T = rustc_span::def_id::DefId;
+    fn internal(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        self.0.internal(tables)
+    }
+}
+
 impl<'tcx> RustcInternal<'tcx> for DefId {
     type T = rustc_span::def_id::DefId;
     fn internal(&self, tables: &mut Tables<'tcx>) -> Self::T {
@@ -38,8 +55,9 @@ impl<'tcx> RustcInternal<'tcx> for GenericArgKind {
 
 impl<'tcx> RustcInternal<'tcx> for Region {
     type T = rustc_ty::Region<'tcx>;
-    fn internal(&self, _tables: &mut Tables<'tcx>) -> Self::T {
-        todo!()
+    fn internal(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        // Cannot recover region. Use erased instead.
+        tables.tcx.lifetimes.re_erased
     }
 }
 
@@ -50,6 +68,154 @@ impl<'tcx> RustcInternal<'tcx> for Ty {
     }
 }
 
+impl<'tcx> RustcInternal<'tcx> for RigidTy {
+    type T = rustc_ty::TyKind<'tcx>;
+
+    fn internal(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        match self {
+            RigidTy::Bool => rustc_ty::TyKind::Bool,
+            RigidTy::Char => rustc_ty::TyKind::Char,
+            RigidTy::Int(int_ty) => rustc_ty::TyKind::Int(int_ty.internal(tables)),
+            RigidTy::Uint(uint_ty) => rustc_ty::TyKind::Uint(uint_ty.internal(tables)),
+            RigidTy::Float(float_ty) => rustc_ty::TyKind::Float(float_ty.internal(tables)),
+            RigidTy::Never => rustc_ty::TyKind::Never,
+            RigidTy::Array(ty, cnst) => {
+                rustc_ty::TyKind::Array(ty.internal(tables), ty_const(cnst, tables))
+            }
+            RigidTy::Adt(def, args) => {
+                rustc_ty::TyKind::Adt(def.internal(tables), args.internal(tables))
+            }
+            RigidTy::Str => rustc_ty::TyKind::Str,
+            RigidTy::Slice(ty) => rustc_ty::TyKind::Slice(ty.internal(tables)),
+            RigidTy::RawPtr(ty, mutability) => rustc_ty::TyKind::RawPtr(rustc_ty::TypeAndMut {
+                ty: ty.internal(tables),
+                mutbl: mutability.internal(tables),
+            }),
+            RigidTy::Ref(region, ty, mutability) => rustc_ty::TyKind::Ref(
+                region.internal(tables),
+                ty.internal(tables),
+                mutability.internal(tables),
+            ),
+            RigidTy::Foreign(def) => rustc_ty::TyKind::Foreign(def.0.internal(tables)),
+            RigidTy::FnDef(def, args) => {
+                rustc_ty::TyKind::FnDef(def.0.internal(tables), args.internal(tables))
+            }
+            RigidTy::FnPtr(sig) => rustc_ty::TyKind::FnPtr(sig.internal(tables)),
+            RigidTy::Closure(def, args) => {
+                rustc_ty::TyKind::Closure(def.0.internal(tables), args.internal(tables))
+            }
+            RigidTy::Coroutine(def, args, mov) => rustc_ty::TyKind::Coroutine(
+                def.0.internal(tables),
+                args.internal(tables),
+                mov.internal(tables),
+            ),
+            RigidTy::CoroutineWitness(def, args) => {
+                rustc_ty::TyKind::CoroutineWitness(def.0.internal(tables), args.internal(tables))
+            }
+            RigidTy::Dynamic(predicate, region, dyn_kind) => rustc_ty::TyKind::Dynamic(
+                tables.tcx.mk_poly_existential_predicates(&predicate.internal(tables)),
+                region.internal(tables),
+                dyn_kind.internal(tables),
+            ),
+            RigidTy::Tuple(tys) => {
+                rustc_ty::TyKind::Tuple(tables.tcx.mk_type_list(&tys.internal(tables)))
+            }
+        }
+    }
+}
+
+impl<'tcx> RustcInternal<'tcx> for IntTy {
+    type T = rustc_ty::IntTy;
+
+    fn internal(&self, _tables: &mut Tables<'tcx>) -> Self::T {
+        match self {
+            IntTy::Isize => rustc_ty::IntTy::Isize,
+            IntTy::I8 => rustc_ty::IntTy::I8,
+            IntTy::I16 => rustc_ty::IntTy::I16,
+            IntTy::I32 => rustc_ty::IntTy::I32,
+            IntTy::I64 => rustc_ty::IntTy::I64,
+            IntTy::I128 => rustc_ty::IntTy::I128,
+        }
+    }
+}
+
+impl<'tcx> RustcInternal<'tcx> for UintTy {
+    type T = rustc_ty::UintTy;
+
+    fn internal(&self, _tables: &mut Tables<'tcx>) -> Self::T {
+        match self {
+            UintTy::Usize => rustc_ty::UintTy::Usize,
+            UintTy::U8 => rustc_ty::UintTy::U8,
+            UintTy::U16 => rustc_ty::UintTy::U16,
+            UintTy::U32 => rustc_ty::UintTy::U32,
+            UintTy::U64 => rustc_ty::UintTy::U64,
+            UintTy::U128 => rustc_ty::UintTy::U128,
+        }
+    }
+}
+
+impl<'tcx> RustcInternal<'tcx> for FloatTy {
+    type T = rustc_ty::FloatTy;
+
+    fn internal(&self, _tables: &mut Tables<'tcx>) -> Self::T {
+        match self {
+            FloatTy::F32 => rustc_ty::FloatTy::F32,
+            FloatTy::F64 => rustc_ty::FloatTy::F64,
+        }
+    }
+}
+
+impl<'tcx> RustcInternal<'tcx> for Mutability {
+    type T = rustc_ty::Mutability;
+
+    fn internal(&self, _tables: &mut Tables<'tcx>) -> Self::T {
+        match self {
+            Mutability::Not => rustc_ty::Mutability::Not,
+            Mutability::Mut => rustc_ty::Mutability::Mut,
+        }
+    }
+}
+
+impl<'tcx> RustcInternal<'tcx> for Movability {
+    type T = rustc_ty::Movability;
+
+    fn internal(&self, _tables: &mut Tables<'tcx>) -> Self::T {
+        match self {
+            Movability::Static => rustc_ty::Movability::Static,
+            Movability::Movable => rustc_ty::Movability::Movable,
+        }
+    }
+}
+
+impl<'tcx> RustcInternal<'tcx> for FnSig {
+    type T = rustc_ty::FnSig<'tcx>;
+
+    fn internal(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        rustc_ty::FnSig {
+            inputs_and_output: tables.tcx.mk_type_list(&self.inputs_and_output.internal(tables)),
+            c_variadic: self.c_variadic,
+            unsafety: self.unsafety.internal(tables),
+            abi: self.abi.internal(tables),
+        }
+    }
+}
+
+impl<'tcx> RustcInternal<'tcx> for VariantIdx {
+    type T = rustc_target::abi::VariantIdx;
+
+    fn internal(&self, _tables: &mut Tables<'tcx>) -> Self::T {
+        rustc_target::abi::VariantIdx::from(self.to_index())
+    }
+}
+
+impl<'tcx> RustcInternal<'tcx> for VariantDef {
+    type T = &'tcx rustc_ty::VariantDef;
+
+    fn internal(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        self.adt_def.internal(tables).variant(self.idx.internal(tables))
+    }
+}
+
 fn ty_const<'tcx>(constant: &Const, tables: &mut Tables<'tcx>) -> rustc_ty::Const<'tcx> {
     match constant.internal(tables) {
         rustc_middle::mir::Const::Ty(c) => c,
@@ -65,3 +231,273 @@ impl<'tcx> RustcInternal<'tcx> for Const {
         tables.constants[self.id]
     }
 }
+
+impl<'tcx> RustcInternal<'tcx> for MonoItem {
+    type T = rustc_middle::mir::mono::MonoItem<'tcx>;
+
+    fn internal(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        use rustc_middle::mir::mono as rustc_mono;
+        match self {
+            MonoItem::Fn(instance) => rustc_mono::MonoItem::Fn(instance.internal(tables)),
+            MonoItem::Static(def) => rustc_mono::MonoItem::Static(def.internal(tables)),
+            MonoItem::GlobalAsm(_) => {
+                unimplemented!()
+            }
+        }
+    }
+}
+
+impl<'tcx> RustcInternal<'tcx> for Instance {
+    type T = rustc_ty::Instance<'tcx>;
+
+    fn internal(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        tables.instances[self.def]
+    }
+}
+
+impl<'tcx> RustcInternal<'tcx> for StaticDef {
+    type T = rustc_span::def_id::DefId;
+
+    fn internal(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        self.0.internal(tables)
+    }
+}
+
+#[allow(rustc::usage_of_qualified_ty)]
+impl<'tcx, T> RustcInternal<'tcx> for Binder<T>
+where
+    T: RustcInternal<'tcx>,
+    T::T: rustc_ty::TypeVisitable<rustc_ty::TyCtxt<'tcx>>,
+{
+    type T = rustc_ty::Binder<'tcx, T::T>;
+
+    fn internal(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        rustc_ty::Binder::bind_with_vars(
+            self.value.internal(tables),
+            tables.tcx.mk_bound_variable_kinds_from_iter(
+                self.bound_vars.iter().map(|bound| bound.internal(tables)),
+            ),
+        )
+    }
+}
+
+impl<'tcx> RustcInternal<'tcx> for BoundVariableKind {
+    type T = rustc_ty::BoundVariableKind;
+
+    fn internal(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        match self {
+            BoundVariableKind::Ty(kind) => rustc_ty::BoundVariableKind::Ty(match kind {
+                BoundTyKind::Anon => rustc_ty::BoundTyKind::Anon,
+                BoundTyKind::Param(def, symbol) => {
+                    rustc_ty::BoundTyKind::Param(def.0.internal(tables), Symbol::intern(symbol))
+                }
+            }),
+            BoundVariableKind::Region(kind) => rustc_ty::BoundVariableKind::Region(match kind {
+                BoundRegionKind::BrAnon => rustc_ty::BoundRegionKind::BrAnon,
+                BoundRegionKind::BrNamed(def, symbol) => rustc_ty::BoundRegionKind::BrNamed(
+                    def.0.internal(tables),
+                    Symbol::intern(symbol),
+                ),
+                BoundRegionKind::BrEnv => rustc_ty::BoundRegionKind::BrEnv,
+            }),
+            BoundVariableKind::Const => rustc_ty::BoundVariableKind::Const,
+        }
+    }
+}
+
+impl<'tcx> RustcInternal<'tcx> for DynKind {
+    type T = rustc_ty::DynKind;
+
+    fn internal(&self, _tables: &mut Tables<'tcx>) -> Self::T {
+        match self {
+            DynKind::Dyn => rustc_ty::DynKind::Dyn,
+            DynKind::DynStar => rustc_ty::DynKind::DynStar,
+        }
+    }
+}
+
+impl<'tcx> RustcInternal<'tcx> for ExistentialPredicate {
+    type T = rustc_ty::ExistentialPredicate<'tcx>;
+
+    fn internal(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        match self {
+            ExistentialPredicate::Trait(trait_ref) => {
+                rustc_ty::ExistentialPredicate::Trait(trait_ref.internal(tables))
+            }
+            ExistentialPredicate::Projection(proj) => {
+                rustc_ty::ExistentialPredicate::Projection(proj.internal(tables))
+            }
+            ExistentialPredicate::AutoTrait(trait_def) => {
+                rustc_ty::ExistentialPredicate::AutoTrait(trait_def.0.internal(tables))
+            }
+        }
+    }
+}
+
+impl<'tcx> RustcInternal<'tcx> for ExistentialProjection {
+    type T = rustc_ty::ExistentialProjection<'tcx>;
+
+    fn internal(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        rustc_ty::ExistentialProjection {
+            def_id: self.def_id.0.internal(tables),
+            args: self.generic_args.internal(tables),
+            term: self.term.internal(tables),
+        }
+    }
+}
+
+impl<'tcx> RustcInternal<'tcx> for TermKind {
+    type T = rustc_ty::Term<'tcx>;
+
+    fn internal(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        match self {
+            TermKind::Type(ty) => ty.internal(tables).into(),
+            TermKind::Const(const_) => ty_const(const_, tables).into(),
+        }
+    }
+}
+
+impl<'tcx> RustcInternal<'tcx> for ExistentialTraitRef {
+    type T = rustc_ty::ExistentialTraitRef<'tcx>;
+
+    fn internal(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        rustc_ty::ExistentialTraitRef {
+            def_id: self.def_id.0.internal(tables),
+            args: self.generic_args.internal(tables),
+        }
+    }
+}
+
+impl<'tcx> RustcInternal<'tcx> for TraitRef {
+    type T = rustc_ty::TraitRef<'tcx>;
+
+    fn internal(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        rustc_ty::TraitRef::new(
+            tables.tcx,
+            self.def_id.0.internal(tables),
+            self.args().internal(tables),
+        )
+    }
+}
+
+impl<'tcx> RustcInternal<'tcx> for AllocId {
+    type T = rustc_middle::mir::interpret::AllocId;
+    fn internal(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        tables.alloc_ids[*self]
+    }
+}
+
+impl<'tcx> RustcInternal<'tcx> for ClosureKind {
+    type T = rustc_ty::ClosureKind;
+
+    fn internal(&self, _tables: &mut Tables<'tcx>) -> Self::T {
+        match self {
+            ClosureKind::Fn => rustc_ty::ClosureKind::Fn,
+            ClosureKind::FnMut => rustc_ty::ClosureKind::FnMut,
+            ClosureKind::FnOnce => rustc_ty::ClosureKind::FnOnce,
+        }
+    }
+}
+
+impl<'tcx> RustcInternal<'tcx> for AdtDef {
+    type T = rustc_ty::AdtDef<'tcx>;
+    fn internal(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        tables.tcx.adt_def(self.0.internal(&mut *tables))
+    }
+}
+
+impl<'tcx> RustcInternal<'tcx> for Abi {
+    type T = rustc_target::spec::abi::Abi;
+
+    fn internal(&self, _tables: &mut Tables<'tcx>) -> Self::T {
+        match *self {
+            Abi::Rust => rustc_target::spec::abi::Abi::Rust,
+            Abi::C { unwind } => rustc_target::spec::abi::Abi::C { unwind },
+            Abi::Cdecl { unwind } => rustc_target::spec::abi::Abi::Cdecl { unwind },
+            Abi::Stdcall { unwind } => rustc_target::spec::abi::Abi::Stdcall { unwind },
+            Abi::Fastcall { unwind } => rustc_target::spec::abi::Abi::Fastcall { unwind },
+            Abi::Vectorcall { unwind } => rustc_target::spec::abi::Abi::Vectorcall { unwind },
+            Abi::Thiscall { unwind } => rustc_target::spec::abi::Abi::Thiscall { unwind },
+            Abi::Aapcs { unwind } => rustc_target::spec::abi::Abi::Aapcs { unwind },
+            Abi::Win64 { unwind } => rustc_target::spec::abi::Abi::Win64 { unwind },
+            Abi::SysV64 { unwind } => rustc_target::spec::abi::Abi::SysV64 { unwind },
+            Abi::PtxKernel => rustc_target::spec::abi::Abi::PtxKernel,
+            Abi::Msp430Interrupt => rustc_target::spec::abi::Abi::Msp430Interrupt,
+            Abi::X86Interrupt => rustc_target::spec::abi::Abi::X86Interrupt,
+            Abi::AmdGpuKernel => rustc_target::spec::abi::Abi::AmdGpuKernel,
+            Abi::EfiApi => rustc_target::spec::abi::Abi::EfiApi,
+            Abi::AvrInterrupt => rustc_target::spec::abi::Abi::AvrInterrupt,
+            Abi::AvrNonBlockingInterrupt => rustc_target::spec::abi::Abi::AvrNonBlockingInterrupt,
+            Abi::CCmseNonSecureCall => rustc_target::spec::abi::Abi::CCmseNonSecureCall,
+            Abi::Wasm => rustc_target::spec::abi::Abi::Wasm,
+            Abi::System { unwind } => rustc_target::spec::abi::Abi::System { unwind },
+            Abi::RustIntrinsic => rustc_target::spec::abi::Abi::RustIntrinsic,
+            Abi::RustCall => rustc_target::spec::abi::Abi::RustCall,
+            Abi::PlatformIntrinsic => rustc_target::spec::abi::Abi::PlatformIntrinsic,
+            Abi::Unadjusted => rustc_target::spec::abi::Abi::Unadjusted,
+            Abi::RustCold => rustc_target::spec::abi::Abi::RustCold,
+            Abi::RiscvInterruptM => rustc_target::spec::abi::Abi::RiscvInterruptM,
+            Abi::RiscvInterruptS => rustc_target::spec::abi::Abi::RiscvInterruptS,
+        }
+    }
+}
+
+impl<'tcx> RustcInternal<'tcx> for Safety {
+    type T = rustc_hir::Unsafety;
+
+    fn internal(&self, _tables: &mut Tables<'tcx>) -> Self::T {
+        match self {
+            Safety::Unsafe => rustc_hir::Unsafety::Unsafe,
+            Safety::Normal => rustc_hir::Unsafety::Normal,
+        }
+    }
+}
+
+impl<'tcx> RustcInternal<'tcx> for Span {
+    type T = rustc_span::Span;
+
+    fn internal(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        tables[*self]
+    }
+}
+
+impl<'tcx> RustcInternal<'tcx> for Layout {
+    type T = rustc_target::abi::Layout<'tcx>;
+
+    fn internal(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        tables.layouts[*self]
+    }
+}
+
+impl<'tcx, T> RustcInternal<'tcx> for &T
+where
+    T: RustcInternal<'tcx>,
+{
+    type T = T::T;
+
+    fn internal(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        (*self).internal(tables)
+    }
+}
+
+impl<'tcx, T> RustcInternal<'tcx> for Option<T>
+where
+    T: RustcInternal<'tcx>,
+{
+    type T = Option<T::T>;
+
+    fn internal(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        self.as_ref().map(|inner| inner.internal(tables))
+    }
+}
+
+impl<'tcx, T> RustcInternal<'tcx> for Vec<T>
+where
+    T: RustcInternal<'tcx>,
+{
+    type T = Vec<T::T>;
+
+    fn internal(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        self.iter().map(|e| e.internal(tables)).collect()
+    }
+}
diff --git a/compiler/rustc_smir/src/rustc_internal/mod.rs b/compiler/rustc_smir/src/rustc_internal/mod.rs
index 4d2a51822..4bac98909 100644
--- a/compiler/rustc_smir/src/rustc_internal/mod.rs
+++ b/compiler/rustc_smir/src/rustc_internal/mod.rs
@@ -3,7 +3,7 @@
 //! For that, we define APIs that will temporarily be public to 3P that exposes rustc internal APIs
 //! until stable MIR is complete.
 
-use crate::rustc_smir::{Stable, Tables, TablesWrapper};
+use crate::rustc_smir::{context::TablesWrapper, Stable, Tables};
 use rustc_data_structures::fx;
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_middle::mir::interpret::AllocId;
@@ -12,7 +12,9 @@ use rustc_middle::ty::TyCtxt;
 use rustc_span::def_id::{CrateNum, DefId};
 use rustc_span::Span;
 use scoped_tls::scoped_thread_local;
+use stable_mir::abi::Layout;
 use stable_mir::ty::IndexedVal;
+use stable_mir::Error;
 use std::cell::Cell;
 use std::cell::RefCell;
 use std::fmt::Debug;
@@ -20,12 +22,13 @@ use std::hash::Hash;
 use std::ops::Index;
 
 mod internal;
+pub mod pretty;
 
-pub fn stable<'tcx, S: Stable<'tcx>>(item: &S) -> S::T {
+pub fn stable<'tcx, S: Stable<'tcx>>(item: S) -> S::T {
     with_tables(|tables| item.stable(tables))
 }
 
-pub fn internal<'tcx, S: RustcInternal<'tcx>>(item: &S) -> S::T {
+pub fn internal<'tcx, S: RustcInternal<'tcx>>(item: S) -> S::T {
     with_tables(|tables| item.internal(tables))
 }
 
@@ -104,6 +107,10 @@ impl<'tcx> Tables<'tcx> {
         stable_mir::ty::RegionDef(self.create_def_id(did))
     }
 
+    pub fn coroutine_witness_def(&mut self, did: DefId) -> stable_mir::ty::CoroutineWitnessDef {
+        stable_mir::ty::CoroutineWitnessDef(self.create_def_id(did))
+    }
+
     pub fn prov(&mut self, aid: AllocId) -> stable_mir::ty::Prov {
         stable_mir::ty::Prov(self.create_alloc_id(aid))
     }
@@ -112,7 +119,7 @@ impl<'tcx> Tables<'tcx> {
         self.def_ids.create_or_fetch(did)
     }
 
-    fn create_alloc_id(&mut self, aid: AllocId) -> stable_mir::AllocId {
+    pub(crate) fn create_alloc_id(&mut self, aid: AllocId) -> stable_mir::mir::alloc::AllocId {
         self.alloc_ids.create_or_fetch(aid)
     }
 
@@ -130,6 +137,10 @@ impl<'tcx> Tables<'tcx> {
     pub(crate) fn static_def(&mut self, did: DefId) -> stable_mir::mir::mono::StaticDef {
         stable_mir::mir::mono::StaticDef(self.create_def_id(did))
     }
+
+    pub(crate) fn layout_id(&mut self, layout: rustc_target::abi::Layout<'tcx>) -> Layout {
+        self.layouts.create_or_fetch(layout)
+    }
 }
 
 pub fn crate_num(item: &stable_mir::Crate) -> CrateNum {
@@ -140,12 +151,13 @@ pub fn crate_num(item: &stable_mir::Crate) -> CrateNum {
 // datastructures and stable MIR datastructures
 scoped_thread_local! (static TLV: Cell<*const ()>);
 
-pub(crate) fn init<'tcx>(tables: &TablesWrapper<'tcx>, f: impl FnOnce()) {
+pub(crate) fn init<'tcx, F, T>(tables: &TablesWrapper<'tcx>, f: F) -> T
+where
+    F: FnOnce() -> T,
+{
     assert!(!TLV.is_set());
     let ptr = tables as *const _ as *const ();
-    TLV.set(&Cell::new(ptr), || {
-        f();
-    });
+    TLV.set(&Cell::new(ptr), || f())
 }
 
 /// Loads the current context and calls a function with it.
@@ -161,7 +173,10 @@ pub(crate) fn with_tables<'tcx, R>(f: impl FnOnce(&mut Tables<'tcx>) -> R) -> R
     })
 }
 
-pub fn run(tcx: TyCtxt<'_>, f: impl FnOnce()) {
+pub fn run<F, T>(tcx: TyCtxt<'_>, f: F) -> Result<T, Error>
+where
+    F: FnOnce() -> T,
+{
     let tables = TablesWrapper(RefCell::new(Tables {
         tcx,
         def_ids: IndexMap::default(),
@@ -170,8 +185,9 @@ pub fn run(tcx: TyCtxt<'_>, f: impl FnOnce()) {
         types: IndexMap::default(),
         instances: IndexMap::default(),
         constants: IndexMap::default(),
+        layouts: IndexMap::default(),
     }));
-    stable_mir::run(&tables, || init(&tables, f));
+    stable_mir::compiler_interface::run(&tables, || init(&tables, f))
 }
 
 #[macro_export]
@@ -237,7 +253,8 @@ macro_rules! run {
                 queries.global_ctxt().unwrap().enter(|tcx| {
                     rustc_internal::run(tcx, || {
                         self.result = Some((self.callback)(tcx));
-                    });
+                    })
+                    .unwrap();
                     if self.result.as_ref().is_some_and(|val| val.is_continue()) {
                         Compilation::Continue
                     } else {
diff --git a/compiler/rustc_smir/src/rustc_internal/pretty.rs b/compiler/rustc_smir/src/rustc_internal/pretty.rs
new file mode 100644
index 000000000..3ef2d28ea
--- /dev/null
+++ b/compiler/rustc_smir/src/rustc_internal/pretty.rs
@@ -0,0 +1,20 @@
+use std::io;
+
+use super::run;
+use rustc_middle::ty::TyCtxt;
+
+pub fn write_smir_pretty<'tcx, W: io::Write>(tcx: TyCtxt<'tcx>, w: &mut W) -> io::Result<()> {
+    writeln!(
+        w,
+        "// WARNING: This is highly experimental output it's intended for stable-mir developers only."
+    )?;
+    writeln!(
+        w,
+        "// If you find a bug or want to improve the output open a issue at https://github.com/rust-lang/project-stable-mir."
+    )?;
+    let _ = run(tcx, || {
+        let items = stable_mir::all_local_items();
+        let _ = items.iter().map(|item| -> io::Result<()> { item.dump(w) }).collect::<Vec<_>>();
+    });
+    Ok(())
+}
diff --git a/compiler/rustc_smir/src/rustc_smir/alloc.rs b/compiler/rustc_smir/src/rustc_smir/alloc.rs
index 63a2a1450..48cb164c3 100644
--- a/compiler/rustc_smir/src/rustc_smir/alloc.rs
+++ b/compiler/rustc_smir/src/rustc_smir/alloc.rs
@@ -2,6 +2,7 @@ use rustc_middle::mir::{
     interpret::{alloc_range, AllocRange, Pointer},
     ConstValue,
 };
+use stable_mir::Error;
 
 use crate::rustc_smir::{Stable, Tables};
 use stable_mir::mir::Mutability;
@@ -26,33 +27,47 @@ pub fn new_allocation<'tcx>(
     const_value: ConstValue<'tcx>,
     tables: &mut Tables<'tcx>,
 ) -> Allocation {
-    match const_value {
+    try_new_allocation(ty, const_value, tables).unwrap()
+}
+
+#[allow(rustc::usage_of_qualified_ty)]
+pub fn try_new_allocation<'tcx>(
+    ty: rustc_middle::ty::Ty<'tcx>,
+    const_value: ConstValue<'tcx>,
+    tables: &mut Tables<'tcx>,
+) -> Result<Allocation, Error> {
+    Ok(match const_value {
         ConstValue::Scalar(scalar) => {
             let size = scalar.size();
             let align = tables
                 .tcx
                 .layout_of(rustc_middle::ty::ParamEnv::reveal_all().and(ty))
-                .unwrap()
+                .map_err(|e| e.stable(tables))?
                 .align;
             let mut allocation = rustc_middle::mir::interpret::Allocation::uninit(size, align.abi);
             allocation
                 .write_scalar(&tables.tcx, alloc_range(rustc_target::abi::Size::ZERO, size), scalar)
-                .unwrap();
+                .map_err(|e| e.stable(tables))?;
             allocation.stable(tables)
         }
         ConstValue::ZeroSized => {
-            let align =
-                tables.tcx.layout_of(rustc_middle::ty::ParamEnv::empty().and(ty)).unwrap().align;
+            let align = tables
+                .tcx
+                .layout_of(rustc_middle::ty::ParamEnv::empty().and(ty))
+                .map_err(|e| e.stable(tables))?
+                .align;
             new_empty_allocation(align.abi)
         }
         ConstValue::Slice { data, meta } => {
             let alloc_id = tables.tcx.reserve_and_set_memory_alloc(data);
-            let ptr = Pointer::new(alloc_id, rustc_target::abi::Size::ZERO);
+            let ptr = Pointer::new(alloc_id.into(), rustc_target::abi::Size::ZERO);
             let scalar_ptr = rustc_middle::mir::interpret::Scalar::from_pointer(ptr, &tables.tcx);
             let scalar_meta =
                 rustc_middle::mir::interpret::Scalar::from_target_usize(meta, &tables.tcx);
-            let layout =
-                tables.tcx.layout_of(rustc_middle::ty::ParamEnv::reveal_all().and(ty)).unwrap();
+            let layout = tables
+                .tcx
+                .layout_of(rustc_middle::ty::ParamEnv::reveal_all().and(ty))
+                .map_err(|e| e.stable(tables))?;
             let mut allocation =
                 rustc_middle::mir::interpret::Allocation::uninit(layout.size, layout.align.abi);
             allocation
@@ -61,14 +76,14 @@ pub fn new_allocation<'tcx>(
                     alloc_range(rustc_target::abi::Size::ZERO, tables.tcx.data_layout.pointer_size),
                     scalar_ptr,
                 )
-                .unwrap();
+                .map_err(|e| e.stable(tables))?;
             allocation
                 .write_scalar(
                     &tables.tcx,
                     alloc_range(tables.tcx.data_layout.pointer_size, scalar_meta.size()),
                     scalar_meta,
                 )
-                .unwrap();
+                .map_err(|e| e.stable(tables))?;
             allocation.stable(tables)
         }
         ConstValue::Indirect { alloc_id, offset } => {
@@ -76,11 +91,11 @@ pub fn new_allocation<'tcx>(
             let ty_size = tables
                 .tcx
                 .layout_of(rustc_middle::ty::ParamEnv::reveal_all().and(ty))
-                .unwrap()
+                .map_err(|e| e.stable(tables))?
                 .size;
             allocation_filter(&alloc.0, alloc_range(offset, ty_size), tables)
         }
-    }
+    })
 }
 
 /// Creates an `Allocation` only from information within the `AllocRange`.
@@ -112,7 +127,10 @@ pub(super) fn allocation_filter<'tcx>(
         .iter()
         .filter(|a| a.0 >= alloc_range.start && a.0 <= alloc_range.end())
     {
-        ptrs.push((offset.bytes_usize() - alloc_range.start.bytes_usize(), tables.prov(*prov)));
+        ptrs.push((
+            offset.bytes_usize() - alloc_range.start.bytes_usize(),
+            tables.prov(prov.alloc_id()),
+        ));
     }
     Allocation {
         bytes: bytes,
diff --git a/compiler/rustc_smir/src/rustc_smir/builder.rs b/compiler/rustc_smir/src/rustc_smir/builder.rs
index 8ff3958da..039bdec4c 100644
--- a/compiler/rustc_smir/src/rustc_smir/builder.rs
+++ b/compiler/rustc_smir/src/rustc_smir/builder.rs
@@ -6,7 +6,7 @@
 use crate::rustc_smir::{Stable, Tables};
 use rustc_middle::mir;
 use rustc_middle::mir::visit::MutVisitor;
-use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_middle::ty::{self, GenericArgsRef, Ty, TyCtxt};
 
 /// Builds a monomorphic body for a given instance.
 pub struct BodyBuilder<'tcx> {
@@ -19,10 +19,15 @@ impl<'tcx> BodyBuilder<'tcx> {
         BodyBuilder { tcx, instance }
     }
 
+    /// Build a stable monomorphic body for a given instance based on the MIR body.
+    ///
+    /// Note that we skip instantiation for static and constants. Trying to do so can cause ICE.
+    ///
+    /// We do monomorphize non-generic functions to eval unevaluated constants.
     pub fn build(mut self, tables: &mut Tables<'tcx>) -> stable_mir::mir::Body {
         let mut body = self.tcx.instance_mir(self.instance.def).clone();
-        let generics = self.tcx.generics_of(self.instance.def_id());
-        if generics.requires_monomorphization(self.tcx) {
+        if self.tcx.def_kind(self.instance.def_id()).is_fn_like() || !self.instance.args.is_empty()
+        {
             self.visit_body(&mut body);
         }
         body.stable(tables)
@@ -49,6 +54,24 @@ impl<'tcx> MutVisitor<'tcx> for BodyBuilder<'tcx> {
         *ty = self.monomorphize(*ty);
     }
 
+    fn visit_constant(&mut self, constant: &mut mir::ConstOperand<'tcx>, location: mir::Location) {
+        let const_ = self.monomorphize(constant.const_);
+        let val = match const_.eval(self.tcx, ty::ParamEnv::reveal_all(), None) {
+            Ok(v) => v,
+            Err(mir::interpret::ErrorHandled::Reported(..)) => return,
+            Err(mir::interpret::ErrorHandled::TooGeneric(..)) => {
+                unreachable!("Failed to evaluate instance constant: {:?}", const_)
+            }
+        };
+        let ty = constant.ty();
+        constant.const_ = mir::Const::Val(val, ty);
+        self.super_constant(constant, location);
+    }
+
+    fn visit_args(&mut self, args: &mut GenericArgsRef<'tcx>, _: mir::Location) {
+        *args = self.monomorphize(*args);
+    }
+
     fn tcx(&self) -> TyCtxt<'tcx> {
         self.tcx
     }
diff --git a/compiler/rustc_smir/src/rustc_smir/context.rs b/compiler/rustc_smir/src/rustc_smir/context.rs
new file mode 100644
index 000000000..f84c466cc
--- /dev/null
+++ b/compiler/rustc_smir/src/rustc_smir/context.rs
@@ -0,0 +1,555 @@
+//! Implementation of `[stable_mir::compiler_interface::Context]` trait.
+//!
+//! This trait is currently the main interface between the Rust compiler,
+//! and the `stable_mir` crate.
+
+#![allow(rustc::usage_of_qualified_ty)]
+
+use rustc_abi::HasDataLayout;
+use rustc_middle::ty;
+use rustc_middle::ty::layout::{
+    FnAbiOf, FnAbiOfHelpers, HasParamEnv, HasTyCtxt, LayoutOf, LayoutOfHelpers,
+};
+use rustc_middle::ty::print::{with_forced_trimmed_paths, with_no_trimmed_paths};
+use rustc_middle::ty::{
+    GenericPredicates, Instance, List, ParamEnv, ScalarInt, TyCtxt, TypeVisitableExt, ValTree,
+};
+use rustc_span::def_id::LOCAL_CRATE;
+use stable_mir::abi::{FnAbi, Layout, LayoutShape};
+use stable_mir::compiler_interface::Context;
+use stable_mir::mir::alloc::GlobalAlloc;
+use stable_mir::mir::mono::{InstanceDef, StaticDef};
+use stable_mir::mir::Body;
+use stable_mir::target::{MachineInfo, MachineSize};
+use stable_mir::ty::{
+    AdtDef, AdtKind, Allocation, ClosureDef, ClosureKind, Const, FieldDef, FnDef, GenericArgs,
+    LineInfo, PolyFnSig, RigidTy, Span, Ty, TyKind, VariantDef,
+};
+use stable_mir::{Crate, CrateItem, DefId, Error, Filename, ItemKind, Symbol};
+use std::cell::RefCell;
+
+use crate::rustc_internal::{internal, RustcInternal};
+use crate::rustc_smir::builder::BodyBuilder;
+use crate::rustc_smir::{alloc, new_item_kind, smir_crate, Stable, Tables};
+
+impl<'tcx> Context for TablesWrapper<'tcx> {
+    fn target_info(&self) -> MachineInfo {
+        let mut tables = self.0.borrow_mut();
+        MachineInfo {
+            endian: tables.tcx.data_layout.endian.stable(&mut *tables),
+            pointer_width: MachineSize::from_bits(
+                tables.tcx.data_layout.pointer_size.bits().try_into().unwrap(),
+            ),
+        }
+    }
+
+    fn entry_fn(&self) -> Option<stable_mir::CrateItem> {
+        let mut tables = self.0.borrow_mut();
+        let tcx = tables.tcx;
+        Some(tables.crate_item(tcx.entry_fn(())?.0))
+    }
+
+    fn all_local_items(&self) -> stable_mir::CrateItems {
+        let mut tables = self.0.borrow_mut();
+        tables.tcx.mir_keys(()).iter().map(|item| tables.crate_item(item.to_def_id())).collect()
+    }
+
+    fn mir_body(&self, item: stable_mir::DefId) -> stable_mir::mir::Body {
+        let mut tables = self.0.borrow_mut();
+        let def_id = tables[item];
+        tables.tcx.instance_mir(rustc_middle::ty::InstanceDef::Item(def_id)).stable(&mut tables)
+    }
+
+    fn has_body(&self, def: DefId) -> bool {
+        let tables = self.0.borrow();
+        let def_id = tables[def];
+        tables.tcx.is_mir_available(def_id)
+    }
+
+    fn all_trait_decls(&self) -> stable_mir::TraitDecls {
+        let mut tables = self.0.borrow_mut();
+        tables
+            .tcx
+            .traits(LOCAL_CRATE)
+            .iter()
+            .map(|trait_def_id| tables.trait_def(*trait_def_id))
+            .collect()
+    }
+
+    fn trait_decl(&self, trait_def: &stable_mir::ty::TraitDef) -> stable_mir::ty::TraitDecl {
+        let mut tables = self.0.borrow_mut();
+        let def_id = tables[trait_def.0];
+        let trait_def = tables.tcx.trait_def(def_id);
+        trait_def.stable(&mut *tables)
+    }
+
+    fn all_trait_impls(&self) -> stable_mir::ImplTraitDecls {
+        let mut tables = self.0.borrow_mut();
+        tables
+            .tcx
+            .trait_impls_in_crate(LOCAL_CRATE)
+            .iter()
+            .map(|impl_def_id| tables.impl_def(*impl_def_id))
+            .collect()
+    }
+
+    fn trait_impl(&self, impl_def: &stable_mir::ty::ImplDef) -> stable_mir::ty::ImplTrait {
+        let mut tables = self.0.borrow_mut();
+        let def_id = tables[impl_def.0];
+        let impl_trait = tables.tcx.impl_trait_ref(def_id).unwrap();
+        impl_trait.stable(&mut *tables)
+    }
+
+    fn generics_of(&self, def_id: stable_mir::DefId) -> stable_mir::ty::Generics {
+        let mut tables = self.0.borrow_mut();
+        let def_id = tables[def_id];
+        let generics = tables.tcx.generics_of(def_id);
+        generics.stable(&mut *tables)
+    }
+
+    fn predicates_of(&self, def_id: stable_mir::DefId) -> stable_mir::ty::GenericPredicates {
+        let mut tables = self.0.borrow_mut();
+        let def_id = tables[def_id];
+        let GenericPredicates { parent, predicates } = tables.tcx.predicates_of(def_id);
+        stable_mir::ty::GenericPredicates {
+            parent: parent.map(|did| tables.trait_def(did)),
+            predicates: predicates
+                .iter()
+                .map(|(clause, span)| {
+                    (
+                        clause.as_predicate().kind().skip_binder().stable(&mut *tables),
+                        span.stable(&mut *tables),
+                    )
+                })
+                .collect(),
+        }
+    }
+
+    fn explicit_predicates_of(
+        &self,
+        def_id: stable_mir::DefId,
+    ) -> stable_mir::ty::GenericPredicates {
+        let mut tables = self.0.borrow_mut();
+        let def_id = tables[def_id];
+        let GenericPredicates { parent, predicates } = tables.tcx.explicit_predicates_of(def_id);
+        stable_mir::ty::GenericPredicates {
+            parent: parent.map(|did| tables.trait_def(did)),
+            predicates: predicates
+                .iter()
+                .map(|(clause, span)| {
+                    (
+                        clause.as_predicate().kind().skip_binder().stable(&mut *tables),
+                        span.stable(&mut *tables),
+                    )
+                })
+                .collect(),
+        }
+    }
+
+    fn local_crate(&self) -> stable_mir::Crate {
+        let tables = self.0.borrow();
+        smir_crate(tables.tcx, LOCAL_CRATE)
+    }
+
+    fn external_crates(&self) -> Vec<stable_mir::Crate> {
+        let tables = self.0.borrow();
+        tables.tcx.crates(()).iter().map(|crate_num| smir_crate(tables.tcx, *crate_num)).collect()
+    }
+
+    fn find_crates(&self, name: &str) -> Vec<stable_mir::Crate> {
+        let tables = self.0.borrow();
+        let crates: Vec<stable_mir::Crate> = [LOCAL_CRATE]
+            .iter()
+            .chain(tables.tcx.crates(()).iter())
+            .map(|crate_num| {
+                let crate_name = tables.tcx.crate_name(*crate_num).to_string();
+                (name == crate_name).then(|| smir_crate(tables.tcx, *crate_num))
+            })
+            .flatten()
+            .collect();
+        crates
+    }
+
+    fn def_name(&self, def_id: stable_mir::DefId, trimmed: bool) -> Symbol {
+        let tables = self.0.borrow();
+        if trimmed {
+            with_forced_trimmed_paths!(tables.tcx.def_path_str(tables[def_id]))
+        } else {
+            with_no_trimmed_paths!(tables.tcx.def_path_str(tables[def_id]))
+        }
+    }
+
+    fn span_to_string(&self, span: stable_mir::ty::Span) -> String {
+        let tables = self.0.borrow();
+        tables.tcx.sess.source_map().span_to_diagnostic_string(tables[span])
+    }
+
+    fn get_filename(&self, span: &Span) -> Filename {
+        let tables = self.0.borrow();
+        tables
+            .tcx
+            .sess
+            .source_map()
+            .span_to_filename(tables[*span])
+            .display(rustc_span::FileNameDisplayPreference::Local)
+            .to_string()
+    }
+
+    fn get_lines(&self, span: &Span) -> LineInfo {
+        let tables = self.0.borrow();
+        let lines = &tables.tcx.sess.source_map().span_to_location_info(tables[*span]);
+        LineInfo { start_line: lines.1, start_col: lines.2, end_line: lines.3, end_col: lines.4 }
+    }
+
+    fn item_kind(&self, item: CrateItem) -> ItemKind {
+        let tables = self.0.borrow();
+        new_item_kind(tables.tcx.def_kind(tables[item.0]))
+    }
+
+    fn is_foreign_item(&self, item: DefId) -> bool {
+        let tables = self.0.borrow();
+        tables.tcx.is_foreign_item(tables[item])
+    }
+
+    fn adt_kind(&self, def: AdtDef) -> AdtKind {
+        let mut tables = self.0.borrow_mut();
+        def.internal(&mut *tables).adt_kind().stable(&mut *tables)
+    }
+
+    fn adt_is_box(&self, def: AdtDef) -> bool {
+        let mut tables = self.0.borrow_mut();
+        def.internal(&mut *tables).is_box()
+    }
+
+    fn adt_is_simd(&self, def: AdtDef) -> bool {
+        let mut tables = self.0.borrow_mut();
+        def.internal(&mut *tables).repr().simd()
+    }
+
+    fn adt_is_cstr(&self, def: AdtDef) -> bool {
+        let mut tables = self.0.borrow_mut();
+        let def_id = def.0.internal(&mut *tables);
+        tables.tcx.lang_items().c_str() == Some(def_id)
+    }
+
+    fn fn_sig(&self, def: FnDef, args: &GenericArgs) -> PolyFnSig {
+        let mut tables = self.0.borrow_mut();
+        let def_id = def.0.internal(&mut *tables);
+        let sig = tables.tcx.fn_sig(def_id).instantiate(tables.tcx, args.internal(&mut *tables));
+        sig.stable(&mut *tables)
+    }
+
+    fn closure_sig(&self, args: &GenericArgs) -> PolyFnSig {
+        let mut tables = self.0.borrow_mut();
+        let args_ref = args.internal(&mut *tables);
+        let sig = args_ref.as_closure().sig();
+        sig.stable(&mut *tables)
+    }
+
+    fn adt_variants_len(&self, def: AdtDef) -> usize {
+        let mut tables = self.0.borrow_mut();
+        def.internal(&mut *tables).variants().len()
+    }
+
+    fn variant_name(&self, def: VariantDef) -> Symbol {
+        let mut tables = self.0.borrow_mut();
+        def.internal(&mut *tables).name.to_string()
+    }
+
+    fn variant_fields(&self, def: VariantDef) -> Vec<FieldDef> {
+        let mut tables = self.0.borrow_mut();
+        def.internal(&mut *tables).fields.iter().map(|f| f.stable(&mut *tables)).collect()
+    }
+
+    fn eval_target_usize(&self, cnst: &Const) -> Result<u64, Error> {
+        let mut tables = self.0.borrow_mut();
+        let mir_const = cnst.internal(&mut *tables);
+        mir_const
+            .try_eval_target_usize(tables.tcx, ParamEnv::empty())
+            .ok_or_else(|| Error::new(format!("Const `{cnst:?}` cannot be encoded as u64")))
+    }
+
+    fn usize_to_const(&self, val: u64) -> Result<Const, Error> {
+        let mut tables = self.0.borrow_mut();
+        let ty = tables.tcx.types.usize;
+        let size = tables.tcx.layout_of(ParamEnv::empty().and(ty)).unwrap().size;
+
+        let scalar = ScalarInt::try_from_uint(val, size).ok_or_else(|| {
+            Error::new(format!("Value overflow: cannot convert `{val}` to usize."))
+        })?;
+        Ok(rustc_middle::ty::Const::new_value(tables.tcx, ValTree::from_scalar_int(scalar), ty)
+            .stable(&mut *tables))
+    }
+
+    fn new_rigid_ty(&self, kind: RigidTy) -> stable_mir::ty::Ty {
+        let mut tables = self.0.borrow_mut();
+        let internal_kind = kind.internal(&mut *tables);
+        tables.tcx.mk_ty_from_kind(internal_kind).stable(&mut *tables)
+    }
+
+    fn new_box_ty(&self, ty: stable_mir::ty::Ty) -> stable_mir::ty::Ty {
+        let mut tables = self.0.borrow_mut();
+        let inner = ty.internal(&mut *tables);
+        ty::Ty::new_box(tables.tcx, inner).stable(&mut *tables)
+    }
+
+    fn def_ty(&self, item: stable_mir::DefId) -> stable_mir::ty::Ty {
+        let mut tables = self.0.borrow_mut();
+        tables.tcx.type_of(item.internal(&mut *tables)).instantiate_identity().stable(&mut *tables)
+    }
+
+    fn def_ty_with_args(&self, item: stable_mir::DefId, args: &GenericArgs) -> stable_mir::ty::Ty {
+        let mut tables = self.0.borrow_mut();
+        let args = args.internal(&mut *tables);
+        let def_ty = tables.tcx.type_of(item.internal(&mut *tables));
+        def_ty.instantiate(tables.tcx, args).stable(&mut *tables)
+    }
+
+    fn const_literal(&self, cnst: &stable_mir::ty::Const) -> String {
+        internal(cnst).to_string()
+    }
+
+    fn span_of_an_item(&self, def_id: stable_mir::DefId) -> Span {
+        let mut tables = self.0.borrow_mut();
+        tables.tcx.def_span(tables[def_id]).stable(&mut *tables)
+    }
+
+    fn ty_kind(&self, ty: stable_mir::ty::Ty) -> TyKind {
+        let mut tables = self.0.borrow_mut();
+        tables.types[ty].kind().stable(&mut *tables)
+    }
+
+    fn rigid_ty_discriminant_ty(&self, ty: &RigidTy) -> stable_mir::ty::Ty {
+        let mut tables = self.0.borrow_mut();
+        let internal_kind = ty.internal(&mut *tables);
+        let internal_ty = tables.tcx.mk_ty_from_kind(internal_kind);
+        internal_ty.discriminant_ty(tables.tcx).stable(&mut *tables)
+    }
+
+    fn instance_body(&self, def: InstanceDef) -> Option<Body> {
+        let mut tables = self.0.borrow_mut();
+        let instance = tables.instances[def];
+        tables
+            .has_body(instance)
+            .then(|| BodyBuilder::new(tables.tcx, instance).build(&mut *tables))
+    }
+
+    fn instance_ty(&self, def: InstanceDef) -> stable_mir::ty::Ty {
+        let mut tables = self.0.borrow_mut();
+        let instance = tables.instances[def];
+        assert!(!instance.has_non_region_param(), "{instance:?} needs further substitution");
+        instance.ty(tables.tcx, ParamEnv::reveal_all()).stable(&mut *tables)
+    }
+
+    fn instance_args(&self, def: InstanceDef) -> GenericArgs {
+        let mut tables = self.0.borrow_mut();
+        let instance = tables.instances[def];
+        instance.args.stable(&mut *tables)
+    }
+
+    fn instance_abi(&self, def: InstanceDef) -> Result<FnAbi, Error> {
+        let mut tables = self.0.borrow_mut();
+        let instance = tables.instances[def];
+        Ok(tables.fn_abi_of_instance(instance, List::empty())?.stable(&mut *tables))
+    }
+
+    fn instance_def_id(&self, def: InstanceDef) -> stable_mir::DefId {
+        let mut tables = self.0.borrow_mut();
+        let def_id = tables.instances[def].def_id();
+        tables.create_def_id(def_id)
+    }
+
+    fn instance_mangled_name(&self, instance: InstanceDef) -> Symbol {
+        let tables = self.0.borrow_mut();
+        let instance = tables.instances[instance];
+        tables.tcx.symbol_name(instance).name.to_string()
+    }
+
+    fn is_empty_drop_shim(&self, def: InstanceDef) -> bool {
+        let tables = self.0.borrow_mut();
+        let instance = tables.instances[def];
+        matches!(instance.def, ty::InstanceDef::DropGlue(_, None))
+    }
+
+    fn mono_instance(&self, def_id: stable_mir::DefId) -> stable_mir::mir::mono::Instance {
+        let mut tables = self.0.borrow_mut();
+        let def_id = tables[def_id];
+        Instance::mono(tables.tcx, def_id).stable(&mut *tables)
+    }
+
+    fn requires_monomorphization(&self, def_id: stable_mir::DefId) -> bool {
+        let tables = self.0.borrow();
+        let def_id = tables[def_id];
+        let generics = tables.tcx.generics_of(def_id);
+        let result = generics.requires_monomorphization(tables.tcx);
+        result
+    }
+
+    fn resolve_instance(
+        &self,
+        def: stable_mir::ty::FnDef,
+        args: &stable_mir::ty::GenericArgs,
+    ) -> Option<stable_mir::mir::mono::Instance> {
+        let mut tables = self.0.borrow_mut();
+        let def_id = def.0.internal(&mut *tables);
+        let args_ref = args.internal(&mut *tables);
+        match Instance::resolve(tables.tcx, ParamEnv::reveal_all(), def_id, args_ref) {
+            Ok(Some(instance)) => Some(instance.stable(&mut *tables)),
+            Ok(None) | Err(_) => None,
+        }
+    }
+
+    fn resolve_drop_in_place(&self, ty: stable_mir::ty::Ty) -> stable_mir::mir::mono::Instance {
+        let mut tables = self.0.borrow_mut();
+        let internal_ty = ty.internal(&mut *tables);
+        let instance = Instance::resolve_drop_in_place(tables.tcx, internal_ty);
+        instance.stable(&mut *tables)
+    }
+
+    fn resolve_for_fn_ptr(
+        &self,
+        def: FnDef,
+        args: &GenericArgs,
+    ) -> Option<stable_mir::mir::mono::Instance> {
+        let mut tables = self.0.borrow_mut();
+        let def_id = def.0.internal(&mut *tables);
+        let args_ref = args.internal(&mut *tables);
+        Instance::resolve_for_fn_ptr(tables.tcx, ParamEnv::reveal_all(), def_id, args_ref)
+            .stable(&mut *tables)
+    }
+
+    fn resolve_closure(
+        &self,
+        def: ClosureDef,
+        args: &GenericArgs,
+        kind: ClosureKind,
+    ) -> Option<stable_mir::mir::mono::Instance> {
+        let mut tables = self.0.borrow_mut();
+        let def_id = def.0.internal(&mut *tables);
+        let args_ref = args.internal(&mut *tables);
+        let closure_kind = kind.internal(&mut *tables);
+        Instance::resolve_closure(tables.tcx, def_id, args_ref, closure_kind).stable(&mut *tables)
+    }
+
+    fn eval_instance(&self, def: InstanceDef, const_ty: Ty) -> Result<Allocation, Error> {
+        let mut tables = self.0.borrow_mut();
+        let instance = tables.instances[def];
+        let result = tables.tcx.const_eval_instance(
+            ParamEnv::reveal_all(),
+            instance,
+            Some(tables.tcx.def_span(instance.def_id())),
+        );
+        result
+            .map(|const_val| {
+                alloc::try_new_allocation(const_ty.internal(&mut *tables), const_val, &mut *tables)
+            })
+            .map_err(|e| e.stable(&mut *tables))?
+    }
+
+    fn eval_static_initializer(&self, def: StaticDef) -> Result<Allocation, Error> {
+        let mut tables = self.0.borrow_mut();
+        let def_id = def.0.internal(&mut *tables);
+        tables.tcx.eval_static_initializer(def_id).stable(&mut *tables)
+    }
+
+    fn global_alloc(&self, alloc: stable_mir::mir::alloc::AllocId) -> GlobalAlloc {
+        let mut tables = self.0.borrow_mut();
+        let alloc_id = alloc.internal(&mut *tables);
+        tables.tcx.global_alloc(alloc_id).stable(&mut *tables)
+    }
+
+    fn vtable_allocation(
+        &self,
+        global_alloc: &GlobalAlloc,
+    ) -> Option<stable_mir::mir::alloc::AllocId> {
+        let mut tables = self.0.borrow_mut();
+        let GlobalAlloc::VTable(ty, trait_ref) = global_alloc else { return None };
+        let alloc_id = tables
+            .tcx
+            .vtable_allocation((ty.internal(&mut *tables), trait_ref.internal(&mut *tables)));
+        Some(alloc_id.stable(&mut *tables))
+    }
+
+    fn krate(&self, def_id: stable_mir::DefId) -> Crate {
+        let tables = self.0.borrow();
+        smir_crate(tables.tcx, tables[def_id].krate)
+    }
+
+    /// Retrieve the instance name for diagnostic messages.
+    ///
+    /// This will return the specialized name, e.g., `Vec<char>::new`.
+    fn instance_name(&self, def: InstanceDef, trimmed: bool) -> Symbol {
+        let tables = self.0.borrow_mut();
+        let instance = tables.instances[def];
+        if trimmed {
+            with_forced_trimmed_paths!(
+                tables.tcx.def_path_str_with_args(instance.def_id(), instance.args)
+            )
+        } else {
+            with_no_trimmed_paths!(
+                tables.tcx.def_path_str_with_args(instance.def_id(), instance.args)
+            )
+        }
+    }
+
+    fn ty_layout(&self, ty: Ty) -> Result<Layout, Error> {
+        let mut tables = self.0.borrow_mut();
+        let ty = ty.internal(&mut *tables);
+        let layout = tables.layout_of(ty)?.layout;
+        Ok(layout.stable(&mut *tables))
+    }
+
+    fn layout_shape(&self, id: Layout) -> LayoutShape {
+        let mut tables = self.0.borrow_mut();
+        id.internal(&mut *tables).0.stable(&mut *tables)
+    }
+}
+
+pub struct TablesWrapper<'tcx>(pub RefCell<Tables<'tcx>>);
+
+/// Implement error handling for extracting function ABI information.
+impl<'tcx> FnAbiOfHelpers<'tcx> for Tables<'tcx> {
+    type FnAbiOfResult = Result<&'tcx rustc_target::abi::call::FnAbi<'tcx, ty::Ty<'tcx>>, Error>;
+
+    #[inline]
+    fn handle_fn_abi_err(
+        &self,
+        err: ty::layout::FnAbiError<'tcx>,
+        _span: rustc_span::Span,
+        fn_abi_request: ty::layout::FnAbiRequest<'tcx>,
+    ) -> Error {
+        Error::new(format!("Failed to get ABI for `{fn_abi_request:?}`: {err:?}"))
+    }
+}
+
+impl<'tcx> LayoutOfHelpers<'tcx> for Tables<'tcx> {
+    type LayoutOfResult = Result<ty::layout::TyAndLayout<'tcx>, Error>;
+
+    #[inline]
+    fn handle_layout_err(
+        &self,
+        err: ty::layout::LayoutError<'tcx>,
+        _span: rustc_span::Span,
+        ty: ty::Ty<'tcx>,
+    ) -> Error {
+        Error::new(format!("Failed to get layout for `{ty}`: {err}"))
+    }
+}
+
+impl<'tcx> HasParamEnv<'tcx> for Tables<'tcx> {
+    fn param_env(&self) -> ty::ParamEnv<'tcx> {
+        ty::ParamEnv::reveal_all()
+    }
+}
+
+impl<'tcx> HasTyCtxt<'tcx> for Tables<'tcx> {
+    fn tcx(&self) -> TyCtxt<'tcx> {
+        self.tcx
+    }
+}
+
+impl<'tcx> HasDataLayout for Tables<'tcx> {
+    fn data_layout(&self) -> &rustc_abi::TargetDataLayout {
+        self.tcx.data_layout()
+    }
+}
diff --git a/compiler/rustc_smir/src/rustc_smir/convert/abi.rs b/compiler/rustc_smir/src/rustc_smir/convert/abi.rs
new file mode 100644
index 000000000..632e97b32
--- /dev/null
+++ b/compiler/rustc_smir/src/rustc_smir/convert/abi.rs
@@ -0,0 +1,242 @@
+//! Conversion of internal Rust compiler `rustc_target::abi` and `rustc_abi` items to stable ones.
+
+#![allow(rustc::usage_of_qualified_ty)]
+
+use crate::rustc_smir::{Stable, Tables};
+use rustc_middle::ty;
+use rustc_target::abi::call::Conv;
+use stable_mir::abi::{
+    ArgAbi, CallConvention, FieldsShape, FnAbi, Layout, LayoutShape, PassMode, TagEncoding,
+    TyAndLayout, ValueAbi, VariantsShape,
+};
+use stable_mir::ty::{Align, IndexedVal, Size, VariantIdx};
+use stable_mir::{opaque, Opaque};
+
+impl<'tcx> Stable<'tcx> for rustc_target::abi::VariantIdx {
+    type T = VariantIdx;
+    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
+        VariantIdx::to_val(self.as_usize())
+    }
+}
+
+impl<'tcx> Stable<'tcx> for rustc_abi::Endian {
+    type T = stable_mir::target::Endian;
+
+    fn stable(&self, _tables: &mut Tables<'tcx>) -> Self::T {
+        match self {
+            rustc_abi::Endian::Little => stable_mir::target::Endian::Little,
+            rustc_abi::Endian::Big => stable_mir::target::Endian::Big,
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for rustc_target::abi::TyAndLayout<'tcx, ty::Ty<'tcx>> {
+    type T = TyAndLayout;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        TyAndLayout { ty: self.ty.stable(tables), layout: self.layout.stable(tables) }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for rustc_target::abi::Layout<'tcx> {
+    type T = Layout;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        tables.layout_id(*self)
+    }
+}
+
+impl<'tcx> Stable<'tcx>
+    for rustc_abi::LayoutS<rustc_target::abi::FieldIdx, rustc_target::abi::VariantIdx>
+{
+    type T = LayoutShape;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        LayoutShape {
+            fields: self.fields.stable(tables),
+            variants: self.variants.stable(tables),
+            abi: self.abi.stable(tables),
+            abi_align: self.align.abi.stable(tables),
+            size: self.size.stable(tables),
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for rustc_target::abi::call::FnAbi<'tcx, ty::Ty<'tcx>> {
+    type T = FnAbi;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        assert!(self.args.len() >= self.fixed_count as usize);
+        assert!(!self.c_variadic || matches!(self.conv, Conv::C));
+        FnAbi {
+            args: self.args.as_ref().stable(tables),
+            ret: self.ret.stable(tables),
+            fixed_count: self.fixed_count,
+            conv: self.conv.stable(tables),
+            c_variadic: self.c_variadic,
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for rustc_target::abi::call::ArgAbi<'tcx, ty::Ty<'tcx>> {
+    type T = ArgAbi;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        ArgAbi {
+            ty: self.layout.ty.stable(tables),
+            layout: self.layout.layout.stable(tables),
+            mode: self.mode.stable(tables),
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for rustc_target::abi::call::Conv {
+    type T = CallConvention;
+
+    fn stable(&self, _tables: &mut Tables<'tcx>) -> Self::T {
+        match self {
+            Conv::C => CallConvention::C,
+            Conv::Rust => CallConvention::Rust,
+            Conv::Cold => CallConvention::Cold,
+            Conv::PreserveMost => CallConvention::PreserveMost,
+            Conv::PreserveAll => CallConvention::PreserveAll,
+            Conv::ArmAapcs => CallConvention::ArmAapcs,
+            Conv::CCmseNonSecureCall => CallConvention::CCmseNonSecureCall,
+            Conv::Msp430Intr => CallConvention::Msp430Intr,
+            Conv::PtxKernel => CallConvention::PtxKernel,
+            Conv::X86Fastcall => CallConvention::X86Fastcall,
+            Conv::X86Intr => CallConvention::X86Intr,
+            Conv::X86Stdcall => CallConvention::X86Stdcall,
+            Conv::X86ThisCall => CallConvention::X86ThisCall,
+            Conv::X86VectorCall => CallConvention::X86VectorCall,
+            Conv::X86_64SysV => CallConvention::X86_64SysV,
+            Conv::X86_64Win64 => CallConvention::X86_64Win64,
+            Conv::AmdGpuKernel => CallConvention::AmdGpuKernel,
+            Conv::AvrInterrupt => CallConvention::AvrInterrupt,
+            Conv::AvrNonBlockingInterrupt => CallConvention::AvrNonBlockingInterrupt,
+            Conv::RiscvInterrupt { .. } => CallConvention::RiscvInterrupt,
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for rustc_target::abi::call::PassMode {
+    type T = PassMode;
+
+    fn stable(&self, _tables: &mut Tables<'tcx>) -> Self::T {
+        match self {
+            rustc_target::abi::call::PassMode::Ignore => PassMode::Ignore,
+            rustc_target::abi::call::PassMode::Direct(attr) => PassMode::Direct(opaque(attr)),
+            rustc_target::abi::call::PassMode::Pair(first, second) => {
+                PassMode::Pair(opaque(first), opaque(second))
+            }
+            rustc_target::abi::call::PassMode::Cast { pad_i32, cast } => {
+                PassMode::Cast { pad_i32: *pad_i32, cast: opaque(cast) }
+            }
+            rustc_target::abi::call::PassMode::Indirect { attrs, meta_attrs, on_stack } => {
+                PassMode::Indirect {
+                    attrs: opaque(attrs),
+                    meta_attrs: opaque(meta_attrs),
+                    on_stack: *on_stack,
+                }
+            }
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for rustc_abi::FieldsShape<rustc_target::abi::FieldIdx> {
+    type T = FieldsShape;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        match self {
+            rustc_abi::FieldsShape::Primitive => FieldsShape::Primitive,
+            rustc_abi::FieldsShape::Union(count) => FieldsShape::Union(*count),
+            rustc_abi::FieldsShape::Array { stride, count } => {
+                FieldsShape::Array { stride: stride.stable(tables), count: *count }
+            }
+            rustc_abi::FieldsShape::Arbitrary { offsets, .. } => {
+                FieldsShape::Arbitrary { offsets: offsets.iter().as_slice().stable(tables) }
+            }
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx>
+    for rustc_abi::Variants<rustc_target::abi::FieldIdx, rustc_target::abi::VariantIdx>
+{
+    type T = VariantsShape;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        match self {
+            rustc_abi::Variants::Single { index } => {
+                VariantsShape::Single { index: index.stable(tables) }
+            }
+            rustc_abi::Variants::Multiple { tag, tag_encoding, tag_field, variants } => {
+                VariantsShape::Multiple {
+                    tag: tag.stable(tables),
+                    tag_encoding: tag_encoding.stable(tables),
+                    tag_field: *tag_field,
+                    variants: variants.iter().as_slice().stable(tables),
+                }
+            }
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for rustc_abi::TagEncoding<rustc_target::abi::VariantIdx> {
+    type T = TagEncoding;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        match self {
+            rustc_abi::TagEncoding::Direct => TagEncoding::Direct,
+            rustc_abi::TagEncoding::Niche { untagged_variant, niche_variants, niche_start } => {
+                TagEncoding::Niche {
+                    untagged_variant: untagged_variant.stable(tables),
+                    niche_variants: niche_variants.stable(tables),
+                    niche_start: *niche_start,
+                }
+            }
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for rustc_abi::Abi {
+    type T = ValueAbi;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        match *self {
+            rustc_abi::Abi::Uninhabited => ValueAbi::Uninhabited,
+            rustc_abi::Abi::Scalar(scalar) => ValueAbi::Scalar(scalar.stable(tables)),
+            rustc_abi::Abi::ScalarPair(first, second) => {
+                ValueAbi::ScalarPair(first.stable(tables), second.stable(tables))
+            }
+            rustc_abi::Abi::Vector { element, count } => {
+                ValueAbi::Vector { element: element.stable(tables), count }
+            }
+            rustc_abi::Abi::Aggregate { sized } => ValueAbi::Aggregate { sized },
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for rustc_abi::Size {
+    type T = Size;
+
+    fn stable(&self, _tables: &mut Tables<'tcx>) -> Self::T {
+        self.bytes_usize()
+    }
+}
+
+impl<'tcx> Stable<'tcx> for rustc_abi::Align {
+    type T = Align;
+
+    fn stable(&self, _tables: &mut Tables<'tcx>) -> Self::T {
+        self.bytes()
+    }
+}
+
+impl<'tcx> Stable<'tcx> for rustc_abi::Scalar {
+    type T = Opaque;
+
+    fn stable(&self, _tables: &mut Tables<'tcx>) -> Self::T {
+        opaque(self)
+    }
+}
diff --git a/compiler/rustc_smir/src/rustc_smir/convert/error.rs b/compiler/rustc_smir/src/rustc_smir/convert/error.rs
new file mode 100644
index 000000000..6c582b799
--- /dev/null
+++ b/compiler/rustc_smir/src/rustc_smir/convert/error.rs
@@ -0,0 +1,22 @@
+//! Handle the conversion of different internal errors into a stable version.
+//!
+//! Currently we encode everything as [stable_mir::Error], which is represented as a string.
+use crate::rustc_smir::{Stable, Tables};
+use rustc_middle::mir::interpret::AllocError;
+use rustc_middle::ty::layout::LayoutError;
+
+impl<'tcx> Stable<'tcx> for LayoutError<'tcx> {
+    type T = stable_mir::Error;
+
+    fn stable(&self, _tables: &mut Tables<'tcx>) -> Self::T {
+        stable_mir::Error::new(format!("{self:?}"))
+    }
+}
+
+impl<'tcx> Stable<'tcx> for AllocError {
+    type T = stable_mir::Error;
+
+    fn stable(&self, _tables: &mut Tables<'tcx>) -> Self::T {
+        stable_mir::Error::new(format!("{self:?}"))
+    }
+}
diff --git a/compiler/rustc_smir/src/rustc_smir/convert/mir.rs b/compiler/rustc_smir/src/rustc_smir/convert/mir.rs
new file mode 100644
index 000000000..49bf2192f
--- /dev/null
+++ b/compiler/rustc_smir/src/rustc_smir/convert/mir.rs
@@ -0,0 +1,746 @@
+//! Conversion of internal Rust compiler `mir` items to stable ones.
+
+use rustc_middle::mir;
+use rustc_middle::mir::interpret::alloc_range;
+use rustc_middle::mir::mono::MonoItem;
+use stable_mir::mir::alloc::GlobalAlloc;
+use stable_mir::mir::{ConstOperand, Statement, UserTypeProjection, VarDebugInfoFragment};
+use stable_mir::ty::{Allocation, Const, ConstantKind};
+use stable_mir::{opaque, Error};
+
+use crate::rustc_smir::{alloc, Stable, Tables};
+
+impl<'tcx> Stable<'tcx> for mir::Body<'tcx> {
+    type T = stable_mir::mir::Body;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        stable_mir::mir::Body::new(
+            self.basic_blocks
+                .iter()
+                .map(|block| stable_mir::mir::BasicBlock {
+                    terminator: block.terminator().stable(tables),
+                    statements: block
+                        .statements
+                        .iter()
+                        .map(|statement| statement.stable(tables))
+                        .collect(),
+                })
+                .collect(),
+            self.local_decls
+                .iter()
+                .map(|decl| stable_mir::mir::LocalDecl {
+                    ty: decl.ty.stable(tables),
+                    span: decl.source_info.span.stable(tables),
+                    mutability: decl.mutability.stable(tables),
+                })
+                .collect(),
+            self.arg_count,
+            self.var_debug_info.iter().map(|info| info.stable(tables)).collect(),
+            self.spread_arg.stable(tables),
+            self.span.stable(tables),
+        )
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::VarDebugInfo<'tcx> {
+    type T = stable_mir::mir::VarDebugInfo;
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        stable_mir::mir::VarDebugInfo {
+            name: self.name.to_string(),
+            source_info: self.source_info.stable(tables),
+            composite: self.composite.as_ref().map(|composite| composite.stable(tables)),
+            value: self.value.stable(tables),
+            argument_index: self.argument_index,
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::Statement<'tcx> {
+    type T = stable_mir::mir::Statement;
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        Statement { kind: self.kind.stable(tables), span: self.source_info.span.stable(tables) }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::SourceInfo {
+    type T = stable_mir::mir::SourceInfo;
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        stable_mir::mir::SourceInfo { span: self.span.stable(tables), scope: self.scope.into() }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::VarDebugInfoFragment<'tcx> {
+    type T = stable_mir::mir::VarDebugInfoFragment;
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        VarDebugInfoFragment {
+            ty: self.ty.stable(tables),
+            projection: self.projection.iter().map(|e| e.stable(tables)).collect(),
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::VarDebugInfoContents<'tcx> {
+    type T = stable_mir::mir::VarDebugInfoContents;
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        match self {
+            mir::VarDebugInfoContents::Place(place) => {
+                stable_mir::mir::VarDebugInfoContents::Place(place.stable(tables))
+            }
+            mir::VarDebugInfoContents::Const(const_operand) => {
+                let op = ConstOperand {
+                    span: const_operand.span.stable(tables),
+                    user_ty: const_operand.user_ty.map(|index| index.as_usize()),
+                    const_: const_operand.const_.stable(tables),
+                };
+                stable_mir::mir::VarDebugInfoContents::Const(op)
+            }
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::StatementKind<'tcx> {
+    type T = stable_mir::mir::StatementKind;
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        match self {
+            mir::StatementKind::Assign(assign) => stable_mir::mir::StatementKind::Assign(
+                assign.0.stable(tables),
+                assign.1.stable(tables),
+            ),
+            mir::StatementKind::FakeRead(fake_read_place) => {
+                stable_mir::mir::StatementKind::FakeRead(
+                    fake_read_place.0.stable(tables),
+                    fake_read_place.1.stable(tables),
+                )
+            }
+            mir::StatementKind::SetDiscriminant { place, variant_index } => {
+                stable_mir::mir::StatementKind::SetDiscriminant {
+                    place: place.as_ref().stable(tables),
+                    variant_index: variant_index.stable(tables),
+                }
+            }
+            mir::StatementKind::Deinit(place) => {
+                stable_mir::mir::StatementKind::Deinit(place.stable(tables))
+            }
+
+            mir::StatementKind::StorageLive(place) => {
+                stable_mir::mir::StatementKind::StorageLive(place.stable(tables))
+            }
+
+            mir::StatementKind::StorageDead(place) => {
+                stable_mir::mir::StatementKind::StorageDead(place.stable(tables))
+            }
+            mir::StatementKind::Retag(retag, place) => {
+                stable_mir::mir::StatementKind::Retag(retag.stable(tables), place.stable(tables))
+            }
+            mir::StatementKind::PlaceMention(place) => {
+                stable_mir::mir::StatementKind::PlaceMention(place.stable(tables))
+            }
+            mir::StatementKind::AscribeUserType(place_projection, variance) => {
+                stable_mir::mir::StatementKind::AscribeUserType {
+                    place: place_projection.as_ref().0.stable(tables),
+                    projections: place_projection.as_ref().1.stable(tables),
+                    variance: variance.stable(tables),
+                }
+            }
+            mir::StatementKind::Coverage(coverage) => {
+                stable_mir::mir::StatementKind::Coverage(opaque(coverage))
+            }
+            mir::StatementKind::Intrinsic(intrinstic) => {
+                stable_mir::mir::StatementKind::Intrinsic(intrinstic.stable(tables))
+            }
+            mir::StatementKind::ConstEvalCounter => {
+                stable_mir::mir::StatementKind::ConstEvalCounter
+            }
+            mir::StatementKind::Nop => stable_mir::mir::StatementKind::Nop,
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::Rvalue<'tcx> {
+    type T = stable_mir::mir::Rvalue;
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        use rustc_middle::mir::Rvalue::*;
+        match self {
+            Use(op) => stable_mir::mir::Rvalue::Use(op.stable(tables)),
+            Repeat(op, len) => {
+                let len = len.stable(tables);
+                stable_mir::mir::Rvalue::Repeat(op.stable(tables), len)
+            }
+            Ref(region, kind, place) => stable_mir::mir::Rvalue::Ref(
+                region.stable(tables),
+                kind.stable(tables),
+                place.stable(tables),
+            ),
+            ThreadLocalRef(def_id) => {
+                stable_mir::mir::Rvalue::ThreadLocalRef(tables.crate_item(*def_id))
+            }
+            AddressOf(mutability, place) => {
+                stable_mir::mir::Rvalue::AddressOf(mutability.stable(tables), place.stable(tables))
+            }
+            Len(place) => stable_mir::mir::Rvalue::Len(place.stable(tables)),
+            Cast(cast_kind, op, ty) => stable_mir::mir::Rvalue::Cast(
+                cast_kind.stable(tables),
+                op.stable(tables),
+                ty.stable(tables),
+            ),
+            BinaryOp(bin_op, ops) => stable_mir::mir::Rvalue::BinaryOp(
+                bin_op.stable(tables),
+                ops.0.stable(tables),
+                ops.1.stable(tables),
+            ),
+            CheckedBinaryOp(bin_op, ops) => stable_mir::mir::Rvalue::CheckedBinaryOp(
+                bin_op.stable(tables),
+                ops.0.stable(tables),
+                ops.1.stable(tables),
+            ),
+            NullaryOp(null_op, ty) => {
+                stable_mir::mir::Rvalue::NullaryOp(null_op.stable(tables), ty.stable(tables))
+            }
+            UnaryOp(un_op, op) => {
+                stable_mir::mir::Rvalue::UnaryOp(un_op.stable(tables), op.stable(tables))
+            }
+            Discriminant(place) => stable_mir::mir::Rvalue::Discriminant(place.stable(tables)),
+            Aggregate(agg_kind, operands) => {
+                let operands = operands.iter().map(|op| op.stable(tables)).collect();
+                stable_mir::mir::Rvalue::Aggregate(agg_kind.stable(tables), operands)
+            }
+            ShallowInitBox(op, ty) => {
+                stable_mir::mir::Rvalue::ShallowInitBox(op.stable(tables), ty.stable(tables))
+            }
+            CopyForDeref(place) => stable_mir::mir::Rvalue::CopyForDeref(place.stable(tables)),
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::Mutability {
+    type T = stable_mir::mir::Mutability;
+    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
+        use rustc_hir::Mutability::*;
+        match *self {
+            Not => stable_mir::mir::Mutability::Not,
+            Mut => stable_mir::mir::Mutability::Mut,
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::BorrowKind {
+    type T = stable_mir::mir::BorrowKind;
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        use rustc_middle::mir::BorrowKind::*;
+        match *self {
+            Shared => stable_mir::mir::BorrowKind::Shared,
+            Fake => stable_mir::mir::BorrowKind::Fake,
+            Mut { kind } => stable_mir::mir::BorrowKind::Mut { kind: kind.stable(tables) },
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::MutBorrowKind {
+    type T = stable_mir::mir::MutBorrowKind;
+    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
+        use rustc_middle::mir::MutBorrowKind::*;
+        match *self {
+            Default => stable_mir::mir::MutBorrowKind::Default,
+            TwoPhaseBorrow => stable_mir::mir::MutBorrowKind::TwoPhaseBorrow,
+            ClosureCapture => stable_mir::mir::MutBorrowKind::ClosureCapture,
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::NullOp<'tcx> {
+    type T = stable_mir::mir::NullOp;
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        use rustc_middle::mir::NullOp::*;
+        match self {
+            SizeOf => stable_mir::mir::NullOp::SizeOf,
+            AlignOf => stable_mir::mir::NullOp::AlignOf,
+            OffsetOf(indices) => stable_mir::mir::NullOp::OffsetOf(
+                indices.iter().map(|idx| idx.stable(tables)).collect(),
+            ),
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::CastKind {
+    type T = stable_mir::mir::CastKind;
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        use rustc_middle::mir::CastKind::*;
+        match self {
+            PointerExposeAddress => stable_mir::mir::CastKind::PointerExposeAddress,
+            PointerFromExposedAddress => stable_mir::mir::CastKind::PointerFromExposedAddress,
+            PointerCoercion(c) => stable_mir::mir::CastKind::PointerCoercion(c.stable(tables)),
+            DynStar => stable_mir::mir::CastKind::DynStar,
+            IntToInt => stable_mir::mir::CastKind::IntToInt,
+            FloatToInt => stable_mir::mir::CastKind::FloatToInt,
+            FloatToFloat => stable_mir::mir::CastKind::FloatToFloat,
+            IntToFloat => stable_mir::mir::CastKind::IntToFloat,
+            PtrToPtr => stable_mir::mir::CastKind::PtrToPtr,
+            FnPtrToPtr => stable_mir::mir::CastKind::FnPtrToPtr,
+            Transmute => stable_mir::mir::CastKind::Transmute,
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::FakeReadCause {
+    type T = stable_mir::mir::FakeReadCause;
+    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
+        use rustc_middle::mir::FakeReadCause::*;
+        match self {
+            ForMatchGuard => stable_mir::mir::FakeReadCause::ForMatchGuard,
+            ForMatchedPlace(local_def_id) => {
+                stable_mir::mir::FakeReadCause::ForMatchedPlace(opaque(local_def_id))
+            }
+            ForGuardBinding => stable_mir::mir::FakeReadCause::ForGuardBinding,
+            ForLet(local_def_id) => stable_mir::mir::FakeReadCause::ForLet(opaque(local_def_id)),
+            ForIndex => stable_mir::mir::FakeReadCause::ForIndex,
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::Operand<'tcx> {
+    type T = stable_mir::mir::Operand;
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        use rustc_middle::mir::Operand::*;
+        match self {
+            Copy(place) => stable_mir::mir::Operand::Copy(place.stable(tables)),
+            Move(place) => stable_mir::mir::Operand::Move(place.stable(tables)),
+            Constant(c) => stable_mir::mir::Operand::Constant(c.stable(tables)),
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::ConstOperand<'tcx> {
+    type T = stable_mir::mir::Constant;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        stable_mir::mir::Constant {
+            span: self.span.stable(tables),
+            user_ty: self.user_ty.map(|u| u.as_usize()).or(None),
+            literal: self.const_.stable(tables),
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::Place<'tcx> {
+    type T = stable_mir::mir::Place;
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        stable_mir::mir::Place {
+            local: self.local.as_usize(),
+            projection: self.projection.iter().map(|e| e.stable(tables)).collect(),
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::PlaceElem<'tcx> {
+    type T = stable_mir::mir::ProjectionElem;
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        use rustc_middle::mir::ProjectionElem::*;
+        match self {
+            Deref => stable_mir::mir::ProjectionElem::Deref,
+            Field(idx, ty) => {
+                stable_mir::mir::ProjectionElem::Field(idx.stable(tables), ty.stable(tables))
+            }
+            Index(local) => stable_mir::mir::ProjectionElem::Index(local.stable(tables)),
+            ConstantIndex { offset, min_length, from_end } => {
+                stable_mir::mir::ProjectionElem::ConstantIndex {
+                    offset: *offset,
+                    min_length: *min_length,
+                    from_end: *from_end,
+                }
+            }
+            Subslice { from, to, from_end } => stable_mir::mir::ProjectionElem::Subslice {
+                from: *from,
+                to: *to,
+                from_end: *from_end,
+            },
+            // MIR includes an `Option<Symbol>` argument for `Downcast` that is the name of the
+            // variant, used for printing MIR. However this information should also be accessible
+            // via a lookup using the `VariantIdx`. The `Option<Symbol>` argument is therefore
+            // dropped when converting to Stable MIR. A brief justification for this decision can be
+            // found at https://github.com/rust-lang/rust/pull/117517#issuecomment-1811683486
+            Downcast(_, idx) => stable_mir::mir::ProjectionElem::Downcast(idx.stable(tables)),
+            OpaqueCast(ty) => stable_mir::mir::ProjectionElem::OpaqueCast(ty.stable(tables)),
+            Subtype(ty) => stable_mir::mir::ProjectionElem::Subtype(ty.stable(tables)),
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::UserTypeProjection {
+    type T = stable_mir::mir::UserTypeProjection;
+
+    fn stable(&self, _tables: &mut Tables<'tcx>) -> Self::T {
+        UserTypeProjection { base: self.base.as_usize(), projection: opaque(&self.projs) }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::Local {
+    type T = stable_mir::mir::Local;
+    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
+        self.as_usize()
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::RetagKind {
+    type T = stable_mir::mir::RetagKind;
+    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
+        use rustc_middle::mir::RetagKind;
+        match self {
+            RetagKind::FnEntry => stable_mir::mir::RetagKind::FnEntry,
+            RetagKind::TwoPhase => stable_mir::mir::RetagKind::TwoPhase,
+            RetagKind::Raw => stable_mir::mir::RetagKind::Raw,
+            RetagKind::Default => stable_mir::mir::RetagKind::Default,
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::UnwindAction {
+    type T = stable_mir::mir::UnwindAction;
+    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
+        use rustc_middle::mir::UnwindAction;
+        match self {
+            UnwindAction::Continue => stable_mir::mir::UnwindAction::Continue,
+            UnwindAction::Unreachable => stable_mir::mir::UnwindAction::Unreachable,
+            UnwindAction::Terminate(_) => stable_mir::mir::UnwindAction::Terminate,
+            UnwindAction::Cleanup(bb) => stable_mir::mir::UnwindAction::Cleanup(bb.as_usize()),
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::NonDivergingIntrinsic<'tcx> {
+    type T = stable_mir::mir::NonDivergingIntrinsic;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        use rustc_middle::mir::NonDivergingIntrinsic;
+        use stable_mir::mir::CopyNonOverlapping;
+        match self {
+            NonDivergingIntrinsic::Assume(op) => {
+                stable_mir::mir::NonDivergingIntrinsic::Assume(op.stable(tables))
+            }
+            NonDivergingIntrinsic::CopyNonOverlapping(copy_non_overlapping) => {
+                stable_mir::mir::NonDivergingIntrinsic::CopyNonOverlapping(CopyNonOverlapping {
+                    src: copy_non_overlapping.src.stable(tables),
+                    dst: copy_non_overlapping.dst.stable(tables),
+                    count: copy_non_overlapping.count.stable(tables),
+                })
+            }
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::AssertMessage<'tcx> {
+    type T = stable_mir::mir::AssertMessage;
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        use rustc_middle::mir::AssertKind;
+        match self {
+            AssertKind::BoundsCheck { len, index } => stable_mir::mir::AssertMessage::BoundsCheck {
+                len: len.stable(tables),
+                index: index.stable(tables),
+            },
+            AssertKind::Overflow(bin_op, op1, op2) => stable_mir::mir::AssertMessage::Overflow(
+                bin_op.stable(tables),
+                op1.stable(tables),
+                op2.stable(tables),
+            ),
+            AssertKind::OverflowNeg(op) => {
+                stable_mir::mir::AssertMessage::OverflowNeg(op.stable(tables))
+            }
+            AssertKind::DivisionByZero(op) => {
+                stable_mir::mir::AssertMessage::DivisionByZero(op.stable(tables))
+            }
+            AssertKind::RemainderByZero(op) => {
+                stable_mir::mir::AssertMessage::RemainderByZero(op.stable(tables))
+            }
+            AssertKind::ResumedAfterReturn(coroutine) => {
+                stable_mir::mir::AssertMessage::ResumedAfterReturn(coroutine.stable(tables))
+            }
+            AssertKind::ResumedAfterPanic(coroutine) => {
+                stable_mir::mir::AssertMessage::ResumedAfterPanic(coroutine.stable(tables))
+            }
+            AssertKind::MisalignedPointerDereference { required, found } => {
+                stable_mir::mir::AssertMessage::MisalignedPointerDereference {
+                    required: required.stable(tables),
+                    found: found.stable(tables),
+                }
+            }
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::BinOp {
+    type T = stable_mir::mir::BinOp;
+    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
+        use rustc_middle::mir::BinOp;
+        match self {
+            BinOp::Add => stable_mir::mir::BinOp::Add,
+            BinOp::AddUnchecked => stable_mir::mir::BinOp::AddUnchecked,
+            BinOp::Sub => stable_mir::mir::BinOp::Sub,
+            BinOp::SubUnchecked => stable_mir::mir::BinOp::SubUnchecked,
+            BinOp::Mul => stable_mir::mir::BinOp::Mul,
+            BinOp::MulUnchecked => stable_mir::mir::BinOp::MulUnchecked,
+            BinOp::Div => stable_mir::mir::BinOp::Div,
+            BinOp::Rem => stable_mir::mir::BinOp::Rem,
+            BinOp::BitXor => stable_mir::mir::BinOp::BitXor,
+            BinOp::BitAnd => stable_mir::mir::BinOp::BitAnd,
+            BinOp::BitOr => stable_mir::mir::BinOp::BitOr,
+            BinOp::Shl => stable_mir::mir::BinOp::Shl,
+            BinOp::ShlUnchecked => stable_mir::mir::BinOp::ShlUnchecked,
+            BinOp::Shr => stable_mir::mir::BinOp::Shr,
+            BinOp::ShrUnchecked => stable_mir::mir::BinOp::ShrUnchecked,
+            BinOp::Eq => stable_mir::mir::BinOp::Eq,
+            BinOp::Lt => stable_mir::mir::BinOp::Lt,
+            BinOp::Le => stable_mir::mir::BinOp::Le,
+            BinOp::Ne => stable_mir::mir::BinOp::Ne,
+            BinOp::Ge => stable_mir::mir::BinOp::Ge,
+            BinOp::Gt => stable_mir::mir::BinOp::Gt,
+            BinOp::Offset => stable_mir::mir::BinOp::Offset,
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::UnOp {
+    type T = stable_mir::mir::UnOp;
+    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
+        use rustc_middle::mir::UnOp;
+        match self {
+            UnOp::Not => stable_mir::mir::UnOp::Not,
+            UnOp::Neg => stable_mir::mir::UnOp::Neg,
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::AggregateKind<'tcx> {
+    type T = stable_mir::mir::AggregateKind;
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        match self {
+            mir::AggregateKind::Array(ty) => {
+                stable_mir::mir::AggregateKind::Array(ty.stable(tables))
+            }
+            mir::AggregateKind::Tuple => stable_mir::mir::AggregateKind::Tuple,
+            mir::AggregateKind::Adt(def_id, var_idx, generic_arg, user_ty_index, field_idx) => {
+                stable_mir::mir::AggregateKind::Adt(
+                    tables.adt_def(*def_id),
+                    var_idx.stable(tables),
+                    generic_arg.stable(tables),
+                    user_ty_index.map(|idx| idx.index()),
+                    field_idx.map(|idx| idx.index()),
+                )
+            }
+            mir::AggregateKind::Closure(def_id, generic_arg) => {
+                stable_mir::mir::AggregateKind::Closure(
+                    tables.closure_def(*def_id),
+                    generic_arg.stable(tables),
+                )
+            }
+            mir::AggregateKind::Coroutine(def_id, generic_arg, movability) => {
+                stable_mir::mir::AggregateKind::Coroutine(
+                    tables.coroutine_def(*def_id),
+                    generic_arg.stable(tables),
+                    movability.stable(tables),
+                )
+            }
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::InlineAsmOperand<'tcx> {
+    type T = stable_mir::mir::InlineAsmOperand;
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        use rustc_middle::mir::InlineAsmOperand;
+
+        let (in_value, out_place) = match self {
+            InlineAsmOperand::In { value, .. } => (Some(value.stable(tables)), None),
+            InlineAsmOperand::Out { place, .. } => (None, place.map(|place| place.stable(tables))),
+            InlineAsmOperand::InOut { in_value, out_place, .. } => {
+                (Some(in_value.stable(tables)), out_place.map(|place| place.stable(tables)))
+            }
+            InlineAsmOperand::Const { .. }
+            | InlineAsmOperand::SymFn { .. }
+            | InlineAsmOperand::SymStatic { .. } => (None, None),
+        };
+
+        stable_mir::mir::InlineAsmOperand { in_value, out_place, raw_rpr: format!("{self:?}") }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::Terminator<'tcx> {
+    type T = stable_mir::mir::Terminator;
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        use stable_mir::mir::Terminator;
+        Terminator { kind: self.kind.stable(tables), span: self.source_info.span.stable(tables) }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::TerminatorKind<'tcx> {
+    type T = stable_mir::mir::TerminatorKind;
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        use stable_mir::mir::TerminatorKind;
+        match self {
+            mir::TerminatorKind::Goto { target } => {
+                TerminatorKind::Goto { target: target.as_usize() }
+            }
+            mir::TerminatorKind::SwitchInt { discr, targets } => TerminatorKind::SwitchInt {
+                discr: discr.stable(tables),
+                targets: {
+                    let branches = targets.iter().map(|(val, target)| (val, target.as_usize()));
+                    stable_mir::mir::SwitchTargets::new(
+                        branches.collect(),
+                        targets.otherwise().as_usize(),
+                    )
+                },
+            },
+            mir::TerminatorKind::UnwindResume => TerminatorKind::Resume,
+            mir::TerminatorKind::UnwindTerminate(_) => TerminatorKind::Abort,
+            mir::TerminatorKind::Return => TerminatorKind::Return,
+            mir::TerminatorKind::Unreachable => TerminatorKind::Unreachable,
+            mir::TerminatorKind::Drop { place, target, unwind, replace: _ } => {
+                TerminatorKind::Drop {
+                    place: place.stable(tables),
+                    target: target.as_usize(),
+                    unwind: unwind.stable(tables),
+                }
+            }
+            mir::TerminatorKind::Call {
+                func,
+                args,
+                destination,
+                target,
+                unwind,
+                call_source: _,
+                fn_span: _,
+            } => TerminatorKind::Call {
+                func: func.stable(tables),
+                args: args.iter().map(|arg| arg.stable(tables)).collect(),
+                destination: destination.stable(tables),
+                target: target.map(|t| t.as_usize()),
+                unwind: unwind.stable(tables),
+            },
+            mir::TerminatorKind::Assert { cond, expected, msg, target, unwind } => {
+                TerminatorKind::Assert {
+                    cond: cond.stable(tables),
+                    expected: *expected,
+                    msg: msg.stable(tables),
+                    target: target.as_usize(),
+                    unwind: unwind.stable(tables),
+                }
+            }
+            mir::TerminatorKind::InlineAsm {
+                template,
+                operands,
+                options,
+                line_spans,
+                destination,
+                unwind,
+            } => TerminatorKind::InlineAsm {
+                template: format!("{template:?}"),
+                operands: operands.iter().map(|operand| operand.stable(tables)).collect(),
+                options: format!("{options:?}"),
+                line_spans: format!("{line_spans:?}"),
+                destination: destination.map(|d| d.as_usize()),
+                unwind: unwind.stable(tables),
+            },
+            mir::TerminatorKind::Yield { .. }
+            | mir::TerminatorKind::CoroutineDrop
+            | mir::TerminatorKind::FalseEdge { .. }
+            | mir::TerminatorKind::FalseUnwind { .. } => unreachable!(),
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::interpret::ConstAllocation<'tcx> {
+    type T = Allocation;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        self.inner().stable(tables)
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::interpret::Allocation {
+    type T = stable_mir::ty::Allocation;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        alloc::allocation_filter(
+            self,
+            alloc_range(rustc_target::abi::Size::ZERO, self.size()),
+            tables,
+        )
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::interpret::AllocId {
+    type T = stable_mir::mir::alloc::AllocId;
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        tables.create_alloc_id(*self)
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::interpret::GlobalAlloc<'tcx> {
+    type T = GlobalAlloc;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        match self {
+            mir::interpret::GlobalAlloc::Function(instance) => {
+                GlobalAlloc::Function(instance.stable(tables))
+            }
+            mir::interpret::GlobalAlloc::VTable(ty, trait_ref) => {
+                GlobalAlloc::VTable(ty.stable(tables), trait_ref.stable(tables))
+            }
+            mir::interpret::GlobalAlloc::Static(def) => {
+                GlobalAlloc::Static(tables.static_def(*def))
+            }
+            mir::interpret::GlobalAlloc::Memory(alloc) => GlobalAlloc::Memory(alloc.stable(tables)),
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for rustc_middle::mir::Const<'tcx> {
+    type T = stable_mir::ty::Const;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        match *self {
+            mir::Const::Ty(c) => c.stable(tables),
+            mir::Const::Unevaluated(unev_const, ty) => {
+                let kind =
+                    stable_mir::ty::ConstantKind::Unevaluated(stable_mir::ty::UnevaluatedConst {
+                        def: tables.const_def(unev_const.def),
+                        args: unev_const.args.stable(tables),
+                        promoted: unev_const.promoted.map(|u| u.as_u32()),
+                    });
+                let ty = ty.stable(tables);
+                let id = tables.intern_const(*self);
+                Const::new(kind, ty, id)
+            }
+            mir::Const::Val(mir::ConstValue::ZeroSized, ty) => {
+                let ty = ty.stable(tables);
+                let id = tables.intern_const(*self);
+                Const::new(ConstantKind::ZeroSized, ty, id)
+            }
+            mir::Const::Val(val, ty) => {
+                let kind = ConstantKind::Allocated(alloc::new_allocation(ty, val, tables));
+                let ty = ty.stable(tables);
+                let id = tables.intern_const(*self);
+                Const::new(kind, ty, id)
+            }
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for mir::interpret::ErrorHandled {
+    type T = Error;
+
+    fn stable(&self, _tables: &mut Tables<'tcx>) -> Self::T {
+        Error::new(format!("{self:?}"))
+    }
+}
+
+impl<'tcx> Stable<'tcx> for MonoItem<'tcx> {
+    type T = stable_mir::mir::mono::MonoItem;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        use stable_mir::mir::mono::MonoItem as StableMonoItem;
+        match self {
+            MonoItem::Fn(instance) => StableMonoItem::Fn(instance.stable(tables)),
+            MonoItem::Static(def_id) => StableMonoItem::Static(tables.static_def(*def_id)),
+            MonoItem::GlobalAsm(item_id) => StableMonoItem::GlobalAsm(opaque(item_id)),
+        }
+    }
+}
diff --git a/compiler/rustc_smir/src/rustc_smir/convert/mod.rs b/compiler/rustc_smir/src/rustc_smir/convert/mod.rs
new file mode 100644
index 000000000..8b7b26f96
--- /dev/null
+++ b/compiler/rustc_smir/src/rustc_smir/convert/mod.rs
@@ -0,0 +1,74 @@
+//! Conversion of internal Rust compiler items to stable ones.
+
+use rustc_target::abi::FieldIdx;
+
+use crate::rustc_smir::{Stable, Tables};
+
+mod abi;
+mod error;
+mod mir;
+mod ty;
+
+impl<'tcx> Stable<'tcx> for rustc_hir::Unsafety {
+    type T = stable_mir::mir::Safety;
+    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
+        match self {
+            rustc_hir::Unsafety::Unsafe => stable_mir::mir::Safety::Unsafe,
+            rustc_hir::Unsafety::Normal => stable_mir::mir::Safety::Normal,
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for FieldIdx {
+    type T = usize;
+    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
+        self.as_usize()
+    }
+}
+
+impl<'tcx> Stable<'tcx> for rustc_hir::CoroutineSource {
+    type T = stable_mir::mir::CoroutineSource;
+    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
+        use rustc_hir::CoroutineSource;
+        match self {
+            CoroutineSource::Block => stable_mir::mir::CoroutineSource::Block,
+            CoroutineSource::Closure => stable_mir::mir::CoroutineSource::Closure,
+            CoroutineSource::Fn => stable_mir::mir::CoroutineSource::Fn,
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for rustc_hir::CoroutineKind {
+    type T = stable_mir::mir::CoroutineKind;
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        use rustc_hir::CoroutineKind;
+        match self {
+            CoroutineKind::Async(source) => {
+                stable_mir::mir::CoroutineKind::Async(source.stable(tables))
+            }
+            CoroutineKind::Gen(source) => {
+                stable_mir::mir::CoroutineKind::Gen(source.stable(tables))
+            }
+            CoroutineKind::Coroutine => stable_mir::mir::CoroutineKind::Coroutine,
+            CoroutineKind::AsyncGen(source) => {
+                stable_mir::mir::CoroutineKind::AsyncGen(source.stable(tables))
+            }
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for rustc_span::Symbol {
+    type T = stable_mir::Symbol;
+
+    fn stable(&self, _tables: &mut Tables<'tcx>) -> Self::T {
+        self.to_string()
+    }
+}
+
+impl<'tcx> Stable<'tcx> for rustc_span::Span {
+    type T = stable_mir::ty::Span;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        tables.create_span(*self)
+    }
+}
diff --git a/compiler/rustc_smir/src/rustc_smir/convert/ty.rs b/compiler/rustc_smir/src/rustc_smir/convert/ty.rs
new file mode 100644
index 000000000..cbdddc300
--- /dev/null
+++ b/compiler/rustc_smir/src/rustc_smir/convert/ty.rs
@@ -0,0 +1,829 @@
+//! Conversion of internal Rust compiler `ty` items to stable ones.
+
+use rustc_middle::ty::Ty;
+use rustc_middle::{mir, ty};
+use stable_mir::ty::{
+    AdtKind, Const, ConstantKind, FloatTy, GenericArgs, GenericParamDef, IntTy, Region, RigidTy,
+    TyKind, UintTy,
+};
+
+use crate::rustc_smir::{alloc, Stable, Tables};
+
+impl<'tcx> Stable<'tcx> for ty::AliasKind {
+    type T = stable_mir::ty::AliasKind;
+    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
+        use rustc_middle::ty::AliasKind::*;
+        match self {
+            Projection => stable_mir::ty::AliasKind::Projection,
+            Inherent => stable_mir::ty::AliasKind::Inherent,
+            Opaque => stable_mir::ty::AliasKind::Opaque,
+            Weak => stable_mir::ty::AliasKind::Weak,
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::AliasTy<'tcx> {
+    type T = stable_mir::ty::AliasTy;
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        let ty::AliasTy { args, def_id, .. } = self;
+        stable_mir::ty::AliasTy { def_id: tables.alias_def(*def_id), args: args.stable(tables) }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::DynKind {
+    type T = stable_mir::ty::DynKind;
+
+    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
+        use rustc_middle::ty::DynKind;
+        match self {
+            DynKind::Dyn => stable_mir::ty::DynKind::Dyn,
+            DynKind::DynStar => stable_mir::ty::DynKind::DynStar,
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::ExistentialPredicate<'tcx> {
+    type T = stable_mir::ty::ExistentialPredicate;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        use stable_mir::ty::ExistentialPredicate::*;
+        match self {
+            ty::ExistentialPredicate::Trait(existential_trait_ref) => {
+                Trait(existential_trait_ref.stable(tables))
+            }
+            ty::ExistentialPredicate::Projection(existential_projection) => {
+                Projection(existential_projection.stable(tables))
+            }
+            ty::ExistentialPredicate::AutoTrait(def_id) => AutoTrait(tables.trait_def(*def_id)),
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::ExistentialTraitRef<'tcx> {
+    type T = stable_mir::ty::ExistentialTraitRef;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        let ty::ExistentialTraitRef { def_id, args } = self;
+        stable_mir::ty::ExistentialTraitRef {
+            def_id: tables.trait_def(*def_id),
+            generic_args: args.stable(tables),
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::TermKind<'tcx> {
+    type T = stable_mir::ty::TermKind;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        use stable_mir::ty::TermKind;
+        match self {
+            ty::TermKind::Ty(ty) => TermKind::Type(ty.stable(tables)),
+            ty::TermKind::Const(cnst) => {
+                let cnst = cnst.stable(tables);
+                TermKind::Const(cnst)
+            }
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::ExistentialProjection<'tcx> {
+    type T = stable_mir::ty::ExistentialProjection;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        let ty::ExistentialProjection { def_id, args, term } = self;
+        stable_mir::ty::ExistentialProjection {
+            def_id: tables.trait_def(*def_id),
+            generic_args: args.stable(tables),
+            term: term.unpack().stable(tables),
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::adjustment::PointerCoercion {
+    type T = stable_mir::mir::PointerCoercion;
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        use rustc_middle::ty::adjustment::PointerCoercion;
+        match self {
+            PointerCoercion::ReifyFnPointer => stable_mir::mir::PointerCoercion::ReifyFnPointer,
+            PointerCoercion::UnsafeFnPointer => stable_mir::mir::PointerCoercion::UnsafeFnPointer,
+            PointerCoercion::ClosureFnPointer(unsafety) => {
+                stable_mir::mir::PointerCoercion::ClosureFnPointer(unsafety.stable(tables))
+            }
+            PointerCoercion::MutToConstPointer => {
+                stable_mir::mir::PointerCoercion::MutToConstPointer
+            }
+            PointerCoercion::ArrayToPointer => stable_mir::mir::PointerCoercion::ArrayToPointer,
+            PointerCoercion::Unsize => stable_mir::mir::PointerCoercion::Unsize,
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::UserTypeAnnotationIndex {
+    type T = usize;
+    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
+        self.as_usize()
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::AdtKind {
+    type T = AdtKind;
+
+    fn stable(&self, _tables: &mut Tables<'tcx>) -> Self::T {
+        match self {
+            ty::AdtKind::Struct => AdtKind::Struct,
+            ty::AdtKind::Union => AdtKind::Union,
+            ty::AdtKind::Enum => AdtKind::Enum,
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::FieldDef {
+    type T = stable_mir::ty::FieldDef;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        stable_mir::ty::FieldDef {
+            def: tables.create_def_id(self.did),
+            name: self.name.stable(tables),
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::GenericArgs<'tcx> {
+    type T = stable_mir::ty::GenericArgs;
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        GenericArgs(self.iter().map(|arg| arg.unpack().stable(tables)).collect())
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::GenericArgKind<'tcx> {
+    type T = stable_mir::ty::GenericArgKind;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        use stable_mir::ty::GenericArgKind;
+        match self {
+            ty::GenericArgKind::Lifetime(region) => GenericArgKind::Lifetime(region.stable(tables)),
+            ty::GenericArgKind::Type(ty) => GenericArgKind::Type(ty.stable(tables)),
+            ty::GenericArgKind::Const(cnst) => GenericArgKind::Const(cnst.stable(tables)),
+        }
+    }
+}
+
+impl<'tcx, S, V> Stable<'tcx> for ty::Binder<'tcx, S>
+where
+    S: Stable<'tcx, T = V>,
+{
+    type T = stable_mir::ty::Binder<V>;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        use stable_mir::ty::Binder;
+
+        Binder {
+            value: self.as_ref().skip_binder().stable(tables),
+            bound_vars: self
+                .bound_vars()
+                .iter()
+                .map(|bound_var| bound_var.stable(tables))
+                .collect(),
+        }
+    }
+}
+
+impl<'tcx, S, V> Stable<'tcx> for ty::EarlyBinder<S>
+where
+    S: Stable<'tcx, T = V>,
+{
+    type T = stable_mir::ty::EarlyBinder<V>;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        use stable_mir::ty::EarlyBinder;
+
+        EarlyBinder { value: self.as_ref().skip_binder().stable(tables) }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::FnSig<'tcx> {
+    type T = stable_mir::ty::FnSig;
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        use rustc_target::spec::abi;
+        use stable_mir::ty::{Abi, FnSig};
+
+        FnSig {
+            inputs_and_output: self.inputs_and_output.iter().map(|ty| ty.stable(tables)).collect(),
+            c_variadic: self.c_variadic,
+            unsafety: self.unsafety.stable(tables),
+            abi: match self.abi {
+                abi::Abi::Rust => Abi::Rust,
+                abi::Abi::C { unwind } => Abi::C { unwind },
+                abi::Abi::Cdecl { unwind } => Abi::Cdecl { unwind },
+                abi::Abi::Stdcall { unwind } => Abi::Stdcall { unwind },
+                abi::Abi::Fastcall { unwind } => Abi::Fastcall { unwind },
+                abi::Abi::Vectorcall { unwind } => Abi::Vectorcall { unwind },
+                abi::Abi::Thiscall { unwind } => Abi::Thiscall { unwind },
+                abi::Abi::Aapcs { unwind } => Abi::Aapcs { unwind },
+                abi::Abi::Win64 { unwind } => Abi::Win64 { unwind },
+                abi::Abi::SysV64 { unwind } => Abi::SysV64 { unwind },
+                abi::Abi::PtxKernel => Abi::PtxKernel,
+                abi::Abi::Msp430Interrupt => Abi::Msp430Interrupt,
+                abi::Abi::X86Interrupt => Abi::X86Interrupt,
+                abi::Abi::AmdGpuKernel => Abi::AmdGpuKernel,
+                abi::Abi::EfiApi => Abi::EfiApi,
+                abi::Abi::AvrInterrupt => Abi::AvrInterrupt,
+                abi::Abi::AvrNonBlockingInterrupt => Abi::AvrNonBlockingInterrupt,
+                abi::Abi::CCmseNonSecureCall => Abi::CCmseNonSecureCall,
+                abi::Abi::Wasm => Abi::Wasm,
+                abi::Abi::System { unwind } => Abi::System { unwind },
+                abi::Abi::RustIntrinsic => Abi::RustIntrinsic,
+                abi::Abi::RustCall => Abi::RustCall,
+                abi::Abi::PlatformIntrinsic => Abi::PlatformIntrinsic,
+                abi::Abi::Unadjusted => Abi::Unadjusted,
+                abi::Abi::RustCold => Abi::RustCold,
+                abi::Abi::RiscvInterruptM => Abi::RiscvInterruptM,
+                abi::Abi::RiscvInterruptS => Abi::RiscvInterruptS,
+            },
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::BoundTyKind {
+    type T = stable_mir::ty::BoundTyKind;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        use stable_mir::ty::BoundTyKind;
+
+        match self {
+            ty::BoundTyKind::Anon => BoundTyKind::Anon,
+            ty::BoundTyKind::Param(def_id, symbol) => {
+                BoundTyKind::Param(tables.param_def(*def_id), symbol.to_string())
+            }
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::BoundRegionKind {
+    type T = stable_mir::ty::BoundRegionKind;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        use stable_mir::ty::BoundRegionKind;
+
+        match self {
+            ty::BoundRegionKind::BrAnon => BoundRegionKind::BrAnon,
+            ty::BoundRegionKind::BrNamed(def_id, symbol) => {
+                BoundRegionKind::BrNamed(tables.br_named_def(*def_id), symbol.to_string())
+            }
+            ty::BoundRegionKind::BrEnv => BoundRegionKind::BrEnv,
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::BoundVariableKind {
+    type T = stable_mir::ty::BoundVariableKind;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        use stable_mir::ty::BoundVariableKind;
+
+        match self {
+            ty::BoundVariableKind::Ty(bound_ty_kind) => {
+                BoundVariableKind::Ty(bound_ty_kind.stable(tables))
+            }
+            ty::BoundVariableKind::Region(bound_region_kind) => {
+                BoundVariableKind::Region(bound_region_kind.stable(tables))
+            }
+            ty::BoundVariableKind::Const => BoundVariableKind::Const,
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::IntTy {
+    type T = IntTy;
+
+    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
+        match self {
+            ty::IntTy::Isize => IntTy::Isize,
+            ty::IntTy::I8 => IntTy::I8,
+            ty::IntTy::I16 => IntTy::I16,
+            ty::IntTy::I32 => IntTy::I32,
+            ty::IntTy::I64 => IntTy::I64,
+            ty::IntTy::I128 => IntTy::I128,
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::UintTy {
+    type T = UintTy;
+
+    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
+        match self {
+            ty::UintTy::Usize => UintTy::Usize,
+            ty::UintTy::U8 => UintTy::U8,
+            ty::UintTy::U16 => UintTy::U16,
+            ty::UintTy::U32 => UintTy::U32,
+            ty::UintTy::U64 => UintTy::U64,
+            ty::UintTy::U128 => UintTy::U128,
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::FloatTy {
+    type T = FloatTy;
+
+    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
+        match self {
+            ty::FloatTy::F32 => FloatTy::F32,
+            ty::FloatTy::F64 => FloatTy::F64,
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for Ty<'tcx> {
+    type T = stable_mir::ty::Ty;
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        tables.intern_ty(*self)
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::TyKind<'tcx> {
+    type T = stable_mir::ty::TyKind;
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        match self {
+            ty::Bool => TyKind::RigidTy(RigidTy::Bool),
+            ty::Char => TyKind::RigidTy(RigidTy::Char),
+            ty::Int(int_ty) => TyKind::RigidTy(RigidTy::Int(int_ty.stable(tables))),
+            ty::Uint(uint_ty) => TyKind::RigidTy(RigidTy::Uint(uint_ty.stable(tables))),
+            ty::Float(float_ty) => TyKind::RigidTy(RigidTy::Float(float_ty.stable(tables))),
+            ty::Adt(adt_def, generic_args) => TyKind::RigidTy(RigidTy::Adt(
+                tables.adt_def(adt_def.did()),
+                generic_args.stable(tables),
+            )),
+            ty::Foreign(def_id) => TyKind::RigidTy(RigidTy::Foreign(tables.foreign_def(*def_id))),
+            ty::Str => TyKind::RigidTy(RigidTy::Str),
+            ty::Array(ty, constant) => {
+                TyKind::RigidTy(RigidTy::Array(ty.stable(tables), constant.stable(tables)))
+            }
+            ty::Slice(ty) => TyKind::RigidTy(RigidTy::Slice(ty.stable(tables))),
+            ty::RawPtr(ty::TypeAndMut { ty, mutbl }) => {
+                TyKind::RigidTy(RigidTy::RawPtr(ty.stable(tables), mutbl.stable(tables)))
+            }
+            ty::Ref(region, ty, mutbl) => TyKind::RigidTy(RigidTy::Ref(
+                region.stable(tables),
+                ty.stable(tables),
+                mutbl.stable(tables),
+            )),
+            ty::FnDef(def_id, generic_args) => {
+                TyKind::RigidTy(RigidTy::FnDef(tables.fn_def(*def_id), generic_args.stable(tables)))
+            }
+            ty::FnPtr(poly_fn_sig) => TyKind::RigidTy(RigidTy::FnPtr(poly_fn_sig.stable(tables))),
+            ty::Dynamic(existential_predicates, region, dyn_kind) => {
+                TyKind::RigidTy(RigidTy::Dynamic(
+                    existential_predicates
+                        .iter()
+                        .map(|existential_predicate| existential_predicate.stable(tables))
+                        .collect(),
+                    region.stable(tables),
+                    dyn_kind.stable(tables),
+                ))
+            }
+            ty::Closure(def_id, generic_args) => TyKind::RigidTy(RigidTy::Closure(
+                tables.closure_def(*def_id),
+                generic_args.stable(tables),
+            )),
+            ty::Coroutine(def_id, generic_args, movability) => TyKind::RigidTy(RigidTy::Coroutine(
+                tables.coroutine_def(*def_id),
+                generic_args.stable(tables),
+                movability.stable(tables),
+            )),
+            ty::Never => TyKind::RigidTy(RigidTy::Never),
+            ty::Tuple(fields) => {
+                TyKind::RigidTy(RigidTy::Tuple(fields.iter().map(|ty| ty.stable(tables)).collect()))
+            }
+            ty::Alias(alias_kind, alias_ty) => {
+                TyKind::Alias(alias_kind.stable(tables), alias_ty.stable(tables))
+            }
+            ty::Param(param_ty) => TyKind::Param(param_ty.stable(tables)),
+            ty::Bound(debruijn_idx, bound_ty) => {
+                TyKind::Bound(debruijn_idx.as_usize(), bound_ty.stable(tables))
+            }
+            ty::CoroutineWitness(def_id, args) => TyKind::RigidTy(RigidTy::CoroutineWitness(
+                tables.coroutine_witness_def(*def_id),
+                args.stable(tables),
+            )),
+            ty::Placeholder(..) | ty::Infer(_) | ty::Error(_) => {
+                unreachable!();
+            }
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::Const<'tcx> {
+    type T = stable_mir::ty::Const;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        let kind = match self.kind() {
+            ty::Value(val) => {
+                let const_val = tables.tcx.valtree_to_const_val((self.ty(), val));
+                if matches!(const_val, mir::ConstValue::ZeroSized) {
+                    ConstantKind::ZeroSized
+                } else {
+                    stable_mir::ty::ConstantKind::Allocated(alloc::new_allocation(
+                        self.ty(),
+                        const_val,
+                        tables,
+                    ))
+                }
+            }
+            ty::ParamCt(param) => stable_mir::ty::ConstantKind::Param(param.stable(tables)),
+            ty::ErrorCt(_) => unreachable!(),
+            ty::InferCt(_) => unreachable!(),
+            ty::BoundCt(_, _) => unimplemented!(),
+            ty::PlaceholderCt(_) => unimplemented!(),
+            ty::Unevaluated(uv) => {
+                stable_mir::ty::ConstantKind::Unevaluated(stable_mir::ty::UnevaluatedConst {
+                    def: tables.const_def(uv.def),
+                    args: uv.args.stable(tables),
+                    promoted: None,
+                })
+            }
+            ty::ExprCt(_) => unimplemented!(),
+        };
+        let ty = self.ty().stable(tables);
+        let id = tables.intern_const(mir::Const::Ty(*self));
+        Const::new(kind, ty, id)
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::ParamConst {
+    type T = stable_mir::ty::ParamConst;
+    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
+        use stable_mir::ty::ParamConst;
+        ParamConst { index: self.index, name: self.name.to_string() }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::ParamTy {
+    type T = stable_mir::ty::ParamTy;
+    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
+        use stable_mir::ty::ParamTy;
+        ParamTy { index: self.index, name: self.name.to_string() }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::BoundTy {
+    type T = stable_mir::ty::BoundTy;
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        use stable_mir::ty::BoundTy;
+        BoundTy { var: self.var.as_usize(), kind: self.kind.stable(tables) }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::trait_def::TraitSpecializationKind {
+    type T = stable_mir::ty::TraitSpecializationKind;
+    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
+        use stable_mir::ty::TraitSpecializationKind;
+
+        match self {
+            ty::trait_def::TraitSpecializationKind::None => TraitSpecializationKind::None,
+            ty::trait_def::TraitSpecializationKind::Marker => TraitSpecializationKind::Marker,
+            ty::trait_def::TraitSpecializationKind::AlwaysApplicable => {
+                TraitSpecializationKind::AlwaysApplicable
+            }
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::TraitDef {
+    type T = stable_mir::ty::TraitDecl;
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        use stable_mir::opaque;
+        use stable_mir::ty::TraitDecl;
+
+        TraitDecl {
+            def_id: tables.trait_def(self.def_id),
+            unsafety: self.unsafety.stable(tables),
+            paren_sugar: self.paren_sugar,
+            has_auto_impl: self.has_auto_impl,
+            is_marker: self.is_marker,
+            is_coinductive: self.is_coinductive,
+            skip_array_during_method_dispatch: self.skip_array_during_method_dispatch,
+            specialization_kind: self.specialization_kind.stable(tables),
+            must_implement_one_of: self
+                .must_implement_one_of
+                .as_ref()
+                .map(|idents| idents.iter().map(|ident| opaque(ident)).collect()),
+            implement_via_object: self.implement_via_object,
+            deny_explicit_impl: self.deny_explicit_impl,
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::TraitRef<'tcx> {
+    type T = stable_mir::ty::TraitRef;
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        use stable_mir::ty::TraitRef;
+
+        TraitRef::try_new(tables.trait_def(self.def_id), self.args.stable(tables)).unwrap()
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::Generics {
+    type T = stable_mir::ty::Generics;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        use stable_mir::ty::Generics;
+
+        let params: Vec<_> = self.params.iter().map(|param| param.stable(tables)).collect();
+        let param_def_id_to_index =
+            params.iter().map(|param| (param.def_id, param.index)).collect();
+
+        Generics {
+            parent: self.parent.map(|did| tables.generic_def(did)),
+            parent_count: self.parent_count,
+            params,
+            param_def_id_to_index,
+            has_self: self.has_self,
+            has_late_bound_regions: self
+                .has_late_bound_regions
+                .as_ref()
+                .map(|late_bound_regions| late_bound_regions.stable(tables)),
+            host_effect_index: self.host_effect_index,
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for rustc_middle::ty::GenericParamDefKind {
+    type T = stable_mir::ty::GenericParamDefKind;
+
+    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
+        use stable_mir::ty::GenericParamDefKind;
+        match self {
+            ty::GenericParamDefKind::Lifetime => GenericParamDefKind::Lifetime,
+            ty::GenericParamDefKind::Type { has_default, synthetic } => {
+                GenericParamDefKind::Type { has_default: *has_default, synthetic: *synthetic }
+            }
+            ty::GenericParamDefKind::Const { has_default, is_host_effect: _ } => {
+                GenericParamDefKind::Const { has_default: *has_default }
+            }
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for rustc_middle::ty::GenericParamDef {
+    type T = stable_mir::ty::GenericParamDef;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        GenericParamDef {
+            name: self.name.to_string(),
+            def_id: tables.generic_def(self.def_id),
+            index: self.index,
+            pure_wrt_drop: self.pure_wrt_drop,
+            kind: self.kind.stable(tables),
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::PredicateKind<'tcx> {
+    type T = stable_mir::ty::PredicateKind;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        use rustc_middle::ty::PredicateKind;
+        match self {
+            PredicateKind::Clause(clause_kind) => {
+                stable_mir::ty::PredicateKind::Clause(clause_kind.stable(tables))
+            }
+            PredicateKind::ObjectSafe(did) => {
+                stable_mir::ty::PredicateKind::ObjectSafe(tables.trait_def(*did))
+            }
+            PredicateKind::Subtype(subtype_predicate) => {
+                stable_mir::ty::PredicateKind::SubType(subtype_predicate.stable(tables))
+            }
+            PredicateKind::Coerce(coerce_predicate) => {
+                stable_mir::ty::PredicateKind::Coerce(coerce_predicate.stable(tables))
+            }
+            PredicateKind::ConstEquate(a, b) => {
+                stable_mir::ty::PredicateKind::ConstEquate(a.stable(tables), b.stable(tables))
+            }
+            PredicateKind::Ambiguous => stable_mir::ty::PredicateKind::Ambiguous,
+            PredicateKind::NormalizesTo(_pred) => unimplemented!(),
+            PredicateKind::AliasRelate(a, b, alias_relation_direction) => {
+                stable_mir::ty::PredicateKind::AliasRelate(
+                    a.unpack().stable(tables),
+                    b.unpack().stable(tables),
+                    alias_relation_direction.stable(tables),
+                )
+            }
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::ClauseKind<'tcx> {
+    type T = stable_mir::ty::ClauseKind;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        use rustc_middle::ty::ClauseKind;
+        match *self {
+            ClauseKind::Trait(trait_object) => {
+                stable_mir::ty::ClauseKind::Trait(trait_object.stable(tables))
+            }
+            ClauseKind::RegionOutlives(region_outlives) => {
+                stable_mir::ty::ClauseKind::RegionOutlives(region_outlives.stable(tables))
+            }
+            ClauseKind::TypeOutlives(type_outlives) => {
+                let ty::OutlivesPredicate::<_, _>(a, b) = type_outlives;
+                stable_mir::ty::ClauseKind::TypeOutlives(stable_mir::ty::OutlivesPredicate(
+                    a.stable(tables),
+                    b.stable(tables),
+                ))
+            }
+            ClauseKind::Projection(projection_predicate) => {
+                stable_mir::ty::ClauseKind::Projection(projection_predicate.stable(tables))
+            }
+            ClauseKind::ConstArgHasType(const_, ty) => stable_mir::ty::ClauseKind::ConstArgHasType(
+                const_.stable(tables),
+                ty.stable(tables),
+            ),
+            ClauseKind::WellFormed(generic_arg) => {
+                stable_mir::ty::ClauseKind::WellFormed(generic_arg.unpack().stable(tables))
+            }
+            ClauseKind::ConstEvaluatable(const_) => {
+                stable_mir::ty::ClauseKind::ConstEvaluatable(const_.stable(tables))
+            }
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::ClosureKind {
+    type T = stable_mir::ty::ClosureKind;
+
+    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
+        use rustc_middle::ty::ClosureKind::*;
+        match self {
+            Fn => stable_mir::ty::ClosureKind::Fn,
+            FnMut => stable_mir::ty::ClosureKind::FnMut,
+            FnOnce => stable_mir::ty::ClosureKind::FnOnce,
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::SubtypePredicate<'tcx> {
+    type T = stable_mir::ty::SubtypePredicate;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        let ty::SubtypePredicate { a, b, a_is_expected: _ } = self;
+        stable_mir::ty::SubtypePredicate { a: a.stable(tables), b: b.stable(tables) }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::CoercePredicate<'tcx> {
+    type T = stable_mir::ty::CoercePredicate;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        let ty::CoercePredicate { a, b } = self;
+        stable_mir::ty::CoercePredicate { a: a.stable(tables), b: b.stable(tables) }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::AliasRelationDirection {
+    type T = stable_mir::ty::AliasRelationDirection;
+
+    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
+        use rustc_middle::ty::AliasRelationDirection::*;
+        match self {
+            Equate => stable_mir::ty::AliasRelationDirection::Equate,
+            Subtype => stable_mir::ty::AliasRelationDirection::Subtype,
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::TraitPredicate<'tcx> {
+    type T = stable_mir::ty::TraitPredicate;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        let ty::TraitPredicate { trait_ref, polarity } = self;
+        stable_mir::ty::TraitPredicate {
+            trait_ref: trait_ref.stable(tables),
+            polarity: polarity.stable(tables),
+        }
+    }
+}
+
+impl<'tcx, A, B, U, V> Stable<'tcx> for ty::OutlivesPredicate<A, B>
+where
+    A: Stable<'tcx, T = U>,
+    B: Stable<'tcx, T = V>,
+{
+    type T = stable_mir::ty::OutlivesPredicate<U, V>;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        let ty::OutlivesPredicate(a, b) = self;
+        stable_mir::ty::OutlivesPredicate(a.stable(tables), b.stable(tables))
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::ProjectionPredicate<'tcx> {
+    type T = stable_mir::ty::ProjectionPredicate;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        let ty::ProjectionPredicate { projection_ty, term } = self;
+        stable_mir::ty::ProjectionPredicate {
+            projection_ty: projection_ty.stable(tables),
+            term: term.unpack().stable(tables),
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::ImplPolarity {
+    type T = stable_mir::ty::ImplPolarity;
+
+    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
+        use rustc_middle::ty::ImplPolarity::*;
+        match self {
+            Positive => stable_mir::ty::ImplPolarity::Positive,
+            Negative => stable_mir::ty::ImplPolarity::Negative,
+            Reservation => stable_mir::ty::ImplPolarity::Reservation,
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::Region<'tcx> {
+    type T = stable_mir::ty::Region;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        Region { kind: self.kind().stable(tables) }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::RegionKind<'tcx> {
+    type T = stable_mir::ty::RegionKind;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        use stable_mir::ty::{BoundRegion, EarlyParamRegion, RegionKind};
+        match self {
+            ty::ReEarlyParam(early_reg) => RegionKind::ReEarlyParam(EarlyParamRegion {
+                def_id: tables.region_def(early_reg.def_id),
+                index: early_reg.index,
+                name: early_reg.name.to_string(),
+            }),
+            ty::ReBound(db_index, bound_reg) => RegionKind::ReBound(
+                db_index.as_u32(),
+                BoundRegion { var: bound_reg.var.as_u32(), kind: bound_reg.kind.stable(tables) },
+            ),
+            ty::ReStatic => RegionKind::ReStatic,
+            ty::RePlaceholder(place_holder) => {
+                RegionKind::RePlaceholder(stable_mir::ty::Placeholder {
+                    universe: place_holder.universe.as_u32(),
+                    bound: BoundRegion {
+                        var: place_holder.bound.var.as_u32(),
+                        kind: place_holder.bound.kind.stable(tables),
+                    },
+                })
+            }
+            ty::ReErased => RegionKind::ReErased,
+            _ => unreachable!("{self:?}"),
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::Instance<'tcx> {
+    type T = stable_mir::mir::mono::Instance;
+
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        let def = tables.instance_def(*self);
+        let kind = match self.def {
+            ty::InstanceDef::Item(..) => stable_mir::mir::mono::InstanceKind::Item,
+            ty::InstanceDef::Intrinsic(..) => stable_mir::mir::mono::InstanceKind::Intrinsic,
+            ty::InstanceDef::Virtual(_def_id, idx) => {
+                stable_mir::mir::mono::InstanceKind::Virtual { idx }
+            }
+            ty::InstanceDef::VTableShim(..)
+            | ty::InstanceDef::ReifyShim(..)
+            | ty::InstanceDef::FnPtrAddrShim(..)
+            | ty::InstanceDef::ClosureOnceShim { .. }
+            | ty::InstanceDef::ThreadLocalShim(..)
+            | ty::InstanceDef::DropGlue(..)
+            | ty::InstanceDef::CloneShim(..)
+            | ty::InstanceDef::FnPtrShim(..) => stable_mir::mir::mono::InstanceKind::Shim,
+        };
+        stable_mir::mir::mono::Instance { def, kind }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::Variance {
+    type T = stable_mir::mir::Variance;
+    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
+        match self {
+            ty::Variance::Bivariant => stable_mir::mir::Variance::Bivariant,
+            ty::Variance::Contravariant => stable_mir::mir::Variance::Contravariant,
+            ty::Variance::Covariant => stable_mir::mir::Variance::Covariant,
+            ty::Variance::Invariant => stable_mir::mir::Variance::Invariant,
+        }
+    }
+}
+
+impl<'tcx> Stable<'tcx> for ty::Movability {
+    type T = stable_mir::ty::Movability;
+
+    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
+        match self {
+            ty::Movability::Static => stable_mir::ty::Movability::Static,
+            ty::Movability::Movable => stable_mir::ty::Movability::Movable,
+        }
+    }
+}
diff --git a/compiler/rustc_smir/src/rustc_smir/mod.rs b/compiler/rustc_smir/src/rustc_smir/mod.rs
index 27596c08f..e1ee40c0b 100644
--- a/compiler/rustc_smir/src/rustc_smir/mod.rs
+++ b/compiler/rustc_smir/src/rustc_smir/mod.rs
@@ -7,280 +7,100 @@
 //!
 //! For now, we are developing everything inside `rustc`, thus, we keep this module private.
 
-use crate::rustc_internal::{IndexMap, RustcInternal};
-use crate::rustc_smir::hir::def::DefKind;
-use crate::rustc_smir::stable_mir::ty::{BoundRegion, EarlyBoundRegion, Region};
-use rustc_hir as hir;
+use rustc_hir::def::DefKind;
 use rustc_middle::mir;
-use rustc_middle::mir::interpret::{alloc_range, AllocId};
-use rustc_middle::mir::mono::MonoItem;
-use rustc_middle::ty::{self, Instance, ParamEnv, Ty, TyCtxt, Variance};
+use rustc_middle::mir::interpret::AllocId;
+use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
 use rustc_span::def_id::{CrateNum, DefId, LOCAL_CRATE};
-use rustc_target::abi::FieldIdx;
+use stable_mir::abi::Layout;
 use stable_mir::mir::mono::InstanceDef;
-use stable_mir::mir::{Body, CopyNonOverlapping, Statement, UserTypeProjection, VariantIdx};
-use stable_mir::ty::{
-    Const, ConstId, ConstantKind, FloatTy, GenericParamDef, IntTy, LineInfo, Movability, RigidTy,
-    Span, TyKind, UintTy,
-};
-use stable_mir::{self, opaque, Context, Filename};
-use std::cell::RefCell;
+use stable_mir::ty::{ConstId, Span};
+use stable_mir::{CtorKind, ItemKind};
+use std::ops::RangeInclusive;
 use tracing::debug;
 
+use crate::rustc_internal::IndexMap;
+
 mod alloc;
 mod builder;
-
-impl<'tcx> Context for TablesWrapper<'tcx> {
-    fn local_crate(&self) -> stable_mir::Crate {
-        let tables = self.0.borrow();
-        smir_crate(tables.tcx, LOCAL_CRATE)
-    }
-
-    fn external_crates(&self) -> Vec<stable_mir::Crate> {
-        let tables = self.0.borrow();
-        tables.tcx.crates(()).iter().map(|crate_num| smir_crate(tables.tcx, *crate_num)).collect()
-    }
-
-    fn find_crates(&self, name: &str) -> Vec<stable_mir::Crate> {
-        let tables = self.0.borrow();
-        let crates: Vec<stable_mir::Crate> = [LOCAL_CRATE]
-            .iter()
-            .chain(tables.tcx.crates(()).iter())
-            .map(|crate_num| {
-                let crate_name = tables.tcx.crate_name(*crate_num).to_string();
-                (name == crate_name).then(|| smir_crate(tables.tcx, *crate_num))
-            })
-            .into_iter()
-            .filter_map(|c| c)
-            .collect();
-        crates
-    }
-
-    fn name_of_def_id(&self, def_id: stable_mir::DefId) -> String {
-        let tables = self.0.borrow();
-        tables.tcx.def_path_str(tables[def_id])
-    }
-
-    fn span_to_string(&self, span: stable_mir::ty::Span) -> String {
-        let tables = self.0.borrow();
-        tables.tcx.sess.source_map().span_to_diagnostic_string(tables[span])
-    }
-
-    fn get_filename(&self, span: &Span) -> Filename {
-        let tables = self.0.borrow();
-        opaque(
-            &tables
-                .tcx
-                .sess
-                .source_map()
-                .span_to_filename(tables[*span])
-                .display(rustc_span::FileNameDisplayPreference::Local)
-                .to_string(),
-        )
-    }
-
-    fn get_lines(&self, span: &Span) -> LineInfo {
-        let tables = self.0.borrow();
-        let lines = &tables.tcx.sess.source_map().span_to_location_info(tables[*span]);
-        LineInfo { start_line: lines.1, start_col: lines.2, end_line: lines.3, end_col: lines.4 }
-    }
-
-    fn def_kind(&self, def_id: stable_mir::DefId) -> stable_mir::DefKind {
-        let mut tables = self.0.borrow_mut();
-        tables.tcx.def_kind(tables[def_id]).stable(&mut *tables)
-    }
-
-    fn span_of_an_item(&self, def_id: stable_mir::DefId) -> Span {
-        let mut tables = self.0.borrow_mut();
-        tables.tcx.def_span(tables[def_id]).stable(&mut *tables)
-    }
-
-    fn all_local_items(&self) -> stable_mir::CrateItems {
-        let mut tables = self.0.borrow_mut();
-        tables.tcx.mir_keys(()).iter().map(|item| tables.crate_item(item.to_def_id())).collect()
-    }
-
-    fn entry_fn(&self) -> Option<stable_mir::CrateItem> {
-        let mut tables = self.0.borrow_mut();
-        let tcx = tables.tcx;
-        Some(tables.crate_item(tcx.entry_fn(())?.0))
-    }
-
-    fn all_trait_decls(&self) -> stable_mir::TraitDecls {
-        let mut tables = self.0.borrow_mut();
-        tables
-            .tcx
-            .traits(LOCAL_CRATE)
-            .iter()
-            .map(|trait_def_id| tables.trait_def(*trait_def_id))
-            .collect()
-    }
-
-    fn trait_decl(&self, trait_def: &stable_mir::ty::TraitDef) -> stable_mir::ty::TraitDecl {
-        let mut tables = self.0.borrow_mut();
-        let def_id = tables[trait_def.0];
-        let trait_def = tables.tcx.trait_def(def_id);
-        trait_def.stable(&mut *tables)
-    }
-
-    fn all_trait_impls(&self) -> stable_mir::ImplTraitDecls {
-        let mut tables = self.0.borrow_mut();
-        tables
-            .tcx
-            .trait_impls_in_crate(LOCAL_CRATE)
-            .iter()
-            .map(|impl_def_id| tables.impl_def(*impl_def_id))
-            .collect()
-    }
-
-    fn trait_impl(&self, impl_def: &stable_mir::ty::ImplDef) -> stable_mir::ty::ImplTrait {
-        let mut tables = self.0.borrow_mut();
-        let def_id = tables[impl_def.0];
-        let impl_trait = tables.tcx.impl_trait_ref(def_id).unwrap();
-        impl_trait.stable(&mut *tables)
-    }
-
-    fn mir_body(&self, item: stable_mir::DefId) -> stable_mir::mir::Body {
-        let mut tables = self.0.borrow_mut();
-        let def_id = tables[item];
-        tables.tcx.instance_mir(ty::InstanceDef::Item(def_id)).stable(&mut tables)
-    }
-
-    fn ty_kind(&self, ty: stable_mir::ty::Ty) -> TyKind {
-        let mut tables = self.0.borrow_mut();
-        tables.types[ty].kind().stable(&mut *tables)
-    }
-
-    fn generics_of(&self, def_id: stable_mir::DefId) -> stable_mir::ty::Generics {
-        let mut tables = self.0.borrow_mut();
-        let def_id = tables[def_id];
-        let generics = tables.tcx.generics_of(def_id);
-        generics.stable(&mut *tables)
-    }
-
-    fn predicates_of(&self, def_id: stable_mir::DefId) -> stable_mir::ty::GenericPredicates {
-        let mut tables = self.0.borrow_mut();
-        let def_id = tables[def_id];
-        let ty::GenericPredicates { parent, predicates } = tables.tcx.predicates_of(def_id);
-        stable_mir::ty::GenericPredicates {
-            parent: parent.map(|did| tables.trait_def(did)),
-            predicates: predicates
-                .iter()
-                .map(|(clause, span)| {
-                    (
-                        clause.as_predicate().kind().skip_binder().stable(&mut *tables),
-                        span.stable(&mut *tables),
-                    )
-                })
-                .collect(),
-        }
-    }
-
-    fn explicit_predicates_of(
-        &self,
-        def_id: stable_mir::DefId,
-    ) -> stable_mir::ty::GenericPredicates {
-        let mut tables = self.0.borrow_mut();
-        let def_id = tables[def_id];
-        let ty::GenericPredicates { parent, predicates } =
-            tables.tcx.explicit_predicates_of(def_id);
-        stable_mir::ty::GenericPredicates {
-            parent: parent.map(|did| tables.trait_def(did)),
-            predicates: predicates
-                .iter()
-                .map(|(clause, span)| {
-                    (
-                        clause.as_predicate().kind().skip_binder().stable(&mut *tables),
-                        span.stable(&mut *tables),
-                    )
-                })
-                .collect(),
-        }
-    }
-
-    fn instance_body(&self, def: InstanceDef) -> Body {
-        let mut tables = self.0.borrow_mut();
-        let instance = tables.instances[def];
-        builder::BodyBuilder::new(tables.tcx, instance).build(&mut *tables)
-    }
-
-    fn instance_ty(&self, def: InstanceDef) -> stable_mir::ty::Ty {
-        let mut tables = self.0.borrow_mut();
-        let instance = tables.instances[def];
-        instance.ty(tables.tcx, ParamEnv::empty()).stable(&mut *tables)
-    }
-
-    fn instance_def_id(&self, def: InstanceDef) -> stable_mir::DefId {
-        let mut tables = self.0.borrow_mut();
-        let def_id = tables.instances[def].def_id();
-        tables.create_def_id(def_id)
-    }
-
-    fn instance_mangled_name(&self, def: InstanceDef) -> String {
-        let tables = self.0.borrow_mut();
-        let instance = tables.instances[def];
-        tables.tcx.symbol_name(instance).name.to_string()
-    }
-
-    fn mono_instance(&self, item: stable_mir::CrateItem) -> stable_mir::mir::mono::Instance {
-        let mut tables = self.0.borrow_mut();
-        let def_id = tables[item.0];
-        Instance::mono(tables.tcx, def_id).stable(&mut *tables)
-    }
-
-    fn requires_monomorphization(&self, def_id: stable_mir::DefId) -> bool {
-        let tables = self.0.borrow();
-        let def_id = tables[def_id];
-        let generics = tables.tcx.generics_of(def_id);
-        let result = generics.requires_monomorphization(tables.tcx);
-        result
-    }
-
-    fn resolve_instance(
-        &self,
-        def: stable_mir::ty::FnDef,
-        args: &stable_mir::ty::GenericArgs,
-    ) -> Option<stable_mir::mir::mono::Instance> {
-        let mut tables = self.0.borrow_mut();
-        let def_id = def.0.internal(&mut *tables);
-        let args_ref = args.internal(&mut *tables);
-        match Instance::resolve(tables.tcx, ParamEnv::reveal_all(), def_id, args_ref) {
-            Ok(Some(instance)) => Some(instance.stable(&mut *tables)),
-            Ok(None) | Err(_) => None,
-        }
-    }
-}
-
-pub(crate) struct TablesWrapper<'tcx>(pub(crate) RefCell<Tables<'tcx>>);
+pub(crate) mod context;
+mod convert;
 
 pub struct Tables<'tcx> {
     pub(crate) tcx: TyCtxt<'tcx>,
     pub(crate) def_ids: IndexMap<DefId, stable_mir::DefId>,
-    pub(crate) alloc_ids: IndexMap<AllocId, stable_mir::AllocId>,
+    pub(crate) alloc_ids: IndexMap<AllocId, stable_mir::mir::alloc::AllocId>,
     pub(crate) spans: IndexMap<rustc_span::Span, Span>,
     pub(crate) types: IndexMap<Ty<'tcx>, stable_mir::ty::Ty>,
     pub(crate) instances: IndexMap<ty::Instance<'tcx>, InstanceDef>,
     pub(crate) constants: IndexMap<mir::Const<'tcx>, ConstId>,
+    pub(crate) layouts: IndexMap<rustc_target::abi::Layout<'tcx>, Layout>,
 }
 
 impl<'tcx> Tables<'tcx> {
-    fn intern_ty(&mut self, ty: Ty<'tcx>) -> stable_mir::ty::Ty {
+    pub(crate) fn intern_ty(&mut self, ty: Ty<'tcx>) -> stable_mir::ty::Ty {
         self.types.create_or_fetch(ty)
     }
 
-    fn intern_const(&mut self, constant: mir::Const<'tcx>) -> ConstId {
+    pub(crate) fn intern_const(&mut self, constant: mir::Const<'tcx>) -> ConstId {
         self.constants.create_or_fetch(constant)
     }
+
+    pub(crate) fn has_body(&self, instance: Instance<'tcx>) -> bool {
+        let def_id = instance.def_id();
+        self.tcx.is_mir_available(def_id)
+            || !matches!(
+                instance.def,
+                ty::InstanceDef::Virtual(..)
+                    | ty::InstanceDef::Intrinsic(..)
+                    | ty::InstanceDef::Item(..)
+            )
+    }
 }
 
 /// Build a stable mir crate from a given crate number.
-fn smir_crate(tcx: TyCtxt<'_>, crate_num: CrateNum) -> stable_mir::Crate {
+pub(crate) fn smir_crate(tcx: TyCtxt<'_>, crate_num: CrateNum) -> stable_mir::Crate {
     let crate_name = tcx.crate_name(crate_num).to_string();
     let is_local = crate_num == LOCAL_CRATE;
     debug!(?crate_name, ?crate_num, "smir_crate");
     stable_mir::Crate { id: crate_num.into(), name: crate_name, is_local }
 }
 
+pub(crate) fn new_item_kind(kind: DefKind) -> ItemKind {
+    match kind {
+        DefKind::Mod
+        | DefKind::Struct
+        | DefKind::Union
+        | DefKind::Enum
+        | DefKind::Variant
+        | DefKind::Trait
+        | DefKind::TyAlias
+        | DefKind::ForeignTy
+        | DefKind::TraitAlias
+        | DefKind::AssocTy
+        | DefKind::TyParam
+        | DefKind::ConstParam
+        | DefKind::Macro(_)
+        | DefKind::ExternCrate
+        | DefKind::Use
+        | DefKind::ForeignMod
+        | DefKind::OpaqueTy
+        | DefKind::Field
+        | DefKind::LifetimeParam
+        | DefKind::Impl { .. }
+        | DefKind::GlobalAsm => {
+            unreachable!("Not a valid item kind: {kind:?}");
+        }
+        DefKind::Closure | DefKind::AssocFn | DefKind::Fn => ItemKind::Fn,
+        DefKind::Const | DefKind::InlineConst | DefKind::AssocConst | DefKind::AnonConst => {
+            ItemKind::Const
+        }
+        DefKind::Static(_) => ItemKind::Static,
+        DefKind::Ctor(_, rustc_hir::def::CtorKind::Const) => ItemKind::Ctor(CtorKind::Const),
+        DefKind::Ctor(_, rustc_hir::def::CtorKind::Fn) => ItemKind::Ctor(CtorKind::Fn),
+    }
+}
+
 /// Trait used to convert between an internal MIR type to a Stable MIR type.
 pub trait Stable<'tcx> {
     /// The stable representation of the type implementing Stable.
@@ -289,1481 +109,70 @@ pub trait Stable<'tcx> {
     fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T;
 }
 
-impl<'tcx> Stable<'tcx> for mir::Body<'tcx> {
-    type T = stable_mir::mir::Body;
-
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        stable_mir::mir::Body::new(
-            self.basic_blocks
-                .iter()
-                .map(|block| stable_mir::mir::BasicBlock {
-                    terminator: block.terminator().stable(tables),
-                    statements: block
-                        .statements
-                        .iter()
-                        .map(|statement| statement.stable(tables))
-                        .collect(),
-                })
-                .collect(),
-            self.local_decls
-                .iter()
-                .map(|decl| stable_mir::mir::LocalDecl {
-                    ty: decl.ty.stable(tables),
-                    span: decl.source_info.span.stable(tables),
-                })
-                .collect(),
-            self.arg_count,
-        )
-    }
-}
-
-impl<'tcx> Stable<'tcx> for mir::Statement<'tcx> {
-    type T = stable_mir::mir::Statement;
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        Statement { kind: self.kind.stable(tables), span: self.source_info.span.stable(tables) }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for mir::StatementKind<'tcx> {
-    type T = stable_mir::mir::StatementKind;
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        match self {
-            mir::StatementKind::Assign(assign) => stable_mir::mir::StatementKind::Assign(
-                assign.0.stable(tables),
-                assign.1.stable(tables),
-            ),
-            mir::StatementKind::FakeRead(fake_read_place) => {
-                stable_mir::mir::StatementKind::FakeRead(
-                    fake_read_place.0.stable(tables),
-                    fake_read_place.1.stable(tables),
-                )
-            }
-            mir::StatementKind::SetDiscriminant { place, variant_index } => {
-                stable_mir::mir::StatementKind::SetDiscriminant {
-                    place: place.as_ref().stable(tables),
-                    variant_index: variant_index.stable(tables),
-                }
-            }
-            mir::StatementKind::Deinit(place) => {
-                stable_mir::mir::StatementKind::Deinit(place.stable(tables))
-            }
-
-            mir::StatementKind::StorageLive(place) => {
-                stable_mir::mir::StatementKind::StorageLive(place.stable(tables))
-            }
-
-            mir::StatementKind::StorageDead(place) => {
-                stable_mir::mir::StatementKind::StorageDead(place.stable(tables))
-            }
-            mir::StatementKind::Retag(retag, place) => {
-                stable_mir::mir::StatementKind::Retag(retag.stable(tables), place.stable(tables))
-            }
-            mir::StatementKind::PlaceMention(place) => {
-                stable_mir::mir::StatementKind::PlaceMention(place.stable(tables))
-            }
-            mir::StatementKind::AscribeUserType(place_projection, variance) => {
-                stable_mir::mir::StatementKind::AscribeUserType {
-                    place: place_projection.as_ref().0.stable(tables),
-                    projections: place_projection.as_ref().1.stable(tables),
-                    variance: variance.stable(tables),
-                }
-            }
-            mir::StatementKind::Coverage(coverage) => {
-                stable_mir::mir::StatementKind::Coverage(opaque(coverage))
-            }
-            mir::StatementKind::Intrinsic(intrinstic) => {
-                stable_mir::mir::StatementKind::Intrinsic(intrinstic.stable(tables))
-            }
-            mir::StatementKind::ConstEvalCounter => {
-                stable_mir::mir::StatementKind::ConstEvalCounter
-            }
-            mir::StatementKind::Nop => stable_mir::mir::StatementKind::Nop,
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for mir::Rvalue<'tcx> {
-    type T = stable_mir::mir::Rvalue;
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        use mir::Rvalue::*;
-        match self {
-            Use(op) => stable_mir::mir::Rvalue::Use(op.stable(tables)),
-            Repeat(op, len) => {
-                let len = len.stable(tables);
-                stable_mir::mir::Rvalue::Repeat(op.stable(tables), len)
-            }
-            Ref(region, kind, place) => stable_mir::mir::Rvalue::Ref(
-                region.stable(tables),
-                kind.stable(tables),
-                place.stable(tables),
-            ),
-            ThreadLocalRef(def_id) => {
-                stable_mir::mir::Rvalue::ThreadLocalRef(tables.crate_item(*def_id))
-            }
-            AddressOf(mutability, place) => {
-                stable_mir::mir::Rvalue::AddressOf(mutability.stable(tables), place.stable(tables))
-            }
-            Len(place) => stable_mir::mir::Rvalue::Len(place.stable(tables)),
-            Cast(cast_kind, op, ty) => stable_mir::mir::Rvalue::Cast(
-                cast_kind.stable(tables),
-                op.stable(tables),
-                ty.stable(tables),
-            ),
-            BinaryOp(bin_op, ops) => stable_mir::mir::Rvalue::BinaryOp(
-                bin_op.stable(tables),
-                ops.0.stable(tables),
-                ops.1.stable(tables),
-            ),
-            CheckedBinaryOp(bin_op, ops) => stable_mir::mir::Rvalue::CheckedBinaryOp(
-                bin_op.stable(tables),
-                ops.0.stable(tables),
-                ops.1.stable(tables),
-            ),
-            NullaryOp(null_op, ty) => {
-                stable_mir::mir::Rvalue::NullaryOp(null_op.stable(tables), ty.stable(tables))
-            }
-            UnaryOp(un_op, op) => {
-                stable_mir::mir::Rvalue::UnaryOp(un_op.stable(tables), op.stable(tables))
-            }
-            Discriminant(place) => stable_mir::mir::Rvalue::Discriminant(place.stable(tables)),
-            Aggregate(agg_kind, operands) => {
-                let operands = operands.iter().map(|op| op.stable(tables)).collect();
-                stable_mir::mir::Rvalue::Aggregate(agg_kind.stable(tables), operands)
-            }
-            ShallowInitBox(op, ty) => {
-                stable_mir::mir::Rvalue::ShallowInitBox(op.stable(tables), ty.stable(tables))
-            }
-            CopyForDeref(place) => stable_mir::mir::Rvalue::CopyForDeref(place.stable(tables)),
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for mir::Mutability {
-    type T = stable_mir::mir::Mutability;
-    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
-        use mir::Mutability::*;
-        match *self {
-            Not => stable_mir::mir::Mutability::Not,
-            Mut => stable_mir::mir::Mutability::Mut,
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for mir::BorrowKind {
-    type T = stable_mir::mir::BorrowKind;
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        use mir::BorrowKind::*;
-        match *self {
-            Shared => stable_mir::mir::BorrowKind::Shared,
-            Fake => stable_mir::mir::BorrowKind::Fake,
-            Mut { kind } => stable_mir::mir::BorrowKind::Mut { kind: kind.stable(tables) },
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for mir::MutBorrowKind {
-    type T = stable_mir::mir::MutBorrowKind;
-    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
-        use mir::MutBorrowKind::*;
-        match *self {
-            Default => stable_mir::mir::MutBorrowKind::Default,
-            TwoPhaseBorrow => stable_mir::mir::MutBorrowKind::TwoPhaseBorrow,
-            ClosureCapture => stable_mir::mir::MutBorrowKind::ClosureCapture,
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for mir::NullOp<'tcx> {
-    type T = stable_mir::mir::NullOp;
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        use mir::NullOp::*;
-        match self {
-            SizeOf => stable_mir::mir::NullOp::SizeOf,
-            AlignOf => stable_mir::mir::NullOp::AlignOf,
-            OffsetOf(indices) => stable_mir::mir::NullOp::OffsetOf(
-                indices.iter().map(|idx| idx.stable(tables)).collect(),
-            ),
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for mir::CastKind {
-    type T = stable_mir::mir::CastKind;
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        use mir::CastKind::*;
-        match self {
-            PointerExposeAddress => stable_mir::mir::CastKind::PointerExposeAddress,
-            PointerFromExposedAddress => stable_mir::mir::CastKind::PointerFromExposedAddress,
-            PointerCoercion(c) => stable_mir::mir::CastKind::PointerCoercion(c.stable(tables)),
-            DynStar => stable_mir::mir::CastKind::DynStar,
-            IntToInt => stable_mir::mir::CastKind::IntToInt,
-            FloatToInt => stable_mir::mir::CastKind::FloatToInt,
-            FloatToFloat => stable_mir::mir::CastKind::FloatToFloat,
-            IntToFloat => stable_mir::mir::CastKind::IntToFloat,
-            PtrToPtr => stable_mir::mir::CastKind::PtrToPtr,
-            FnPtrToPtr => stable_mir::mir::CastKind::FnPtrToPtr,
-            Transmute => stable_mir::mir::CastKind::Transmute,
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::AliasKind {
-    type T = stable_mir::ty::AliasKind;
-    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
-        use ty::AliasKind::*;
-        match self {
-            Projection => stable_mir::ty::AliasKind::Projection,
-            Inherent => stable_mir::ty::AliasKind::Inherent,
-            Opaque => stable_mir::ty::AliasKind::Opaque,
-            Weak => stable_mir::ty::AliasKind::Weak,
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::AliasTy<'tcx> {
-    type T = stable_mir::ty::AliasTy;
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        let ty::AliasTy { args, def_id, .. } = self;
-        stable_mir::ty::AliasTy { def_id: tables.alias_def(*def_id), args: args.stable(tables) }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::DynKind {
-    type T = stable_mir::ty::DynKind;
-
-    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
-        use ty::DynKind;
-        match self {
-            DynKind::Dyn => stable_mir::ty::DynKind::Dyn,
-            DynKind::DynStar => stable_mir::ty::DynKind::DynStar,
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::ExistentialPredicate<'tcx> {
-    type T = stable_mir::ty::ExistentialPredicate;
-
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        use stable_mir::ty::ExistentialPredicate::*;
-        match self {
-            ty::ExistentialPredicate::Trait(existential_trait_ref) => {
-                Trait(existential_trait_ref.stable(tables))
-            }
-            ty::ExistentialPredicate::Projection(existential_projection) => {
-                Projection(existential_projection.stable(tables))
-            }
-            ty::ExistentialPredicate::AutoTrait(def_id) => AutoTrait(tables.trait_def(*def_id)),
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::ExistentialTraitRef<'tcx> {
-    type T = stable_mir::ty::ExistentialTraitRef;
-
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        let ty::ExistentialTraitRef { def_id, args } = self;
-        stable_mir::ty::ExistentialTraitRef {
-            def_id: tables.trait_def(*def_id),
-            generic_args: args.stable(tables),
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::TermKind<'tcx> {
-    type T = stable_mir::ty::TermKind;
-
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        use stable_mir::ty::TermKind;
-        match self {
-            ty::TermKind::Ty(ty) => TermKind::Type(ty.stable(tables)),
-            ty::TermKind::Const(cnst) => {
-                let cnst = cnst.stable(tables);
-                TermKind::Const(cnst)
-            }
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::ExistentialProjection<'tcx> {
-    type T = stable_mir::ty::ExistentialProjection;
-
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        let ty::ExistentialProjection { def_id, args, term } = self;
-        stable_mir::ty::ExistentialProjection {
-            def_id: tables.trait_def(*def_id),
-            generic_args: args.stable(tables),
-            term: term.unpack().stable(tables),
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::adjustment::PointerCoercion {
-    type T = stable_mir::mir::PointerCoercion;
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        use ty::adjustment::PointerCoercion;
-        match self {
-            PointerCoercion::ReifyFnPointer => stable_mir::mir::PointerCoercion::ReifyFnPointer,
-            PointerCoercion::UnsafeFnPointer => stable_mir::mir::PointerCoercion::UnsafeFnPointer,
-            PointerCoercion::ClosureFnPointer(unsafety) => {
-                stable_mir::mir::PointerCoercion::ClosureFnPointer(unsafety.stable(tables))
-            }
-            PointerCoercion::MutToConstPointer => {
-                stable_mir::mir::PointerCoercion::MutToConstPointer
-            }
-            PointerCoercion::ArrayToPointer => stable_mir::mir::PointerCoercion::ArrayToPointer,
-            PointerCoercion::Unsize => stable_mir::mir::PointerCoercion::Unsize,
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for rustc_hir::Unsafety {
-    type T = stable_mir::mir::Safety;
-    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
-        match self {
-            rustc_hir::Unsafety::Unsafe => stable_mir::mir::Safety::Unsafe,
-            rustc_hir::Unsafety::Normal => stable_mir::mir::Safety::Normal,
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for mir::FakeReadCause {
-    type T = stable_mir::mir::FakeReadCause;
-    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
-        use mir::FakeReadCause::*;
-        match self {
-            ForMatchGuard => stable_mir::mir::FakeReadCause::ForMatchGuard,
-            ForMatchedPlace(local_def_id) => {
-                stable_mir::mir::FakeReadCause::ForMatchedPlace(opaque(local_def_id))
-            }
-            ForGuardBinding => stable_mir::mir::FakeReadCause::ForGuardBinding,
-            ForLet(local_def_id) => stable_mir::mir::FakeReadCause::ForLet(opaque(local_def_id)),
-            ForIndex => stable_mir::mir::FakeReadCause::ForIndex,
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for FieldIdx {
-    type T = usize;
-    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
-        self.as_usize()
-    }
-}
-
-impl<'tcx> Stable<'tcx> for (rustc_target::abi::VariantIdx, FieldIdx) {
-    type T = (usize, usize);
-    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
-        (self.0.as_usize(), self.1.as_usize())
-    }
-}
-
-impl<'tcx> Stable<'tcx> for mir::Operand<'tcx> {
-    type T = stable_mir::mir::Operand;
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        use mir::Operand::*;
-        match self {
-            Copy(place) => stable_mir::mir::Operand::Copy(place.stable(tables)),
-            Move(place) => stable_mir::mir::Operand::Move(place.stable(tables)),
-            Constant(c) => stable_mir::mir::Operand::Constant(c.stable(tables)),
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for mir::ConstOperand<'tcx> {
-    type T = stable_mir::mir::Constant;
-
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        stable_mir::mir::Constant {
-            span: self.span.stable(tables),
-            user_ty: self.user_ty.map(|u| u.as_usize()).or(None),
-            literal: self.const_.stable(tables),
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for mir::Place<'tcx> {
-    type T = stable_mir::mir::Place;
-    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
-        stable_mir::mir::Place {
-            local: self.local.as_usize(),
-            projection: format!("{:?}", self.projection),
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for mir::UserTypeProjection {
-    type T = stable_mir::mir::UserTypeProjection;
-
-    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
-        UserTypeProjection { base: self.base.as_usize(), projection: format!("{:?}", self.projs) }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for mir::Local {
-    type T = stable_mir::mir::Local;
-    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
-        self.as_usize()
-    }
-}
-
-impl<'tcx> Stable<'tcx> for rustc_target::abi::VariantIdx {
-    type T = VariantIdx;
-    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
-        self.as_usize()
-    }
-}
-
-impl<'tcx> Stable<'tcx> for Variance {
-    type T = stable_mir::mir::Variance;
-    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
-        match self {
-            Variance::Bivariant => stable_mir::mir::Variance::Bivariant,
-            Variance::Contravariant => stable_mir::mir::Variance::Contravariant,
-            Variance::Covariant => stable_mir::mir::Variance::Covariant,
-            Variance::Invariant => stable_mir::mir::Variance::Invariant,
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for mir::RetagKind {
-    type T = stable_mir::mir::RetagKind;
-    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
-        use rustc_middle::mir::RetagKind;
-        match self {
-            RetagKind::FnEntry => stable_mir::mir::RetagKind::FnEntry,
-            RetagKind::TwoPhase => stable_mir::mir::RetagKind::TwoPhase,
-            RetagKind::Raw => stable_mir::mir::RetagKind::Raw,
-            RetagKind::Default => stable_mir::mir::RetagKind::Default,
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::UserTypeAnnotationIndex {
-    type T = usize;
-    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
-        self.as_usize()
-    }
-}
-
-impl<'tcx> Stable<'tcx> for mir::UnwindAction {
-    type T = stable_mir::mir::UnwindAction;
-    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
-        use rustc_middle::mir::UnwindAction;
-        match self {
-            UnwindAction::Continue => stable_mir::mir::UnwindAction::Continue,
-            UnwindAction::Unreachable => stable_mir::mir::UnwindAction::Unreachable,
-            UnwindAction::Terminate(_) => stable_mir::mir::UnwindAction::Terminate,
-            UnwindAction::Cleanup(bb) => stable_mir::mir::UnwindAction::Cleanup(bb.as_usize()),
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for mir::NonDivergingIntrinsic<'tcx> {
-    type T = stable_mir::mir::NonDivergingIntrinsic;
-
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        use rustc_middle::mir::NonDivergingIntrinsic;
-        match self {
-            NonDivergingIntrinsic::Assume(op) => {
-                stable_mir::mir::NonDivergingIntrinsic::Assume(op.stable(tables))
-            }
-            NonDivergingIntrinsic::CopyNonOverlapping(copy_non_overlapping) => {
-                stable_mir::mir::NonDivergingIntrinsic::CopyNonOverlapping(CopyNonOverlapping {
-                    src: copy_non_overlapping.src.stable(tables),
-                    dst: copy_non_overlapping.dst.stable(tables),
-                    count: copy_non_overlapping.count.stable(tables),
-                })
-            }
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for mir::AssertMessage<'tcx> {
-    type T = stable_mir::mir::AssertMessage;
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        use rustc_middle::mir::AssertKind;
-        match self {
-            AssertKind::BoundsCheck { len, index } => stable_mir::mir::AssertMessage::BoundsCheck {
-                len: len.stable(tables),
-                index: index.stable(tables),
-            },
-            AssertKind::Overflow(bin_op, op1, op2) => stable_mir::mir::AssertMessage::Overflow(
-                bin_op.stable(tables),
-                op1.stable(tables),
-                op2.stable(tables),
-            ),
-            AssertKind::OverflowNeg(op) => {
-                stable_mir::mir::AssertMessage::OverflowNeg(op.stable(tables))
-            }
-            AssertKind::DivisionByZero(op) => {
-                stable_mir::mir::AssertMessage::DivisionByZero(op.stable(tables))
-            }
-            AssertKind::RemainderByZero(op) => {
-                stable_mir::mir::AssertMessage::RemainderByZero(op.stable(tables))
-            }
-            AssertKind::ResumedAfterReturn(coroutine) => {
-                stable_mir::mir::AssertMessage::ResumedAfterReturn(coroutine.stable(tables))
-            }
-            AssertKind::ResumedAfterPanic(coroutine) => {
-                stable_mir::mir::AssertMessage::ResumedAfterPanic(coroutine.stable(tables))
-            }
-            AssertKind::MisalignedPointerDereference { required, found } => {
-                stable_mir::mir::AssertMessage::MisalignedPointerDereference {
-                    required: required.stable(tables),
-                    found: found.stable(tables),
-                }
-            }
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for mir::BinOp {
-    type T = stable_mir::mir::BinOp;
-    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
-        use mir::BinOp;
-        match self {
-            BinOp::Add => stable_mir::mir::BinOp::Add,
-            BinOp::AddUnchecked => stable_mir::mir::BinOp::AddUnchecked,
-            BinOp::Sub => stable_mir::mir::BinOp::Sub,
-            BinOp::SubUnchecked => stable_mir::mir::BinOp::SubUnchecked,
-            BinOp::Mul => stable_mir::mir::BinOp::Mul,
-            BinOp::MulUnchecked => stable_mir::mir::BinOp::MulUnchecked,
-            BinOp::Div => stable_mir::mir::BinOp::Div,
-            BinOp::Rem => stable_mir::mir::BinOp::Rem,
-            BinOp::BitXor => stable_mir::mir::BinOp::BitXor,
-            BinOp::BitAnd => stable_mir::mir::BinOp::BitAnd,
-            BinOp::BitOr => stable_mir::mir::BinOp::BitOr,
-            BinOp::Shl => stable_mir::mir::BinOp::Shl,
-            BinOp::ShlUnchecked => stable_mir::mir::BinOp::ShlUnchecked,
-            BinOp::Shr => stable_mir::mir::BinOp::Shr,
-            BinOp::ShrUnchecked => stable_mir::mir::BinOp::ShrUnchecked,
-            BinOp::Eq => stable_mir::mir::BinOp::Eq,
-            BinOp::Lt => stable_mir::mir::BinOp::Lt,
-            BinOp::Le => stable_mir::mir::BinOp::Le,
-            BinOp::Ne => stable_mir::mir::BinOp::Ne,
-            BinOp::Ge => stable_mir::mir::BinOp::Ge,
-            BinOp::Gt => stable_mir::mir::BinOp::Gt,
-            BinOp::Offset => stable_mir::mir::BinOp::Offset,
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for mir::UnOp {
-    type T = stable_mir::mir::UnOp;
-    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
-        use mir::UnOp;
-        match self {
-            UnOp::Not => stable_mir::mir::UnOp::Not,
-            UnOp::Neg => stable_mir::mir::UnOp::Neg,
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for mir::AggregateKind<'tcx> {
-    type T = stable_mir::mir::AggregateKind;
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        match self {
-            mir::AggregateKind::Array(ty) => {
-                stable_mir::mir::AggregateKind::Array(ty.stable(tables))
-            }
-            mir::AggregateKind::Tuple => stable_mir::mir::AggregateKind::Tuple,
-            mir::AggregateKind::Adt(def_id, var_idx, generic_arg, user_ty_index, field_idx) => {
-                stable_mir::mir::AggregateKind::Adt(
-                    tables.adt_def(*def_id),
-                    var_idx.index(),
-                    generic_arg.stable(tables),
-                    user_ty_index.map(|idx| idx.index()),
-                    field_idx.map(|idx| idx.index()),
-                )
-            }
-            mir::AggregateKind::Closure(def_id, generic_arg) => {
-                stable_mir::mir::AggregateKind::Closure(
-                    tables.closure_def(*def_id),
-                    generic_arg.stable(tables),
-                )
-            }
-            mir::AggregateKind::Coroutine(def_id, generic_arg, movability) => {
-                stable_mir::mir::AggregateKind::Coroutine(
-                    tables.coroutine_def(*def_id),
-                    generic_arg.stable(tables),
-                    movability.stable(tables),
-                )
-            }
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for rustc_hir::CoroutineSource {
-    type T = stable_mir::mir::CoroutineSource;
-    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
-        use rustc_hir::CoroutineSource;
-        match self {
-            CoroutineSource::Block => stable_mir::mir::CoroutineSource::Block,
-            CoroutineSource::Closure => stable_mir::mir::CoroutineSource::Closure,
-            CoroutineSource::Fn => stable_mir::mir::CoroutineSource::Fn,
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for rustc_hir::CoroutineKind {
-    type T = stable_mir::mir::CoroutineKind;
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        use rustc_hir::CoroutineKind;
-        match self {
-            CoroutineKind::Async(source) => {
-                stable_mir::mir::CoroutineKind::Async(source.stable(tables))
-            }
-            CoroutineKind::Gen(source) => {
-                stable_mir::mir::CoroutineKind::Gen(source.stable(tables))
-            }
-            CoroutineKind::Coroutine => stable_mir::mir::CoroutineKind::Coroutine,
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for mir::InlineAsmOperand<'tcx> {
-    type T = stable_mir::mir::InlineAsmOperand;
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        use rustc_middle::mir::InlineAsmOperand;
-
-        let (in_value, out_place) = match self {
-            InlineAsmOperand::In { value, .. } => (Some(value.stable(tables)), None),
-            InlineAsmOperand::Out { place, .. } => (None, place.map(|place| place.stable(tables))),
-            InlineAsmOperand::InOut { in_value, out_place, .. } => {
-                (Some(in_value.stable(tables)), out_place.map(|place| place.stable(tables)))
-            }
-            InlineAsmOperand::Const { .. }
-            | InlineAsmOperand::SymFn { .. }
-            | InlineAsmOperand::SymStatic { .. } => (None, None),
-        };
-
-        stable_mir::mir::InlineAsmOperand { in_value, out_place, raw_rpr: format!("{self:?}") }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for mir::Terminator<'tcx> {
-    type T = stable_mir::mir::Terminator;
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        use stable_mir::mir::Terminator;
-        Terminator { kind: self.kind.stable(tables), span: self.source_info.span.stable(tables) }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for mir::TerminatorKind<'tcx> {
-    type T = stable_mir::mir::TerminatorKind;
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        use stable_mir::mir::TerminatorKind;
-        match self {
-            mir::TerminatorKind::Goto { target } => {
-                TerminatorKind::Goto { target: target.as_usize() }
-            }
-            mir::TerminatorKind::SwitchInt { discr, targets } => TerminatorKind::SwitchInt {
-                discr: discr.stable(tables),
-                targets: targets
-                    .iter()
-                    .map(|(value, target)| stable_mir::mir::SwitchTarget {
-                        value,
-                        target: target.as_usize(),
-                    })
-                    .collect(),
-                otherwise: targets.otherwise().as_usize(),
-            },
-            mir::TerminatorKind::UnwindResume => TerminatorKind::Resume,
-            mir::TerminatorKind::UnwindTerminate(_) => TerminatorKind::Abort,
-            mir::TerminatorKind::Return => TerminatorKind::Return,
-            mir::TerminatorKind::Unreachable => TerminatorKind::Unreachable,
-            mir::TerminatorKind::Drop { place, target, unwind, replace: _ } => {
-                TerminatorKind::Drop {
-                    place: place.stable(tables),
-                    target: target.as_usize(),
-                    unwind: unwind.stable(tables),
-                }
-            }
-            mir::TerminatorKind::Call {
-                func,
-                args,
-                destination,
-                target,
-                unwind,
-                call_source: _,
-                fn_span: _,
-            } => TerminatorKind::Call {
-                func: func.stable(tables),
-                args: args.iter().map(|arg| arg.stable(tables)).collect(),
-                destination: destination.stable(tables),
-                target: target.map(|t| t.as_usize()),
-                unwind: unwind.stable(tables),
-            },
-            mir::TerminatorKind::Assert { cond, expected, msg, target, unwind } => {
-                TerminatorKind::Assert {
-                    cond: cond.stable(tables),
-                    expected: *expected,
-                    msg: msg.stable(tables),
-                    target: target.as_usize(),
-                    unwind: unwind.stable(tables),
-                }
-            }
-            mir::TerminatorKind::InlineAsm {
-                template,
-                operands,
-                options,
-                line_spans,
-                destination,
-                unwind,
-            } => TerminatorKind::InlineAsm {
-                template: format!("{template:?}"),
-                operands: operands.iter().map(|operand| operand.stable(tables)).collect(),
-                options: format!("{options:?}"),
-                line_spans: format!("{line_spans:?}"),
-                destination: destination.map(|d| d.as_usize()),
-                unwind: unwind.stable(tables),
-            },
-            mir::TerminatorKind::Yield { .. }
-            | mir::TerminatorKind::CoroutineDrop
-            | mir::TerminatorKind::FalseEdge { .. }
-            | mir::TerminatorKind::FalseUnwind { .. } => unreachable!(),
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::GenericArgs<'tcx> {
-    type T = stable_mir::ty::GenericArgs;
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        use stable_mir::ty::GenericArgs;
-
-        GenericArgs(self.iter().map(|arg| arg.unpack().stable(tables)).collect())
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::GenericArgKind<'tcx> {
-    type T = stable_mir::ty::GenericArgKind;
-
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        use stable_mir::ty::GenericArgKind;
-        match self {
-            ty::GenericArgKind::Lifetime(region) => GenericArgKind::Lifetime(region.stable(tables)),
-            ty::GenericArgKind::Type(ty) => GenericArgKind::Type(ty.stable(tables)),
-            ty::GenericArgKind::Const(cnst) => GenericArgKind::Const(cnst.stable(tables)),
-        }
-    }
-}
-
-impl<'tcx, S, V> Stable<'tcx> for ty::Binder<'tcx, S>
+impl<'tcx, T> Stable<'tcx> for &T
 where
-    S: Stable<'tcx, T = V>,
+    T: Stable<'tcx>,
 {
-    type T = stable_mir::ty::Binder<V>;
+    type T = T::T;
 
     fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        use stable_mir::ty::Binder;
-
-        Binder {
-            value: self.as_ref().skip_binder().stable(tables),
-            bound_vars: self
-                .bound_vars()
-                .iter()
-                .map(|bound_var| bound_var.stable(tables))
-                .collect(),
-        }
+        (*self).stable(tables)
     }
 }
 
-impl<'tcx, S, V> Stable<'tcx> for ty::EarlyBinder<S>
+impl<'tcx, T> Stable<'tcx> for Option<T>
 where
-    S: Stable<'tcx, T = V>,
+    T: Stable<'tcx>,
 {
-    type T = stable_mir::ty::EarlyBinder<V>;
-
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        use stable_mir::ty::EarlyBinder;
-
-        EarlyBinder { value: self.as_ref().skip_binder().stable(tables) }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::FnSig<'tcx> {
-    type T = stable_mir::ty::FnSig;
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        use rustc_target::spec::abi;
-        use stable_mir::ty::{Abi, FnSig};
-
-        FnSig {
-            inputs_and_output: self.inputs_and_output.iter().map(|ty| ty.stable(tables)).collect(),
-            c_variadic: self.c_variadic,
-            unsafety: self.unsafety.stable(tables),
-            abi: match self.abi {
-                abi::Abi::Rust => Abi::Rust,
-                abi::Abi::C { unwind } => Abi::C { unwind },
-                abi::Abi::Cdecl { unwind } => Abi::Cdecl { unwind },
-                abi::Abi::Stdcall { unwind } => Abi::Stdcall { unwind },
-                abi::Abi::Fastcall { unwind } => Abi::Fastcall { unwind },
-                abi::Abi::Vectorcall { unwind } => Abi::Vectorcall { unwind },
-                abi::Abi::Thiscall { unwind } => Abi::Thiscall { unwind },
-                abi::Abi::Aapcs { unwind } => Abi::Aapcs { unwind },
-                abi::Abi::Win64 { unwind } => Abi::Win64 { unwind },
-                abi::Abi::SysV64 { unwind } => Abi::SysV64 { unwind },
-                abi::Abi::PtxKernel => Abi::PtxKernel,
-                abi::Abi::Msp430Interrupt => Abi::Msp430Interrupt,
-                abi::Abi::X86Interrupt => Abi::X86Interrupt,
-                abi::Abi::AmdGpuKernel => Abi::AmdGpuKernel,
-                abi::Abi::EfiApi => Abi::EfiApi,
-                abi::Abi::AvrInterrupt => Abi::AvrInterrupt,
-                abi::Abi::AvrNonBlockingInterrupt => Abi::AvrNonBlockingInterrupt,
-                abi::Abi::CCmseNonSecureCall => Abi::CCmseNonSecureCall,
-                abi::Abi::Wasm => Abi::Wasm,
-                abi::Abi::System { unwind } => Abi::System { unwind },
-                abi::Abi::RustIntrinsic => Abi::RustIntrinsic,
-                abi::Abi::RustCall => Abi::RustCall,
-                abi::Abi::PlatformIntrinsic => Abi::PlatformIntrinsic,
-                abi::Abi::Unadjusted => Abi::Unadjusted,
-                abi::Abi::RustCold => Abi::RustCold,
-                abi::Abi::RiscvInterruptM => Abi::RiscvInterruptM,
-                abi::Abi::RiscvInterruptS => Abi::RiscvInterruptS,
-            },
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::BoundTyKind {
-    type T = stable_mir::ty::BoundTyKind;
-
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        use stable_mir::ty::BoundTyKind;
-
-        match self {
-            ty::BoundTyKind::Anon => BoundTyKind::Anon,
-            ty::BoundTyKind::Param(def_id, symbol) => {
-                BoundTyKind::Param(tables.param_def(*def_id), symbol.to_string())
-            }
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::BoundRegionKind {
-    type T = stable_mir::ty::BoundRegionKind;
-
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        use stable_mir::ty::BoundRegionKind;
-
-        match self {
-            ty::BoundRegionKind::BrAnon => BoundRegionKind::BrAnon,
-            ty::BoundRegionKind::BrNamed(def_id, symbol) => {
-                BoundRegionKind::BrNamed(tables.br_named_def(*def_id), symbol.to_string())
-            }
-            ty::BoundRegionKind::BrEnv => BoundRegionKind::BrEnv,
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::BoundVariableKind {
-    type T = stable_mir::ty::BoundVariableKind;
-
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        use stable_mir::ty::BoundVariableKind;
-
-        match self {
-            ty::BoundVariableKind::Ty(bound_ty_kind) => {
-                BoundVariableKind::Ty(bound_ty_kind.stable(tables))
-            }
-            ty::BoundVariableKind::Region(bound_region_kind) => {
-                BoundVariableKind::Region(bound_region_kind.stable(tables))
-            }
-            ty::BoundVariableKind::Const => BoundVariableKind::Const,
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::IntTy {
-    type T = IntTy;
-
-    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
-        match self {
-            ty::IntTy::Isize => IntTy::Isize,
-            ty::IntTy::I8 => IntTy::I8,
-            ty::IntTy::I16 => IntTy::I16,
-            ty::IntTy::I32 => IntTy::I32,
-            ty::IntTy::I64 => IntTy::I64,
-            ty::IntTy::I128 => IntTy::I128,
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::UintTy {
-    type T = UintTy;
-
-    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
-        match self {
-            ty::UintTy::Usize => UintTy::Usize,
-            ty::UintTy::U8 => UintTy::U8,
-            ty::UintTy::U16 => UintTy::U16,
-            ty::UintTy::U32 => UintTy::U32,
-            ty::UintTy::U64 => UintTy::U64,
-            ty::UintTy::U128 => UintTy::U128,
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::FloatTy {
-    type T = FloatTy;
-
-    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
-        match self {
-            ty::FloatTy::F32 => FloatTy::F32,
-            ty::FloatTy::F64 => FloatTy::F64,
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for hir::Movability {
-    type T = Movability;
-
-    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
-        match self {
-            hir::Movability::Static => Movability::Static,
-            hir::Movability::Movable => Movability::Movable,
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for Ty<'tcx> {
-    type T = stable_mir::ty::Ty;
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        tables.intern_ty(*self)
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::TyKind<'tcx> {
-    type T = stable_mir::ty::TyKind;
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        match self {
-            ty::Bool => TyKind::RigidTy(RigidTy::Bool),
-            ty::Char => TyKind::RigidTy(RigidTy::Char),
-            ty::Int(int_ty) => TyKind::RigidTy(RigidTy::Int(int_ty.stable(tables))),
-            ty::Uint(uint_ty) => TyKind::RigidTy(RigidTy::Uint(uint_ty.stable(tables))),
-            ty::Float(float_ty) => TyKind::RigidTy(RigidTy::Float(float_ty.stable(tables))),
-            ty::Adt(adt_def, generic_args) => TyKind::RigidTy(RigidTy::Adt(
-                tables.adt_def(adt_def.did()),
-                generic_args.stable(tables),
-            )),
-            ty::Foreign(def_id) => TyKind::RigidTy(RigidTy::Foreign(tables.foreign_def(*def_id))),
-            ty::Str => TyKind::RigidTy(RigidTy::Str),
-            ty::Array(ty, constant) => {
-                TyKind::RigidTy(RigidTy::Array(ty.stable(tables), constant.stable(tables)))
-            }
-            ty::Slice(ty) => TyKind::RigidTy(RigidTy::Slice(ty.stable(tables))),
-            ty::RawPtr(ty::TypeAndMut { ty, mutbl }) => {
-                TyKind::RigidTy(RigidTy::RawPtr(ty.stable(tables), mutbl.stable(tables)))
-            }
-            ty::Ref(region, ty, mutbl) => TyKind::RigidTy(RigidTy::Ref(
-                region.stable(tables),
-                ty.stable(tables),
-                mutbl.stable(tables),
-            )),
-            ty::FnDef(def_id, generic_args) => {
-                TyKind::RigidTy(RigidTy::FnDef(tables.fn_def(*def_id), generic_args.stable(tables)))
-            }
-            ty::FnPtr(poly_fn_sig) => TyKind::RigidTy(RigidTy::FnPtr(poly_fn_sig.stable(tables))),
-            ty::Dynamic(existential_predicates, region, dyn_kind) => {
-                TyKind::RigidTy(RigidTy::Dynamic(
-                    existential_predicates
-                        .iter()
-                        .map(|existential_predicate| existential_predicate.stable(tables))
-                        .collect(),
-                    region.stable(tables),
-                    dyn_kind.stable(tables),
-                ))
-            }
-            ty::Closure(def_id, generic_args) => TyKind::RigidTy(RigidTy::Closure(
-                tables.closure_def(*def_id),
-                generic_args.stable(tables),
-            )),
-            ty::Coroutine(def_id, generic_args, movability) => TyKind::RigidTy(RigidTy::Coroutine(
-                tables.coroutine_def(*def_id),
-                generic_args.stable(tables),
-                movability.stable(tables),
-            )),
-            ty::Never => TyKind::RigidTy(RigidTy::Never),
-            ty::Tuple(fields) => {
-                TyKind::RigidTy(RigidTy::Tuple(fields.iter().map(|ty| ty.stable(tables)).collect()))
-            }
-            ty::Alias(alias_kind, alias_ty) => {
-                TyKind::Alias(alias_kind.stable(tables), alias_ty.stable(tables))
-            }
-            ty::Param(param_ty) => TyKind::Param(param_ty.stable(tables)),
-            ty::Bound(debruijn_idx, bound_ty) => {
-                TyKind::Bound(debruijn_idx.as_usize(), bound_ty.stable(tables))
-            }
-            ty::Placeholder(..) | ty::CoroutineWitness(..) | ty::Infer(_) | ty::Error(_) => {
-                unreachable!();
-            }
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::Const<'tcx> {
-    type T = stable_mir::ty::Const;
-
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        let kind = match self.kind() {
-            ty::Value(val) => {
-                let const_val = tables.tcx.valtree_to_const_val((self.ty(), val));
-                if matches!(const_val, mir::ConstValue::ZeroSized) {
-                    ConstantKind::ZeroSized
-                } else {
-                    stable_mir::ty::ConstantKind::Allocated(alloc::new_allocation(
-                        self.ty(),
-                        const_val,
-                        tables,
-                    ))
-                }
-            }
-            ty::ParamCt(param) => stable_mir::ty::ConstantKind::Param(param.stable(tables)),
-            ty::ErrorCt(_) => unreachable!(),
-            ty::InferCt(_) => unreachable!(),
-            ty::BoundCt(_, _) => unimplemented!(),
-            ty::PlaceholderCt(_) => unimplemented!(),
-            ty::Unevaluated(uv) => {
-                stable_mir::ty::ConstantKind::Unevaluated(stable_mir::ty::UnevaluatedConst {
-                    def: tables.const_def(uv.def),
-                    args: uv.args.stable(tables),
-                    promoted: None,
-                })
-            }
-            ty::ExprCt(_) => unimplemented!(),
-        };
-        let ty = self.ty().stable(tables);
-        let id = tables.intern_const(mir::Const::Ty(*self));
-        Const::new(kind, ty, id)
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::ParamConst {
-    type T = stable_mir::ty::ParamConst;
-    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
-        use stable_mir::ty::ParamConst;
-        ParamConst { index: self.index, name: self.name.to_string() }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::ParamTy {
-    type T = stable_mir::ty::ParamTy;
-    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
-        use stable_mir::ty::ParamTy;
-        ParamTy { index: self.index, name: self.name.to_string() }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::BoundTy {
-    type T = stable_mir::ty::BoundTy;
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        use stable_mir::ty::BoundTy;
-        BoundTy { var: self.var.as_usize(), kind: self.kind.stable(tables) }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for mir::interpret::Allocation {
-    type T = stable_mir::ty::Allocation;
-
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        alloc::allocation_filter(
-            self,
-            alloc_range(rustc_target::abi::Size::ZERO, self.size()),
-            tables,
-        )
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::trait_def::TraitSpecializationKind {
-    type T = stable_mir::ty::TraitSpecializationKind;
-    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
-        use stable_mir::ty::TraitSpecializationKind;
-
-        match self {
-            ty::trait_def::TraitSpecializationKind::None => TraitSpecializationKind::None,
-            ty::trait_def::TraitSpecializationKind::Marker => TraitSpecializationKind::Marker,
-            ty::trait_def::TraitSpecializationKind::AlwaysApplicable => {
-                TraitSpecializationKind::AlwaysApplicable
-            }
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::TraitDef {
-    type T = stable_mir::ty::TraitDecl;
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        use stable_mir::ty::TraitDecl;
-
-        TraitDecl {
-            def_id: tables.trait_def(self.def_id),
-            unsafety: self.unsafety.stable(tables),
-            paren_sugar: self.paren_sugar,
-            has_auto_impl: self.has_auto_impl,
-            is_marker: self.is_marker,
-            is_coinductive: self.is_coinductive,
-            skip_array_during_method_dispatch: self.skip_array_during_method_dispatch,
-            specialization_kind: self.specialization_kind.stable(tables),
-            must_implement_one_of: self
-                .must_implement_one_of
-                .as_ref()
-                .map(|idents| idents.iter().map(|ident| opaque(ident)).collect()),
-            implement_via_object: self.implement_via_object,
-            deny_explicit_impl: self.deny_explicit_impl,
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for rustc_middle::mir::Const<'tcx> {
-    type T = stable_mir::ty::Const;
-
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        match *self {
-            mir::Const::Ty(c) => c.stable(tables),
-            mir::Const::Unevaluated(unev_const, ty) => {
-                let kind =
-                    stable_mir::ty::ConstantKind::Unevaluated(stable_mir::ty::UnevaluatedConst {
-                        def: tables.const_def(unev_const.def),
-                        args: unev_const.args.stable(tables),
-                        promoted: unev_const.promoted.map(|u| u.as_u32()),
-                    });
-                let ty = ty.stable(tables);
-                let id = tables.intern_const(*self);
-                Const::new(kind, ty, id)
-            }
-            mir::Const::Val(val, ty) if matches!(val, mir::ConstValue::ZeroSized) => {
-                let ty = ty.stable(tables);
-                let id = tables.intern_const(*self);
-                Const::new(ConstantKind::ZeroSized, ty, id)
-            }
-            mir::Const::Val(val, ty) => {
-                let kind = ConstantKind::Allocated(alloc::new_allocation(ty, val, tables));
-                let ty = ty.stable(tables);
-                let id = tables.intern_const(*self);
-                Const::new(kind, ty, id)
-            }
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::TraitRef<'tcx> {
-    type T = stable_mir::ty::TraitRef;
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        use stable_mir::ty::TraitRef;
-
-        TraitRef { def_id: tables.trait_def(self.def_id), args: self.args.stable(tables) }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::Generics {
-    type T = stable_mir::ty::Generics;
-
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        use stable_mir::ty::Generics;
-
-        let params: Vec<_> = self.params.iter().map(|param| param.stable(tables)).collect();
-        let param_def_id_to_index =
-            params.iter().map(|param| (param.def_id, param.index)).collect();
-
-        Generics {
-            parent: self.parent.map(|did| tables.generic_def(did)),
-            parent_count: self.parent_count,
-            params,
-            param_def_id_to_index,
-            has_self: self.has_self,
-            has_late_bound_regions: self
-                .has_late_bound_regions
-                .as_ref()
-                .map(|late_bound_regions| late_bound_regions.stable(tables)),
-            host_effect_index: self.host_effect_index,
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for rustc_middle::ty::GenericParamDefKind {
-    type T = stable_mir::ty::GenericParamDefKind;
-
-    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
-        use stable_mir::ty::GenericParamDefKind;
-        match self {
-            ty::GenericParamDefKind::Lifetime => GenericParamDefKind::Lifetime,
-            ty::GenericParamDefKind::Type { has_default, synthetic } => {
-                GenericParamDefKind::Type { has_default: *has_default, synthetic: *synthetic }
-            }
-            ty::GenericParamDefKind::Const { has_default, is_host_effect: _ } => {
-                GenericParamDefKind::Const { has_default: *has_default }
-            }
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for rustc_middle::ty::GenericParamDef {
-    type T = stable_mir::ty::GenericParamDef;
-
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        GenericParamDef {
-            name: self.name.to_string(),
-            def_id: tables.generic_def(self.def_id),
-            index: self.index,
-            pure_wrt_drop: self.pure_wrt_drop,
-            kind: self.kind.stable(tables),
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::PredicateKind<'tcx> {
-    type T = stable_mir::ty::PredicateKind;
-
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        use ty::PredicateKind;
-        match self {
-            PredicateKind::Clause(clause_kind) => {
-                stable_mir::ty::PredicateKind::Clause(clause_kind.stable(tables))
-            }
-            PredicateKind::ObjectSafe(did) => {
-                stable_mir::ty::PredicateKind::ObjectSafe(tables.trait_def(*did))
-            }
-            PredicateKind::ClosureKind(did, generic_args, closure_kind) => {
-                stable_mir::ty::PredicateKind::ClosureKind(
-                    tables.closure_def(*did),
-                    generic_args.stable(tables),
-                    closure_kind.stable(tables),
-                )
-            }
-            PredicateKind::Subtype(subtype_predicate) => {
-                stable_mir::ty::PredicateKind::SubType(subtype_predicate.stable(tables))
-            }
-            PredicateKind::Coerce(coerce_predicate) => {
-                stable_mir::ty::PredicateKind::Coerce(coerce_predicate.stable(tables))
-            }
-            PredicateKind::ConstEquate(a, b) => {
-                stable_mir::ty::PredicateKind::ConstEquate(a.stable(tables), b.stable(tables))
-            }
-            PredicateKind::Ambiguous => stable_mir::ty::PredicateKind::Ambiguous,
-            PredicateKind::AliasRelate(a, b, alias_relation_direction) => {
-                stable_mir::ty::PredicateKind::AliasRelate(
-                    a.unpack().stable(tables),
-                    b.unpack().stable(tables),
-                    alias_relation_direction.stable(tables),
-                )
-            }
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::ClauseKind<'tcx> {
-    type T = stable_mir::ty::ClauseKind;
-
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        use ty::ClauseKind;
-        match *self {
-            ClauseKind::Trait(trait_object) => {
-                stable_mir::ty::ClauseKind::Trait(trait_object.stable(tables))
-            }
-            ClauseKind::RegionOutlives(region_outlives) => {
-                stable_mir::ty::ClauseKind::RegionOutlives(region_outlives.stable(tables))
-            }
-            ClauseKind::TypeOutlives(type_outlives) => {
-                let ty::OutlivesPredicate::<_, _>(a, b) = type_outlives;
-                stable_mir::ty::ClauseKind::TypeOutlives(stable_mir::ty::OutlivesPredicate(
-                    a.stable(tables),
-                    b.stable(tables),
-                ))
-            }
-            ClauseKind::Projection(projection_predicate) => {
-                stable_mir::ty::ClauseKind::Projection(projection_predicate.stable(tables))
-            }
-            ClauseKind::ConstArgHasType(const_, ty) => stable_mir::ty::ClauseKind::ConstArgHasType(
-                const_.stable(tables),
-                ty.stable(tables),
-            ),
-            ClauseKind::WellFormed(generic_arg) => {
-                stable_mir::ty::ClauseKind::WellFormed(generic_arg.unpack().stable(tables))
-            }
-            ClauseKind::ConstEvaluatable(const_) => {
-                stable_mir::ty::ClauseKind::ConstEvaluatable(const_.stable(tables))
-            }
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::ClosureKind {
-    type T = stable_mir::ty::ClosureKind;
-
-    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
-        use ty::ClosureKind::*;
-        match self {
-            Fn => stable_mir::ty::ClosureKind::Fn,
-            FnMut => stable_mir::ty::ClosureKind::FnMut,
-            FnOnce => stable_mir::ty::ClosureKind::FnOnce,
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::SubtypePredicate<'tcx> {
-    type T = stable_mir::ty::SubtypePredicate;
+    type T = Option<T::T>;
 
     fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        let ty::SubtypePredicate { a, b, a_is_expected: _ } = self;
-        stable_mir::ty::SubtypePredicate { a: a.stable(tables), b: b.stable(tables) }
+        self.as_ref().map(|value| value.stable(tables))
     }
 }
 
-impl<'tcx> Stable<'tcx> for ty::CoercePredicate<'tcx> {
-    type T = stable_mir::ty::CoercePredicate;
-
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        let ty::CoercePredicate { a, b } = self;
-        stable_mir::ty::CoercePredicate { a: a.stable(tables), b: b.stable(tables) }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::AliasRelationDirection {
-    type T = stable_mir::ty::AliasRelationDirection;
-
-    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
-        use ty::AliasRelationDirection::*;
-        match self {
-            Equate => stable_mir::ty::AliasRelationDirection::Equate,
-            Subtype => stable_mir::ty::AliasRelationDirection::Subtype,
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::TraitPredicate<'tcx> {
-    type T = stable_mir::ty::TraitPredicate;
-
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        let ty::TraitPredicate { trait_ref, polarity } = self;
-        stable_mir::ty::TraitPredicate {
-            trait_ref: trait_ref.stable(tables),
-            polarity: polarity.stable(tables),
-        }
-    }
-}
-
-impl<'tcx, A, B, U, V> Stable<'tcx> for ty::OutlivesPredicate<A, B>
+impl<'tcx, T, E> Stable<'tcx> for Result<T, E>
 where
-    A: Stable<'tcx, T = U>,
-    B: Stable<'tcx, T = V>,
+    T: Stable<'tcx>,
+    E: Stable<'tcx>,
 {
-    type T = stable_mir::ty::OutlivesPredicate<U, V>;
-
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        let ty::OutlivesPredicate(a, b) = self;
-        stable_mir::ty::OutlivesPredicate(a.stable(tables), b.stable(tables))
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::ProjectionPredicate<'tcx> {
-    type T = stable_mir::ty::ProjectionPredicate;
-
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        let ty::ProjectionPredicate { projection_ty, term } = self;
-        stable_mir::ty::ProjectionPredicate {
-            projection_ty: projection_ty.stable(tables),
-            term: term.unpack().stable(tables),
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::ImplPolarity {
-    type T = stable_mir::ty::ImplPolarity;
-
-    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
-        use ty::ImplPolarity::*;
-        match self {
-            Positive => stable_mir::ty::ImplPolarity::Positive,
-            Negative => stable_mir::ty::ImplPolarity::Negative,
-            Reservation => stable_mir::ty::ImplPolarity::Reservation,
-        }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::Region<'tcx> {
-    type T = stable_mir::ty::Region;
-
-    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        Region { kind: self.kind().stable(tables) }
-    }
-}
-
-impl<'tcx> Stable<'tcx> for ty::RegionKind<'tcx> {
-    type T = stable_mir::ty::RegionKind;
+    type T = Result<T::T, E::T>;
 
     fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        use stable_mir::ty::RegionKind;
         match self {
-            ty::ReEarlyBound(early_reg) => RegionKind::ReEarlyBound(EarlyBoundRegion {
-                def_id: tables.region_def(early_reg.def_id),
-                index: early_reg.index,
-                name: early_reg.name.to_string(),
-            }),
-            ty::ReLateBound(db_index, bound_reg) => RegionKind::ReLateBound(
-                db_index.as_u32(),
-                BoundRegion { var: bound_reg.var.as_u32(), kind: bound_reg.kind.stable(tables) },
-            ),
-            ty::ReStatic => RegionKind::ReStatic,
-            ty::RePlaceholder(place_holder) => {
-                RegionKind::RePlaceholder(stable_mir::ty::Placeholder {
-                    universe: place_holder.universe.as_u32(),
-                    bound: BoundRegion {
-                        var: place_holder.bound.var.as_u32(),
-                        kind: place_holder.bound.kind.stable(tables),
-                    },
-                })
-            }
-            ty::ReErased => RegionKind::ReErased,
-            _ => unreachable!("{self:?}"),
+            Ok(val) => Ok(val.stable(tables)),
+            Err(error) => Err(error.stable(tables)),
         }
     }
 }
 
-impl<'tcx> Stable<'tcx> for rustc_span::Span {
-    type T = stable_mir::ty::Span;
-
+impl<'tcx, T> Stable<'tcx> for &[T]
+where
+    T: Stable<'tcx>,
+{
+    type T = Vec<T::T>;
     fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        tables.create_span(*self)
-    }
-}
-
-impl<'tcx> Stable<'tcx> for DefKind {
-    type T = stable_mir::DefKind;
-
-    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
-        // FIXME: add a real implementation of stable DefKind
-        opaque(self)
+        self.iter().map(|e| e.stable(tables)).collect()
     }
 }
 
-impl<'tcx> Stable<'tcx> for ty::Instance<'tcx> {
-    type T = stable_mir::mir::mono::Instance;
-
+impl<'tcx, T, U> Stable<'tcx> for (T, U)
+where
+    T: Stable<'tcx>,
+    U: Stable<'tcx>,
+{
+    type T = (T::T, U::T);
     fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        let def = tables.instance_def(*self);
-        let kind = match self.def {
-            ty::InstanceDef::Item(..) => stable_mir::mir::mono::InstanceKind::Item,
-            ty::InstanceDef::Intrinsic(..) => stable_mir::mir::mono::InstanceKind::Intrinsic,
-            ty::InstanceDef::Virtual(..) => stable_mir::mir::mono::InstanceKind::Virtual,
-            ty::InstanceDef::VTableShim(..)
-            | ty::InstanceDef::ReifyShim(..)
-            | ty::InstanceDef::FnPtrAddrShim(..)
-            | ty::InstanceDef::ClosureOnceShim { .. }
-            | ty::InstanceDef::ThreadLocalShim(..)
-            | ty::InstanceDef::DropGlue(..)
-            | ty::InstanceDef::CloneShim(..)
-            | ty::InstanceDef::FnPtrShim(..) => stable_mir::mir::mono::InstanceKind::Shim,
-        };
-        stable_mir::mir::mono::Instance { def, kind }
+        (self.0.stable(tables), self.1.stable(tables))
     }
 }
 
-impl<'tcx> Stable<'tcx> for MonoItem<'tcx> {
-    type T = stable_mir::mir::mono::MonoItem;
-
+impl<'tcx, T> Stable<'tcx> for RangeInclusive<T>
+where
+    T: Stable<'tcx>,
+{
+    type T = RangeInclusive<T::T>;
     fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
-        use stable_mir::mir::mono::MonoItem as StableMonoItem;
-        match self {
-            MonoItem::Fn(instance) => StableMonoItem::Fn(instance.stable(tables)),
-            MonoItem::Static(def_id) => StableMonoItem::Static(tables.static_def(*def_id)),
-            MonoItem::GlobalAsm(item_id) => StableMonoItem::GlobalAsm(opaque(item_id)),
-        }
+        RangeInclusive::new(self.start().stable(tables), self.end().stable(tables))
     }
 }
diff --git a/compiler/rustc_span/src/caching_source_map_view.rs b/compiler/rustc_span/src/caching_source_map_view.rs
index fbfc5c22f..4c7029c4e 100644
--- a/compiler/rustc_span/src/caching_source_map_view.rs
+++ b/compiler/rustc_span/src/caching_source_map_view.rs
@@ -117,7 +117,7 @@ impl<'sm> CachingSourceMapView<'sm> {
         self.time_stamp += 1;
 
         // Check if lo and hi are in the cached lines.
-        let lo_cache_idx = self.cache_entry_index(span_data.lo);
+        let lo_cache_idx: isize = self.cache_entry_index(span_data.lo);
         let hi_cache_idx = self.cache_entry_index(span_data.hi);
 
         if lo_cache_idx != -1 && hi_cache_idx != -1 {
@@ -205,7 +205,9 @@ impl<'sm> CachingSourceMapView<'sm> {
                 (lo_cache_idx as usize, oldest)
             }
             _ => {
-                panic!();
+                panic!(
+                    "the case of neither value being equal to -1 was handled above and the function returns."
+                );
             }
         };
 
diff --git a/compiler/rustc_span/src/def_id.rs b/compiler/rustc_span/src/def_id.rs
index 595babc26..b2d51ac6c 100644
--- a/compiler/rustc_span/src/def_id.rs
+++ b/compiler/rustc_span/src/def_id.rs
@@ -13,7 +13,7 @@ pub type StableCrateIdMap =
     indexmap::IndexMap<StableCrateId, CrateNum, BuildHasherDefault<Unhasher>>;
 
 rustc_index::newtype_index! {
-    #[custom_encodable]
+    #[orderable]
     #[debug_format = "crate{}"]
     pub struct CrateNum {}
 }
@@ -213,7 +213,7 @@ rustc_index::newtype_index! {
     /// A DefIndex is an index into the hir-map for a crate, identifying a
     /// particular definition. It should really be considered an interned
     /// shorthand for a particular DefPath.
-    #[custom_encodable] // (only encodable in metadata)
+    #[orderable]
     #[debug_format = "DefIndex({})"]
     pub struct DefIndex {
         /// The crate root is always assigned index 0 by the AST Map code,
diff --git a/compiler/rustc_span/src/edit_distance.rs b/compiler/rustc_span/src/edit_distance.rs
index 96a118e59..14cb1d6d3 100644
--- a/compiler/rustc_span/src/edit_distance.rs
+++ b/compiler/rustc_span/src/edit_distance.rs
@@ -188,7 +188,11 @@ fn find_best_match_for_name_impl(
         return Some(*c);
     }
 
-    let mut dist = dist.unwrap_or_else(|| cmp::max(lookup.len(), 3) / 3);
+    // `fn edit_distance()` use `chars()` to calculate edit distance, so we must
+    // also use `chars()` (and not `str::len()`) to calculate length here.
+    let lookup_len = lookup.chars().count();
+
+    let mut dist = dist.unwrap_or_else(|| cmp::max(lookup_len, 3) / 3);
     let mut best = None;
     // store the candidates with the same distance, only for `use_substring_score` current.
     let mut next_candidates = vec![];
diff --git a/compiler/rustc_span/src/edition.rs b/compiler/rustc_span/src/edition.rs
index 608b8c24b..78ac61fa3 100644
--- a/compiler/rustc_span/src/edition.rs
+++ b/compiler/rustc_span/src/edition.rs
@@ -1,4 +1,3 @@
-use crate::symbol::{sym, Symbol};
 use std::fmt;
 use std::str::FromStr;
 
@@ -58,15 +57,6 @@ impl Edition {
         }
     }
 
-    pub fn feature_name(self) -> Symbol {
-        match self {
-            Edition::Edition2015 => sym::rust_2015_preview,
-            Edition::Edition2018 => sym::rust_2018_preview,
-            Edition::Edition2021 => sym::rust_2021_preview,
-            Edition::Edition2024 => sym::rust_2024_preview,
-        }
-    }
-
     pub fn is_stable(self) -> bool {
         match self {
             Edition::Edition2015 => true,
diff --git a/compiler/rustc_span/src/hygiene.rs b/compiler/rustc_span/src/hygiene.rs
index 36731d0fe..b717229b6 100644
--- a/compiler/rustc_span/src/hygiene.rs
+++ b/compiler/rustc_span/src/hygiene.rs
@@ -60,7 +60,7 @@ pub struct SyntaxContextData {
 
 rustc_index::newtype_index! {
     /// A unique ID associated with a macro invocation and expansion.
-    #[custom_encodable]
+    #[orderable]
     pub struct ExpnIndex {}
 }
 
@@ -80,8 +80,6 @@ impl fmt::Debug for ExpnId {
 
 rustc_index::newtype_index! {
     /// A unique ID associated with a macro invocation and expansion.
-    #[custom_encodable]
-    #[no_ord_impl]
     #[debug_format = "expn{}"]
     pub struct LocalExpnId {}
 }
@@ -569,10 +567,6 @@ impl HygieneData {
     }
 }
 
-pub fn clear_syntax_context_map() {
-    HygieneData::with(|data| data.syntax_context_map = FxHashMap::default());
-}
-
 pub fn walk_chain(span: Span, to: SyntaxContext) -> Span {
     HygieneData::with(|data| data.walk_chain(span, to))
 }
diff --git a/compiler/rustc_span/src/lib.rs b/compiler/rustc_span/src/lib.rs
index 489c8d189..cc3f0962d 100644
--- a/compiler/rustc_span/src/lib.rs
+++ b/compiler/rustc_span/src/lib.rs
@@ -17,11 +17,10 @@
 
 // tidy-alphabetical-start
 #![allow(internal_features)]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
 #![deny(rustc::diagnostic_outside_of_impl)]
 #![deny(rustc::untranslatable_diagnostic)]
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
+#![doc(rust_logo)]
 #![feature(array_windows)]
 #![feature(cfg_match)]
 #![feature(core_io_borrowed_buf)]
@@ -33,6 +32,7 @@
 #![feature(read_buf)]
 #![feature(round_char_boundary)]
 #![feature(rustc_attrs)]
+#![feature(rustdoc_internals)]
 // tidy-alphabetical-end
 
 #[macro_use]
@@ -139,13 +139,6 @@ pub fn set_session_globals_then<R>(session_globals: &SessionGlobals, f: impl FnO
     SESSION_GLOBALS.set(session_globals, f)
 }
 
-pub fn create_default_session_if_not_set_then<R, F>(f: F) -> R
-where
-    F: FnOnce(&SessionGlobals) -> R,
-{
-    create_session_if_not_set_then(edition::DEFAULT_EDITION, f)
-}
-
 pub fn create_session_if_not_set_then<R, F>(edition: Edition, f: F) -> R
 where
     F: FnOnce(&SessionGlobals) -> R,
@@ -2254,7 +2247,7 @@ pub struct ErrorGuaranteed(());
 impl ErrorGuaranteed {
     /// To be used only if you really know what you are doing... ideally, we would find a way to
     /// eliminate all calls to this method.
-    #[deprecated = "`Session::delay_span_bug` should be preferred over this function"]
+    #[deprecated = "`Session::span_delayed_bug` should be preferred over this function"]
     pub fn unchecked_claim_error_was_emitted() -> Self {
         ErrorGuaranteed(())
     }
diff --git a/compiler/rustc_span/src/source_map.rs b/compiler/rustc_span/src/source_map.rs
index dcf346acb..cb10e6bf2 100644
--- a/compiler/rustc_span/src/source_map.rs
+++ b/compiler/rustc_span/src/source_map.rs
@@ -11,13 +11,11 @@
 
 use crate::*;
 use rustc_data_structures::fx::FxHashMap;
-use rustc_data_structures::stable_hasher::{Hash128, Hash64, StableHasher};
-use rustc_data_structures::sync::{IntoDynSyncSend, Lrc, MappedReadGuard, ReadGuard, RwLock};
-use std::cmp;
+use rustc_data_structures::sync::{IntoDynSyncSend, MappedReadGuard, ReadGuard, RwLock};
 use std::fs;
 use std::hash::Hash;
 use std::io::{self, BorrowedBuf, Read};
-use std::path::{self, Path, PathBuf};
+use std::path::{self};
 
 #[cfg(test)]
 mod tests;
diff --git a/compiler/rustc_span/src/source_map/tests.rs b/compiler/rustc_span/src/source_map/tests.rs
index 5697969dd..113ca493d 100644
--- a/compiler/rustc_span/src/source_map/tests.rs
+++ b/compiler/rustc_span/src/source_map/tests.rs
@@ -1,7 +1,5 @@
 use super::*;
 
-use rustc_data_structures::sync::{FreezeLock, Lrc};
-
 fn init_source_map() -> SourceMap {
     let sm = SourceMap::new(FilePathMapping::empty());
     sm.new_source_file(PathBuf::from("blork.rs").into(), "first line.\nsecond line".to_string());
diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
index f287862cc..0333b5f04 100644
--- a/compiler/rustc_span/src/symbol.rs
+++ b/compiler/rustc_span/src/symbol.rs
@@ -124,22 +124,24 @@ symbols! {
     // There is currently no checking that all symbols are used; that would be
     // nice to have.
     Symbols {
+        Abi,
         AcqRel,
         Acquire,
         AddToDiagnostic,
-        Alignment,
         Any,
         Arc,
         ArcWeak,
         Argument,
         ArgumentMethods,
-        Arguments,
         ArrayIntoIter,
         AsMut,
         AsRef,
         AssertParamIsClone,
         AssertParamIsCopy,
         AssertParamIsEq,
+        AsyncGenFinished,
+        AsyncGenPending,
+        AsyncGenReady,
         AtomicBool,
         AtomicI128,
         AtomicI16,
@@ -163,16 +165,15 @@ symbols! {
         Break,
         C,
         CStr,
-        CString,
         Capture,
         Center,
+        Cleanup,
         Clone,
         Command,
         ConstParamTy,
         Context,
         Continue,
         Copy,
-        Count,
         Cow,
         Debug,
         DebugStruct,
@@ -197,7 +198,6 @@ symbols! {
         Fn,
         FnMut,
         FnOnce,
-        FormatSpec,
         Formatter,
         From,
         FromIterator,
@@ -206,8 +206,6 @@ symbols! {
         FsPermissions,
         Future,
         FutureOutput,
-        FxHashMap,
-        FxHashSet,
         GlobalAlloc,
         Hash,
         HashMap,
@@ -215,6 +213,7 @@ symbols! {
         HashSet,
         Hasher,
         Implied,
+        InCleanup,
         IndexOutput,
         Input,
         Instant,
@@ -250,7 +249,6 @@ symbols! {
         NonZeroI32,
         NonZeroI64,
         NonZeroI8,
-        NonZeroIsize,
         NonZeroU128,
         NonZeroU16,
         NonZeroU32,
@@ -258,6 +256,7 @@ symbols! {
         NonZeroU8,
         NonZeroUsize,
         None,
+        Normal,
         Ok,
         Option,
         Ord,
@@ -271,7 +270,6 @@ symbols! {
         Path,
         PathBuf,
         Pending,
-        Pin,
         Pointer,
         Poll,
         ProcMacro,
@@ -329,7 +327,6 @@ symbols! {
         TyCtxt,
         TyKind,
         Unknown,
-        UnsafeArg,
         Vec,
         VecDeque,
         Wrapper,
@@ -385,7 +382,6 @@ symbols! {
         allow_fail,
         allow_internal_unsafe,
         allow_internal_unstable,
-        allowed,
         alu32,
         always,
         and,
@@ -401,8 +397,6 @@ symbols! {
         arm,
         arm_target_feature,
         array,
-        arrays,
-        as_mut_ptr,
         as_ptr,
         as_ref,
         as_str,
@@ -432,6 +426,7 @@ symbols! {
         async_closure,
         async_fn_in_trait,
         async_fn_track_caller,
+        async_iterator,
         atomic,
         atomic_mod,
         atomics,
@@ -525,6 +520,8 @@ symbols! {
         cmp,
         cmp_max,
         cmp_min,
+        cmp_partialeq_eq,
+        cmp_partialeq_ne,
         cmpxchg16b_target_feature,
         cmse_nonsecure_entry,
         coerce_unsized,
@@ -585,7 +582,6 @@ symbols! {
         const_try,
         constant,
         constructor,
-        context,
         convert_identity,
         copy,
         copy_closures,
@@ -651,7 +647,6 @@ symbols! {
         default_method_body_is_const,
         default_type_parameter_fallback,
         default_type_params,
-        delay_span_bug_from_inside_query,
         deny,
         deprecated,
         deprecated_safe,
@@ -772,8 +767,6 @@ symbols! {
         field,
         field_init_shorthand,
         file,
-        fill,
-        flags,
         float,
         float_to_int_unchecked,
         floorf32,
@@ -783,6 +776,7 @@ symbols! {
         fmt,
         fmul_fast,
         fn_align,
+        fn_delegation,
         fn_must_use,
         fn_mut,
         fn_once,
@@ -941,10 +935,12 @@ symbols! {
         lib,
         libc,
         lifetime,
+        lifetime_capture_rules_2024,
         lifetimes,
         likely,
         line,
         link,
+        link_arg_attribute,
         link_args,
         link_cfg,
         link_llvm_intrinsics,
@@ -984,6 +980,7 @@ symbols! {
         managed_boxes,
         manually_drop,
         map,
+        map_err,
         marker,
         marker_trait_attr,
         masked,
@@ -1023,8 +1020,37 @@ symbols! {
         minnumf32,
         minnumf64,
         mips_target_feature,
+        mir_basic_block,
+        mir_call,
+        mir_cast_transmute,
+        mir_checked,
+        mir_copy_for_deref,
+        mir_debuginfo,
+        mir_deinit,
+        mir_discriminant,
+        mir_drop,
+        mir_field,
+        mir_goto,
+        mir_len,
+        mir_make_place,
+        mir_move,
+        mir_offset,
+        mir_retag,
+        mir_return,
+        mir_set_discriminant,
+        mir_static,
+        mir_static_mut,
+        mir_storage_dead,
+        mir_storage_live,
+        mir_unreachable,
+        mir_unwind_cleanup,
+        mir_unwind_continue,
+        mir_unwind_resume,
+        mir_unwind_terminate,
+        mir_unwind_terminate_reason,
+        mir_unwind_unreachable,
+        mir_variant,
         miri,
-        misc,
         mmx_reg,
         modifiers,
         module,
@@ -1062,6 +1088,7 @@ symbols! {
         negative_impls,
         neon,
         never,
+        never_patterns,
         never_type,
         never_type_fallback,
         new,
@@ -1118,19 +1145,17 @@ symbols! {
         offset,
         offset_of,
         offset_of_enum,
+        ok_or_else,
         omit_gdb_pretty_printer_section,
         on,
         on_unimplemented,
-        oom,
         opaque,
-        ops,
         opt_out_copy,
         optimize,
         optimize_attribute,
         optin_builtin_traits,
         option,
         option_env,
-        option_payload_ptr,
         options,
         or,
         or_patterns,
@@ -1182,7 +1207,7 @@ symbols! {
         pointer,
         pointer_like,
         poll,
-        position,
+        poll_next,
         post_dash_lto: "post-lto",
         powerpc_target_feature,
         powf32,
@@ -1191,7 +1216,6 @@ symbols! {
         powif64,
         pre_dash_lto: "pre-lto",
         precise_pointer_size_matching,
-        precision,
         pref_align_of,
         prefetch_read_data,
         prefetch_read_instruction,
@@ -1201,7 +1225,6 @@ symbols! {
         prelude,
         prelude_import,
         preserves_flags,
-        primitive,
         print_macro,
         println_macro,
         proc_dash_macro: "proc-macro",
@@ -1225,7 +1248,6 @@ symbols! {
         ptr_const_is_null,
         ptr_copy,
         ptr_copy_nonoverlapping,
-        ptr_drop_in_place,
         ptr_eq,
         ptr_from_ref,
         ptr_guaranteed_cmp,
@@ -1330,13 +1352,10 @@ symbols! {
         rtm_target_feature,
         rust,
         rust_2015,
-        rust_2015_preview,
         rust_2018,
         rust_2018_preview,
         rust_2021,
-        rust_2021_preview,
         rust_2024,
-        rust_2024_preview,
         rust_begin_unwind,
         rust_cold_cc,
         rust_eh_catch_typeinfo,
@@ -1381,7 +1400,6 @@ symbols! {
         rustc_expected_cgu_reuse,
         rustc_has_incoherent_inherent_impls,
         rustc_hidden_type_of_opaques,
-        rustc_host,
         rustc_if_this_changed,
         rustc_inherit_overflow_checks,
         rustc_insignificant_dtor,
@@ -1498,6 +1516,8 @@ symbols! {
         simd_insert,
         simd_le,
         simd_lt,
+        simd_masked_load,
+        simd_masked_store,
         simd_mul,
         simd_ne,
         simd_neg,
@@ -1544,6 +1564,7 @@ symbols! {
         slice_patterns,
         slicing_syntax,
         soft,
+        span_delayed_bug_from_inside_query,
         specialization,
         speed,
         spotlight,
@@ -1587,7 +1608,6 @@ symbols! {
         structural_match,
         structural_peq,
         structural_teq,
-        sty,
         sub,
         sub_assign,
         sub_with_overflow,
@@ -1709,7 +1729,6 @@ symbols! {
         unrestricted_attribute_tokens,
         unsafe_block_in_unsafe_fn,
         unsafe_cell,
-        unsafe_cell_from_mut,
         unsafe_cell_raw_get,
         unsafe_no_drop_flag,
         unsafe_pin_internals,
@@ -1734,7 +1753,6 @@ symbols! {
         used_with_arg,
         using,
         usize,
-        v1,
         va_arg,
         va_copy,
         va_end,
@@ -1766,7 +1784,6 @@ symbols! {
         wasm_import_module,
         wasm_target_feature,
         while_let,
-        width,
         windows,
         windows_subsystem,
         with_negative_coherence,
@@ -1985,6 +2002,7 @@ impl fmt::Display for MacroRulesNormalizedIdent {
 pub struct Symbol(SymbolIndex);
 
 rustc_index::newtype_index! {
+    #[orderable]
     struct SymbolIndex {}
 }
 
@@ -2108,11 +2126,7 @@ impl Interner {
             return Symbol::new(idx as u32);
         }
 
-        // SAFETY: we convert from `&str` to `&[u8]`, clone it into the arena,
-        // and immediately convert the clone back to `&[u8]`, all because there
-        // is no `inner.arena.alloc_str()` method. This is clearly safe.
-        let string: &str =
-            unsafe { str::from_utf8_unchecked(inner.arena.alloc_slice(string.as_bytes())) };
+        let string: &str = inner.arena.alloc_str(string);
 
         // SAFETY: we can extend the arena allocation to `'static` because we
         // only access these while the arena is still alive.
diff --git a/compiler/rustc_symbol_mangling/src/errors.rs b/compiler/rustc_symbol_mangling/src/errors.rs
index 2e081e555..ff253b6f4 100644
--- a/compiler/rustc_symbol_mangling/src/errors.rs
+++ b/compiler/rustc_symbol_mangling/src/errors.rs
@@ -16,12 +16,12 @@ pub struct TestOutput {
 impl IntoDiagnostic<'_> for TestOutput {
     fn into_diagnostic(
         self,
-        handler: &'_ rustc_errors::Handler,
+        dcx: &'_ rustc_errors::DiagCtxt,
     ) -> rustc_errors::DiagnosticBuilder<'_, ErrorGuaranteed> {
         let TestOutput { span, kind, content } = self;
 
         #[allow(rustc::untranslatable_diagnostic)]
-        let mut diag = handler.struct_err(format!("{kind}({content})"));
+        let mut diag = dcx.struct_err(format!("{kind}({content})"));
         diag.set_span(span);
         diag
     }
diff --git a/compiler/rustc_symbol_mangling/src/legacy.rs b/compiler/rustc_symbol_mangling/src/legacy.rs
index 53925eeaa..0178ff53b 100644
--- a/compiler/rustc_symbol_mangling/src/legacy.rs
+++ b/compiler/rustc_symbol_mangling/src/legacy.rs
@@ -4,7 +4,6 @@ use rustc_hir::definitions::{DefPathData, DisambiguatedDefPathData};
 use rustc_middle::ty::print::{PrettyPrinter, Print, PrintError, Printer};
 use rustc_middle::ty::{self, Instance, Ty, TyCtxt, TypeVisitableExt};
 use rustc_middle::ty::{GenericArg, GenericArgKind};
-use rustc_middle::util::common::record_time;
 
 use std::fmt::{self, Write};
 use std::mem::{self, discriminant};
@@ -101,40 +100,38 @@ fn get_symbol_hash<'tcx>(
     tcx.with_stable_hashing_context(|mut hcx| {
         let mut hasher = StableHasher::new();
 
-        record_time(&tcx.sess.perf_stats.symbol_hash_time, || {
-            // the main symbol name is not necessarily unique; hash in the
-            // compiler's internal def-path, guaranteeing each symbol has a
-            // truly unique path
-            tcx.def_path_hash(def_id).hash_stable(&mut hcx, &mut hasher);
-
-            // Include the main item-type. Note that, in this case, the
-            // assertions about `has_param` may not hold, but this item-type
-            // ought to be the same for every reference anyway.
-            assert!(!item_type.has_erasable_regions());
-            hcx.while_hashing_spans(false, |hcx| {
-                item_type.hash_stable(hcx, &mut hasher);
-
-                // If this is a function, we hash the signature as well.
-                // This is not *strictly* needed, but it may help in some
-                // situations, see the `run-make/a-b-a-linker-guard` test.
-                if let ty::FnDef(..) = item_type.kind() {
-                    item_type.fn_sig(tcx).hash_stable(hcx, &mut hasher);
-                }
+        // the main symbol name is not necessarily unique; hash in the
+        // compiler's internal def-path, guaranteeing each symbol has a
+        // truly unique path
+        tcx.def_path_hash(def_id).hash_stable(&mut hcx, &mut hasher);
+
+        // Include the main item-type. Note that, in this case, the
+        // assertions about `has_param` may not hold, but this item-type
+        // ought to be the same for every reference anyway.
+        assert!(!item_type.has_erasable_regions());
+        hcx.while_hashing_spans(false, |hcx| {
+            item_type.hash_stable(hcx, &mut hasher);
+
+            // If this is a function, we hash the signature as well.
+            // This is not *strictly* needed, but it may help in some
+            // situations, see the `run-make/a-b-a-linker-guard` test.
+            if let ty::FnDef(..) = item_type.kind() {
+                item_type.fn_sig(tcx).hash_stable(hcx, &mut hasher);
+            }
 
-                // also include any type parameters (for generic items)
-                args.hash_stable(hcx, &mut hasher);
+            // also include any type parameters (for generic items)
+            args.hash_stable(hcx, &mut hasher);
 
-                if let Some(instantiating_crate) = instantiating_crate {
-                    tcx.def_path_hash(instantiating_crate.as_def_id())
-                        .stable_crate_id()
-                        .hash_stable(hcx, &mut hasher);
-                }
+            if let Some(instantiating_crate) = instantiating_crate {
+                tcx.def_path_hash(instantiating_crate.as_def_id())
+                    .stable_crate_id()
+                    .hash_stable(hcx, &mut hasher);
+            }
 
-                // We want to avoid accidental collision between different types of instances.
-                // Especially, `VTableShim`s and `ReifyShim`s may overlap with their original
-                // instances without this.
-                discriminant(&instance.def).hash_stable(hcx, &mut hasher);
-            });
+            // We want to avoid accidental collision between different types of instances.
+            // Especially, `VTableShim`s and `ReifyShim`s may overlap with their original
+            // instances without this.
+            discriminant(&instance.def).hash_stable(hcx, &mut hasher);
         });
 
         // 64 bits should be enough to avoid collisions.
diff --git a/compiler/rustc_symbol_mangling/src/lib.rs b/compiler/rustc_symbol_mangling/src/lib.rs
index 14dd8b4e5..8c035ba94 100644
--- a/compiler/rustc_symbol_mangling/src/lib.rs
+++ b/compiler/rustc_symbol_mangling/src/lib.rs
@@ -88,12 +88,11 @@
 //! DefPaths which are much more robust in the face of changes to the code base.
 
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
-#![cfg_attr(not(bootstrap), allow(internal_features))]
+#![doc(rust_logo)]
+#![feature(rustdoc_internals)]
+#![allow(internal_features)]
 #![feature(never_type)]
 #![recursion_limit = "256"]
-#![allow(rustc::potential_query_instability)]
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
 
@@ -234,7 +233,7 @@ fn compute_symbol_name<'tcx>(
     // and we want to be sure to avoid any symbol conflicts here.
     let is_globally_shared_function = matches!(
         tcx.def_kind(instance.def_id()),
-        DefKind::Fn | DefKind::AssocFn | DefKind::Closure | DefKind::Coroutine | DefKind::Ctor(..)
+        DefKind::Fn | DefKind::AssocFn | DefKind::Closure | DefKind::Ctor(..)
     ) && matches!(
         MonoItem::Fn(instance).instantiation_mode(tcx),
         InstantiationMode::GloballyShared { may_conflict: true }
diff --git a/compiler/rustc_symbol_mangling/src/typeid/typeid_itanium_cxx_abi.rs b/compiler/rustc_symbol_mangling/src/typeid/typeid_itanium_cxx_abi.rs
index 5ce188488..3d673f2f1 100644
--- a/compiler/rustc_symbol_mangling/src/typeid/typeid_itanium_cxx_abi.rs
+++ b/compiler/rustc_symbol_mangling/src/typeid/typeid_itanium_cxx_abi.rs
@@ -270,7 +270,7 @@ fn encode_region<'tcx>(
     // u6region[I[<region-disambiguator>][<region-index>]E] as vendor extended type
     let mut s = String::new();
     match region.kind() {
-        RegionKind::ReLateBound(debruijn, r) => {
+        RegionKind::ReBound(debruijn, r) => {
             s.push_str("u6regionI");
             // Debruijn index, which identifies the binder, as region disambiguator
             let num = debruijn.index() as u64;
@@ -282,11 +282,12 @@ fn encode_region<'tcx>(
             s.push('E');
             compress(dict, DictKey::Region(region), &mut s);
         }
-        RegionKind::ReEarlyBound(..) | RegionKind::ReErased => {
+        // FIXME(@lcnr): Why is `ReEarlyParam` reachable here.
+        RegionKind::ReEarlyParam(..) | RegionKind::ReErased => {
             s.push_str("u6region");
             compress(dict, DictKey::Region(region), &mut s);
         }
-        RegionKind::ReFree(..)
+        RegionKind::ReLateParam(..)
         | RegionKind::ReStatic
         | RegionKind::ReError(_)
         | RegionKind::ReVar(..)
@@ -363,7 +364,7 @@ fn encode_ty_name(tcx: TyCtxt<'_>, def_id: DefId) -> String {
     //     _ZTSFvu27NvNtC1234_5crate6Trait13fooIu22NtC1234_5crate7Struct1Iu3i32ES_EE
     //
     // The reason for not using v0's extended form of paths is to use a consistent and simpler
-    // encoding, as the reasoning for using it isn't relevand for type metadata identifiers (i.e.,
+    // encoding, as the reasoning for using it isn't relevant for type metadata identifiers (i.e.,
     // keep symbol names close to how methods are represented in error messages). See
     // https://rust-lang.github.io/rfcs/2603-rust-symbol-name-mangling-v0.html#methods.
     let mut s = String::new();
@@ -378,14 +379,13 @@ fn encode_ty_name(tcx: TyCtxt<'_>, def_id: DefId) -> String {
             hir::definitions::DefPathData::ForeignMod => "F", // Not specified in v0's <namespace>
             hir::definitions::DefPathData::TypeNs(..) => "t",
             hir::definitions::DefPathData::ValueNs(..) => "v",
-            hir::definitions::DefPathData::ClosureExpr => "C",
+            hir::definitions::DefPathData::Closure => "C",
             hir::definitions::DefPathData::Ctor => "c",
             hir::definitions::DefPathData::AnonConst => "k",
-            hir::definitions::DefPathData::ImplTrait => "i",
+            hir::definitions::DefPathData::OpaqueTy => "i",
             hir::definitions::DefPathData::CrateRoot
             | hir::definitions::DefPathData::Use
             | hir::definitions::DefPathData::GlobalAsm
-            | hir::definitions::DefPathData::ImplTraitAssocTy
             | hir::definitions::DefPathData::MacroNs(..)
             | hir::definitions::DefPathData::LifetimeNs(..) => {
                 bug!("encode_ty_name: unexpected `{:?}`", disambiguated_data.data);
@@ -551,7 +551,7 @@ fn encode_ty<'tcx>(
                 // Use user-defined CFI encoding for type
                 if let Some(value_str) = cfi_encoding.value_str() {
                     if !value_str.to_string().trim().is_empty() {
-                        s.push_str(&value_str.to_string().trim());
+                        s.push_str(value_str.to_string().trim());
                     } else {
                         #[allow(
                             rustc::diagnostic_outside_of_impl,
@@ -603,7 +603,7 @@ fn encode_ty<'tcx>(
                 // Use user-defined CFI encoding for type
                 if let Some(value_str) = cfi_encoding.value_str() {
                     if !value_str.to_string().trim().is_empty() {
-                        s.push_str(&value_str.to_string().trim());
+                        s.push_str(value_str.to_string().trim());
                     } else {
                         #[allow(
                             rustc::diagnostic_outside_of_impl,
@@ -773,12 +773,7 @@ fn transform_ty<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>, options: TransformTyOptio
     let mut ty = ty;
 
     match ty.kind() {
-        ty::Float(..)
-        | ty::Char
-        | ty::Str
-        | ty::Never
-        | ty::Foreign(..)
-        | ty::CoroutineWitness(..) => {}
+        ty::Float(..) | ty::Str | ty::Never | ty::Foreign(..) | ty::CoroutineWitness(..) => {}
 
         ty::Bool => {
             if options.contains(EncodeTyOptions::NORMALIZE_INTEGERS) {
@@ -792,6 +787,14 @@ fn transform_ty<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>, options: TransformTyOptio
             }
         }
 
+        ty::Char => {
+            if options.contains(EncodeTyOptions::NORMALIZE_INTEGERS) {
+                // Since #118032, char is guaranteed to have the same size, alignment, and function
+                // call ABI as u32 on all platforms.
+                ty = tcx.types.u32;
+            }
+        }
+
         ty::Int(..) | ty::Uint(..) => {
             if options.contains(EncodeTyOptions::NORMALIZE_INTEGERS) {
                 // Note: C99 7.18.2.4 requires uintptr_t and intptr_t to be at least 16-bit wide.
@@ -1144,5 +1147,5 @@ pub fn typeid_for_instance<'tcx>(
         }
     }
 
-    typeid_for_fnabi(tcx, &fn_abi, options)
+    typeid_for_fnabi(tcx, fn_abi, options)
 }
diff --git a/compiler/rustc_symbol_mangling/src/v0.rs b/compiler/rustc_symbol_mangling/src/v0.rs
index ad3d291df..e002e345a 100644
--- a/compiler/rustc_symbol_mangling/src/v0.rs
+++ b/compiler/rustc_symbol_mangling/src/v0.rs
@@ -189,7 +189,7 @@ impl<'tcx> SymbolMangler<'tcx> {
         self.push("N");
         self.out.push(ns);
         print_prefix(self)?;
-        self.push_disambiguator(disambiguator as u64);
+        self.push_disambiguator(disambiguator);
         self.push_ident(name);
         Ok(())
     }
@@ -319,9 +319,9 @@ impl<'tcx> Printer<'tcx> for SymbolMangler<'tcx> {
             // shorter mangling of `L_`.
             ty::ReErased => 0,
 
-            // Late-bound lifetimes use indices starting at 1,
+            // Bound lifetimes use indices starting at 1,
             // see `BinderLevel` for more details.
-            ty::ReLateBound(debruijn, ty::BoundRegion { var, kind: ty::BrAnon }) => {
+            ty::ReBound(debruijn, ty::BoundRegion { var, kind: ty::BrAnon }) => {
                 let binder = &self.binders[self.binders.len() - 1 - debruijn.index()];
                 let depth = binder.lifetime_depths.start + var.as_u32();
 
@@ -770,17 +770,16 @@ impl<'tcx> Printer<'tcx> for SymbolMangler<'tcx> {
             // Uppercase categories are more stable than lowercase ones.
             DefPathData::TypeNs(_) => 't',
             DefPathData::ValueNs(_) => 'v',
-            DefPathData::ClosureExpr => 'C',
+            DefPathData::Closure => 'C',
             DefPathData::Ctor => 'c',
             DefPathData::AnonConst => 'k',
-            DefPathData::ImplTrait => 'i',
+            DefPathData::OpaqueTy => 'i',
 
             // These should never show up as `path_append` arguments.
             DefPathData::CrateRoot
             | DefPathData::Use
             | DefPathData::GlobalAsm
             | DefPathData::Impl
-            | DefPathData::ImplTraitAssocTy
             | DefPathData::MacroNs(_)
             | DefPathData::LifetimeNs(_) => {
                 bug!("symbol_names: unexpected DefPathData: {:?}", disambiguated_data.data)
diff --git a/compiler/rustc_target/src/abi/call/aarch64.rs b/compiler/rustc_target/src/abi/call/aarch64.rs
index b4c7b0f12..f99f6a3b7 100644
--- a/compiler/rustc_target/src/abi/call/aarch64.rs
+++ b/compiler/rustc_target/src/abi/call/aarch64.rs
@@ -40,6 +40,10 @@ where
     Ty: TyAbiInterface<'a, C> + Copy,
     C: HasDataLayout,
 {
+    if !ret.layout.is_sized() {
+        // Not touching this...
+        return;
+    }
     if !ret.layout.is_aggregate() {
         if kind == AbiKind::DarwinPCS {
             // On Darwin, when returning an i8/i16, it must be sign-extended to 32 bits,
@@ -67,6 +71,10 @@ where
     Ty: TyAbiInterface<'a, C> + Copy,
     C: HasDataLayout,
 {
+    if !arg.layout.is_sized() {
+        // Not touching this...
+        return;
+    }
     if !arg.layout.is_aggregate() {
         if kind == AbiKind::DarwinPCS {
             // On Darwin, when passing an i8/i16, it must be sign-extended to 32 bits,
diff --git a/compiler/rustc_target/src/abi/call/arm.rs b/compiler/rustc_target/src/abi/call/arm.rs
index 1923ea588..95f6691d4 100644
--- a/compiler/rustc_target/src/abi/call/arm.rs
+++ b/compiler/rustc_target/src/abi/call/arm.rs
@@ -30,6 +30,10 @@ where
     Ty: TyAbiInterface<'a, C> + Copy,
     C: HasDataLayout,
 {
+    if !ret.layout.is_sized() {
+        // Not touching this...
+        return;
+    }
     if !ret.layout.is_aggregate() {
         ret.extend_integer_width_to(32);
         return;
@@ -56,6 +60,10 @@ where
     Ty: TyAbiInterface<'a, C> + Copy,
     C: HasDataLayout,
 {
+    if !arg.layout.is_sized() {
+        // Not touching this...
+        return;
+    }
     if !arg.layout.is_aggregate() {
         arg.extend_integer_width_to(32);
         return;
diff --git a/compiler/rustc_target/src/abi/call/csky.rs b/compiler/rustc_target/src/abi/call/csky.rs
index 706493b0a..8b4328db5 100644
--- a/compiler/rustc_target/src/abi/call/csky.rs
+++ b/compiler/rustc_target/src/abi/call/csky.rs
@@ -7,6 +7,10 @@
 use crate::abi::call::{ArgAbi, FnAbi, Reg, Uniform};
 
 fn classify_ret<Ty>(arg: &mut ArgAbi<'_, Ty>) {
+    if !arg.layout.is_sized() {
+        // Not touching this...
+        return;
+    }
     // For return type, aggregate which <= 2*XLen will be returned in registers.
     // Otherwise, aggregate will be returned indirectly.
     if arg.layout.is_aggregate() {
@@ -24,6 +28,10 @@ fn classify_ret<Ty>(arg: &mut ArgAbi<'_, Ty>) {
 }
 
 fn classify_arg<Ty>(arg: &mut ArgAbi<'_, Ty>) {
+    if !arg.layout.is_sized() {
+        // Not touching this...
+        return;
+    }
     // For argument type, the first 4*XLen parts of aggregate will be passed
     // in registers, and the rest will be passed in stack.
     // So we can coerce to integers directly and let backend handle it correctly.
diff --git a/compiler/rustc_target/src/abi/call/loongarch.rs b/compiler/rustc_target/src/abi/call/loongarch.rs
index e649d58bb..647b6500c 100644
--- a/compiler/rustc_target/src/abi/call/loongarch.rs
+++ b/compiler/rustc_target/src/abi/call/loongarch.rs
@@ -152,6 +152,10 @@ fn classify_ret<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>, xlen: u64, flen: u6
 where
     Ty: TyAbiInterface<'a, C> + Copy,
 {
+    if !arg.layout.is_sized() {
+        // Not touching this...
+        return false; // I guess? return value of this function is not documented
+    }
     if let Some(conv) = should_use_fp_conv(cx, &arg.layout, xlen, flen) {
         match conv {
             FloatConv::Float(f) => {
@@ -214,6 +218,10 @@ fn classify_arg<'a, Ty, C>(
 ) where
     Ty: TyAbiInterface<'a, C> + Copy,
 {
+    if !arg.layout.is_sized() {
+        // Not touching this...
+        return;
+    }
     if !is_vararg {
         match should_use_fp_conv(cx, &arg.layout, xlen, flen) {
             Some(FloatConv::Float(f)) if *avail_fprs >= 1 => {
diff --git a/compiler/rustc_target/src/abi/call/m68k.rs b/compiler/rustc_target/src/abi/call/m68k.rs
index 1d4649ed8..06697bdd8 100644
--- a/compiler/rustc_target/src/abi/call/m68k.rs
+++ b/compiler/rustc_target/src/abi/call/m68k.rs
@@ -9,6 +9,10 @@ fn classify_ret<Ty>(ret: &mut ArgAbi<'_, Ty>) {
 }
 
 fn classify_arg<Ty>(arg: &mut ArgAbi<'_, Ty>) {
+    if !arg.layout.is_sized() {
+        // Not touching this...
+        return;
+    }
     if arg.layout.is_aggregate() {
         arg.make_indirect_byval(None);
     } else {
diff --git a/compiler/rustc_target/src/abi/call/mips.rs b/compiler/rustc_target/src/abi/call/mips.rs
index edcd1bab8..57ccfe215 100644
--- a/compiler/rustc_target/src/abi/call/mips.rs
+++ b/compiler/rustc_target/src/abi/call/mips.rs
@@ -17,6 +17,10 @@ fn classify_arg<Ty, C>(cx: &C, arg: &mut ArgAbi<'_, Ty>, offset: &mut Size)
 where
     C: HasDataLayout,
 {
+    if !arg.layout.is_sized() {
+        // Not touching this...
+        return;
+    }
     let dl = cx.data_layout();
     let size = arg.layout.size;
     let align = arg.layout.align.max(dl.i32_align).min(dl.i64_align).abi;
diff --git a/compiler/rustc_target/src/abi/call/mod.rs b/compiler/rustc_target/src/abi/call/mod.rs
index 5efd171b9..e97309473 100644
--- a/compiler/rustc_target/src/abi/call/mod.rs
+++ b/compiler/rustc_target/src/abi/call/mod.rs
@@ -382,6 +382,7 @@ impl HomogeneousAggregate {
 }
 
 impl<'a, Ty> TyAndLayout<'a, Ty> {
+    /// Returns `true` if this is an aggregate type (including a ScalarPair!)
     fn is_aggregate(&self) -> bool {
         match self.abi {
             Abi::Uninhabited | Abi::Scalar(_) | Abi::Vector { .. } => false,
@@ -422,7 +423,7 @@ impl<'a, Ty> TyAndLayout<'a, Ty> {
                 }))
             }
 
-            Abi::ScalarPair(..) | Abi::Aggregate { .. } => {
+            Abi::ScalarPair(..) | Abi::Aggregate { sized: true } => {
                 // Helper for computing `homogeneous_aggregate`, allowing a custom
                 // starting offset (used below for handling variants).
                 let from_fields_at =
@@ -520,6 +521,7 @@ impl<'a, Ty> TyAndLayout<'a, Ty> {
                     Ok(result)
                 }
             }
+            Abi::Aggregate { sized: false } => Err(Heterogeneous),
         }
     }
 }
@@ -555,8 +557,7 @@ impl<'a, Ty> ArgAbi<'a, Ty> {
                 scalar_attrs(&layout, b, a.size(cx).align_to(b.align(cx).abi)),
             ),
             Abi::Vector { .. } => PassMode::Direct(ArgAttributes::new()),
-            // The `Aggregate` ABI should always be adjusted later.
-            Abi::Aggregate { .. } => PassMode::Direct(ArgAttributes::new()),
+            Abi::Aggregate { .. } => Self::indirect_pass_mode(&layout),
         };
         ArgAbi { layout, mode }
     }
@@ -580,14 +581,30 @@ impl<'a, Ty> ArgAbi<'a, Ty> {
         PassMode::Indirect { attrs, meta_attrs, on_stack: false }
     }
 
+    /// Pass this argument directly instead. Should NOT be used!
+    /// Only exists because of past ABI mistakes that will take time to fix
+    /// (see <https://github.com/rust-lang/rust/issues/115666>).
+    pub fn make_direct_deprecated(&mut self) {
+        match self.mode {
+            PassMode::Indirect { .. } => {
+                self.mode = PassMode::Direct(ArgAttributes::new());
+            }
+            PassMode::Ignore | PassMode::Direct(_) | PassMode::Pair(_, _) => return, // already direct
+            _ => panic!("Tried to make {:?} direct", self.mode),
+        }
+    }
+
     pub fn make_indirect(&mut self) {
         match self.mode {
-            PassMode::Direct(_) | PassMode::Pair(_, _) => {}
-            PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: false } => return,
+            PassMode::Direct(_) | PassMode::Pair(_, _) => {
+                self.mode = Self::indirect_pass_mode(&self.layout);
+            }
+            PassMode::Indirect { attrs: _, meta_attrs: _, on_stack: false } => {
+                // already indirect
+                return;
+            }
             _ => panic!("Tried to make {:?} indirect", self.mode),
         }
-
-        self.mode = Self::indirect_pass_mode(&self.layout);
     }
 
     pub fn make_indirect_byval(&mut self, byval_align: Option<Align>) {
@@ -836,7 +853,6 @@ impl<'a, Ty> FnAbi<'a, Ty> {
                     wasm::compute_c_abi_info(cx, self)
                 }
             }
-            "asmjs" => wasm::compute_c_abi_info(cx, self),
             "bpf" => bpf::compute_abi_info(self),
             arch => {
                 return Err(AdjustForForeignAbiError::Unsupported {
diff --git a/compiler/rustc_target/src/abi/call/nvptx64.rs b/compiler/rustc_target/src/abi/call/nvptx64.rs
index 4abe51cd6..5c040ce9c 100644
--- a/compiler/rustc_target/src/abi/call/nvptx64.rs
+++ b/compiler/rustc_target/src/abi/call/nvptx64.rs
@@ -4,12 +4,18 @@ use crate::abi::{HasDataLayout, TyAbiInterface};
 fn classify_ret<Ty>(ret: &mut ArgAbi<'_, Ty>) {
     if ret.layout.is_aggregate() && ret.layout.size.bits() > 64 {
         ret.make_indirect();
+    } else {
+        // FIXME: this is wrong! Need to decide which ABI we really want here.
+        ret.make_direct_deprecated();
     }
 }
 
 fn classify_arg<Ty>(arg: &mut ArgAbi<'_, Ty>) {
     if arg.layout.is_aggregate() && arg.layout.size.bits() > 64 {
         arg.make_indirect();
+    } else {
+        // FIXME: this is wrong! Need to decide which ABI we really want here.
+        arg.make_direct_deprecated();
     }
 }
 
@@ -30,6 +36,9 @@ where
             _ => unreachable!("Align is given as power of 2 no larger than 16 bytes"),
         };
         arg.cast_to(Uniform { unit, total: Size::from_bytes(2 * align_bytes) });
+    } else {
+        // FIXME: find a better way to do this. See https://github.com/rust-lang/rust/issues/117271.
+        arg.make_direct_deprecated();
     }
 }
 
diff --git a/compiler/rustc_target/src/abi/call/powerpc64.rs b/compiler/rustc_target/src/abi/call/powerpc64.rs
index 359bb8fc0..2d41f77e5 100644
--- a/compiler/rustc_target/src/abi/call/powerpc64.rs
+++ b/compiler/rustc_target/src/abi/call/powerpc64.rs
@@ -46,6 +46,10 @@ where
     Ty: TyAbiInterface<'a, C> + Copy,
     C: HasDataLayout,
 {
+    if !ret.layout.is_sized() {
+        // Not touching this...
+        return;
+    }
     if !ret.layout.is_aggregate() {
         ret.extend_integer_width_to(64);
         return;
@@ -89,6 +93,10 @@ where
     Ty: TyAbiInterface<'a, C> + Copy,
     C: HasDataLayout,
 {
+    if !arg.layout.is_sized() {
+        // Not touching this...
+        return;
+    }
     if !arg.layout.is_aggregate() {
         arg.extend_integer_width_to(64);
         return;
diff --git a/compiler/rustc_target/src/abi/call/riscv.rs b/compiler/rustc_target/src/abi/call/riscv.rs
index 93a204563..cbde234d3 100644
--- a/compiler/rustc_target/src/abi/call/riscv.rs
+++ b/compiler/rustc_target/src/abi/call/riscv.rs
@@ -158,6 +158,10 @@ fn classify_ret<'a, Ty, C>(cx: &C, arg: &mut ArgAbi<'a, Ty>, xlen: u64, flen: u6
 where
     Ty: TyAbiInterface<'a, C> + Copy,
 {
+    if !arg.layout.is_sized() {
+        // Not touching this...
+        return false; // I guess? return value of this function is not documented
+    }
     if let Some(conv) = should_use_fp_conv(cx, &arg.layout, xlen, flen) {
         match conv {
             FloatConv::Float(f) => {
@@ -220,6 +224,10 @@ fn classify_arg<'a, Ty, C>(
 ) where
     Ty: TyAbiInterface<'a, C> + Copy,
 {
+    if !arg.layout.is_sized() {
+        // Not touching this...
+        return;
+    }
     if !is_vararg {
         match should_use_fp_conv(cx, &arg.layout, xlen, flen) {
             Some(FloatConv::Float(f)) if *avail_fprs >= 1 => {
diff --git a/compiler/rustc_target/src/abi/call/s390x.rs b/compiler/rustc_target/src/abi/call/s390x.rs
index ea2369281..1a2191082 100644
--- a/compiler/rustc_target/src/abi/call/s390x.rs
+++ b/compiler/rustc_target/src/abi/call/s390x.rs
@@ -17,6 +17,10 @@ where
     Ty: TyAbiInterface<'a, C> + Copy,
     C: HasDataLayout,
 {
+    if !arg.layout.is_sized() {
+        // Not touching this...
+        return;
+    }
     if !arg.layout.is_aggregate() && arg.layout.size.bits() <= 64 {
         arg.extend_integer_width_to(64);
         return;
diff --git a/compiler/rustc_target/src/abi/call/sparc.rs b/compiler/rustc_target/src/abi/call/sparc.rs
index edcd1bab8..57ccfe215 100644
--- a/compiler/rustc_target/src/abi/call/sparc.rs
+++ b/compiler/rustc_target/src/abi/call/sparc.rs
@@ -17,6 +17,10 @@ fn classify_arg<Ty, C>(cx: &C, arg: &mut ArgAbi<'_, Ty>, offset: &mut Size)
 where
     C: HasDataLayout,
 {
+    if !arg.layout.is_sized() {
+        // Not touching this...
+        return;
+    }
     let dl = cx.data_layout();
     let size = arg.layout.size;
     let align = arg.layout.align.max(dl.i32_align).min(dl.i64_align).abi;
diff --git a/compiler/rustc_target/src/abi/call/wasm.rs b/compiler/rustc_target/src/abi/call/wasm.rs
index 796b752ff..a7a2b314a 100644
--- a/compiler/rustc_target/src/abi/call/wasm.rs
+++ b/compiler/rustc_target/src/abi/call/wasm.rs
@@ -34,6 +34,10 @@ where
     Ty: TyAbiInterface<'a, C> + Copy,
     C: HasDataLayout,
 {
+    if !arg.layout.is_sized() {
+        // Not touching this...
+        return;
+    }
     arg.extend_integer_width_to(32);
     if arg.layout.is_aggregate() && !unwrap_trivial_aggregate(cx, arg) {
         arg.make_indirect_byval(None);
@@ -67,21 +71,33 @@ where
 /// Also see <https://github.com/rust-lang/rust/issues/115666>.
 pub fn compute_wasm_abi_info<Ty>(fn_abi: &mut FnAbi<'_, Ty>) {
     if !fn_abi.ret.is_ignore() {
-        classify_ret(&mut fn_abi.ret);
+        classify_ret_wasm_abi(&mut fn_abi.ret);
     }
 
     for arg in fn_abi.args.iter_mut() {
         if arg.is_ignore() {
             continue;
         }
-        classify_arg(arg);
+        classify_arg_wasm_abi(arg);
     }
 
-    fn classify_ret<Ty>(ret: &mut ArgAbi<'_, Ty>) {
+    fn classify_ret_wasm_abi<Ty>(ret: &mut ArgAbi<'_, Ty>) {
+        if !ret.layout.is_sized() {
+            // Not touching this...
+            return;
+        }
+        // FIXME: this is bad! https://github.com/rust-lang/rust/issues/115666
+        ret.make_direct_deprecated();
         ret.extend_integer_width_to(32);
     }
 
-    fn classify_arg<Ty>(arg: &mut ArgAbi<'_, Ty>) {
+    fn classify_arg_wasm_abi<Ty>(arg: &mut ArgAbi<'_, Ty>) {
+        if !arg.layout.is_sized() {
+            // Not touching this...
+            return;
+        }
+        // FIXME: this is bad! https://github.com/rust-lang/rust/issues/115666
+        arg.make_direct_deprecated();
         arg.extend_integer_width_to(32);
     }
 }
diff --git a/compiler/rustc_target/src/abi/call/x86.rs b/compiler/rustc_target/src/abi/call/x86.rs
index c27f1e6dd..e9aedc3d2 100644
--- a/compiler/rustc_target/src/abi/call/x86.rs
+++ b/compiler/rustc_target/src/abi/call/x86.rs
@@ -14,7 +14,7 @@ where
     C: HasDataLayout + HasTargetSpec,
 {
     if !fn_abi.ret.is_ignore() {
-        if fn_abi.ret.layout.is_aggregate() {
+        if fn_abi.ret.layout.is_aggregate() && fn_abi.ret.layout.is_sized() {
             // Returning a structure. Most often, this will use
             // a hidden first argument. On some platforms, though,
             // small structs are returned as integers.
@@ -50,7 +50,7 @@ where
     }
 
     for arg in fn_abi.args.iter_mut() {
-        if arg.is_ignore() {
+        if arg.is_ignore() || !arg.layout.is_sized() {
             continue;
         }
 
diff --git a/compiler/rustc_target/src/abi/call/x86_64.rs b/compiler/rustc_target/src/abi/call/x86_64.rs
index d1efe9776..6c34585a1 100644
--- a/compiler/rustc_target/src/abi/call/x86_64.rs
+++ b/compiler/rustc_target/src/abi/call/x86_64.rs
@@ -153,9 +153,9 @@ fn reg_component(cls: &[Option<Class>], i: &mut usize, size: Size) -> Option<Reg
     }
 }
 
-fn cast_target(cls: &[Option<Class>], size: Size) -> Option<CastTarget> {
+fn cast_target(cls: &[Option<Class>], size: Size) -> CastTarget {
     let mut i = 0;
-    let lo = reg_component(cls, &mut i, size)?;
+    let lo = reg_component(cls, &mut i, size).unwrap();
     let offset = Size::from_bytes(8) * (i as u64);
     let mut target = CastTarget::from(lo);
     if size > offset {
@@ -164,7 +164,7 @@ fn cast_target(cls: &[Option<Class>], size: Size) -> Option<CastTarget> {
         }
     }
     assert_eq!(reg_component(cls, &mut i, Size::ZERO), None);
-    Some(target)
+    target
 }
 
 const MAX_INT_REGS: usize = 6; // RDI, RSI, RDX, RCX, R8, R9
@@ -179,6 +179,10 @@ where
     let mut sse_regs = MAX_SSE_REGS;
 
     let mut x86_64_arg_or_ret = |arg: &mut ArgAbi<'a, Ty>, is_arg: bool| {
+        if !arg.layout.is_sized() {
+            // Not touching this...
+            return;
+        }
         let mut cls_or_mem = classify_arg(cx, arg);
 
         if is_arg {
@@ -227,9 +231,7 @@ where
                 // split into sized chunks passed individually
                 if arg.layout.is_aggregate() {
                     let size = arg.layout.size;
-                    if let Some(cast_target) = cast_target(cls, size) {
-                        arg.cast_to(cast_target);
-                    }
+                    arg.cast_to(cast_target(cls, size));
                 } else {
                     arg.extend_integer_width_to(32);
                 }
diff --git a/compiler/rustc_target/src/abi/call/x86_win64.rs b/compiler/rustc_target/src/abi/call/x86_win64.rs
index 1aaf0e511..90de1a42b 100644
--- a/compiler/rustc_target/src/abi/call/x86_win64.rs
+++ b/compiler/rustc_target/src/abi/call/x86_win64.rs
@@ -6,8 +6,8 @@ use crate::abi::Abi;
 pub fn compute_abi_info<Ty>(fn_abi: &mut FnAbi<'_, Ty>) {
     let fixup = |a: &mut ArgAbi<'_, Ty>| {
         match a.layout.abi {
-            Abi::Uninhabited => {}
-            Abi::ScalarPair(..) | Abi::Aggregate { .. } => match a.layout.size.bits() {
+            Abi::Uninhabited | Abi::Aggregate { sized: false } => {}
+            Abi::ScalarPair(..) | Abi::Aggregate { sized: true } => match a.layout.size.bits() {
                 8 => a.cast_to(Reg::i8()),
                 16 => a.cast_to(Reg::i16()),
                 32 => a.cast_to(Reg::i32()),
diff --git a/compiler/rustc_target/src/abi/mod.rs b/compiler/rustc_target/src/abi/mod.rs
index b00567e87..a274790bf 100644
--- a/compiler/rustc_target/src/abi/mod.rs
+++ b/compiler/rustc_target/src/abi/mod.rs
@@ -42,6 +42,8 @@ rustc_index::newtype_index! {
     /// `d` is `FieldIdx(1)` in `VariantIdx(1)`, and
     /// `f` is `FieldIdx(1)` in `VariantIdx(0)`.
     #[derive(HashStable_Generic)]
+    #[encodable]
+    #[orderable]
     pub struct FieldIdx {}
 }
 
@@ -57,6 +59,8 @@ rustc_index::newtype_index! {
     /// `struct`s, `tuples`, and `unions`s are considered to have a single variant
     /// with variant index zero, aka [`FIRST_VARIANT`].
     #[derive(HashStable_Generic)]
+    #[encodable]
+    #[orderable]
     pub struct VariantIdx {
         /// Equivalent to `VariantIdx(0)`.
         const FIRST_VARIANT = 0;
diff --git a/compiler/rustc_target/src/lib.rs b/compiler/rustc_target/src/lib.rs
index 9c5ce8894..257c67779 100644
--- a/compiler/rustc_target/src/lib.rs
+++ b/compiler/rustc_target/src/lib.rs
@@ -8,8 +8,8 @@
 //! LLVM.
 
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
+#![doc(rust_logo)]
+#![feature(rustdoc_internals)]
 #![feature(assert_matches)]
 #![feature(associated_type_bounds)]
 #![feature(exhaustive_patterns)]
@@ -35,6 +35,7 @@ pub mod abi;
 pub mod asm;
 pub mod json;
 pub mod spec;
+pub mod target_features;
 
 #[cfg(test)]
 mod tests;
diff --git a/compiler/rustc_target/src/spec/base/android.rs b/compiler/rustc_target/src/spec/base/android.rs
index af15c16a5..5320f1b4b 100644
--- a/compiler/rustc_target/src/spec/base/android.rs
+++ b/compiler/rustc_target/src/spec/base/android.rs
@@ -1,10 +1,11 @@
-use crate::spec::{base, SanitizerSet, TargetOptions};
+use crate::spec::{base, SanitizerSet, TargetOptions, TlsModel};
 
 pub fn opts() -> TargetOptions {
     let mut base = base::linux::opts();
     base.os = "android".into();
     base.is_like_android = true;
     base.default_dwarf_version = 2;
+    base.tls_model = TlsModel::Emulated;
     base.has_thread_local = false;
     base.supported_sanitizers = SanitizerSet::ADDRESS;
     // This is for backward compatibility, see https://github.com/rust-lang/rust/issues/49867
diff --git a/compiler/rustc_target/src/spec/base/apple/mod.rs b/compiler/rustc_target/src/spec/base/apple/mod.rs
index 99e64503e..afd60b40c 100644
--- a/compiler/rustc_target/src/spec/base/apple/mod.rs
+++ b/compiler/rustc_target/src/spec/base/apple/mod.rs
@@ -1,19 +1,20 @@
 use std::{borrow::Cow, env};
 
 use crate::spec::{add_link_args, add_link_args_iter};
-use crate::spec::{cvs, Cc, DebuginfoKind, FramePointer, LinkArgs};
-use crate::spec::{LinkerFlavor, Lld, SplitDebuginfo, StaticCow, Target, TargetOptions};
+use crate::spec::{cvs, Cc, DebuginfoKind, FramePointer, LinkArgs, LinkerFlavor, Lld};
+use crate::spec::{SplitDebuginfo, StackProbeType, StaticCow, Target, TargetOptions};
 
 #[cfg(test)]
 mod tests;
 
 use Arch::*;
 #[allow(non_camel_case_types)]
-#[derive(Copy, Clone)]
+#[derive(Copy, Clone, PartialEq)]
 pub enum Arch {
     Armv7k,
     Armv7s,
     Arm64,
+    Arm64e,
     Arm64_32,
     I386,
     I686,
@@ -31,6 +32,7 @@ impl Arch {
             Armv7k => "armv7k",
             Armv7s => "armv7s",
             Arm64 | Arm64_macabi | Arm64_sim => "arm64",
+            Arm64e => "arm64e",
             Arm64_32 => "arm64_32",
             I386 => "i386",
             I686 => "i686",
@@ -42,7 +44,7 @@ impl Arch {
     pub fn target_arch(self) -> Cow<'static, str> {
         Cow::Borrowed(match self {
             Armv7k | Armv7s => "arm",
-            Arm64 | Arm64_32 | Arm64_macabi | Arm64_sim => "aarch64",
+            Arm64 | Arm64e | Arm64_32 | Arm64_macabi | Arm64_sim => "aarch64",
             I386 | I686 => "x86",
             X86_64 | X86_64_sim | X86_64_macabi | X86_64h => "x86_64",
         })
@@ -50,7 +52,7 @@ impl Arch {
 
     fn target_abi(self) -> &'static str {
         match self {
-            Armv7k | Armv7s | Arm64 | Arm64_32 | I386 | I686 | X86_64 | X86_64h => "",
+            Armv7k | Armv7s | Arm64 | Arm64e | Arm64_32 | I386 | I686 | X86_64 | X86_64h => "",
             X86_64_macabi | Arm64_macabi => "macabi",
             // x86_64-apple-ios is a simulator target, even though it isn't
             // declared that way in the target like the other ones...
@@ -63,6 +65,7 @@ impl Arch {
             Armv7k => "cortex-a8",
             Armv7s => "swift", // iOS 10 is only supported on iPhone 5 or higher.
             Arm64 => "apple-a7",
+            Arm64e => "apple-a12",
             Arm64_32 => "apple-s4",
             // Only macOS 10.12+ is supported, which means
             // all x86_64/x86 CPUs must be running at least penryn
@@ -78,6 +81,14 @@ impl Arch {
             Arm64_sim => "apple-a12",
         }
     }
+
+    fn stack_probes(self) -> StackProbeType {
+        match self {
+            Armv7k | Armv7s => StackProbeType::None,
+            Arm64 | Arm64e | Arm64_32 | I386 | I686 | X86_64 | X86_64h | X86_64_sim
+            | X86_64_macabi | Arm64_macabi | Arm64_sim => StackProbeType::Inline,
+        }
+    }
 }
 
 fn pre_link_args(os: &'static str, arch: Arch, abi: &'static str) -> LinkArgs {
@@ -88,7 +99,7 @@ fn pre_link_args(os: &'static str, arch: Arch, abi: &'static str) -> LinkArgs {
     };
 
     let platform_version: StaticCow<str> = match os {
-        "ios" => ios_lld_platform_version(),
+        "ios" => ios_lld_platform_version(arch),
         "tvos" => tvos_lld_platform_version(),
         "watchos" => watchos_lld_platform_version(),
         "macos" => macos_lld_platform_version(arch),
@@ -144,6 +155,7 @@ pub fn opts(os: &'static str, arch: Arch) -> TargetOptions {
         abi_return_struct_as_int: true,
         emit_debug_gdb_scripts: false,
         eh_frame_header: false,
+        stack_probes: arch.stack_probes(),
 
         debuginfo_kind: DebuginfoKind::DwarfDsym,
         // The historical default for macOS targets is to run `dsymutil` which
@@ -202,12 +214,22 @@ pub fn deployment_target(target: &Target) -> Option<(u32, u32)> {
     let (major, minor) = match &*target.os {
         "macos" => {
             // This does not need to be specific. It just needs to handle x86 vs M1.
-            let arch = if target.arch == "x86" || target.arch == "x86_64" { X86_64 } else { Arm64 };
+            let arch = match target.arch.as_ref() {
+                "x86" | "x86_64" => X86_64,
+                "arm64e" => Arm64e,
+                _ => Arm64,
+            };
             macos_deployment_target(arch)
         }
         "ios" => match &*target.abi {
             "macabi" => mac_catalyst_deployment_target(),
-            _ => ios_deployment_target(),
+            _ => {
+                let arch = match target.arch.as_ref() {
+                    "arm64e" => Arm64e,
+                    _ => Arm64,
+                };
+                ios_deployment_target(arch)
+            }
         },
         "watchos" => watchos_deployment_target(),
         "tvos" => tvos_deployment_target(),
@@ -228,7 +250,7 @@ fn from_set_deployment_target(var_name: &str) -> Option<(u32, u32)> {
 fn macos_default_deployment_target(arch: Arch) -> (u32, u32) {
     match arch {
         // Note: Arm64_sim is not included since macOS has no simulator.
-        Arm64 | Arm64_macabi => (11, 0),
+        Arm64 | Arm64e | Arm64_macabi => (11, 0),
         _ => (10, 12),
     }
 }
@@ -280,8 +302,8 @@ fn link_env_remove(arch: Arch, os: &'static str) -> StaticCow<[StaticCow<str>]>
         // Otherwise if cross-compiling for a different OS/SDK, remove any part
         // of the linking environment that's wrong and reversed.
         match arch {
-            Armv7k | Armv7s | Arm64 | Arm64_32 | I386 | I686 | X86_64 | X86_64_sim | X86_64h
-            | Arm64_sim => {
+            Armv7k | Armv7s | Arm64 | Arm64e | Arm64_32 | I386 | I686 | X86_64 | X86_64_sim
+            | X86_64h | Arm64_sim => {
                 cvs!["MACOSX_DEPLOYMENT_TARGET"]
             }
             X86_64_macabi | Arm64_macabi => cvs!["IPHONEOS_DEPLOYMENT_TARGET"],
@@ -289,9 +311,10 @@ fn link_env_remove(arch: Arch, os: &'static str) -> StaticCow<[StaticCow<str>]>
     }
 }
 
-fn ios_deployment_target() -> (u32, u32) {
+fn ios_deployment_target(arch: Arch) -> (u32, u32) {
     // If you are looking for the default deployment target, prefer `rustc --print deployment-target`.
-    from_set_deployment_target("IPHONEOS_DEPLOYMENT_TARGET").unwrap_or((10, 0))
+    let (major, minor) = if arch == Arm64e { (14, 0) } else { (10, 0) };
+    from_set_deployment_target("IPHONEOS_DEPLOYMENT_TARGET").unwrap_or((major, minor))
 }
 
 fn mac_catalyst_deployment_target() -> (u32, u32) {
@@ -306,17 +329,17 @@ pub fn ios_llvm_target(arch: Arch) -> String {
     // set high enough. Luckily one LC_BUILD_VERSION is enough, for Xcode
     // to pick it up (since std and core are still built with the fallback
     // of version 7.0 and hence emit the old LC_IPHONE_MIN_VERSION).
-    let (major, minor) = ios_deployment_target();
+    let (major, minor) = ios_deployment_target(arch);
     format!("{}-apple-ios{}.{}.0", arch.target_name(), major, minor)
 }
 
-fn ios_lld_platform_version() -> String {
-    let (major, minor) = ios_deployment_target();
+fn ios_lld_platform_version(arch: Arch) -> String {
+    let (major, minor) = ios_deployment_target(arch);
     format!("{major}.{minor}")
 }
 
 pub fn ios_sim_llvm_target(arch: Arch) -> String {
-    let (major, minor) = ios_deployment_target();
+    let (major, minor) = ios_deployment_target(arch);
     format!("{}-apple-ios{}.{}.0-simulator", arch.target_name(), major, minor)
 }
 
diff --git a/compiler/rustc_target/src/spec/base/linux_ohos.rs b/compiler/rustc_target/src/spec/base/linux_ohos.rs
index 273e6a98d..8272cda05 100644
--- a/compiler/rustc_target/src/spec/base/linux_ohos.rs
+++ b/compiler/rustc_target/src/spec/base/linux_ohos.rs
@@ -1,11 +1,11 @@
-use crate::spec::{base, TargetOptions};
+use crate::spec::{base, TargetOptions, TlsModel};
 
 pub fn opts() -> TargetOptions {
     let mut base = base::linux::opts();
 
     base.env = "ohos".into();
     base.crt_static_default = false;
-    base.force_emulated_tls = true;
+    base.tls_model = TlsModel::Emulated;
     base.has_thread_local = false;
 
     base
diff --git a/compiler/rustc_target/src/spec/base/openbsd.rs b/compiler/rustc_target/src/spec/base/openbsd.rs
index e7db14e05..bc3aecb5c 100644
--- a/compiler/rustc_target/src/spec/base/openbsd.rs
+++ b/compiler/rustc_target/src/spec/base/openbsd.rs
@@ -1,4 +1,4 @@
-use crate::spec::{cvs, FramePointer, RelroLevel, TargetOptions};
+use crate::spec::{cvs, FramePointer, RelroLevel, TargetOptions, TlsModel};
 
 pub fn opts() -> TargetOptions {
     TargetOptions {
@@ -11,6 +11,7 @@ pub fn opts() -> TargetOptions {
         frame_pointer: FramePointer::Always, // FIXME 43575: should be MayOmit...
         relro_level: RelroLevel::Full,
         default_dwarf_version: 2,
+        tls_model: TlsModel::Emulated,
         ..Default::default()
     }
 }
diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs
index f04799482..b688c9731 100644
--- a/compiler/rustc_target/src/spec/mod.rs
+++ b/compiler/rustc_target/src/spec/mod.rs
@@ -929,6 +929,7 @@ pub enum TlsModel {
     LocalDynamic,
     InitialExec,
     LocalExec,
+    Emulated,
 }
 
 impl FromStr for TlsModel {
@@ -942,6 +943,7 @@ impl FromStr for TlsModel {
             "local-dynamic" => TlsModel::LocalDynamic,
             "initial-exec" => TlsModel::InitialExec,
             "local-exec" => TlsModel::LocalExec,
+            "emulated" => TlsModel::Emulated,
             _ => return Err(()),
         })
     }
@@ -954,6 +956,7 @@ impl ToJson for TlsModel {
             TlsModel::LocalDynamic => "local-dynamic",
             TlsModel::InitialExec => "initial-exec",
             TlsModel::LocalExec => "local-exec",
+            TlsModel::Emulated => "emulated",
         }
         .to_json()
     }
@@ -1155,10 +1158,6 @@ pub enum StackProbeType {
 }
 
 impl StackProbeType {
-    // LLVM X86 targets (ix86 and x86_64) can use inline-asm stack probes starting with LLVM 16.
-    // Notable past issues were rust#83139 (fixed in 14) and rust#84667 (fixed in 16).
-    const X86: Self = Self::InlineOrCall { min_llvm_version_for_inline: (16, 0, 0) };
-
     fn from_json(json: &Json) -> Result<Self, String> {
         let object = json.as_object().ok_or_else(|| "expected a JSON object")?;
         let kind = object
@@ -1544,6 +1543,7 @@ supported_targets! {
     ("i686-unknown-hurd-gnu", i686_unknown_hurd_gnu),
 
     ("aarch64-apple-darwin", aarch64_apple_darwin),
+    ("arm64e-apple-darwin", arm64e_apple_darwin),
     ("x86_64-apple-darwin", x86_64_apple_darwin),
     ("x86_64h-apple-darwin", x86_64h_apple_darwin),
     ("i686-apple-darwin", i686_apple_darwin),
@@ -1566,6 +1566,7 @@ supported_targets! {
     ("i386-apple-ios", i386_apple_ios),
     ("x86_64-apple-ios", x86_64_apple_ios),
     ("aarch64-apple-ios", aarch64_apple_ios),
+    ("arm64e-apple-ios", arm64e_apple_ios),
     ("armv7s-apple-ios", armv7s_apple_ios),
     ("x86_64-apple-ios-macabi", x86_64_apple_ios_macabi),
     ("aarch64-apple-ios-macabi", aarch64_apple_ios_macabi),
@@ -1577,6 +1578,7 @@ supported_targets! {
     ("armv7k-apple-watchos", armv7k_apple_watchos),
     ("arm64_32-apple-watchos", arm64_32_apple_watchos),
     ("x86_64-apple-watchos-sim", x86_64_apple_watchos_sim),
+    ("aarch64-apple-watchos", aarch64_apple_watchos),
     ("aarch64-apple-watchos-sim", aarch64_apple_watchos_sim),
 
     ("armebv7r-none-eabi", armebv7r_none_eabi),
@@ -1585,7 +1587,6 @@ supported_targets! {
     ("armv7r-none-eabihf", armv7r_none_eabihf),
 
     ("x86_64-pc-solaris", x86_64_pc_solaris),
-    ("x86_64-sun-solaris", x86_64_sun_solaris),
     ("sparcv9-sun-solaris", sparcv9_sun_solaris),
 
     ("x86_64-unknown-illumos", x86_64_unknown_illumos),
@@ -1603,13 +1604,14 @@ supported_targets! {
     ("aarch64-uwp-windows-msvc", aarch64_uwp_windows_msvc),
     ("x86_64-pc-windows-msvc", x86_64_pc_windows_msvc),
     ("x86_64-uwp-windows-msvc", x86_64_uwp_windows_msvc),
+    ("x86_64-win7-windows-msvc", x86_64_win7_windows_msvc),
     ("i686-pc-windows-msvc", i686_pc_windows_msvc),
     ("i686-uwp-windows-msvc", i686_uwp_windows_msvc),
+    ("i686-win7-windows-msvc", i686_win7_windows_msvc),
     ("i586-pc-windows-msvc", i586_pc_windows_msvc),
     ("thumbv7a-pc-windows-msvc", thumbv7a_pc_windows_msvc),
     ("thumbv7a-uwp-windows-msvc", thumbv7a_uwp_windows_msvc),
 
-    ("asmjs-unknown-emscripten", asmjs_unknown_emscripten),
     ("wasm32-unknown-emscripten", wasm32_unknown_emscripten),
     ("wasm32-unknown-unknown", wasm32_unknown_unknown),
     ("wasm32-wasi", wasm32_wasi),
@@ -1641,6 +1643,7 @@ supported_targets! {
     ("riscv32imc-esp-espidf", riscv32imc_esp_espidf),
     ("riscv32imac-esp-espidf", riscv32imac_esp_espidf),
     ("riscv32imac-unknown-none-elf", riscv32imac_unknown_none_elf),
+    ("riscv32imafc-unknown-none-elf", riscv32imafc_unknown_none_elf),
     ("riscv32imac-unknown-xous-elf", riscv32imac_unknown_xous_elf),
     ("riscv32gc-unknown-linux-gnu", riscv32gc_unknown_linux_gnu),
     ("riscv32gc-unknown-linux-musl", riscv32gc_unknown_linux_musl),
@@ -2092,7 +2095,11 @@ pub struct TargetOptions {
     pub no_builtins: bool,
 
     /// The default visibility for symbols in this target should be "hidden"
-    /// rather than "default"
+    /// rather than "default".
+    ///
+    /// This value typically shouldn't be accessed directly, but through
+    /// the `rustc_session::Session::default_hidden_visibility` method, which
+    /// allows `rustc` users to override this setting using cmdline flags.
     pub default_hidden_visibility: bool,
 
     /// Whether a .debug_gdb_scripts section will be added to the output object file
@@ -2194,9 +2201,6 @@ pub struct TargetOptions {
 
     /// Whether the target supports XRay instrumentation.
     pub supports_xray: bool,
-
-    /// Forces the use of emulated TLS (__emutls_get_address)
-    pub force_emulated_tls: bool,
 }
 
 /// Add arguments for the given flavor and also for its "twin" flavors
@@ -2244,10 +2248,6 @@ impl TargetOptions {
         add_link_args(&mut self.pre_link_args, flavor, args);
     }
 
-    fn add_post_link_args(&mut self, flavor: LinkerFlavor, args: &[&'static str]) {
-        add_link_args(&mut self.post_link_args, flavor, args);
-    }
-
     fn update_from_cli(&mut self) {
         self.linker_flavor = LinkerFlavor::from_cli_json(
             self.linker_flavor_json,
@@ -2416,7 +2416,6 @@ impl Default for TargetOptions {
             entry_name: "main".into(),
             entry_abi: Conv::C,
             supports_xray: false,
-            force_emulated_tls: false,
         }
     }
 }
@@ -3120,7 +3119,6 @@ impl Target {
         key!(entry_name);
         key!(entry_abi, Conv)?;
         key!(supports_xray, bool);
-        key!(force_emulated_tls, bool);
 
         if base.is_builtin {
             // This can cause unfortunate ICEs later down the line.
@@ -3376,7 +3374,6 @@ impl ToJson for Target {
         target_option_val!(entry_name);
         target_option_val!(entry_abi);
         target_option_val!(supports_xray);
-        target_option_val!(force_emulated_tls);
 
         if let Some(abi) = self.default_adjusted_cabi {
             d.insert("default-adjusted-cabi".into(), Abi::name(abi).to_json());
@@ -3503,7 +3500,7 @@ impl TargetTriple {
     /// If this target is a path, a hash of the path is appended to the triple returned
     /// by `triple()`.
     pub fn debug_triple(&self) -> String {
-        use std::collections::hash_map::DefaultHasher;
+        use std::hash::DefaultHasher;
 
         match self {
             TargetTriple::TargetTriple(triple) => triple.to_owned(),
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_ios.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_ios.rs
index 9fc5b5de4..f291ac545 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_apple_ios.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_ios.rs
@@ -18,19 +18,7 @@ pub fn target() -> Target {
         options: TargetOptions {
             features: "+neon,+fp-armv8,+apple-a7".into(),
             max_atomic_width: Some(128),
-            forces_embed_bitcode: true,
             frame_pointer: FramePointer::NonLeaf,
-            // Taken from a clang build on Xcode 11.4.1.
-            // These arguments are not actually invoked - they just have
-            // to look right to pass App Store validation.
-            bitcode_llvm_cmdline: "-triple\0\
-                arm64-apple-ios11.0.0\0\
-                -emit-obj\0\
-                -disable-llvm-passes\0\
-                -target-abi\0\
-                darwinpcs\0\
-                -Os\0"
-                .into(),
             ..base
         },
     }
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_ios_macabi.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_ios_macabi.rs
index 0172a3a9c..78067a138 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_apple_ios_macabi.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_ios_macabi.rs
@@ -17,17 +17,7 @@ pub fn target() -> Target {
         options: TargetOptions {
             features: "+neon,+fp-armv8,+apple-a12".into(),
             max_atomic_width: Some(128),
-            forces_embed_bitcode: true,
             frame_pointer: FramePointer::NonLeaf,
-            // Taken from a clang build on Xcode 11.4.1.
-            // These arguments are not actually invoked - they just have
-            // to look right to pass App Store validation.
-            bitcode_llvm_cmdline: "-triple\0\
-                arm64-apple-ios-macabi\0\
-                -emit-obj\0\
-                -disable-llvm-passes\0\
-                -Os\0"
-                .into(),
             ..base
         },
     }
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_ios_sim.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_ios_sim.rs
index 602a68777..41760e909 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_apple_ios_sim.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_ios_sim.rs
@@ -18,19 +18,7 @@ pub fn target() -> Target {
         options: TargetOptions {
             features: "+neon,+fp-armv8,+apple-a7".into(),
             max_atomic_width: Some(128),
-            forces_embed_bitcode: true,
             frame_pointer: FramePointer::NonLeaf,
-            // Taken from a clang build on Xcode 11.4.1.
-            // These arguments are not actually invoked - they just have
-            // to look right to pass App Store validation.
-            bitcode_llvm_cmdline: "-triple\0\
-                arm64-apple-ios14.0-simulator\0\
-                -emit-obj\0\
-                -disable-llvm-passes\0\
-                -target-abi\0\
-                darwinpcs\0\
-                -Os\0"
-                .into(),
             ..base
         },
     }
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos.rs
index a1a319355..e817308b6 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos.rs
@@ -11,7 +11,6 @@ pub fn target() -> Target {
         options: TargetOptions {
             features: "+neon,+fp-armv8,+apple-a7".into(),
             max_atomic_width: Some(128),
-            forces_embed_bitcode: true,
             frame_pointer: FramePointer::NonLeaf,
             ..opts("tvos", arch)
         },
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos_sim.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos_sim.rs
index 7b0bbb28e..c4aa84792 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos_sim.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_tvos_sim.rs
@@ -11,20 +11,7 @@ pub fn target() -> Target {
         options: TargetOptions {
             features: "+neon,+fp-armv8,+apple-a7".into(),
             max_atomic_width: Some(128),
-            forces_embed_bitcode: true,
             frame_pointer: FramePointer::NonLeaf,
-            // Taken from (and slightly modified) the aarch64-apple-ios-sim spec which says:
-            // Taken from a clang build on Xcode 11.4.1.
-            // These arguments are not actually invoked - they just have
-            // to look right to pass App Store validation.
-            bitcode_llvm_cmdline: "-triple\0\
-                arm64-apple-tvos15.0-simulator\0\
-                -emit-obj\0\
-                -disable-llvm-passes\0\
-                -target-abi\0\
-                darwinpcs\0\
-                -Os\0"
-                .into(),
             ..opts("tvos", arch)
         },
     }
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_watchos.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_watchos.rs
new file mode 100644
index 000000000..b62666dcc
--- /dev/null
+++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_watchos.rs
@@ -0,0 +1,19 @@
+use crate::spec::base::apple::{opts, Arch};
+use crate::spec::{Target, TargetOptions};
+
+pub fn target() -> Target {
+    let base = opts("watchos", Arch::Arm64);
+    Target {
+        llvm_target: "aarch-apple-watchos".into(),
+        pointer_width: 64,
+        data_layout: "e-m:o-i64:64-i128:128-n32:64-S128".into(),
+        arch: "aarch64".into(),
+        options: TargetOptions {
+            features: "+v8a,+neon,+fp-armv8,+apple-a7".into(),
+            max_atomic_width: Some(128),
+            dynamic_linking: false,
+            position_independent_executables: true,
+            ..base
+        },
+    }
+}
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_apple_watchos_sim.rs b/compiler/rustc_target/src/spec/targets/aarch64_apple_watchos_sim.rs
index 014560d22..96d43e6d2 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_apple_watchos_sim.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_apple_watchos_sim.rs
@@ -15,19 +15,7 @@ pub fn target() -> Target {
         options: TargetOptions {
             features: "+neon,+fp-armv8,+apple-a7".into(),
             max_atomic_width: Some(128),
-            forces_embed_bitcode: true,
             frame_pointer: FramePointer::NonLeaf,
-            // Taken from a clang build on Xcode 11.4.1.
-            // These arguments are not actually invoked - they just have
-            // to look right to pass App Store validation.
-            bitcode_llvm_cmdline: "-triple\0\
-                arm64-apple-watchos5.0-simulator\0\
-                -emit-obj\0\
-                -disable-llvm-passes\0\
-                -target-abi\0\
-                darwinpcs\0\
-                -Os\0"
-                .into(),
             ..opts("watchos", arch)
         },
     }
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_be_unknown_linux_gnu.rs b/compiler/rustc_target/src/spec/targets/aarch64_be_unknown_linux_gnu.rs
index bbaa870ec..739a9e4a3 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_be_unknown_linux_gnu.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_be_unknown_linux_gnu.rs
@@ -1,5 +1,5 @@
 use crate::abi::Endian;
-use crate::spec::{base, Target, TargetOptions};
+use crate::spec::{base, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     Target {
@@ -10,6 +10,7 @@ pub fn target() -> Target {
         options: TargetOptions {
             features: "+v8a,+outline-atomics".into(),
             max_atomic_width: Some(128),
+            stack_probes: StackProbeType::Inline,
             mcount: "\u{1}_mcount".into(),
             endian: Endian::Big,
             ..base::linux_gnu::opts()
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_be_unknown_linux_gnu_ilp32.rs b/compiler/rustc_target/src/spec/targets/aarch64_be_unknown_linux_gnu_ilp32.rs
index 7536e507b..1b87c09e3 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_be_unknown_linux_gnu_ilp32.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_be_unknown_linux_gnu_ilp32.rs
@@ -1,5 +1,5 @@
 use crate::abi::Endian;
-use crate::spec::{base, Target, TargetOptions};
+use crate::spec::{base, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = base::linux_gnu::opts();
@@ -13,6 +13,7 @@ pub fn target() -> Target {
         options: TargetOptions {
             abi: "ilp32".into(),
             features: "+v8a,+outline-atomics".into(),
+            stack_probes: StackProbeType::Inline,
             mcount: "\u{1}_mcount".into(),
             endian: Endian::Big,
             ..base
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_be_unknown_netbsd.rs b/compiler/rustc_target/src/spec/targets/aarch64_be_unknown_netbsd.rs
index 21de8a71a..02e156f19 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_be_unknown_netbsd.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_be_unknown_netbsd.rs
@@ -1,5 +1,5 @@
 use crate::abi::Endian;
-use crate::spec::{base, Target, TargetOptions};
+use crate::spec::{base, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     Target {
@@ -10,6 +10,7 @@ pub fn target() -> Target {
         options: TargetOptions {
             mcount: "__mcount".into(),
             max_atomic_width: Some(128),
+            stack_probes: StackProbeType::Inline,
             endian: Endian::Big,
             ..base::netbsd::opts()
         },
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_kmc_solid_asp3.rs b/compiler/rustc_target/src/spec/targets/aarch64_kmc_solid_asp3.rs
index e5e22fd1e..5b4ffc08a 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_kmc_solid_asp3.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_kmc_solid_asp3.rs
@@ -1,4 +1,4 @@
-use crate::spec::{base, RelocModel, Target, TargetOptions};
+use crate::spec::{base, RelocModel, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let base = base::solid::opts("asp3");
@@ -13,6 +13,7 @@ pub fn target() -> Target {
             relocation_model: RelocModel::Static,
             disable_redzone: true,
             max_atomic_width: Some(128),
+            stack_probes: StackProbeType::Inline,
             ..base
         },
     }
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_linux_android.rs b/compiler/rustc_target/src/spec/targets/aarch64_linux_android.rs
index 7c6fed7ab..db9a18085 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_linux_android.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_linux_android.rs
@@ -1,4 +1,4 @@
-use crate::spec::{base, SanitizerSet, Target, TargetOptions};
+use crate::spec::{base, SanitizerSet, StackProbeType, Target, TargetOptions};
 
 // See https://developer.android.com/ndk/guides/abis.html#arm64-v8a
 // for target ABI requirements.
@@ -14,6 +14,7 @@ pub fn target() -> Target {
             // As documented in https://developer.android.com/ndk/guides/cpu-features.html
             // the neon (ASIMD) and FP must exist on all android aarch64 targets.
             features: "+v8a,+neon,+fp-armv8".into(),
+            stack_probes: StackProbeType::Inline,
             supported_sanitizers: SanitizerSet::CFI
                 | SanitizerSet::HWADDRESS
                 | SanitizerSet::MEMTAG
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_nintendo_switch_freestanding.rs b/compiler/rustc_target/src/spec/targets/aarch64_nintendo_switch_freestanding.rs
index 02fcc2bb4..5a92225c4 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_nintendo_switch_freestanding.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_nintendo_switch_freestanding.rs
@@ -1,4 +1,6 @@
-use crate::spec::{Cc, LinkerFlavor, Lld, PanicStrategy, RelroLevel, Target, TargetOptions};
+use crate::spec::{
+    Cc, LinkerFlavor, Lld, PanicStrategy, RelroLevel, StackProbeType, Target, TargetOptions,
+};
 
 const LINKER_SCRIPT: &str = include_str!("./aarch64_nintendo_switch_freestanding_linker_script.ld");
 
@@ -16,6 +18,7 @@ pub fn target() -> Target {
             link_script: Some(LINKER_SCRIPT.into()),
             os: "horizon".into(),
             max_atomic_width: Some(128),
+            stack_probes: StackProbeType::Inline,
             panic_strategy: PanicStrategy::Abort,
             position_independent_executables: true,
             dynamic_linking: true,
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_freebsd.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_freebsd.rs
index 0f95984a3..0ba91a54f 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_freebsd.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_freebsd.rs
@@ -1,4 +1,4 @@
-use crate::spec::{base, SanitizerSet, Target, TargetOptions};
+use crate::spec::{base, SanitizerSet, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     Target {
@@ -9,6 +9,7 @@ pub fn target() -> Target {
         options: TargetOptions {
             features: "+v8a".into(),
             max_atomic_width: Some(128),
+            stack_probes: StackProbeType::Inline,
             supported_sanitizers: SanitizerSet::ADDRESS
                 | SanitizerSet::CFI
                 | SanitizerSet::MEMORY
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_fuchsia.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_fuchsia.rs
index cacc6ea23..430a1c5e7 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_fuchsia.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_fuchsia.rs
@@ -1,4 +1,4 @@
-use crate::spec::{base, SanitizerSet, Target, TargetOptions};
+use crate::spec::{base, SanitizerSet, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     Target {
@@ -9,6 +9,7 @@ pub fn target() -> Target {
         options: TargetOptions {
             features: "+v8a".into(),
             max_atomic_width: Some(128),
+            stack_probes: StackProbeType::Inline,
             supported_sanitizers: SanitizerSet::ADDRESS
                 | SanitizerSet::CFI
                 | SanitizerSet::SHADOWCALLSTACK,
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_hermit.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_hermit.rs
index 3839e0b9c..50813b81b 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_hermit.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_hermit.rs
@@ -1,4 +1,4 @@
-use crate::spec::{base, Target, TargetOptions};
+use crate::spec::{base, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     Target {
@@ -9,6 +9,7 @@ pub fn target() -> Target {
         options: TargetOptions {
             features: "+v8a,+strict-align,+neon,+fp-armv8".into(),
             max_atomic_width: Some(128),
+            stack_probes: StackProbeType::Inline,
             ..base::hermit::opts()
         },
     }
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_gnu.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_gnu.rs
index c0b07db38..5b3a2ffa5 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_gnu.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_gnu.rs
@@ -1,4 +1,4 @@
-use crate::spec::{base, SanitizerSet, Target, TargetOptions};
+use crate::spec::{base, SanitizerSet, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     Target {
@@ -10,6 +10,7 @@ pub fn target() -> Target {
             features: "+v8a,+outline-atomics".into(),
             mcount: "\u{1}_mcount".into(),
             max_atomic_width: Some(128),
+            stack_probes: StackProbeType::Inline,
             supported_sanitizers: SanitizerSet::ADDRESS
                 | SanitizerSet::CFI
                 | SanitizerSet::LEAK
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_gnu_ilp32.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_gnu_ilp32.rs
index d4c6271b2..0ff5c8651 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_gnu_ilp32.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_gnu_ilp32.rs
@@ -1,4 +1,4 @@
-use crate::spec::{base, Target, TargetOptions};
+use crate::spec::{base, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     Target {
@@ -10,6 +10,7 @@ pub fn target() -> Target {
             abi: "ilp32".into(),
             features: "+v8a,+outline-atomics".into(),
             max_atomic_width: Some(128),
+            stack_probes: StackProbeType::Inline,
             mcount: "\u{1}_mcount".into(),
             ..base::linux_gnu::opts()
         },
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_musl.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_musl.rs
index 364df677e..d16ccd803 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_musl.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_musl.rs
@@ -1,10 +1,11 @@
-use crate::spec::{base, SanitizerSet, Target, TargetOptions};
+use crate::spec::{base, SanitizerSet, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = base::linux_musl::opts();
     base.max_atomic_width = Some(128);
     base.supports_xray = true;
     base.features = "+v8a".into();
+    base.stack_probes = StackProbeType::Inline;
     base.supported_sanitizers = SanitizerSet::ADDRESS
         | SanitizerSet::CFI
         | SanitizerSet::LEAK
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_ohos.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_ohos.rs
index 5f3bef63a..1be27a511 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_ohos.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_linux_ohos.rs
@@ -1,5 +1,5 @@
 use crate::spec::SanitizerSet;
-use crate::spec::{base, Target, TargetOptions};
+use crate::spec::{base, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = base::linux_ohos::opts();
@@ -14,6 +14,7 @@ pub fn target() -> Target {
         options: TargetOptions {
             features: "+reserve-x18".into(),
             mcount: "\u{1}_mcount".into(),
+            stack_probes: StackProbeType::Inline,
             supported_sanitizers: SanitizerSet::ADDRESS
                 | SanitizerSet::CFI
                 | SanitizerSet::LEAK
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_netbsd.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_netbsd.rs
index 3194d16ff..32a0816cf 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_netbsd.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_netbsd.rs
@@ -1,4 +1,4 @@
-use crate::spec::{base, Target, TargetOptions};
+use crate::spec::{base, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     Target {
@@ -10,6 +10,7 @@ pub fn target() -> Target {
             features: "+v8a".into(),
             mcount: "__mcount".into(),
             max_atomic_width: Some(128),
+            stack_probes: StackProbeType::Inline,
             ..base::netbsd::opts()
         },
     }
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_none.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_none.rs
index 88fa6d5a7..518597b38 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_none.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_none.rs
@@ -7,18 +7,25 @@
 // For example, `-C target-cpu=cortex-a53`.
 
 use crate::spec::{
-    Cc, LinkerFlavor, Lld, PanicStrategy, RelocModel, SanitizerSet, Target, TargetOptions,
+    Cc, LinkerFlavor, Lld, PanicStrategy, RelocModel, SanitizerSet, StackProbeType, Target,
+    TargetOptions,
 };
 
 pub fn target() -> Target {
     let opts = TargetOptions {
         linker_flavor: LinkerFlavor::Gnu(Cc::No, Lld::Yes),
         linker: Some("rust-lld".into()),
+        // Enable the Cortex-A53 errata 843419 mitigation by default
+        pre_link_args: TargetOptions::link_args(
+            LinkerFlavor::Gnu(Cc::No, Lld::No),
+            &["--fix-cortex-a53-843419"],
+        ),
         features: "+v8a,+strict-align,+neon,+fp-armv8".into(),
         supported_sanitizers: SanitizerSet::KCFI | SanitizerSet::KERNELADDRESS,
         relocation_model: RelocModel::Static,
         disable_redzone: true,
         max_atomic_width: Some(128),
+        stack_probes: StackProbeType::Inline,
         panic_strategy: PanicStrategy::Abort,
         ..Default::default()
     };
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_none_softfloat.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_none_softfloat.rs
index 48b79b0b9..d56fbdbbe 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_none_softfloat.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_none_softfloat.rs
@@ -6,7 +6,9 @@
 //
 // For example, `-C target-cpu=cortex-a53`.
 
-use crate::spec::{Cc, LinkerFlavor, Lld, PanicStrategy, RelocModel, Target, TargetOptions};
+use crate::spec::{
+    Cc, LinkerFlavor, Lld, PanicStrategy, RelocModel, StackProbeType, Target, TargetOptions,
+};
 
 pub fn target() -> Target {
     let opts = TargetOptions {
@@ -17,6 +19,7 @@ pub fn target() -> Target {
         relocation_model: RelocModel::Static,
         disable_redzone: true,
         max_atomic_width: Some(128),
+        stack_probes: StackProbeType::Inline,
         panic_strategy: PanicStrategy::Abort,
         ..Default::default()
     };
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_openbsd.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_openbsd.rs
index e3f5aafc7..d5fca5c31 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_openbsd.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_openbsd.rs
@@ -1,4 +1,4 @@
-use crate::spec::{base, Target, TargetOptions};
+use crate::spec::{base, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     Target {
@@ -9,6 +9,7 @@ pub fn target() -> Target {
         options: TargetOptions {
             features: "+v8a".into(),
             max_atomic_width: Some(128),
+            stack_probes: StackProbeType::Inline,
             ..base::openbsd::opts()
         },
     }
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_redox.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_redox.rs
index de875e29f..844fed472 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_redox.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_redox.rs
@@ -1,8 +1,9 @@
-use crate::spec::{base, Target};
+use crate::spec::{base, StackProbeType, Target};
 
 pub fn target() -> Target {
     let mut base = base::redox::opts();
     base.max_atomic_width = Some(128);
+    base.stack_probes = StackProbeType::Inline;
     base.features = "+v8a".into();
 
     Target {
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_unknown_teeos.rs b/compiler/rustc_target/src/spec/targets/aarch64_unknown_teeos.rs
index eec2668d4..204e9b061 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_unknown_teeos.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_unknown_teeos.rs
@@ -1,10 +1,10 @@
-use crate::spec::{base, Target};
+use crate::spec::{base, StackProbeType, Target};
 
 pub fn target() -> Target {
     let mut base = base::teeos::opts();
     base.features = "+strict-align,+neon,+fp-armv8".into();
     base.max_atomic_width = Some(128);
-    base.linker = Some("aarch64-linux-gnu-ld".into());
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "aarch64-unknown-none".into(),
diff --git a/compiler/rustc_target/src/spec/targets/aarch64_wrs_vxworks.rs b/compiler/rustc_target/src/spec/targets/aarch64_wrs_vxworks.rs
index 29f61d857..48e142347 100644
--- a/compiler/rustc_target/src/spec/targets/aarch64_wrs_vxworks.rs
+++ b/compiler/rustc_target/src/spec/targets/aarch64_wrs_vxworks.rs
@@ -1,4 +1,4 @@
-use crate::spec::{base, Target, TargetOptions};
+use crate::spec::{base, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     Target {
@@ -9,6 +9,7 @@ pub fn target() -> Target {
         options: TargetOptions {
             features: "+v8a".into(),
             max_atomic_width: Some(128),
+            stack_probes: StackProbeType::Inline,
             ..base::vxworks::opts()
         },
     }
diff --git a/compiler/rustc_target/src/spec/targets/arm64_32_apple_watchos.rs b/compiler/rustc_target/src/spec/targets/arm64_32_apple_watchos.rs
index 9931b7b86..59e6022d9 100644
--- a/compiler/rustc_target/src/spec/targets/arm64_32_apple_watchos.rs
+++ b/compiler/rustc_target/src/spec/targets/arm64_32_apple_watchos.rs
@@ -11,19 +11,8 @@ pub fn target() -> Target {
         options: TargetOptions {
             features: "+v8a,+neon,+fp-armv8,+apple-a7".into(),
             max_atomic_width: Some(128),
-            forces_embed_bitcode: true,
             dynamic_linking: false,
             position_independent_executables: true,
-            // These arguments are not actually invoked - they just have
-            // to look right to pass App Store validation.
-            bitcode_llvm_cmdline: "-triple\0\
-                arm64_32-apple-watchos5.0.0\0\
-                -emit-obj\0\
-                -disable-llvm-passes\0\
-                -target-abi\0\
-                darwinpcs\0\
-                -Os\0"
-                .into(),
             ..base
         },
     }
diff --git a/compiler/rustc_target/src/spec/targets/arm64e_apple_darwin.rs b/compiler/rustc_target/src/spec/targets/arm64e_apple_darwin.rs
new file mode 100644
index 000000000..5d1f81158
--- /dev/null
+++ b/compiler/rustc_target/src/spec/targets/arm64e_apple_darwin.rs
@@ -0,0 +1,27 @@
+use crate::spec::base::apple::{macos_llvm_target, opts, Arch};
+use crate::spec::{FramePointer, SanitizerSet, Target, TargetOptions};
+
+pub fn target() -> Target {
+    let arch = Arch::Arm64e;
+    let mut base = opts("macos", arch);
+    base.cpu = "apple-m1".into();
+    base.max_atomic_width = Some(128);
+
+    // FIXME: The leak sanitizer currently fails the tests, see #88132.
+    base.supported_sanitizers = SanitizerSet::ADDRESS | SanitizerSet::CFI | SanitizerSet::THREAD;
+
+    Target {
+        // Clang automatically chooses a more specific target based on
+        // MACOSX_DEPLOYMENT_TARGET. To enable cross-language LTO to work
+        // correctly, we do too.
+        llvm_target: macos_llvm_target(arch).into(),
+        pointer_width: 64,
+        data_layout: "e-m:o-i64:64-i128:128-n32:64-S128".into(),
+        arch: arch.target_arch(),
+        options: TargetOptions {
+            mcount: "\u{1}mcount".into(),
+            frame_pointer: FramePointer::NonLeaf,
+            ..base
+        },
+    }
+}
diff --git a/compiler/rustc_target/src/spec/targets/arm64e_apple_ios.rs b/compiler/rustc_target/src/spec/targets/arm64e_apple_ios.rs
new file mode 100644
index 000000000..38657d7f1
--- /dev/null
+++ b/compiler/rustc_target/src/spec/targets/arm64e_apple_ios.rs
@@ -0,0 +1,25 @@
+use crate::spec::base::apple::{ios_llvm_target, opts, Arch};
+use crate::spec::{FramePointer, SanitizerSet, Target, TargetOptions};
+
+pub fn target() -> Target {
+    let arch = Arch::Arm64e;
+    let mut base = opts("ios", arch);
+    base.supported_sanitizers = SanitizerSet::ADDRESS | SanitizerSet::THREAD;
+
+    Target {
+        // Clang automatically chooses a more specific target based on
+        // IPHONEOS_DEPLOYMENT_TARGET.
+        // This is required for the target to pick the right
+        // MACH-O commands, so we do too.
+        llvm_target: ios_llvm_target(arch).into(),
+        pointer_width: 64,
+        data_layout: "e-m:o-i64:64-i128:128-n32:64-S128".into(),
+        arch: arch.target_arch(),
+        options: TargetOptions {
+            features: "+neon,+fp-armv8,+apple-a12,+v8.3a,+pauth".into(),
+            max_atomic_width: Some(128),
+            frame_pointer: FramePointer::NonLeaf,
+            ..base
+        },
+    }
+}
diff --git a/compiler/rustc_target/src/spec/targets/armv7k_apple_watchos.rs b/compiler/rustc_target/src/spec/targets/armv7k_apple_watchos.rs
index 751fdcb20..7ed71c1ba 100644
--- a/compiler/rustc_target/src/spec/targets/armv7k_apple_watchos.rs
+++ b/compiler/rustc_target/src/spec/targets/armv7k_apple_watchos.rs
@@ -11,19 +11,8 @@ pub fn target() -> Target {
         options: TargetOptions {
             features: "+v7,+vfp4,+neon".into(),
             max_atomic_width: Some(64),
-            forces_embed_bitcode: true,
             dynamic_linking: false,
             position_independent_executables: true,
-            // These arguments are not actually invoked - they just have
-            // to look right to pass App Store validation.
-            bitcode_llvm_cmdline: "-triple\0\
-                armv7k-apple-watchos3.0.0\0\
-                -emit-obj\0\
-                -disable-llvm-passes\0\
-                -target-abi\0\
-                darwinpcs\0\
-                -Os\0"
-                .into(),
             ..opts("watchos", arch)
         },
     }
diff --git a/compiler/rustc_target/src/spec/targets/i386_apple_ios.rs b/compiler/rustc_target/src/spec/targets/i386_apple_ios.rs
index ed2c990ff..13f8b6b5a 100644
--- a/compiler/rustc_target/src/spec/targets/i386_apple_ios.rs
+++ b/compiler/rustc_target/src/spec/targets/i386_apple_ios.rs
@@ -1,5 +1,5 @@
 use crate::spec::base::apple::{ios_sim_llvm_target, opts, Arch};
-use crate::spec::{StackProbeType, Target, TargetOptions};
+use crate::spec::{Target, TargetOptions};
 
 pub fn target() -> Target {
     let arch = Arch::I386;
@@ -14,10 +14,6 @@ pub fn target() -> Target {
             f64:32:64-f80:128-n8:16:32-S128"
             .into(),
         arch: arch.target_arch(),
-        options: TargetOptions {
-            max_atomic_width: Some(64),
-            stack_probes: StackProbeType::X86,
-            ..opts("ios", arch)
-        },
+        options: TargetOptions { max_atomic_width: Some(64), ..opts("ios", arch) },
     }
 }
diff --git a/compiler/rustc_target/src/spec/targets/i386_unknown_linux_gnu.rs b/compiler/rustc_target/src/spec/targets/i386_unknown_linux_gnu.rs
deleted file mode 100644
index 801a88933..000000000
--- a/compiler/rustc_target/src/spec/targets/i386_unknown_linux_gnu.rs
+++ /dev/null
@@ -1,8 +0,0 @@
-use crate::spec::Target;
-
-pub fn target() -> Target {
-    let mut base = super::i686_unknown_linux_gnu::target();
-    base.cpu = "i386".into();
-    base.llvm_target = "i386-unknown-linux-gnu".into();
-    base
-}
diff --git a/compiler/rustc_target/src/spec/targets/i486_unknown_linux_gnu.rs b/compiler/rustc_target/src/spec/targets/i486_unknown_linux_gnu.rs
deleted file mode 100644
index a11fbecc3..000000000
--- a/compiler/rustc_target/src/spec/targets/i486_unknown_linux_gnu.rs
+++ /dev/null
@@ -1,8 +0,0 @@
-use crate::spec::Target;
-
-pub fn target() -> Target {
-    let mut base = super::i686_unknown_linux_gnu::target();
-    base.cpu = "i486".into();
-    base.llvm_target = "i486-unknown-linux-gnu".into();
-    base
-}
diff --git a/compiler/rustc_target/src/spec/targets/i586_pc_nto_qnx700.rs b/compiler/rustc_target/src/spec/targets/i586_pc_nto_qnx700.rs
index 060893916..3ebf4bcf5 100644
--- a/compiler/rustc_target/src/spec/targets/i586_pc_nto_qnx700.rs
+++ b/compiler/rustc_target/src/spec/targets/i586_pc_nto_qnx700.rs
@@ -16,7 +16,7 @@ pub fn target() -> Target {
                 &["-Vgcc_ntox86_cxx"],
             ),
             env: "nto70".into(),
-            stack_probes: StackProbeType::X86,
+            stack_probes: StackProbeType::Inline,
             ..base::nto_qnx::opts()
         },
     }
diff --git a/compiler/rustc_target/src/spec/targets/i686_apple_darwin.rs b/compiler/rustc_target/src/spec/targets/i686_apple_darwin.rs
index 242fe5ed9..59069fe4e 100644
--- a/compiler/rustc_target/src/spec/targets/i686_apple_darwin.rs
+++ b/compiler/rustc_target/src/spec/targets/i686_apple_darwin.rs
@@ -1,5 +1,5 @@
 use crate::spec::base::apple::{macos_llvm_target, opts, Arch};
-use crate::spec::{Cc, FramePointer, LinkerFlavor, Lld, StackProbeType, Target, TargetOptions};
+use crate::spec::{Cc, FramePointer, LinkerFlavor, Lld, Target, TargetOptions};
 
 pub fn target() -> Target {
     // ld64 only understands i386 and not i686
@@ -7,7 +7,6 @@ pub fn target() -> Target {
     let mut base = opts("macos", arch);
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Darwin(Cc::Yes, Lld::No), &["-m32"]);
-    base.stack_probes = StackProbeType::X86;
     base.frame_pointer = FramePointer::Always;
 
     Target {
diff --git a/compiler/rustc_target/src/spec/targets/i686_linux_android.rs b/compiler/rustc_target/src/spec/targets/i686_linux_android.rs
index 987265598..79471040f 100644
--- a/compiler/rustc_target/src/spec/targets/i686_linux_android.rs
+++ b/compiler/rustc_target/src/spec/targets/i686_linux_android.rs
@@ -11,7 +11,7 @@ pub fn target() -> Target {
     // https://developer.android.com/ndk/guides/abis.html#x86
     base.cpu = "pentiumpro".into();
     base.features = "+mmx,+sse,+sse2,+sse3,+ssse3".into();
-    base.stack_probes = StackProbeType::X86;
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "i686-linux-android".into(),
diff --git a/compiler/rustc_target/src/spec/targets/i686_unknown_freebsd.rs b/compiler/rustc_target/src/spec/targets/i686_unknown_freebsd.rs
index 70aa0b479..927b2ab87 100644
--- a/compiler/rustc_target/src/spec/targets/i686_unknown_freebsd.rs
+++ b/compiler/rustc_target/src/spec/targets/i686_unknown_freebsd.rs
@@ -5,7 +5,7 @@ pub fn target() -> Target {
     base.cpu = "pentium4".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gnu(Cc::Yes, Lld::No), &["-m32", "-Wl,-znotext"]);
-    base.stack_probes = StackProbeType::X86;
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "i686-unknown-freebsd".into(),
diff --git a/compiler/rustc_target/src/spec/targets/i686_unknown_haiku.rs b/compiler/rustc_target/src/spec/targets/i686_unknown_haiku.rs
index 9715f6c21..bc7fd6fbc 100644
--- a/compiler/rustc_target/src/spec/targets/i686_unknown_haiku.rs
+++ b/compiler/rustc_target/src/spec/targets/i686_unknown_haiku.rs
@@ -5,7 +5,7 @@ pub fn target() -> Target {
     base.cpu = "pentium4".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gnu(Cc::Yes, Lld::No), &["-m32"]);
-    base.stack_probes = StackProbeType::X86;
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "i686-unknown-haiku".into(),
diff --git a/compiler/rustc_target/src/spec/targets/i686_unknown_linux_gnu.rs b/compiler/rustc_target/src/spec/targets/i686_unknown_linux_gnu.rs
index 0ca058b2f..3b7be48db 100644
--- a/compiler/rustc_target/src/spec/targets/i686_unknown_linux_gnu.rs
+++ b/compiler/rustc_target/src/spec/targets/i686_unknown_linux_gnu.rs
@@ -6,7 +6,7 @@ pub fn target() -> Target {
     base.max_atomic_width = Some(64);
     base.supported_sanitizers = SanitizerSet::ADDRESS;
     base.add_pre_link_args(LinkerFlavor::Gnu(Cc::Yes, Lld::No), &["-m32"]);
-    base.stack_probes = StackProbeType::X86;
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "i686-unknown-linux-gnu".into(),
diff --git a/compiler/rustc_target/src/spec/targets/i686_unknown_linux_musl.rs b/compiler/rustc_target/src/spec/targets/i686_unknown_linux_musl.rs
index ce0df3a72..ef58b4fb4 100644
--- a/compiler/rustc_target/src/spec/targets/i686_unknown_linux_musl.rs
+++ b/compiler/rustc_target/src/spec/targets/i686_unknown_linux_musl.rs
@@ -5,7 +5,7 @@ pub fn target() -> Target {
     base.cpu = "pentium4".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gnu(Cc::Yes, Lld::No), &["-m32", "-Wl,-melf_i386"]);
-    base.stack_probes = StackProbeType::X86;
+    base.stack_probes = StackProbeType::Inline;
 
     // The unwinder used by i686-unknown-linux-musl, the LLVM libunwind
     // implementation, apparently relies on frame pointers existing... somehow.
diff --git a/compiler/rustc_target/src/spec/targets/i686_unknown_netbsd.rs b/compiler/rustc_target/src/spec/targets/i686_unknown_netbsd.rs
index 2a3ac5932..5f3afbe5a 100644
--- a/compiler/rustc_target/src/spec/targets/i686_unknown_netbsd.rs
+++ b/compiler/rustc_target/src/spec/targets/i686_unknown_netbsd.rs
@@ -5,7 +5,7 @@ pub fn target() -> Target {
     base.cpu = "pentium4".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gnu(Cc::Yes, Lld::No), &["-m32"]);
-    base.stack_probes = StackProbeType::X86;
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "i686-unknown-netbsdelf".into(),
diff --git a/compiler/rustc_target/src/spec/targets/i686_unknown_openbsd.rs b/compiler/rustc_target/src/spec/targets/i686_unknown_openbsd.rs
index 45883542b..f44584a10 100644
--- a/compiler/rustc_target/src/spec/targets/i686_unknown_openbsd.rs
+++ b/compiler/rustc_target/src/spec/targets/i686_unknown_openbsd.rs
@@ -5,7 +5,7 @@ pub fn target() -> Target {
     base.cpu = "pentium4".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gnu(Cc::Yes, Lld::No), &["-m32", "-fuse-ld=lld"]);
-    base.stack_probes = StackProbeType::X86;
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "i686-unknown-openbsd".into(),
diff --git a/compiler/rustc_target/src/spec/targets/i686_win7_windows_msvc.rs b/compiler/rustc_target/src/spec/targets/i686_win7_windows_msvc.rs
new file mode 100644
index 000000000..5b91682e1
--- /dev/null
+++ b/compiler/rustc_target/src/spec/targets/i686_win7_windows_msvc.rs
@@ -0,0 +1,33 @@
+use crate::spec::{base, LinkerFlavor, Lld, Target};
+
+pub fn target() -> Target {
+    let mut base = base::windows_msvc::opts();
+    base.cpu = "pentium4".into();
+    base.max_atomic_width = Some(64);
+    base.vendor = "win7".into();
+
+    base.add_pre_link_args(
+        LinkerFlavor::Msvc(Lld::No),
+        &[
+            // Mark all dynamic libraries and executables as compatible with the larger 4GiB address
+            // space available to x86 Windows binaries on x86_64.
+            "/LARGEADDRESSAWARE",
+            // Ensure the linker will only produce an image if it can also produce a table of
+            // the image's safe exception handlers.
+            // https://docs.microsoft.com/en-us/cpp/build/reference/safeseh-image-has-safe-exception-handlers
+            "/SAFESEH",
+        ],
+    );
+    // Workaround for #95429
+    base.has_thread_local = false;
+
+    Target {
+        llvm_target: "i686-pc-windows-msvc".into(),
+        pointer_width: 32,
+        data_layout: "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
+            i64:64-f80:128-n8:16:32-a:0:32-S32"
+            .into(),
+        arch: "x86".into(),
+        options: base,
+    }
+}
diff --git a/compiler/rustc_target/src/spec/targets/i686_wrs_vxworks.rs b/compiler/rustc_target/src/spec/targets/i686_wrs_vxworks.rs
index 965a89597..ee501c516 100644
--- a/compiler/rustc_target/src/spec/targets/i686_wrs_vxworks.rs
+++ b/compiler/rustc_target/src/spec/targets/i686_wrs_vxworks.rs
@@ -5,7 +5,7 @@ pub fn target() -> Target {
     base.cpu = "pentium4".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gnu(Cc::Yes, Lld::No), &["-m32"]);
-    base.stack_probes = StackProbeType::X86;
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "i686-unknown-linux-gnu".into(),
diff --git a/compiler/rustc_target/src/spec/targets/riscv32i_unknown_none_elf.rs b/compiler/rustc_target/src/spec/targets/riscv32i_unknown_none_elf.rs
index 75a65a268..b80a5fbea 100644
--- a/compiler/rustc_target/src/spec/targets/riscv32i_unknown_none_elf.rs
+++ b/compiler/rustc_target/src/spec/targets/riscv32i_unknown_none_elf.rs
@@ -11,8 +11,9 @@ pub fn target() -> Target {
             linker_flavor: LinkerFlavor::Gnu(Cc::No, Lld::Yes),
             linker: Some("rust-lld".into()),
             cpu: "generic-rv32".into(),
-            max_atomic_width: Some(0),
+            max_atomic_width: Some(32),
             atomic_cas: false,
+            features: "+forced-atomics".into(),
             panic_strategy: PanicStrategy::Abort,
             relocation_model: RelocModel::Static,
             emit_debug_gdb_scripts: false,
diff --git a/compiler/rustc_target/src/spec/targets/riscv32im_unknown_none_elf.rs b/compiler/rustc_target/src/spec/targets/riscv32im_unknown_none_elf.rs
index f2242bbe0..6480facb1 100644
--- a/compiler/rustc_target/src/spec/targets/riscv32im_unknown_none_elf.rs
+++ b/compiler/rustc_target/src/spec/targets/riscv32im_unknown_none_elf.rs
@@ -11,9 +11,9 @@ pub fn target() -> Target {
             linker_flavor: LinkerFlavor::Gnu(Cc::No, Lld::Yes),
             linker: Some("rust-lld".into()),
             cpu: "generic-rv32".into(),
-            max_atomic_width: Some(0),
+            max_atomic_width: Some(32),
             atomic_cas: false,
-            features: "+m".into(),
+            features: "+m,+forced-atomics".into(),
             panic_strategy: PanicStrategy::Abort,
             relocation_model: RelocModel::Static,
             emit_debug_gdb_scripts: false,
diff --git a/compiler/rustc_target/src/spec/targets/riscv32imac_unknown_xous_elf.rs b/compiler/rustc_target/src/spec/targets/riscv32imac_unknown_xous_elf.rs
index a263e5d5c..ed0591ad9 100644
--- a/compiler/rustc_target/src/spec/targets/riscv32imac_unknown_xous_elf.rs
+++ b/compiler/rustc_target/src/spec/targets/riscv32imac_unknown_xous_elf.rs
@@ -14,7 +14,7 @@ pub fn target() -> Target {
             cpu: "generic-rv32".into(),
             max_atomic_width: Some(32),
             features: "+m,+a,+c".into(),
-            panic_strategy: PanicStrategy::Abort,
+            panic_strategy: PanicStrategy::Unwind,
             relocation_model: RelocModel::Static,
             ..Default::default()
         },
diff --git a/compiler/rustc_target/src/spec/targets/riscv32imafc_unknown_none_elf.rs b/compiler/rustc_target/src/spec/targets/riscv32imafc_unknown_none_elf.rs
new file mode 100644
index 000000000..9fb68874a
--- /dev/null
+++ b/compiler/rustc_target/src/spec/targets/riscv32imafc_unknown_none_elf.rs
@@ -0,0 +1,24 @@
+use crate::spec::{Cc, LinkerFlavor, Lld, PanicStrategy, RelocModel, Target, TargetOptions};
+
+pub fn target() -> Target {
+    Target {
+        data_layout: "e-m:e-p:32:32-i64:64-n32-S128".into(),
+        llvm_target: "riscv32".into(),
+        pointer_width: 32,
+        arch: "riscv32".into(),
+
+        options: TargetOptions {
+            linker_flavor: LinkerFlavor::Gnu(Cc::No, Lld::Yes),
+            linker: Some("rust-lld".into()),
+            cpu: "generic-rv32".into(),
+            max_atomic_width: Some(32),
+            llvm_abiname: "ilp32f".into(),
+            features: "+m,+a,+c,+f".into(),
+            panic_strategy: PanicStrategy::Abort,
+            relocation_model: RelocModel::Static,
+            emit_debug_gdb_scripts: false,
+            eh_frame_header: false,
+            ..Default::default()
+        },
+    }
+}
diff --git a/compiler/rustc_target/src/spec/targets/riscv32imc_unknown_none_elf.rs b/compiler/rustc_target/src/spec/targets/riscv32imc_unknown_none_elf.rs
index 01e773fae..13a621a66 100644
--- a/compiler/rustc_target/src/spec/targets/riscv32imc_unknown_none_elf.rs
+++ b/compiler/rustc_target/src/spec/targets/riscv32imc_unknown_none_elf.rs
@@ -11,9 +11,9 @@ pub fn target() -> Target {
             linker_flavor: LinkerFlavor::Gnu(Cc::No, Lld::Yes),
             linker: Some("rust-lld".into()),
             cpu: "generic-rv32".into(),
-            max_atomic_width: Some(0),
+            max_atomic_width: Some(32),
             atomic_cas: false,
-            features: "+m,+c".into(),
+            features: "+m,+c,+forced-atomics".into(),
             panic_strategy: PanicStrategy::Abort,
             relocation_model: RelocModel::Static,
             emit_debug_gdb_scripts: false,
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_apple_darwin.rs b/compiler/rustc_target/src/spec/targets/x86_64_apple_darwin.rs
index 9170a10d2..59df3937e 100644
--- a/compiler/rustc_target/src/spec/targets/x86_64_apple_darwin.rs
+++ b/compiler/rustc_target/src/spec/targets/x86_64_apple_darwin.rs
@@ -1,6 +1,6 @@
 use crate::spec::base::apple::{macos_llvm_target, opts, Arch};
 use crate::spec::{Cc, FramePointer, LinkerFlavor, Lld, SanitizerSet};
-use crate::spec::{StackProbeType, Target, TargetOptions};
+use crate::spec::{Target, TargetOptions};
 
 pub fn target() -> Target {
     let arch = Arch::X86_64;
@@ -8,7 +8,6 @@ pub fn target() -> Target {
     base.max_atomic_width = Some(128); // penryn+ supports cmpxchg16b
     base.frame_pointer = FramePointer::Always;
     base.add_pre_link_args(LinkerFlavor::Darwin(Cc::Yes, Lld::No), &["-m64"]);
-    base.stack_probes = StackProbeType::X86;
     base.supported_sanitizers =
         SanitizerSet::ADDRESS | SanitizerSet::CFI | SanitizerSet::LEAK | SanitizerSet::THREAD;
 
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_apple_ios.rs b/compiler/rustc_target/src/spec/targets/x86_64_apple_ios.rs
index d9f3f7de6..d0e2ac44a 100644
--- a/compiler/rustc_target/src/spec/targets/x86_64_apple_ios.rs
+++ b/compiler/rustc_target/src/spec/targets/x86_64_apple_ios.rs
@@ -1,5 +1,5 @@
 use crate::spec::base::apple::{ios_sim_llvm_target, opts, Arch};
-use crate::spec::{SanitizerSet, StackProbeType, Target, TargetOptions};
+use crate::spec::{SanitizerSet, Target, TargetOptions};
 
 pub fn target() -> Target {
     let arch = Arch::X86_64_sim;
@@ -12,10 +12,6 @@ pub fn target() -> Target {
         data_layout: "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
             .into(),
         arch: arch.target_arch(),
-        options: TargetOptions {
-            max_atomic_width: Some(128),
-            stack_probes: StackProbeType::X86,
-            ..base
-        },
+        options: TargetOptions { max_atomic_width: Some(128), ..base },
     }
 }
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_apple_ios_macabi.rs b/compiler/rustc_target/src/spec/targets/x86_64_apple_ios_macabi.rs
index 5e6fbac34..8ef4b88b8 100644
--- a/compiler/rustc_target/src/spec/targets/x86_64_apple_ios_macabi.rs
+++ b/compiler/rustc_target/src/spec/targets/x86_64_apple_ios_macabi.rs
@@ -1,5 +1,5 @@
 use crate::spec::base::apple::{opts, Arch};
-use crate::spec::{Cc, LinkerFlavor, Lld, SanitizerSet, StackProbeType, Target, TargetOptions};
+use crate::spec::{Cc, LinkerFlavor, Lld, SanitizerSet, Target, TargetOptions};
 
 pub fn target() -> Target {
     let llvm_target = "x86_64-apple-ios14.0-macabi";
@@ -15,10 +15,6 @@ pub fn target() -> Target {
         data_layout: "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
             .into(),
         arch: arch.target_arch(),
-        options: TargetOptions {
-            max_atomic_width: Some(128),
-            stack_probes: StackProbeType::X86,
-            ..base
-        },
+        options: TargetOptions { max_atomic_width: Some(128), ..base },
     }
 }
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_apple_tvos.rs b/compiler/rustc_target/src/spec/targets/x86_64_apple_tvos.rs
index 1c4d9196c..17efd437f 100644
--- a/compiler/rustc_target/src/spec/targets/x86_64_apple_tvos.rs
+++ b/compiler/rustc_target/src/spec/targets/x86_64_apple_tvos.rs
@@ -1,5 +1,5 @@
 use crate::spec::base::apple::{opts, tvos_sim_llvm_target, Arch};
-use crate::spec::{StackProbeType, Target, TargetOptions};
+use crate::spec::{Target, TargetOptions};
 
 pub fn target() -> Target {
     let arch = Arch::X86_64_sim;
@@ -9,10 +9,6 @@ pub fn target() -> Target {
         data_layout: "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
             .into(),
         arch: arch.target_arch(),
-        options: TargetOptions {
-            max_atomic_width: Some(128),
-            stack_probes: StackProbeType::X86,
-            ..opts("tvos", arch)
-        },
+        options: TargetOptions { max_atomic_width: Some(128), ..opts("tvos", arch) },
     }
 }
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_apple_watchos_sim.rs b/compiler/rustc_target/src/spec/targets/x86_64_apple_watchos_sim.rs
index 258148677..b1f72ee2f 100644
--- a/compiler/rustc_target/src/spec/targets/x86_64_apple_watchos_sim.rs
+++ b/compiler/rustc_target/src/spec/targets/x86_64_apple_watchos_sim.rs
@@ -1,5 +1,5 @@
 use crate::spec::base::apple::{opts, watchos_sim_llvm_target, Arch};
-use crate::spec::{StackProbeType, Target, TargetOptions};
+use crate::spec::{Target, TargetOptions};
 
 pub fn target() -> Target {
     let arch = Arch::X86_64_sim;
@@ -9,22 +9,6 @@ pub fn target() -> Target {
         data_layout: "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
             .into(),
         arch: arch.target_arch(),
-        options: TargetOptions {
-            max_atomic_width: Some(128),
-            stack_probes: StackProbeType::X86,
-            forces_embed_bitcode: true,
-            // Taken from a clang build on Xcode 11.4.1.
-            // These arguments are not actually invoked - they just have
-            // to look right to pass App Store validation.
-            bitcode_llvm_cmdline: "-triple\0\
-                x86_64-apple-watchos5.0-simulator\0\
-                -emit-obj\0\
-                -disable-llvm-passes\0\
-                -target-abi\0\
-                darwinpcs\0\
-                -Os\0"
-                .into(),
-            ..opts("watchos", arch)
-        },
+        options: TargetOptions { max_atomic_width: Some(128), ..opts("watchos", arch) },
     }
 }
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_linux_android.rs b/compiler/rustc_target/src/spec/targets/x86_64_linux_android.rs
index 6cffda44f..0e6d41abc 100644
--- a/compiler/rustc_target/src/spec/targets/x86_64_linux_android.rs
+++ b/compiler/rustc_target/src/spec/targets/x86_64_linux_android.rs
@@ -10,7 +10,7 @@ pub fn target() -> Target {
     base.features = "+mmx,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gnu(Cc::Yes, Lld::No), &["-m64"]);
-    base.stack_probes = StackProbeType::X86;
+    base.stack_probes = StackProbeType::Inline;
     base.supports_xray = true;
 
     Target {
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_pc_solaris.rs b/compiler/rustc_target/src/spec/targets/x86_64_pc_solaris.rs
index ca6a6dc50..f15ad8dda 100644
--- a/compiler/rustc_target/src/spec/targets/x86_64_pc_solaris.rs
+++ b/compiler/rustc_target/src/spec/targets/x86_64_pc_solaris.rs
@@ -7,7 +7,7 @@ pub fn target() -> Target {
     base.plt_by_default = false;
     base.vendor = "pc".into();
     base.max_atomic_width = Some(64);
-    base.stack_probes = StackProbeType::X86;
+    base.stack_probes = StackProbeType::Inline;
     base.supported_sanitizers = SanitizerSet::ADDRESS | SanitizerSet::CFI | SanitizerSet::THREAD;
 
     Target {
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_sun_solaris.rs b/compiler/rustc_target/src/spec/targets/x86_64_sun_solaris.rs
deleted file mode 100644
index cca099d3b..000000000
--- a/compiler/rustc_target/src/spec/targets/x86_64_sun_solaris.rs
+++ /dev/null
@@ -1,20 +0,0 @@
-use crate::spec::{base, Cc, LinkerFlavor, StackProbeType, Target};
-
-pub fn target() -> Target {
-    let mut base = base::solaris::opts();
-    base.add_pre_link_args(LinkerFlavor::Unix(Cc::Yes), &["-m64"]);
-    base.cpu = "x86-64".into();
-    base.plt_by_default = false;
-    base.vendor = "sun".into();
-    base.max_atomic_width = Some(64);
-    base.stack_probes = StackProbeType::X86;
-
-    Target {
-        llvm_target: "x86_64-pc-solaris".into(),
-        pointer_width: 64,
-        data_layout: "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
-            .into(),
-        arch: "x86_64".into(),
-        options: base,
-    }
-}
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_unikraft_linux_musl.rs b/compiler/rustc_target/src/spec/targets/x86_64_unikraft_linux_musl.rs
index c12cb1ab0..e6159fca3 100644
--- a/compiler/rustc_target/src/spec/targets/x86_64_unikraft_linux_musl.rs
+++ b/compiler/rustc_target/src/spec/targets/x86_64_unikraft_linux_musl.rs
@@ -12,7 +12,7 @@ pub fn target() -> Target {
             plt_by_default: false,
             pre_link_args: TargetOptions::link_args(LinkerFlavor::Gnu(Cc::Yes, Lld::No), &["-m64"]),
             max_atomic_width: Some(64),
-            stack_probes: StackProbeType::X86,
+            stack_probes: StackProbeType::Inline,
             ..base::unikraft_linux_musl::opts()
         },
     }
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_unknown_dragonfly.rs b/compiler/rustc_target/src/spec/targets/x86_64_unknown_dragonfly.rs
index 68d1a755b..80adb8fa2 100644
--- a/compiler/rustc_target/src/spec/targets/x86_64_unknown_dragonfly.rs
+++ b/compiler/rustc_target/src/spec/targets/x86_64_unknown_dragonfly.rs
@@ -6,7 +6,7 @@ pub fn target() -> Target {
     base.plt_by_default = false;
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gnu(Cc::Yes, Lld::No), &["-m64"]);
-    base.stack_probes = StackProbeType::X86;
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "x86_64-unknown-dragonfly".into(),
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_unknown_freebsd.rs b/compiler/rustc_target/src/spec/targets/x86_64_unknown_freebsd.rs
index e3f29fa5a..fa3210353 100644
--- a/compiler/rustc_target/src/spec/targets/x86_64_unknown_freebsd.rs
+++ b/compiler/rustc_target/src/spec/targets/x86_64_unknown_freebsd.rs
@@ -6,7 +6,7 @@ pub fn target() -> Target {
     base.plt_by_default = false;
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gnu(Cc::Yes, Lld::No), &["-m64"]);
-    base.stack_probes = StackProbeType::X86;
+    base.stack_probes = StackProbeType::Inline;
     base.supported_sanitizers =
         SanitizerSet::ADDRESS | SanitizerSet::CFI | SanitizerSet::MEMORY | SanitizerSet::THREAD;
     base.supports_xray = true;
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_unknown_fuchsia.rs b/compiler/rustc_target/src/spec/targets/x86_64_unknown_fuchsia.rs
index c8850d703..f0515615b 100644
--- a/compiler/rustc_target/src/spec/targets/x86_64_unknown_fuchsia.rs
+++ b/compiler/rustc_target/src/spec/targets/x86_64_unknown_fuchsia.rs
@@ -5,7 +5,7 @@ pub fn target() -> Target {
     base.cpu = "x86-64".into();
     base.plt_by_default = false;
     base.max_atomic_width = Some(64);
-    base.stack_probes = StackProbeType::X86;
+    base.stack_probes = StackProbeType::Inline;
     base.supported_sanitizers = SanitizerSet::ADDRESS | SanitizerSet::CFI;
 
     Target {
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_unknown_haiku.rs b/compiler/rustc_target/src/spec/targets/x86_64_unknown_haiku.rs
index ca55e6514..0f927be96 100644
--- a/compiler/rustc_target/src/spec/targets/x86_64_unknown_haiku.rs
+++ b/compiler/rustc_target/src/spec/targets/x86_64_unknown_haiku.rs
@@ -6,7 +6,7 @@ pub fn target() -> Target {
     base.plt_by_default = false;
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gnu(Cc::Yes, Lld::No), &["-m64"]);
-    base.stack_probes = StackProbeType::X86;
+    base.stack_probes = StackProbeType::Inline;
     // This option is required to build executables on Haiku x86_64
     base.position_independent_executables = true;
 
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_unknown_hermit.rs b/compiler/rustc_target/src/spec/targets/x86_64_unknown_hermit.rs
index 3cc50d40d..df191f515 100644
--- a/compiler/rustc_target/src/spec/targets/x86_64_unknown_hermit.rs
+++ b/compiler/rustc_target/src/spec/targets/x86_64_unknown_hermit.rs
@@ -12,7 +12,7 @@ pub fn target() -> Target {
             features: "+rdrnd,+rdseed".into(),
             plt_by_default: false,
             max_atomic_width: Some(64),
-            stack_probes: StackProbeType::X86,
+            stack_probes: StackProbeType::Inline,
             ..base::hermit::opts()
         },
     }
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_unknown_linux_gnu.rs b/compiler/rustc_target/src/spec/targets/x86_64_unknown_linux_gnu.rs
index 80e267c16..bf10f7e5d 100644
--- a/compiler/rustc_target/src/spec/targets/x86_64_unknown_linux_gnu.rs
+++ b/compiler/rustc_target/src/spec/targets/x86_64_unknown_linux_gnu.rs
@@ -6,7 +6,7 @@ pub fn target() -> Target {
     base.plt_by_default = false;
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gnu(Cc::Yes, Lld::No), &["-m64"]);
-    base.stack_probes = StackProbeType::X86;
+    base.stack_probes = StackProbeType::Inline;
     base.static_position_independent_executables = true;
     base.supported_sanitizers = SanitizerSet::ADDRESS
         | SanitizerSet::CFI
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_unknown_linux_gnux32.rs b/compiler/rustc_target/src/spec/targets/x86_64_unknown_linux_gnux32.rs
index 622bfe8bb..1856c6afd 100644
--- a/compiler/rustc_target/src/spec/targets/x86_64_unknown_linux_gnux32.rs
+++ b/compiler/rustc_target/src/spec/targets/x86_64_unknown_linux_gnux32.rs
@@ -6,7 +6,7 @@ pub fn target() -> Target {
     base.abi = "x32".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gnu(Cc::Yes, Lld::No), &["-mx32"]);
-    base.stack_probes = StackProbeType::X86;
+    base.stack_probes = StackProbeType::Inline;
     base.has_thread_local = false;
     // BUG(GabrielMajeri): disabling the PLT on x86_64 Linux with x32 ABI
     // breaks code gen. See LLVM bug 36743
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_unknown_linux_musl.rs b/compiler/rustc_target/src/spec/targets/x86_64_unknown_linux_musl.rs
index d74ff466a..8dc5503e3 100644
--- a/compiler/rustc_target/src/spec/targets/x86_64_unknown_linux_musl.rs
+++ b/compiler/rustc_target/src/spec/targets/x86_64_unknown_linux_musl.rs
@@ -6,7 +6,7 @@ pub fn target() -> Target {
     base.plt_by_default = false;
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gnu(Cc::Yes, Lld::No), &["-m64"]);
-    base.stack_probes = StackProbeType::X86;
+    base.stack_probes = StackProbeType::Inline;
     base.static_position_independent_executables = true;
     base.supported_sanitizers = SanitizerSet::ADDRESS
         | SanitizerSet::CFI
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_unknown_linux_ohos.rs b/compiler/rustc_target/src/spec/targets/x86_64_unknown_linux_ohos.rs
index 87b004df0..35862656a 100644
--- a/compiler/rustc_target/src/spec/targets/x86_64_unknown_linux_ohos.rs
+++ b/compiler/rustc_target/src/spec/targets/x86_64_unknown_linux_ohos.rs
@@ -5,7 +5,7 @@ pub fn target() -> Target {
     base.cpu = "x86-64".into();
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gnu(Cc::Yes, Lld::No), &["-m64"]);
-    base.stack_probes = StackProbeType::X86;
+    base.stack_probes = StackProbeType::Inline;
     base.static_position_independent_executables = true;
     base.supported_sanitizers = SanitizerSet::ADDRESS
         | SanitizerSet::CFI
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_unknown_netbsd.rs b/compiler/rustc_target/src/spec/targets/x86_64_unknown_netbsd.rs
index 04060513a..e2cee0513 100644
--- a/compiler/rustc_target/src/spec/targets/x86_64_unknown_netbsd.rs
+++ b/compiler/rustc_target/src/spec/targets/x86_64_unknown_netbsd.rs
@@ -8,7 +8,7 @@ pub fn target() -> Target {
     base.plt_by_default = false;
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gnu(Cc::Yes, Lld::No), &["-m64"]);
-    base.stack_probes = StackProbeType::X86;
+    base.stack_probes = StackProbeType::Inline;
     base.supported_sanitizers = SanitizerSet::ADDRESS
         | SanitizerSet::CFI
         | SanitizerSet::LEAK
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_unknown_none.rs b/compiler/rustc_target/src/spec/targets/x86_64_unknown_none.rs
index 9aa95a35f..8f7655d8c 100644
--- a/compiler/rustc_target/src/spec/targets/x86_64_unknown_none.rs
+++ b/compiler/rustc_target/src/spec/targets/x86_64_unknown_none.rs
@@ -12,7 +12,7 @@ pub fn target() -> Target {
         cpu: "x86-64".into(),
         plt_by_default: false,
         max_atomic_width: Some(64),
-        stack_probes: StackProbeType::X86,
+        stack_probes: StackProbeType::Inline,
         position_independent_executables: true,
         static_position_independent_executables: true,
         relro_level: RelroLevel::Full,
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_unknown_openbsd.rs b/compiler/rustc_target/src/spec/targets/x86_64_unknown_openbsd.rs
index 5e6e7efb2..1133b50f3 100644
--- a/compiler/rustc_target/src/spec/targets/x86_64_unknown_openbsd.rs
+++ b/compiler/rustc_target/src/spec/targets/x86_64_unknown_openbsd.rs
@@ -6,7 +6,7 @@ pub fn target() -> Target {
     base.plt_by_default = false;
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gnu(Cc::Yes, Lld::No), &["-m64"]);
-    base.stack_probes = StackProbeType::X86;
+    base.stack_probes = StackProbeType::Inline;
     base.supports_xray = true;
 
     Target {
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_unknown_redox.rs b/compiler/rustc_target/src/spec/targets/x86_64_unknown_redox.rs
index 382ff71f0..c1f573112 100644
--- a/compiler/rustc_target/src/spec/targets/x86_64_unknown_redox.rs
+++ b/compiler/rustc_target/src/spec/targets/x86_64_unknown_redox.rs
@@ -6,7 +6,7 @@ pub fn target() -> Target {
     base.plt_by_default = false;
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gnu(Cc::Yes, Lld::No), &["-m64"]);
-    base.stack_probes = StackProbeType::X86;
+    base.stack_probes = StackProbeType::Inline;
 
     Target {
         llvm_target: "x86_64-unknown-redox".into(),
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_win7_windows_msvc.rs b/compiler/rustc_target/src/spec/targets/x86_64_win7_windows_msvc.rs
new file mode 100644
index 000000000..5a59839eb
--- /dev/null
+++ b/compiler/rustc_target/src/spec/targets/x86_64_win7_windows_msvc.rs
@@ -0,0 +1,18 @@
+use crate::spec::{base, Target};
+
+pub fn target() -> Target {
+    let mut base = base::windows_msvc::opts();
+    base.cpu = "x86-64".into();
+    base.plt_by_default = false;
+    base.max_atomic_width = Some(64);
+    base.vendor = "win7".into();
+
+    Target {
+        llvm_target: "x86_64-win7-windows-msvc".into(),
+        pointer_width: 64,
+        data_layout: "e-m:w-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
+            .into(),
+        arch: "x86_64".into(),
+        options: base,
+    }
+}
diff --git a/compiler/rustc_target/src/spec/targets/x86_64_wrs_vxworks.rs b/compiler/rustc_target/src/spec/targets/x86_64_wrs_vxworks.rs
index 4b221fe69..765239bdd 100644
--- a/compiler/rustc_target/src/spec/targets/x86_64_wrs_vxworks.rs
+++ b/compiler/rustc_target/src/spec/targets/x86_64_wrs_vxworks.rs
@@ -6,7 +6,7 @@ pub fn target() -> Target {
     base.plt_by_default = false;
     base.max_atomic_width = Some(64);
     base.add_pre_link_args(LinkerFlavor::Gnu(Cc::Yes, Lld::No), &["-m64"]);
-    base.stack_probes = StackProbeType::X86;
+    base.stack_probes = StackProbeType::Inline;
     base.disable_redzone = true;
 
     Target {
diff --git a/compiler/rustc_target/src/spec/targets/x86_64h_apple_darwin.rs b/compiler/rustc_target/src/spec/targets/x86_64h_apple_darwin.rs
index a30e9d17c..0c731e369 100644
--- a/compiler/rustc_target/src/spec/targets/x86_64h_apple_darwin.rs
+++ b/compiler/rustc_target/src/spec/targets/x86_64h_apple_darwin.rs
@@ -1,6 +1,6 @@
 use crate::spec::base::apple::{macos_llvm_target, opts, Arch};
 use crate::spec::{Cc, FramePointer, LinkerFlavor, Lld, SanitizerSet};
-use crate::spec::{StackProbeType, Target, TargetOptions};
+use crate::spec::{Target, TargetOptions};
 
 pub fn target() -> Target {
     let arch = Arch::X86_64h;
@@ -8,7 +8,6 @@ pub fn target() -> Target {
     base.max_atomic_width = Some(128);
     base.frame_pointer = FramePointer::Always;
     base.add_pre_link_args(LinkerFlavor::Darwin(Cc::Yes, Lld::No), &["-m64"]);
-    base.stack_probes = StackProbeType::X86;
     base.supported_sanitizers =
         SanitizerSet::ADDRESS | SanitizerSet::CFI | SanitizerSet::LEAK | SanitizerSet::THREAD;
 
diff --git a/compiler/rustc_target/src/target_features.rs b/compiler/rustc_target/src/target_features.rs
new file mode 100644
index 000000000..5f5de57dd
--- /dev/null
+++ b/compiler/rustc_target/src/target_features.rs
@@ -0,0 +1,429 @@
+use rustc_span::symbol::sym;
+use rustc_span::symbol::Symbol;
+
+/// Features that control behaviour of rustc, rather than the codegen.
+pub const RUSTC_SPECIFIC_FEATURES: &[&str] = &["crt-static"];
+
+/// Stability information for target features.
+#[derive(Debug, Clone, Copy)]
+pub enum Stability {
+    /// This target feature is stable, it can be used in `#[target_feature]` and
+    /// `#[cfg(target_feature)]`.
+    Stable,
+    /// This target feature is unstable; using it in `#[target_feature]` or `#[cfg(target_feature)]`
+    /// requires enabling the given nightly feature.
+    Unstable(Symbol),
+}
+use Stability::*;
+
+impl Stability {
+    pub fn as_feature_name(self) -> Option<Symbol> {
+        match self {
+            Stable => None,
+            Unstable(s) => Some(s),
+        }
+    }
+
+    pub fn is_stable(self) -> bool {
+        matches!(self, Stable)
+    }
+}
+
+// Here we list target features that rustc "understands": they can be used in `#[target_feature]`
+// and `#[cfg(target_feature)]`. They also do not trigger any warnings when used with
+// `-Ctarget-feature`.
+//
+// When adding features to the below lists
+// check whether they're named already elsewhere in rust
+// e.g. in stdarch and whether the given name matches LLVM's
+// if it doesn't, to_llvm_feature in llvm_util in rustc_codegen_llvm needs to be adapted.
+//
+// Also note that all target features listed here must be purely additive: for target_feature 1.1 to
+// be sound, we can never allow features like `+soft-float` (on x86) to be controlled on a
+// per-function level, since we would then allow safe calls from functions with `+soft-float` to
+// functions without that feature!
+//
+// When adding a new feature, be particularly mindful of features that affect function ABIs. Those
+// need to be treated very carefully to avoid introducing unsoundness! This often affects features
+// that enable/disable hardfloat support (see https://github.com/rust-lang/rust/issues/116344 for an
+// example of this going wrong), but features enabling new SIMD registers are also a concern (see
+// https://github.com/rust-lang/rust/issues/116558 for an example of this going wrong).
+//
+// Stabilizing a target feature requires t-lang approval.
+
+const ARM_ALLOWED_FEATURES: &[(&str, Stability)] = &[
+    // tidy-alphabetical-start
+    ("aclass", Unstable(sym::arm_target_feature)),
+    ("aes", Unstable(sym::arm_target_feature)),
+    ("crc", Unstable(sym::arm_target_feature)),
+    ("d32", Unstable(sym::arm_target_feature)),
+    ("dotprod", Unstable(sym::arm_target_feature)),
+    ("dsp", Unstable(sym::arm_target_feature)),
+    ("fp-armv8", Unstable(sym::arm_target_feature)),
+    ("i8mm", Unstable(sym::arm_target_feature)),
+    ("mclass", Unstable(sym::arm_target_feature)),
+    ("neon", Unstable(sym::arm_target_feature)),
+    ("rclass", Unstable(sym::arm_target_feature)),
+    ("sha2", Unstable(sym::arm_target_feature)),
+    // This is needed for inline assembly, but shouldn't be stabilized as-is
+    // since it should be enabled per-function using #[instruction_set], not
+    // #[target_feature].
+    ("thumb-mode", Unstable(sym::arm_target_feature)),
+    ("thumb2", Unstable(sym::arm_target_feature)),
+    ("trustzone", Unstable(sym::arm_target_feature)),
+    ("v5te", Unstable(sym::arm_target_feature)),
+    ("v6", Unstable(sym::arm_target_feature)),
+    ("v6k", Unstable(sym::arm_target_feature)),
+    ("v6t2", Unstable(sym::arm_target_feature)),
+    ("v7", Unstable(sym::arm_target_feature)),
+    ("v8", Unstable(sym::arm_target_feature)),
+    ("vfp2", Unstable(sym::arm_target_feature)),
+    ("vfp3", Unstable(sym::arm_target_feature)),
+    ("vfp4", Unstable(sym::arm_target_feature)),
+    ("virtualization", Unstable(sym::arm_target_feature)),
+    // tidy-alphabetical-end
+];
+
+const AARCH64_ALLOWED_FEATURES: &[(&str, Stability)] = &[
+    // tidy-alphabetical-start
+    // FEAT_AES
+    ("aes", Stable),
+    // FEAT_BF16
+    ("bf16", Stable),
+    // FEAT_BTI
+    ("bti", Stable),
+    // FEAT_CRC
+    ("crc", Stable),
+    // FEAT_DIT
+    ("dit", Stable),
+    // FEAT_DotProd
+    ("dotprod", Stable),
+    // FEAT_DPB
+    ("dpb", Stable),
+    // FEAT_DPB2
+    ("dpb2", Stable),
+    // FEAT_F32MM
+    ("f32mm", Stable),
+    // FEAT_F64MM
+    ("f64mm", Stable),
+    // FEAT_FCMA
+    ("fcma", Stable),
+    // FEAT_FHM
+    ("fhm", Stable),
+    // FEAT_FLAGM
+    ("flagm", Stable),
+    // FEAT_FP16
+    ("fp16", Stable),
+    // FEAT_FRINTTS
+    ("frintts", Stable),
+    // FEAT_I8MM
+    ("i8mm", Stable),
+    // FEAT_JSCVT
+    ("jsconv", Stable),
+    // FEAT_LOR
+    ("lor", Stable),
+    // FEAT_LSE
+    ("lse", Stable),
+    // FEAT_MTE
+    ("mte", Stable),
+    // FEAT_AdvSimd & FEAT_FP
+    ("neon", Stable),
+    // FEAT_PAUTH (address authentication)
+    ("paca", Stable),
+    // FEAT_PAUTH (generic authentication)
+    ("pacg", Stable),
+    // FEAT_PAN
+    ("pan", Stable),
+    // FEAT_PMUv3
+    ("pmuv3", Stable),
+    // FEAT_RAND
+    ("rand", Stable),
+    // FEAT_RAS
+    ("ras", Stable),
+    // FEAT_RCPC
+    ("rcpc", Stable),
+    // FEAT_RCPC2
+    ("rcpc2", Stable),
+    // FEAT_RDM
+    ("rdm", Stable),
+    // FEAT_SB
+    ("sb", Stable),
+    // FEAT_SHA1 & FEAT_SHA256
+    ("sha2", Stable),
+    // FEAT_SHA512 & FEAT_SHA3
+    ("sha3", Stable),
+    // FEAT_SM3 & FEAT_SM4
+    ("sm4", Stable),
+    // FEAT_SPE
+    ("spe", Stable),
+    // FEAT_SSBS
+    ("ssbs", Stable),
+    // FEAT_SVE
+    ("sve", Stable),
+    // FEAT_SVE2
+    ("sve2", Stable),
+    // FEAT_SVE2_AES
+    ("sve2-aes", Stable),
+    // FEAT_SVE2_BitPerm
+    ("sve2-bitperm", Stable),
+    // FEAT_SVE2_SHA3
+    ("sve2-sha3", Stable),
+    // FEAT_SVE2_SM4
+    ("sve2-sm4", Stable),
+    // FEAT_TME
+    ("tme", Stable),
+    ("v8.1a", Unstable(sym::aarch64_ver_target_feature)),
+    ("v8.2a", Unstable(sym::aarch64_ver_target_feature)),
+    ("v8.3a", Unstable(sym::aarch64_ver_target_feature)),
+    ("v8.4a", Unstable(sym::aarch64_ver_target_feature)),
+    ("v8.5a", Unstable(sym::aarch64_ver_target_feature)),
+    ("v8.6a", Unstable(sym::aarch64_ver_target_feature)),
+    ("v8.7a", Unstable(sym::aarch64_ver_target_feature)),
+    // FEAT_VHE
+    ("vh", Stable),
+    // tidy-alphabetical-end
+];
+
+const AARCH64_TIED_FEATURES: &[&[&str]] = &[
+    &["paca", "pacg"], // Together these represent `pauth` in LLVM
+];
+
+const X86_ALLOWED_FEATURES: &[(&str, Stability)] = &[
+    // tidy-alphabetical-start
+    ("adx", Stable),
+    ("aes", Stable),
+    ("avx", Stable),
+    ("avx2", Stable),
+    ("avx512bf16", Unstable(sym::avx512_target_feature)),
+    ("avx512bitalg", Unstable(sym::avx512_target_feature)),
+    ("avx512bw", Unstable(sym::avx512_target_feature)),
+    ("avx512cd", Unstable(sym::avx512_target_feature)),
+    ("avx512dq", Unstable(sym::avx512_target_feature)),
+    ("avx512er", Unstable(sym::avx512_target_feature)),
+    ("avx512f", Unstable(sym::avx512_target_feature)),
+    ("avx512ifma", Unstable(sym::avx512_target_feature)),
+    ("avx512pf", Unstable(sym::avx512_target_feature)),
+    ("avx512vbmi", Unstable(sym::avx512_target_feature)),
+    ("avx512vbmi2", Unstable(sym::avx512_target_feature)),
+    ("avx512vl", Unstable(sym::avx512_target_feature)),
+    ("avx512vnni", Unstable(sym::avx512_target_feature)),
+    ("avx512vp2intersect", Unstable(sym::avx512_target_feature)),
+    ("avx512vpopcntdq", Unstable(sym::avx512_target_feature)),
+    ("bmi1", Stable),
+    ("bmi2", Stable),
+    ("cmpxchg16b", Stable),
+    ("ermsb", Unstable(sym::ermsb_target_feature)),
+    ("f16c", Stable),
+    ("fma", Stable),
+    ("fxsr", Stable),
+    ("gfni", Unstable(sym::avx512_target_feature)),
+    ("lzcnt", Stable),
+    ("movbe", Stable),
+    ("pclmulqdq", Stable),
+    ("popcnt", Stable),
+    ("rdrand", Stable),
+    ("rdseed", Stable),
+    ("rtm", Unstable(sym::rtm_target_feature)),
+    ("sha", Stable),
+    ("sse", Stable),
+    ("sse2", Stable),
+    ("sse3", Stable),
+    ("sse4.1", Stable),
+    ("sse4.2", Stable),
+    ("sse4a", Unstable(sym::sse4a_target_feature)),
+    ("ssse3", Stable),
+    ("tbm", Unstable(sym::tbm_target_feature)),
+    ("vaes", Unstable(sym::avx512_target_feature)),
+    ("vpclmulqdq", Unstable(sym::avx512_target_feature)),
+    ("xsave", Stable),
+    ("xsavec", Stable),
+    ("xsaveopt", Stable),
+    ("xsaves", Stable),
+    // tidy-alphabetical-end
+];
+
+const HEXAGON_ALLOWED_FEATURES: &[(&str, Stability)] = &[
+    // tidy-alphabetical-start
+    ("hvx", Unstable(sym::hexagon_target_feature)),
+    ("hvx-length128b", Unstable(sym::hexagon_target_feature)),
+    // tidy-alphabetical-end
+];
+
+const POWERPC_ALLOWED_FEATURES: &[(&str, Stability)] = &[
+    // tidy-alphabetical-start
+    ("altivec", Unstable(sym::powerpc_target_feature)),
+    ("power10-vector", Unstable(sym::powerpc_target_feature)),
+    ("power8-altivec", Unstable(sym::powerpc_target_feature)),
+    ("power8-vector", Unstable(sym::powerpc_target_feature)),
+    ("power9-altivec", Unstable(sym::powerpc_target_feature)),
+    ("power9-vector", Unstable(sym::powerpc_target_feature)),
+    ("vsx", Unstable(sym::powerpc_target_feature)),
+    // tidy-alphabetical-end
+];
+
+const MIPS_ALLOWED_FEATURES: &[(&str, Stability)] = &[
+    // tidy-alphabetical-start
+    ("fp64", Unstable(sym::mips_target_feature)),
+    ("msa", Unstable(sym::mips_target_feature)),
+    ("virt", Unstable(sym::mips_target_feature)),
+    // tidy-alphabetical-end
+];
+
+const RISCV_ALLOWED_FEATURES: &[(&str, Stability)] = &[
+    // tidy-alphabetical-start
+    ("a", Stable),
+    ("c", Stable),
+    ("d", Unstable(sym::riscv_target_feature)),
+    ("e", Unstable(sym::riscv_target_feature)),
+    ("f", Unstable(sym::riscv_target_feature)),
+    ("fast-unaligned-access", Unstable(sym::riscv_target_feature)),
+    ("m", Stable),
+    ("relax", Unstable(sym::riscv_target_feature)),
+    ("v", Unstable(sym::riscv_target_feature)),
+    ("zba", Stable),
+    ("zbb", Stable),
+    ("zbc", Stable),
+    ("zbkb", Stable),
+    ("zbkc", Stable),
+    ("zbkx", Stable),
+    ("zbs", Stable),
+    ("zdinx", Unstable(sym::riscv_target_feature)),
+    ("zfh", Unstable(sym::riscv_target_feature)),
+    ("zfhmin", Unstable(sym::riscv_target_feature)),
+    ("zfinx", Unstable(sym::riscv_target_feature)),
+    ("zhinx", Unstable(sym::riscv_target_feature)),
+    ("zhinxmin", Unstable(sym::riscv_target_feature)),
+    ("zk", Stable),
+    ("zkn", Stable),
+    ("zknd", Stable),
+    ("zkne", Stable),
+    ("zknh", Stable),
+    ("zkr", Stable),
+    ("zks", Stable),
+    ("zksed", Stable),
+    ("zksh", Stable),
+    ("zkt", Stable),
+    // tidy-alphabetical-end
+];
+
+const WASM_ALLOWED_FEATURES: &[(&str, Stability)] = &[
+    // tidy-alphabetical-start
+    ("atomics", Unstable(sym::wasm_target_feature)),
+    ("bulk-memory", Unstable(sym::wasm_target_feature)),
+    ("exception-handling", Unstable(sym::wasm_target_feature)),
+    ("multivalue", Unstable(sym::wasm_target_feature)),
+    ("mutable-globals", Unstable(sym::wasm_target_feature)),
+    ("nontrapping-fptoint", Unstable(sym::wasm_target_feature)),
+    ("reference-types", Unstable(sym::wasm_target_feature)),
+    ("relaxed-simd", Unstable(sym::wasm_target_feature)),
+    ("sign-ext", Unstable(sym::wasm_target_feature)),
+    ("simd128", Stable),
+    // tidy-alphabetical-end
+];
+
+const BPF_ALLOWED_FEATURES: &[(&str, Stability)] = &[("alu32", Unstable(sym::bpf_target_feature))];
+
+const CSKY_ALLOWED_FEATURES: &[(&str, Stability)] = &[
+    // tidy-alphabetical-start
+    ("10e60", Unstable(sym::csky_target_feature)),
+    ("2e3", Unstable(sym::csky_target_feature)),
+    ("3e3r1", Unstable(sym::csky_target_feature)),
+    ("3e3r2", Unstable(sym::csky_target_feature)),
+    ("3e3r3", Unstable(sym::csky_target_feature)),
+    ("3e7", Unstable(sym::csky_target_feature)),
+    ("7e10", Unstable(sym::csky_target_feature)),
+    ("cache", Unstable(sym::csky_target_feature)),
+    ("doloop", Unstable(sym::csky_target_feature)),
+    ("dsp1e2", Unstable(sym::csky_target_feature)),
+    ("dspe60", Unstable(sym::csky_target_feature)),
+    ("e1", Unstable(sym::csky_target_feature)),
+    ("e2", Unstable(sym::csky_target_feature)),
+    ("edsp", Unstable(sym::csky_target_feature)),
+    ("elrw", Unstable(sym::csky_target_feature)),
+    ("float1e2", Unstable(sym::csky_target_feature)),
+    ("float1e3", Unstable(sym::csky_target_feature)),
+    ("float3e4", Unstable(sym::csky_target_feature)),
+    ("float7e60", Unstable(sym::csky_target_feature)),
+    ("floate1", Unstable(sym::csky_target_feature)),
+    ("hard-tp", Unstable(sym::csky_target_feature)),
+    ("high-registers", Unstable(sym::csky_target_feature)),
+    ("hwdiv", Unstable(sym::csky_target_feature)),
+    ("mp", Unstable(sym::csky_target_feature)),
+    ("mp1e2", Unstable(sym::csky_target_feature)),
+    ("nvic", Unstable(sym::csky_target_feature)),
+    ("trust", Unstable(sym::csky_target_feature)),
+    ("vdsp2e60f", Unstable(sym::csky_target_feature)),
+    ("vdspv1", Unstable(sym::csky_target_feature)),
+    ("vdspv2", Unstable(sym::csky_target_feature)),
+    // tidy-alphabetical-end
+    //fpu
+    // tidy-alphabetical-start
+    ("fdivdu", Unstable(sym::csky_target_feature)),
+    ("fpuv2_df", Unstable(sym::csky_target_feature)),
+    ("fpuv2_sf", Unstable(sym::csky_target_feature)),
+    ("fpuv3_df", Unstable(sym::csky_target_feature)),
+    ("fpuv3_hf", Unstable(sym::csky_target_feature)),
+    ("fpuv3_hi", Unstable(sym::csky_target_feature)),
+    ("fpuv3_sf", Unstable(sym::csky_target_feature)),
+    ("hard-float", Unstable(sym::csky_target_feature)),
+    ("hard-float-abi", Unstable(sym::csky_target_feature)),
+    // tidy-alphabetical-end
+];
+
+const LOONGARCH_ALLOWED_FEATURES: &[(&str, Stability)] = &[
+    // tidy-alphabetical-start
+    ("d", Unstable(sym::loongarch_target_feature)),
+    ("f", Unstable(sym::loongarch_target_feature)),
+    ("lasx", Unstable(sym::loongarch_target_feature)),
+    ("lbt", Unstable(sym::loongarch_target_feature)),
+    ("lsx", Unstable(sym::loongarch_target_feature)),
+    ("lvz", Unstable(sym::loongarch_target_feature)),
+    ("ual", Unstable(sym::loongarch_target_feature)),
+    // tidy-alphabetical-end
+];
+
+/// When rustdoc is running, provide a list of all known features so that all their respective
+/// primitives may be documented.
+///
+/// IMPORTANT: If you're adding another feature list above, make sure to add it to this iterator!
+pub fn all_known_features() -> impl Iterator<Item = (&'static str, Stability)> {
+    std::iter::empty()
+        .chain(ARM_ALLOWED_FEATURES.iter())
+        .chain(AARCH64_ALLOWED_FEATURES.iter())
+        .chain(X86_ALLOWED_FEATURES.iter())
+        .chain(HEXAGON_ALLOWED_FEATURES.iter())
+        .chain(POWERPC_ALLOWED_FEATURES.iter())
+        .chain(MIPS_ALLOWED_FEATURES.iter())
+        .chain(RISCV_ALLOWED_FEATURES.iter())
+        .chain(WASM_ALLOWED_FEATURES.iter())
+        .chain(BPF_ALLOWED_FEATURES.iter())
+        .chain(CSKY_ALLOWED_FEATURES)
+        .chain(LOONGARCH_ALLOWED_FEATURES)
+        .cloned()
+}
+
+impl super::spec::Target {
+    pub fn supported_target_features(&self) -> &'static [(&'static str, Stability)] {
+        match &*self.arch {
+            "arm" => ARM_ALLOWED_FEATURES,
+            "aarch64" => AARCH64_ALLOWED_FEATURES,
+            "x86" | "x86_64" => X86_ALLOWED_FEATURES,
+            "hexagon" => HEXAGON_ALLOWED_FEATURES,
+            "mips" | "mips32r6" | "mips64" | "mips64r6" => MIPS_ALLOWED_FEATURES,
+            "powerpc" | "powerpc64" => POWERPC_ALLOWED_FEATURES,
+            "riscv32" | "riscv64" => RISCV_ALLOWED_FEATURES,
+            "wasm32" | "wasm64" => WASM_ALLOWED_FEATURES,
+            "bpf" => BPF_ALLOWED_FEATURES,
+            "csky" => CSKY_ALLOWED_FEATURES,
+            "loongarch64" => LOONGARCH_ALLOWED_FEATURES,
+            _ => &[],
+        }
+    }
+
+    pub fn tied_target_features(&self) -> &'static [&'static [&'static str]] {
+        match &*self.arch {
+            "aarch64" => AARCH64_TIED_FEATURES,
+            _ => &[],
+        }
+    }
+}
diff --git a/compiler/rustc_trait_selection/Cargo.toml b/compiler/rustc_trait_selection/Cargo.toml
index 667ee3d4e..29c0d8b5f 100644
--- a/compiler/rustc_trait_selection/Cargo.toml
+++ b/compiler/rustc_trait_selection/Cargo.toml
@@ -15,9 +15,9 @@ rustc_index = { path = "../rustc_index" }
 rustc_infer = { path = "../rustc_infer" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_middle = { path = "../rustc_middle" }
+rustc_next_trait_solver = { path = "../rustc_next_trait_solver" }
 rustc_parse_format = { path = "../rustc_parse_format" }
 rustc_query_system = { path = "../rustc_query_system" }
-rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
diff --git a/compiler/rustc_trait_selection/messages.ftl b/compiler/rustc_trait_selection/messages.ftl
index a9792ca27..41db8059c 100644
--- a/compiler/rustc_trait_selection/messages.ftl
+++ b/compiler/rustc_trait_selection/messages.ftl
@@ -22,6 +22,10 @@ trait_selection_dump_vtable_entries = vtable entries for `{$trait_ref}`: {$entri
 trait_selection_empty_on_clause_in_rustc_on_unimplemented = empty `on`-clause in `#[rustc_on_unimplemented]`
     .label = empty on-clause here
 
+trait_selection_ignored_diagnostic_option = `{$option_name}` is ignored due to previous definition of `{$option_name}`
+    .other_label = `{$option_name}` is first declared here
+    .label = `{$option_name}` is already declared here
+
 trait_selection_inherent_projection_normalization_overflow = overflow evaluating associated type `{$ty}`
 
 trait_selection_invalid_on_clause_in_rustc_on_unimplemented = invalid `on`-clause in `#[rustc_on_unimplemented]`
@@ -51,3 +55,6 @@ trait_selection_trait_has_no_impls = this trait has no implementations, consider
 
 trait_selection_ty_alias_overflow = in case this is a recursive type alias, consider using a struct, enum, or union instead
 trait_selection_unable_to_construct_constant_value = unable to construct a constant value for the unevaluated constant {$unevaluated}
+
+trait_selection_unknown_format_parameter_for_on_unimplemented_attr = there is no parameter `{$argument_name}` on trait `{$trait_name}`
+    .help = expect either a generic argument name or {"`{Self}`"} as format argument
diff --git a/compiler/rustc_trait_selection/src/errors.rs b/compiler/rustc_trait_selection/src/errors.rs
index c1fb287d6..b0ec8b3a4 100644
--- a/compiler/rustc_trait_selection/src/errors.rs
+++ b/compiler/rustc_trait_selection/src/errors.rs
@@ -1,6 +1,6 @@
 use crate::fluent_generated as fluent;
 use rustc_errors::{
-    AddToDiagnostic, Applicability, Diagnostic, ErrorGuaranteed, Handler, IntoDiagnostic,
+    AddToDiagnostic, Applicability, DiagCtxt, Diagnostic, ErrorGuaranteed, IntoDiagnostic,
     SubdiagnosticMessage,
 };
 use rustc_macros::Diagnostic;
@@ -61,9 +61,9 @@ impl IntoDiagnostic<'_> for NegativePositiveConflict<'_> {
     #[track_caller]
     fn into_diagnostic(
         self,
-        handler: &Handler,
+        dcx: &DiagCtxt,
     ) -> rustc_errors::DiagnosticBuilder<'_, ErrorGuaranteed> {
-        let mut diag = handler.struct_err(fluent::trait_selection_negative_positive_conflict);
+        let mut diag = dcx.struct_err(fluent::trait_selection_negative_positive_conflict);
         diag.set_arg("trait_desc", self.trait_desc.print_only_trait_path().to_string());
         diag.set_arg(
             "self_desc",
diff --git a/compiler/rustc_trait_selection/src/infer.rs b/compiler/rustc_trait_selection/src/infer.rs
index 38153cccf..251f0628a 100644
--- a/compiler/rustc_trait_selection/src/infer.rs
+++ b/compiler/rustc_trait_selection/src/infer.rs
@@ -1,8 +1,10 @@
+use crate::solve::FulfillmentCtxt;
 use crate::traits::query::evaluate_obligation::InferCtxtExt as _;
 use crate::traits::{self, DefiningAnchor, ObligationCtxt};
 
 use rustc_hir::def_id::DefId;
 use rustc_hir::lang_items::LangItem;
+use rustc_infer::traits::{TraitEngine, TraitEngineExt};
 use rustc_middle::arena::ArenaAllocatable;
 use rustc_middle::infer::canonical::{Canonical, CanonicalQueryResponse, QueryResponse};
 use rustc_middle::traits::query::NoSolution;
@@ -27,7 +29,7 @@ pub trait InferCtxtExt<'tcx> {
     /// - the parameter environment
     ///
     /// Invokes `evaluate_obligation`, so in the event that evaluating
-    /// `Ty: Trait` causes overflow, EvaluatedToRecur (or EvaluatedToUnknown)
+    /// `Ty: Trait` causes overflow, EvaluatedToErrStackDependent (or EvaluatedToAmbigStackDependent)
     /// will be returned.
     fn type_implements_trait(
         &self,
@@ -35,6 +37,13 @@ pub trait InferCtxtExt<'tcx> {
         params: impl IntoIterator<Item: Into<GenericArg<'tcx>>>,
         param_env: ty::ParamEnv<'tcx>,
     ) -> traits::EvaluationResult;
+
+    fn could_impl_trait(
+        &self,
+        trait_def_id: DefId,
+        ty: Ty<'tcx>,
+        param_env: ty::ParamEnv<'tcx>,
+    ) -> Option<Vec<traits::FulfillmentError<'tcx>>>;
 }
 
 impl<'tcx> InferCtxtExt<'tcx> for InferCtxt<'tcx> {
@@ -76,6 +85,68 @@ impl<'tcx> InferCtxtExt<'tcx> for InferCtxt<'tcx> {
         };
         self.evaluate_obligation(&obligation).unwrap_or(traits::EvaluationResult::EvaluatedToErr)
     }
+
+    fn could_impl_trait(
+        &self,
+        trait_def_id: DefId,
+        ty: Ty<'tcx>,
+        param_env: ty::ParamEnv<'tcx>,
+    ) -> Option<Vec<traits::FulfillmentError<'tcx>>> {
+        self.probe(|_snapshot| {
+            if let ty::Adt(def, args) = ty.kind()
+                && let Some((impl_def_id, _)) = self
+                    .tcx
+                    .all_impls(trait_def_id)
+                    .filter_map(|impl_def_id| {
+                        self.tcx.impl_trait_ref(impl_def_id).map(|r| (impl_def_id, r))
+                    })
+                    .map(|(impl_def_id, imp)| (impl_def_id, imp.skip_binder()))
+                    .find(|(_, imp)| match imp.self_ty().peel_refs().kind() {
+                        ty::Adt(i_def, _) if i_def.did() == def.did() => true,
+                        _ => false,
+                    })
+            {
+                let mut fulfill_cx = FulfillmentCtxt::new(self);
+                // We get all obligations from the impl to talk about specific
+                // trait bounds.
+                let obligations = self
+                    .tcx
+                    .predicates_of(impl_def_id)
+                    .instantiate(self.tcx, args)
+                    .into_iter()
+                    .map(|(clause, span)| {
+                        traits::Obligation::new(
+                            self.tcx,
+                            traits::ObligationCause::dummy_with_span(span),
+                            param_env,
+                            clause,
+                        )
+                    })
+                    .collect::<Vec<_>>();
+                fulfill_cx.register_predicate_obligations(self, obligations);
+                let trait_ref = ty::TraitRef::new(self.tcx, trait_def_id, [ty]);
+                let obligation = traits::Obligation::new(
+                    self.tcx,
+                    traits::ObligationCause::dummy(),
+                    param_env,
+                    trait_ref,
+                );
+                fulfill_cx.register_predicate_obligation(self, obligation);
+                let mut errors = fulfill_cx.select_all_or_error(self);
+                // We remove the last predicate failure, which corresponds to
+                // the top-level obligation, because most of the type we only
+                // care about the other ones, *except* when it is the only one.
+                // This seems to only be relevant for arbitrary self-types.
+                // Look at `tests/ui/moves/move-fn-self-receiver.rs`.
+                if errors.len() > 1 {
+                    errors.truncate(errors.len() - 1);
+                }
+                Some(errors)
+            } else {
+                None
+            }
+        })
+    }
 }
 
 pub trait InferCtxtBuilderExt<'tcx> {
diff --git a/compiler/rustc_trait_selection/src/lib.rs b/compiler/rustc_trait_selection/src/lib.rs
index 5ba29f878..de2577cca 100644
--- a/compiler/rustc_trait_selection/src/lib.rs
+++ b/compiler/rustc_trait_selection/src/lib.rs
@@ -11,9 +11,9 @@
 //! This API is completely unstable and subject to change.
 
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
-#![cfg_attr(not(bootstrap), allow(internal_features))]
+#![doc(rust_logo)]
+#![feature(rustdoc_internals)]
+#![allow(internal_features)]
 #![feature(associated_type_bounds)]
 #![feature(box_patterns)]
 #![feature(control_flow_enum)]
@@ -21,7 +21,6 @@
 #![feature(let_chains)]
 #![feature(if_let_guard)]
 #![feature(never_type)]
-#![feature(result_option_inspect)]
 #![feature(type_alias_impl_trait)]
 #![feature(min_specialization)]
 #![recursion_limit = "512"] // For rustdoc
@@ -38,12 +37,9 @@ extern crate rustc_middle;
 #[macro_use]
 extern crate smallvec;
 
-use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
-use rustc_fluent_macro::fluent_messages;
-
 pub mod errors;
 pub mod infer;
 pub mod solve;
 pub mod traits;
 
-fluent_messages! { "../messages.ftl" }
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
diff --git a/compiler/rustc_trait_selection/src/solve/alias_relate.rs b/compiler/rustc_trait_selection/src/solve/alias_relate.rs
index f7031c5f4..626569fb4 100644
--- a/compiler/rustc_trait_selection/src/solve/alias_relate.rs
+++ b/compiler/rustc_trait_selection/src/solve/alias_relate.rs
@@ -11,18 +11,12 @@
 //! * bidirectional-normalizes-to: If `A` and `B` are both projections, and both
 //!   may apply, then we can compute the "intersection" of both normalizes-to by
 //!   performing them together. This is used specifically to resolve ambiguities.
-use super::{EvalCtxt, SolverMode};
+use super::{EvalCtxt, GoalSource};
+use rustc_infer::infer::DefineOpaqueTypes;
 use rustc_infer::traits::query::NoSolution;
 use rustc_middle::traits::solve::{Certainty, Goal, QueryResult};
 use rustc_middle::ty;
 
-/// We may need to invert the alias relation direction if dealing an alias on the RHS.
-#[derive(Debug)]
-enum Invert {
-    No,
-    Yes,
-}
-
 impl<'tcx> EvalCtxt<'_, 'tcx> {
     #[instrument(level = "debug", skip(self), ret)]
     pub(super) fn compute_alias_relate_goal(
@@ -31,187 +25,128 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
     ) -> QueryResult<'tcx> {
         let tcx = self.tcx();
         let Goal { param_env, predicate: (lhs, rhs, direction) } = goal;
-        if lhs.is_infer() || rhs.is_infer() {
-            bug!(
-                "`AliasRelate` goal with an infer var on lhs or rhs which should have been instantiated"
-            );
-        }
-
-        match (lhs.to_alias_ty(tcx), rhs.to_alias_ty(tcx)) {
-            (None, None) => bug!("`AliasRelate` goal without an alias on either lhs or rhs"),
 
-            // RHS is not a projection, only way this is true is if LHS normalizes-to RHS
-            (Some(alias_lhs), None) => self.assemble_normalizes_to_candidate(
-                param_env,
-                alias_lhs,
-                rhs,
-                direction,
-                Invert::No,
-            ),
+        let Some(lhs) = self.try_normalize_term(param_env, lhs)? else {
+            return self.evaluate_added_goals_and_make_canonical_response(Certainty::OVERFLOW);
+        };
 
-            // LHS is not a projection, only way this is true is if RHS normalizes-to LHS
-            (None, Some(alias_rhs)) => self.assemble_normalizes_to_candidate(
-                param_env,
-                alias_rhs,
-                lhs,
-                direction,
-                Invert::Yes,
-            ),
+        let Some(rhs) = self.try_normalize_term(param_env, rhs)? else {
+            return self.evaluate_added_goals_and_make_canonical_response(Certainty::OVERFLOW);
+        };
 
-            (Some(alias_lhs), Some(alias_rhs)) => {
-                debug!("both sides are aliases");
+        let variance = match direction {
+            ty::AliasRelationDirection::Equate => ty::Variance::Invariant,
+            ty::AliasRelationDirection::Subtype => ty::Variance::Covariant,
+        };
 
-                let mut candidates = Vec::new();
-                // LHS normalizes-to RHS
-                candidates.extend(self.assemble_normalizes_to_candidate(
-                    param_env,
-                    alias_lhs,
-                    rhs,
-                    direction,
-                    Invert::No,
-                ));
-                // RHS normalizes-to RHS
-                candidates.extend(self.assemble_normalizes_to_candidate(
-                    param_env,
-                    alias_rhs,
-                    lhs,
-                    direction,
-                    Invert::Yes,
-                ));
-                // Relate via args
-                candidates.extend(
-                    self.assemble_subst_relate_candidate(
-                        param_env, alias_lhs, alias_rhs, direction,
-                    ),
-                );
-                debug!(?candidates);
+        match (lhs.to_alias_ty(tcx), rhs.to_alias_ty(tcx)) {
+            (None, None) => {
+                self.relate(param_env, lhs, variance, rhs)?;
+                self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+            }
 
-                if let Some(merged) = self.try_merge_responses(&candidates) {
-                    Ok(merged)
+            (Some(alias), None) => {
+                if rhs.is_infer() {
+                    self.relate(param_env, lhs, variance, rhs)?;
+                    self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+                } else if alias.is_opaque(tcx) {
+                    self.define_opaque(param_env, alias, rhs)
                 } else {
-                    // When relating two aliases and we have ambiguity, if both
-                    // aliases can be normalized to something, we prefer
-                    // "bidirectionally normalizing" both of them within the same
-                    // candidate.
-                    //
-                    // See <https://github.com/rust-lang/trait-system-refactor-initiative/issues/25>.
-                    //
-                    // As this is incomplete, we must not do so during coherence.
-                    match self.solver_mode() {
-                        SolverMode::Normal => {
-                            if let Ok(bidirectional_normalizes_to_response) = self
-                                .assemble_bidirectional_normalizes_to_candidate(
-                                    param_env, lhs, rhs, direction,
-                                )
-                            {
-                                Ok(bidirectional_normalizes_to_response)
-                            } else {
-                                self.flounder(&candidates)
-                            }
-                        }
-                        SolverMode::Coherence => self.flounder(&candidates),
-                    }
+                    Err(NoSolution)
+                }
+            }
+            (None, Some(alias)) => {
+                if lhs.is_infer() {
+                    self.relate(param_env, lhs, variance, rhs)?;
+                    self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+                } else if alias.is_opaque(tcx) {
+                    self.define_opaque(param_env, alias, lhs)
+                } else {
+                    Err(NoSolution)
                 }
             }
-        }
-    }
 
-    #[instrument(level = "debug", skip(self), ret)]
-    fn assemble_normalizes_to_candidate(
-        &mut self,
-        param_env: ty::ParamEnv<'tcx>,
-        alias: ty::AliasTy<'tcx>,
-        other: ty::Term<'tcx>,
-        direction: ty::AliasRelationDirection,
-        invert: Invert,
-    ) -> QueryResult<'tcx> {
-        self.probe_misc_candidate("normalizes-to").enter(|ecx| {
-            ecx.normalizes_to_inner(param_env, alias, other, direction, invert)?;
-            ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
-        })
+            (Some(alias_lhs), Some(alias_rhs)) => {
+                self.relate_rigid_alias_or_opaque(param_env, alias_lhs, variance, alias_rhs)
+            }
+        }
     }
 
-    // Computes the normalizes-to branch, with side-effects. This must be performed
-    // in a probe in order to not taint the evaluation context.
-    fn normalizes_to_inner(
+    /// Normalize the `term` to equate it later. This does not define opaque types.
+    #[instrument(level = "debug", skip(self, param_env), ret)]
+    fn try_normalize_term(
         &mut self,
         param_env: ty::ParamEnv<'tcx>,
-        alias: ty::AliasTy<'tcx>,
-        other: ty::Term<'tcx>,
-        direction: ty::AliasRelationDirection,
-        invert: Invert,
-    ) -> Result<(), NoSolution> {
-        let other = match direction {
-            // This is purely an optimization. No need to instantiate a new
-            // infer var and equate the RHS to it.
-            ty::AliasRelationDirection::Equate => other,
-
-            // Instantiate an infer var and subtype our RHS to it, so that we
-            // properly represent a subtype relation between the LHS and RHS
-            // of the goal.
-            ty::AliasRelationDirection::Subtype => {
-                let fresh = self.next_term_infer_of_kind(other);
-                let (sub, sup) = match invert {
-                    Invert::No => (fresh, other),
-                    Invert::Yes => (other, fresh),
-                };
-                self.sub(param_env, sub, sup)?;
-                fresh
+        term: ty::Term<'tcx>,
+    ) -> Result<Option<ty::Term<'tcx>>, NoSolution> {
+        match term.unpack() {
+            ty::TermKind::Ty(ty) => {
+                // We do no define opaque types here but instead do so in `relate_rigid_alias_or_opaque`.
+                Ok(self
+                    .try_normalize_ty_recur(param_env, DefineOpaqueTypes::No, 0, ty)
+                    .map(Into::into))
             }
-        };
-        self.add_goal(Goal::new(
-            self.tcx(),
-            param_env,
-            ty::ProjectionPredicate { projection_ty: alias, term: other },
-        ));
-
-        Ok(())
+            ty::TermKind::Const(_) => {
+                if let Some(alias) = term.to_alias_ty(self.tcx()) {
+                    let term = self.next_term_infer_of_kind(term);
+                    self.add_goal(
+                        GoalSource::Misc,
+                        Goal::new(self.tcx(), param_env, ty::NormalizesTo { alias, term }),
+                    );
+                    self.try_evaluate_added_goals()?;
+                    Ok(Some(self.resolve_vars_if_possible(term)))
+                } else {
+                    Ok(Some(term))
+                }
+            }
+        }
     }
 
-    fn assemble_subst_relate_candidate(
+    fn define_opaque(
         &mut self,
         param_env: ty::ParamEnv<'tcx>,
-        alias_lhs: ty::AliasTy<'tcx>,
-        alias_rhs: ty::AliasTy<'tcx>,
-        direction: ty::AliasRelationDirection,
+        opaque: ty::AliasTy<'tcx>,
+        term: ty::Term<'tcx>,
     ) -> QueryResult<'tcx> {
-        self.probe_misc_candidate("args relate").enter(|ecx| {
-            match direction {
-                ty::AliasRelationDirection::Equate => {
-                    ecx.eq(param_env, alias_lhs, alias_rhs)?;
-                }
-                ty::AliasRelationDirection::Subtype => {
-                    ecx.sub(param_env, alias_lhs, alias_rhs)?;
-                }
-            }
-
-            ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
-        })
+        self.add_goal(
+            GoalSource::Misc,
+            Goal::new(self.tcx(), param_env, ty::NormalizesTo { alias: opaque, term }),
+        );
+        self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
     }
 
-    fn assemble_bidirectional_normalizes_to_candidate(
+    fn relate_rigid_alias_or_opaque(
         &mut self,
         param_env: ty::ParamEnv<'tcx>,
-        lhs: ty::Term<'tcx>,
-        rhs: ty::Term<'tcx>,
-        direction: ty::AliasRelationDirection,
+        lhs: ty::AliasTy<'tcx>,
+        variance: ty::Variance,
+        rhs: ty::AliasTy<'tcx>,
     ) -> QueryResult<'tcx> {
-        self.probe_misc_candidate("bidir normalizes-to").enter(|ecx| {
-            ecx.normalizes_to_inner(
-                param_env,
-                lhs.to_alias_ty(ecx.tcx()).unwrap(),
-                rhs,
-                direction,
-                Invert::No,
-            )?;
-            ecx.normalizes_to_inner(
-                param_env,
-                rhs.to_alias_ty(ecx.tcx()).unwrap(),
-                lhs,
-                direction,
-                Invert::Yes,
-            )?;
+        let tcx = self.tcx();
+        let mut candidates = vec![];
+        if lhs.is_opaque(tcx) {
+            candidates.extend(
+                self.probe_misc_candidate("define-lhs-opaque")
+                    .enter(|ecx| ecx.define_opaque(param_env, lhs, rhs.to_ty(tcx).into())),
+            );
+        }
+
+        if rhs.is_opaque(tcx) {
+            candidates.extend(
+                self.probe_misc_candidate("define-rhs-opaque")
+                    .enter(|ecx| ecx.define_opaque(param_env, rhs, lhs.to_ty(tcx).into())),
+            );
+        }
+
+        candidates.extend(self.probe_misc_candidate("args-relate").enter(|ecx| {
+            ecx.relate(param_env, lhs, variance, rhs)?;
             ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
-        })
+        }));
+
+        if let Some(result) = self.try_merge_responses(&candidates) {
+            Ok(result)
+        } else {
+            self.flounder(&candidates)
+        }
     }
 }
diff --git a/compiler/rustc_trait_selection/src/solve/assembly/mod.rs b/compiler/rustc_trait_selection/src/solve/assembly/mod.rs
index 27d2bdead..81a766f24 100644
--- a/compiler/rustc_trait_selection/src/solve/assembly/mod.rs
+++ b/compiler/rustc_trait_selection/src/solve/assembly/mod.rs
@@ -1,6 +1,7 @@
 //! Code shared by trait and projection goals for candidate assembly.
 
 use super::{EvalCtxt, SolverMode};
+use crate::solve::GoalSource;
 use crate::traits::coherence;
 use rustc_hir::def_id::DefId;
 use rustc_infer::traits::query::NoSolution;
@@ -37,8 +38,6 @@ pub(super) trait GoalKind<'tcx>:
 
     fn trait_ref(self, tcx: TyCtxt<'tcx>) -> ty::TraitRef<'tcx>;
 
-    fn polarity(self) -> ty::ImplPolarity;
-
     fn with_self_ty(self, tcx: TyCtxt<'tcx>, self_ty: Ty<'tcx>) -> Self;
 
     fn trait_def_id(self, tcx: TyCtxt<'tcx>) -> DefId;
@@ -64,7 +63,9 @@ pub(super) trait GoalKind<'tcx>:
         requirements: impl IntoIterator<Item = Goal<'tcx, ty::Predicate<'tcx>>>,
     ) -> QueryResult<'tcx> {
         Self::probe_and_match_goal_against_assumption(ecx, goal, assumption, |ecx| {
-            ecx.add_goals(requirements);
+            // FIXME(-Znext-solver=coinductive): check whether this should be
+            // `GoalSource::ImplWhereBound` for any caller.
+            ecx.add_goals(GoalSource::Misc, requirements);
             ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
         })
     }
@@ -96,12 +97,16 @@ pub(super) trait GoalKind<'tcx>:
             let ty::Dynamic(bounds, _, _) = *goal.predicate.self_ty().kind() else {
                 bug!("expected object type in `consider_object_bound_candidate`");
             };
-            ecx.add_goals(structural_traits::predicates_for_object_candidate(
-                &ecx,
-                goal.param_env,
-                goal.predicate.trait_ref(tcx),
-                bounds,
-            ));
+            // FIXME(-Znext-solver=coinductive): Should this be `GoalSource::ImplWhereBound`?
+            ecx.add_goals(
+                GoalSource::Misc,
+                structural_traits::predicates_for_object_candidate(
+                    ecx,
+                    goal.param_env,
+                    goal.predicate.trait_ref(tcx),
+                    bounds,
+                ),
+            );
             ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
         })
     }
@@ -110,7 +115,7 @@ pub(super) trait GoalKind<'tcx>:
         ecx: &mut EvalCtxt<'_, 'tcx>,
         goal: Goal<'tcx, Self>,
         impl_def_id: DefId,
-    ) -> QueryResult<'tcx>;
+    ) -> Result<Candidate<'tcx>, NoSolution>;
 
     /// If the predicate contained an error, we want to avoid emitting unnecessary trait
     /// errors but still want to emit errors for other trait goals. We have some special
@@ -209,6 +214,11 @@ pub(super) trait GoalKind<'tcx>:
         goal: Goal<'tcx, Self>,
     ) -> QueryResult<'tcx>;
 
+    fn consider_builtin_async_iterator_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx>;
+
     /// A coroutine (that doesn't come from an `async` or `gen` desugaring) is known to
     /// implement `Coroutine<R, Yield = Y, Return = O>`, given the resume, yield,
     /// and return types of the coroutine computed during type-checking.
@@ -263,7 +273,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
     ) -> Vec<Candidate<'tcx>> {
         debug_assert_eq!(goal, self.resolve_vars_if_possible(goal));
         if let Some(ambig) = self.assemble_self_ty_infer_ambiguity_response(goal) {
-            return ambig;
+            return vec![ambig];
         }
 
         let mut candidates = self.assemble_candidates_via_self_ty(goal, 0);
@@ -288,15 +298,20 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
     fn assemble_self_ty_infer_ambiguity_response<G: GoalKind<'tcx>>(
         &mut self,
         goal: Goal<'tcx, G>,
-    ) -> Option<Vec<Candidate<'tcx>>> {
-        goal.predicate.self_ty().is_ty_var().then(|| {
-            vec![Candidate {
-                source: CandidateSource::BuiltinImpl(BuiltinImplSource::Misc),
-                result: self
-                    .evaluate_added_goals_and_make_canonical_response(Certainty::AMBIGUOUS)
-                    .unwrap(),
-            }]
-        })
+    ) -> Option<Candidate<'tcx>> {
+        if goal.predicate.self_ty().is_ty_var() {
+            debug!("adding self_ty_infer_ambiguity_response");
+            let source = CandidateSource::BuiltinImpl(BuiltinImplSource::Misc);
+            let result = self
+                .evaluate_added_goals_and_make_canonical_response(Certainty::AMBIGUOUS)
+                .unwrap();
+            let mut dummy_probe = self.inspect.new_probe();
+            dummy_probe.probe_kind(ProbeKind::TraitCandidate { source, result: Ok(result) });
+            self.inspect.finish_probe(dummy_probe);
+            Some(Candidate { source, result })
+        } else {
+            None
+        }
     }
 
     /// Assemble candidates which apply to the self type. This only looks at candidate which
@@ -310,7 +325,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
     ) -> Vec<Candidate<'tcx>> {
         debug_assert_eq!(goal, self.resolve_vars_if_possible(goal));
         if let Some(ambig) = self.assemble_self_ty_infer_ambiguity_response(goal) {
-            return ambig;
+            return vec![ambig];
         }
 
         let mut candidates = Vec::new();
@@ -349,16 +364,14 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         num_steps: usize,
     ) {
         let tcx = self.tcx();
-        let &ty::Alias(_, projection_ty) = goal.predicate.self_ty().kind() else { return };
+        let &ty::Alias(_, alias) = goal.predicate.self_ty().kind() else { return };
 
         candidates.extend(self.probe(|_| ProbeKind::NormalizedSelfTyAssembly).enter(|ecx| {
-            if num_steps < ecx.local_overflow_limit() {
+            if tcx.recursion_limit().value_within_limit(num_steps) {
                 let normalized_ty = ecx.next_ty_infer();
-                let normalizes_to_goal = goal.with(
-                    tcx,
-                    ty::ProjectionPredicate { projection_ty, term: normalized_ty.into() },
-                );
-                ecx.add_goal(normalizes_to_goal);
+                let normalizes_to_goal =
+                    goal.with(tcx, ty::NormalizesTo { alias, term: normalized_ty.into() });
+                ecx.add_goal(GoalSource::Misc, normalizes_to_goal);
                 if let Err(NoSolution) = ecx.try_evaluate_added_goals() {
                     debug!("self type normalization failed");
                     return vec![];
@@ -395,8 +408,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
             if let Some(impls_for_type) = trait_impls.non_blanket_impls().get(&simp) {
                 for &impl_def_id in impls_for_type {
                     match G::consider_impl_candidate(self, goal, impl_def_id) {
-                        Ok(result) => candidates
-                            .push(Candidate { source: CandidateSource::Impl(impl_def_id), result }),
+                        Ok(candidate) => candidates.push(candidate),
                         Err(NoSolution) => (),
                     }
                 }
@@ -488,6 +500,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         }
     }
 
+    #[instrument(level = "debug", skip_all)]
     fn assemble_unsize_to_dyn_candidate<G: GoalKind<'tcx>>(
         &mut self,
         goal: Goal<'tcx, G>,
@@ -505,6 +518,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         }
     }
 
+    #[instrument(level = "debug", skip_all)]
     fn assemble_blanket_impl_candidates<G: GoalKind<'tcx>>(
         &mut self,
         goal: Goal<'tcx, G>,
@@ -514,8 +528,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         let trait_impls = tcx.trait_impls_of(goal.predicate.trait_def_id(tcx));
         for &impl_def_id in trait_impls.blanket_impls() {
             match G::consider_impl_candidate(self, goal, impl_def_id) {
-                Ok(result) => candidates
-                    .push(Candidate { source: CandidateSource::Impl(impl_def_id), result }),
+                Ok(candidate) => candidates.push(candidate),
                 Err(NoSolution) => (),
             }
         }
@@ -564,6 +577,8 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
             G::consider_builtin_future_candidate(self, goal)
         } else if lang_items.iterator_trait() == Some(trait_def_id) {
             G::consider_builtin_iterator_candidate(self, goal)
+        } else if lang_items.async_iterator_trait() == Some(trait_def_id) {
+            G::consider_builtin_async_iterator_candidate(self, goal)
         } else if lang_items.coroutine_trait() == Some(trait_def_id) {
             G::consider_builtin_coroutine_candidate(self, goal)
         } else if lang_items.discriminant_kind_trait() == Some(trait_def_id) {
@@ -864,23 +879,18 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
 
         let result = self.probe_misc_candidate("coherence unknowable").enter(|ecx| {
             let trait_ref = goal.predicate.trait_ref(tcx);
-
             #[derive(Debug)]
-            enum FailureKind {
-                Overflow,
-                NoSolution(NoSolution),
-            }
+            struct Overflow;
             let lazily_normalize_ty = |ty| match ecx.try_normalize_ty(goal.param_env, ty) {
-                Ok(Some(ty)) => Ok(ty),
-                Ok(None) => Err(FailureKind::Overflow),
-                Err(e) => Err(FailureKind::NoSolution(e)),
+                Some(ty) => Ok(ty),
+                None => Err(Overflow),
             };
 
             match coherence::trait_ref_is_knowable(tcx, trait_ref, lazily_normalize_ty) {
-                Err(FailureKind::Overflow) => {
+                Err(Overflow) => {
                     ecx.evaluate_added_goals_and_make_canonical_response(Certainty::OVERFLOW)
                 }
-                Err(FailureKind::NoSolution(NoSolution)) | Ok(Ok(())) => Err(NoSolution),
+                Ok(Ok(())) => Err(NoSolution),
                 Ok(Err(_)) => {
                     ecx.evaluate_added_goals_and_make_canonical_response(Certainty::AMBIGUOUS)
                 }
diff --git a/compiler/rustc_trait_selection/src/solve/assembly/structural_traits.rs b/compiler/rustc_trait_selection/src/solve/assembly/structural_traits.rs
index 839968b25..f442e2a08 100644
--- a/compiler/rustc_trait_selection/src/solve/assembly/structural_traits.rs
+++ b/compiler/rustc_trait_selection/src/solve/assembly/structural_traits.rs
@@ -50,14 +50,14 @@ pub(in crate::solve) fn instantiate_constituent_tys_for_auto_trait<'tcx>(
 
         ty::Array(element_ty, _) | ty::Slice(element_ty) => Ok(vec![element_ty]),
 
-        ty::Tuple(ref tys) => {
+        ty::Tuple(tys) => {
             // (T1, ..., Tn) -- meets any bound that all of T1...Tn meet
             Ok(tys.iter().collect())
         }
 
-        ty::Closure(_, ref args) => Ok(vec![args.as_closure().tupled_upvars_ty()]),
+        ty::Closure(_, args) => Ok(vec![args.as_closure().tupled_upvars_ty()]),
 
-        ty::Coroutine(_, ref args, _) => {
+        ty::Coroutine(_, args, _) => {
             let coroutine_args = args.as_coroutine();
             Ok(vec![coroutine_args.tupled_upvars_ty(), coroutine_args.witness()])
         }
@@ -91,13 +91,13 @@ pub(in crate::solve) fn replace_erased_lifetimes_with_bound_vars<'tcx>(
     tcx: TyCtxt<'tcx>,
     ty: Ty<'tcx>,
 ) -> ty::Binder<'tcx, Ty<'tcx>> {
-    debug_assert!(!ty.has_late_bound_regions());
+    debug_assert!(!ty.has_bound_regions());
     let mut counter = 0;
     let ty = tcx.fold_regions(ty, |r, current_depth| match r.kind() {
         ty::ReErased => {
             let br = ty::BoundRegion { var: ty::BoundVar::from_u32(counter), kind: ty::BrAnon };
             counter += 1;
-            ty::Region::new_late_bound(tcx, current_depth, br)
+            ty::Region::new_bound(tcx, current_depth, br)
         }
         // All free regions should be erased here.
         r => bug!("unexpected region: {r:?}"),
diff --git a/compiler/rustc_trait_selection/src/solve/eval_ctxt/canonical.rs b/compiler/rustc_trait_selection/src/solve/eval_ctxt/canonical.rs
index b3f9218d7..ecdae2521 100644
--- a/compiler/rustc_trait_selection/src/solve/eval_ctxt/canonical.rs
+++ b/compiler/rustc_trait_selection/src/solve/eval_ctxt/canonical.rs
@@ -9,7 +9,6 @@
 //!
 //! [c]: https://rustc-dev-guide.rust-lang.org/solve/canonicalization.html
 use super::{CanonicalInput, Certainty, EvalCtxt, Goal};
-use crate::solve::canonicalize::{CanonicalizeMode, Canonicalizer};
 use crate::solve::{
     inspect, response_no_constraints_raw, CanonicalResponse, QueryResult, Response,
 };
@@ -18,6 +17,7 @@ use rustc_index::IndexVec;
 use rustc_infer::infer::canonical::query_response::make_query_region_constraints;
 use rustc_infer::infer::canonical::CanonicalVarValues;
 use rustc_infer::infer::canonical::{CanonicalExt, QueryRegionConstraints};
+use rustc_infer::infer::resolve::EagerResolver;
 use rustc_infer::infer::{DefineOpaqueTypes, InferCtxt, InferOk};
 use rustc_middle::infer::canonical::Canonical;
 use rustc_middle::traits::query::NoSolution;
@@ -25,10 +25,8 @@ use rustc_middle::traits::solve::{
     ExternalConstraintsData, MaybeCause, PredefinedOpaquesData, QueryInput,
 };
 use rustc_middle::traits::ObligationCause;
-use rustc_middle::ty::{
-    self, BoundVar, GenericArgKind, Ty, TyCtxt, TypeFoldable, TypeFolder, TypeSuperFoldable,
-    TypeVisitableExt,
-};
+use rustc_middle::ty::{self, BoundVar, GenericArgKind, Ty, TyCtxt, TypeFoldable};
+use rustc_next_trait_solver::canonicalizer::{CanonicalizeMode, Canonicalizer};
 use rustc_span::DUMMY_SP;
 use std::iter;
 use std::ops::Deref;
@@ -58,7 +56,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
     ) -> (Vec<ty::GenericArg<'tcx>>, CanonicalInput<'tcx, T>) {
         let opaque_types = self.infcx.clone_opaque_types_for_query_response();
         let (goal, opaque_types) =
-            (goal, opaque_types).fold_with(&mut EagerResolver { infcx: self.infcx });
+            (goal, opaque_types).fold_with(&mut EagerResolver::new(self.infcx));
 
         let mut orig_values = Default::default();
         let canonical_goal = Canonicalizer::canonicalize(
@@ -96,26 +94,12 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         );
 
         let certainty = certainty.unify_with(goals_certainty);
-        if let Certainty::OVERFLOW = certainty {
-            // If we have overflow, it's probable that we're substituting a type
-            // into itself infinitely and any partial substitutions in the query
-            // response are probably not useful anyways, so just return an empty
-            // query response.
-            //
-            // This may prevent us from potentially useful inference, e.g.
-            // 2 candidates, one ambiguous and one overflow, which both
-            // have the same inference constraints.
-            //
-            // Changing this to retain some constraints in the future
-            // won't be a breaking change, so this is good enough for now.
-            return Ok(self.make_ambiguous_response_no_constraints(MaybeCause::Overflow));
-        }
 
         let var_values = self.var_values;
         let external_constraints = self.compute_external_query_constraints()?;
 
         let (var_values, mut external_constraints) =
-            (var_values, external_constraints).fold_with(&mut EagerResolver { infcx: self.infcx });
+            (var_values, external_constraints).fold_with(&mut EagerResolver::new(self.infcx));
         // Remove any trivial region constraints once we've resolved regions
         external_constraints
             .region_constraints
@@ -262,7 +246,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
                     }
                 }
                 GenericArgKind::Lifetime(r) => {
-                    if let ty::ReLateBound(debruijn, br) = *r {
+                    if let ty::ReBound(debruijn, br) = *r {
                         assert_eq!(debruijn, ty::INNERMOST);
                         opt_values[br.var] = Some(*original_value);
                     }
@@ -364,86 +348,13 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
     }
 }
 
-/// Resolves ty, region, and const vars to their inferred values or their root vars.
-struct EagerResolver<'a, 'tcx> {
-    infcx: &'a InferCtxt<'tcx>,
-}
-
-impl<'tcx> TypeFolder<TyCtxt<'tcx>> for EagerResolver<'_, 'tcx> {
-    fn interner(&self) -> TyCtxt<'tcx> {
-        self.infcx.tcx
-    }
-
-    fn fold_ty(&mut self, t: Ty<'tcx>) -> Ty<'tcx> {
-        match *t.kind() {
-            ty::Infer(ty::TyVar(vid)) => match self.infcx.probe_ty_var(vid) {
-                Ok(t) => t.fold_with(self),
-                Err(_) => Ty::new_var(self.infcx.tcx, self.infcx.root_var(vid)),
-            },
-            ty::Infer(ty::IntVar(vid)) => self.infcx.opportunistic_resolve_int_var(vid),
-            ty::Infer(ty::FloatVar(vid)) => self.infcx.opportunistic_resolve_float_var(vid),
-            _ => {
-                if t.has_infer() {
-                    t.super_fold_with(self)
-                } else {
-                    t
-                }
-            }
-        }
-    }
-
-    fn fold_region(&mut self, r: ty::Region<'tcx>) -> ty::Region<'tcx> {
-        match *r {
-            ty::ReVar(vid) => self
-                .infcx
-                .inner
-                .borrow_mut()
-                .unwrap_region_constraints()
-                .opportunistic_resolve_var(self.infcx.tcx, vid),
-            _ => r,
-        }
-    }
-
-    fn fold_const(&mut self, c: ty::Const<'tcx>) -> ty::Const<'tcx> {
-        match c.kind() {
-            ty::ConstKind::Infer(ty::InferConst::Var(vid)) => {
-                // FIXME: we need to fold the ty too, I think.
-                match self.infcx.probe_const_var(vid) {
-                    Ok(c) => c.fold_with(self),
-                    Err(_) => {
-                        ty::Const::new_var(self.infcx.tcx, self.infcx.root_const_var(vid), c.ty())
-                    }
-                }
-            }
-            ty::ConstKind::Infer(ty::InferConst::EffectVar(vid)) => {
-                debug_assert_eq!(c.ty(), self.infcx.tcx.types.bool);
-                match self.infcx.probe_effect_var(vid) {
-                    Some(c) => c.as_const(self.infcx.tcx),
-                    None => ty::Const::new_infer(
-                        self.infcx.tcx,
-                        ty::InferConst::EffectVar(self.infcx.root_effect_var(vid)),
-                        self.infcx.tcx.types.bool,
-                    ),
-                }
-            }
-            _ => {
-                if c.has_infer() {
-                    c.super_fold_with(self)
-                } else {
-                    c
-                }
-            }
-        }
-    }
-}
-
 impl<'tcx> inspect::ProofTreeBuilder<'tcx> {
     pub fn make_canonical_state<T: TypeFoldable<TyCtxt<'tcx>>>(
         ecx: &EvalCtxt<'_, 'tcx>,
         data: T,
     ) -> inspect::CanonicalState<'tcx, T> {
         let state = inspect::State { var_values: ecx.var_values, data };
-        let state = state.fold_with(&mut EagerResolver { infcx: ecx.infcx });
+        let state = state.fold_with(&mut EagerResolver::new(ecx.infcx));
         Canonicalizer::canonicalize(
             ecx.infcx,
             CanonicalizeMode::Response { max_input_universe: ecx.max_input_universe },
diff --git a/compiler/rustc_trait_selection/src/solve/eval_ctxt/commit_if_ok.rs b/compiler/rustc_trait_selection/src/solve/eval_ctxt/commit_if_ok.rs
new file mode 100644
index 000000000..67b680105
--- /dev/null
+++ b/compiler/rustc_trait_selection/src/solve/eval_ctxt/commit_if_ok.rs
@@ -0,0 +1,45 @@
+use super::{EvalCtxt, NestedGoals};
+use crate::solve::inspect;
+use rustc_middle::traits::query::NoSolution;
+
+impl<'a, 'tcx> EvalCtxt<'a, 'tcx> {
+    pub(in crate::solve) fn commit_if_ok<T>(
+        &mut self,
+        f: impl FnOnce(&mut EvalCtxt<'_, 'tcx>) -> Result<T, NoSolution>,
+    ) -> Result<T, NoSolution> {
+        let mut nested_ecx = EvalCtxt {
+            infcx: self.infcx,
+            variables: self.variables,
+            var_values: self.var_values,
+            predefined_opaques_in_body: self.predefined_opaques_in_body,
+            max_input_universe: self.max_input_universe,
+            search_graph: self.search_graph,
+            nested_goals: NestedGoals::new(),
+            tainted: self.tainted,
+            inspect: self.inspect.new_probe(),
+        };
+
+        let result = nested_ecx.infcx.commit_if_ok(|_| f(&mut nested_ecx));
+        if result.is_ok() {
+            let EvalCtxt {
+                infcx: _,
+                variables: _,
+                var_values: _,
+                predefined_opaques_in_body: _,
+                max_input_universe: _,
+                search_graph: _,
+                nested_goals,
+                tainted,
+                inspect,
+            } = nested_ecx;
+            self.nested_goals.extend(nested_goals);
+            self.tainted = tainted;
+            self.inspect.integrate_snapshot(inspect);
+        } else {
+            nested_ecx.inspect.probe_kind(inspect::ProbeKind::CommitIfOk);
+            self.inspect.finish_probe(nested_ecx.inspect);
+        }
+
+        result
+    }
+}
diff --git a/compiler/rustc_trait_selection/src/solve/eval_ctxt/mod.rs b/compiler/rustc_trait_selection/src/solve/eval_ctxt/mod.rs
index 70235b710..76c50a111 100644
--- a/compiler/rustc_trait_selection/src/solve/eval_ctxt/mod.rs
+++ b/compiler/rustc_trait_selection/src/solve/eval_ctxt/mod.rs
@@ -4,7 +4,7 @@ use rustc_infer::infer::at::ToTrace;
 use rustc_infer::infer::canonical::CanonicalVarValues;
 use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
 use rustc_infer::infer::{
-    DefineOpaqueTypes, InferCtxt, InferOk, LateBoundRegionConversionTime, TyCtxtInferExt,
+    BoundRegionConversionTime, DefineOpaqueTypes, InferCtxt, InferOk, TyCtxtInferExt,
 };
 use rustc_infer::traits::query::NoSolution;
 use rustc_infer::traits::ObligationCause;
@@ -23,17 +23,19 @@ use rustc_middle::ty::{
 use rustc_session::config::DumpSolverProofTree;
 use rustc_span::DUMMY_SP;
 use std::io::Write;
+use std::iter;
 use std::ops::ControlFlow;
 
 use crate::traits::vtable::{count_own_vtable_entries, prepare_vtable_segments, VtblSegment};
 
 use super::inspect::ProofTreeBuilder;
-use super::SolverMode;
 use super::{search_graph, GoalEvaluationKind};
 use super::{search_graph::SearchGraph, Goal};
+use super::{GoalSource, SolverMode};
 pub use select::InferCtxtSelectExt;
 
 mod canonical;
+mod commit_if_ok;
 mod probe;
 mod select;
 
@@ -102,12 +104,12 @@ pub(super) struct NestedGoals<'tcx> {
     /// with a fresh inference variable when we evaluate this goal. That can result
     /// in a trait solver cycle. This would currently result in overflow but can be
     /// can be unsound with more powerful coinduction in the future.
-    pub(super) normalizes_to_hack_goal: Option<Goal<'tcx, ty::ProjectionPredicate<'tcx>>>,
+    pub(super) normalizes_to_hack_goal: Option<Goal<'tcx, ty::NormalizesTo<'tcx>>>,
     /// The rest of the goals which have not yet processed or remain ambiguous.
-    pub(super) goals: Vec<Goal<'tcx, ty::Predicate<'tcx>>>,
+    pub(super) goals: Vec<(GoalSource, Goal<'tcx, ty::Predicate<'tcx>>)>,
 }
 
-impl NestedGoals<'_> {
+impl<'tcx> NestedGoals<'tcx> {
     pub(super) fn new() -> Self {
         Self { normalizes_to_hack_goal: None, goals: Vec::new() }
     }
@@ -115,6 +117,11 @@ impl NestedGoals<'_> {
     pub(super) fn is_empty(&self) -> bool {
         self.normalizes_to_hack_goal.is_none() && self.goals.is_empty()
     }
+
+    pub(super) fn extend(&mut self, other: NestedGoals<'tcx>) {
+        assert_eq!(other.normalizes_to_hack_goal, None);
+        self.goals.extend(other.goals)
+    }
 }
 
 #[derive(PartialEq, Eq, Debug, Hash, HashStable, Clone, Copy)]
@@ -140,7 +147,7 @@ pub trait InferCtxtEvalExt<'tcx> {
 }
 
 impl<'tcx> InferCtxtEvalExt<'tcx> for InferCtxt<'tcx> {
-    #[instrument(level = "debug", skip(self), ret)]
+    #[instrument(level = "debug", skip(self))]
     fn evaluate_root_goal(
         &self,
         goal: Goal<'tcx, ty::Predicate<'tcx>>,
@@ -150,7 +157,7 @@ impl<'tcx> InferCtxtEvalExt<'tcx> for InferCtxt<'tcx> {
         Option<inspect::GoalEvaluation<'tcx>>,
     ) {
         EvalCtxt::enter_root(self, generate_proof_tree, |ecx| {
-            ecx.evaluate_goal(GoalEvaluationKind::Root, goal)
+            ecx.evaluate_goal(GoalEvaluationKind::Root, GoalSource::Misc, goal)
         })
     }
 }
@@ -194,9 +201,10 @@ impl<'a, 'tcx> EvalCtxt<'a, 'tcx> {
         let result = f(&mut ecx);
 
         let tree = ecx.inspect.finalize();
-        if let (Some(tree), DumpSolverProofTree::Always) =
-            (&tree, infcx.tcx.sess.opts.unstable_opts.dump_solver_proof_tree)
-        {
+        if let (Some(tree), DumpSolverProofTree::Always) = (
+            &tree,
+            infcx.tcx.sess.opts.unstable_opts.next_solver.map(|c| c.dump_tree).unwrap_or_default(),
+        ) {
             let mut lock = std::io::stdout().lock();
             let _ = lock.write_fmt(format_args!("{tree:?}\n"));
             let _ = lock.flush();
@@ -327,12 +335,12 @@ impl<'a, 'tcx> EvalCtxt<'a, 'tcx> {
     fn evaluate_goal(
         &mut self,
         goal_evaluation_kind: GoalEvaluationKind,
+        source: GoalSource,
         goal: Goal<'tcx, ty::Predicate<'tcx>>,
     ) -> Result<(bool, Certainty, Vec<Goal<'tcx, ty::Predicate<'tcx>>>), NoSolution> {
         let (orig_values, canonical_goal) = self.canonicalize_goal(goal);
         let mut goal_evaluation =
             self.inspect.new_goal_evaluation(goal, &orig_values, goal_evaluation_kind);
-        let encountered_overflow = self.search_graph.encountered_overflow();
         let canonical_response = EvalCtxt::evaluate_canonical_goal(
             self.tcx(),
             self.search_graph,
@@ -347,13 +355,13 @@ impl<'a, 'tcx> EvalCtxt<'a, 'tcx> {
             Ok(response) => response,
         };
 
-        let has_changed = !canonical_response.value.var_values.is_identity_modulo_regions()
-            || !canonical_response.value.external_constraints.opaque_types.is_empty();
-        let (certainty, nested_goals) = match self.instantiate_and_apply_query_response(
-            goal.param_env,
-            orig_values,
-            canonical_response,
-        ) {
+        let (certainty, has_changed, nested_goals) = match self
+            .instantiate_response_discarding_overflow(
+                goal.param_env,
+                source,
+                orig_values,
+                canonical_response,
+            ) {
             Err(e) => {
                 self.inspect.goal_evaluation(goal_evaluation);
                 return Err(e);
@@ -367,72 +375,54 @@ impl<'a, 'tcx> EvalCtxt<'a, 'tcx> {
             bug!("an unchanged goal shouldn't have any side-effects on instantiation");
         }
 
-        // Check that rerunning this query with its inference constraints applied
-        // doesn't result in new inference constraints and has the same result.
+        // FIXME: We previously had an assert here that checked that recomputing
+        // a goal after applying its constraints did not change its response.
         //
-        // If we have projection goals like `<T as Trait>::Assoc == u32` we recursively
-        // call `exists<U> <T as Trait>::Assoc == U` to enable better caching. This goal
-        // could constrain `U` to `u32` which would cause this check to result in a
-        // solver cycle.
-        if cfg!(debug_assertions)
-            && has_changed
-            && !matches!(
-                goal_evaluation_kind,
-                GoalEvaluationKind::Nested { is_normalizes_to_hack: IsNormalizesToHack::Yes }
-            )
-            && !self.search_graph.in_cycle()
-        {
-            // The nested evaluation has to happen with the original state
-            // of `encountered_overflow`.
-            let from_original_evaluation =
-                self.search_graph.reset_encountered_overflow(encountered_overflow);
-            self.check_evaluate_goal_stable_result(goal, canonical_goal, canonical_response);
-            // In case the evaluation was unstable, we manually make sure that this
-            // debug check does not influence the result of the parent goal.
-            self.search_graph.reset_encountered_overflow(from_original_evaluation);
-        }
+        // This assert was removed as it did not hold for goals constraining
+        // an inference variable to a recursive alias, e.g. in
+        // tests/ui/traits/next-solver/overflow/recursive-self-normalization.rs.
+        //
+        // Once we have decided on how to handle trait-system-refactor-initiative#75,
+        // we should re-add an assert here.
 
         Ok((has_changed, certainty, nested_goals))
     }
 
-    fn check_evaluate_goal_stable_result(
+    fn instantiate_response_discarding_overflow(
         &mut self,
-        goal: Goal<'tcx, ty::Predicate<'tcx>>,
-        original_input: CanonicalInput<'tcx>,
-        original_result: CanonicalResponse<'tcx>,
-    ) {
-        let (_orig_values, canonical_goal) = self.canonicalize_goal(goal);
-        let result = EvalCtxt::evaluate_canonical_goal(
-            self.tcx(),
-            self.search_graph,
-            canonical_goal,
-            // FIXME(-Ztrait-solver=next): we do not track what happens in `evaluate_canonical_goal`
-            &mut ProofTreeBuilder::new_noop(),
-        );
+        param_env: ty::ParamEnv<'tcx>,
+        source: GoalSource,
+        original_values: Vec<ty::GenericArg<'tcx>>,
+        response: CanonicalResponse<'tcx>,
+    ) -> Result<(Certainty, bool, Vec<Goal<'tcx, ty::Predicate<'tcx>>>), NoSolution> {
+        // The old solver did not evaluate nested goals when normalizing.
+        // It returned the selection constraints allowing a `Projection`
+        // obligation to not hold in coherence while avoiding the fatal error
+        // from overflow.
+        //
+        // We match this behavior here by considering all constraints
+        // from nested goals which are not from where-bounds. We will already
+        // need to track which nested goals are required by impl where-bounds
+        // for coinductive cycles, so we simply reuse that here.
+        //
+        // While we could consider overflow constraints in more cases, this should
+        // not be necessary for backcompat and results in better perf. It also
+        // avoids a potential inconsistency which would otherwise require some
+        // tracking for root goals as well. See #119071 for an example.
+        let keep_overflow_constraints = || {
+            self.search_graph.current_goal_is_normalizes_to()
+                && source != GoalSource::ImplWhereBound
+        };
 
-        macro_rules! fail {
-            ($msg:expr) => {{
-                let msg = $msg;
-                warn!(
-                    "unstable result: {msg}\n\
-                    original goal: {original_input:?},\n\
-                    original result: {original_result:?}\n\
-                    re-canonicalized goal: {canonical_goal:?}\n\
-                    second response: {result:?}"
-                );
-                return;
-            }};
-        }
+        if response.value.certainty == Certainty::OVERFLOW && !keep_overflow_constraints() {
+            Ok((Certainty::OVERFLOW, false, Vec::new()))
+        } else {
+            let has_changed = !response.value.var_values.is_identity_modulo_regions()
+                || !response.value.external_constraints.opaque_types.is_empty();
 
-        let Ok(new_canonical_response) = result else { fail!("second response was error") };
-        // We only check for modulo regions as we convert all regions in
-        // the input to new existentials, even if they're expected to be
-        // `'static` or a placeholder region.
-        if !new_canonical_response.value.var_values.is_identity_modulo_regions() {
-            fail!("additional constraints from second response")
-        }
-        if original_result.value.certainty != new_canonical_response.value.certainty {
-            fail!("unstable certainty")
+            let (certainty, nested_goals) =
+                self.instantiate_and_apply_query_response(param_env, original_values, response)?;
+            Ok((certainty, has_changed, nested_goals))
         }
     }
 
@@ -462,8 +452,6 @@ impl<'a, 'tcx> EvalCtxt<'a, 'tcx> {
                 ty::PredicateKind::Coerce(predicate) => {
                     self.compute_coerce_goal(Goal { param_env, predicate })
                 }
-                ty::PredicateKind::ClosureKind(def_id, args, kind) => self
-                    .compute_closure_kind_goal(Goal { param_env, predicate: (def_id, args, kind) }),
                 ty::PredicateKind::ObjectSafe(trait_def_id) => {
                     self.compute_object_safe_goal(trait_def_id)
                 }
@@ -474,7 +462,10 @@ impl<'a, 'tcx> EvalCtxt<'a, 'tcx> {
                     self.compute_const_evaluatable_goal(Goal { param_env, predicate: ct })
                 }
                 ty::PredicateKind::ConstEquate(_, _) => {
-                    bug!("ConstEquate should not be emitted when `-Ztrait-solver=next` is active")
+                    bug!("ConstEquate should not be emitted when `-Znext-solver` is active")
+                }
+                ty::PredicateKind::NormalizesTo(predicate) => {
+                    self.compute_normalizes_to_goal(Goal { param_env, predicate })
                 }
                 ty::PredicateKind::AliasRelate(lhs, rhs, direction) => self
                     .compute_alias_relate_goal(Goal {
@@ -488,7 +479,7 @@ impl<'a, 'tcx> EvalCtxt<'a, 'tcx> {
         } else {
             let kind = self.infcx.instantiate_binder_with_placeholders(kind);
             let goal = goal.with(self.tcx(), ty::Binder::dummy(kind));
-            self.add_goal(goal);
+            self.add_goal(GoalSource::Misc, goal);
             self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
         }
     }
@@ -537,6 +528,13 @@ impl<'a, 'tcx> EvalCtxt<'a, 'tcx> {
         let mut goals = core::mem::replace(&mut self.nested_goals, NestedGoals::new());
 
         self.inspect.evaluate_added_goals_loop_start();
+
+        fn with_misc_source<'tcx>(
+            it: impl IntoIterator<Item = Goal<'tcx, ty::Predicate<'tcx>>>,
+        ) -> impl Iterator<Item = (GoalSource, Goal<'tcx, ty::Predicate<'tcx>>)> {
+            iter::zip(iter::repeat(GoalSource::Misc), it)
+        }
+
         // If this loop did not result in any progress, what's our final certainty.
         let mut unchanged_certainty = Some(Certainty::Yes);
         if let Some(goal) = goals.normalizes_to_hack_goal.take() {
@@ -545,17 +543,15 @@ impl<'a, 'tcx> EvalCtxt<'a, 'tcx> {
             let unconstrained_rhs = self.next_term_infer_of_kind(goal.predicate.term);
             let unconstrained_goal = goal.with(
                 tcx,
-                ty::ProjectionPredicate {
-                    projection_ty: goal.predicate.projection_ty,
-                    term: unconstrained_rhs,
-                },
+                ty::NormalizesTo { alias: goal.predicate.alias, term: unconstrained_rhs },
             );
 
             let (_, certainty, instantiate_goals) = self.evaluate_goal(
                 GoalEvaluationKind::Nested { is_normalizes_to_hack: IsNormalizesToHack::Yes },
+                GoalSource::Misc,
                 unconstrained_goal,
             )?;
-            self.nested_goals.goals.extend(instantiate_goals);
+            self.nested_goals.goals.extend(with_misc_source(instantiate_goals));
 
             // Finally, equate the goal's RHS with the unconstrained var.
             // We put the nested goals from this into goals instead of
@@ -564,15 +560,13 @@ impl<'a, 'tcx> EvalCtxt<'a, 'tcx> {
             // matters in practice, though.
             let eq_goals =
                 self.eq_and_get_goals(goal.param_env, goal.predicate.term, unconstrained_rhs)?;
-            goals.goals.extend(eq_goals);
+            goals.goals.extend(with_misc_source(eq_goals));
 
             // We only look at the `projection_ty` part here rather than
             // looking at the "has changed" return from evaluate_goal,
             // because we expect the `unconstrained_rhs` part of the predicate
             // to have changed -- that means we actually normalized successfully!
-            if goal.predicate.projection_ty
-                != self.resolve_vars_if_possible(goal.predicate.projection_ty)
-            {
+            if goal.predicate.alias != self.resolve_vars_if_possible(goal.predicate.alias) {
                 unchanged_certainty = None;
             }
 
@@ -587,12 +581,13 @@ impl<'a, 'tcx> EvalCtxt<'a, 'tcx> {
             }
         }
 
-        for goal in goals.goals.drain(..) {
+        for (source, goal) in goals.goals.drain(..) {
             let (has_changed, certainty, instantiate_goals) = self.evaluate_goal(
                 GoalEvaluationKind::Nested { is_normalizes_to_hack: IsNormalizesToHack::No },
+                source,
                 goal,
             )?;
-            self.nested_goals.goals.extend(instantiate_goals);
+            self.nested_goals.goals.extend(with_misc_source(instantiate_goals));
             if has_changed {
                 unchanged_certainty = None;
             }
@@ -600,7 +595,7 @@ impl<'a, 'tcx> EvalCtxt<'a, 'tcx> {
             match certainty {
                 Certainty::Yes => {}
                 Certainty::Maybe(_) => {
-                    self.nested_goals.goals.push(goal);
+                    self.nested_goals.goals.push((source, goal));
                     unchanged_certainty = unchanged_certainty.map(|c| c.unify_with(certainty));
                 }
             }
@@ -642,9 +637,10 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
     ///
     /// This is the case if the `term` is an inference variable in the innermost universe
     /// and does not occur in any other part of the predicate.
+    #[instrument(level = "debug", skip(self), ret)]
     pub(super) fn term_is_fully_unconstrained(
         &self,
-        goal: Goal<'tcx, ty::ProjectionPredicate<'tcx>>,
+        goal: Goal<'tcx, ty::NormalizesTo<'tcx>>,
     ) -> bool {
         let term_is_infer = match goal.predicate.term.unpack() {
             ty::TermKind::Ty(ty) => {
@@ -708,7 +704,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         let mut visitor = ContainsTerm { infcx: self.infcx, term: goal.predicate.term };
 
         term_is_infer
-            && goal.predicate.projection_ty.visit_with(&mut visitor).is_continue()
+            && goal.predicate.alias.visit_with(&mut visitor).is_continue()
             && goal.param_env.visit_with(&mut visitor).is_continue()
     }
 
@@ -723,7 +719,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
             .at(&ObligationCause::dummy(), param_env)
             .eq(DefineOpaqueTypes::No, lhs, rhs)
             .map(|InferOk { value: (), obligations }| {
-                self.add_goals(obligations.into_iter().map(|o| o.into()));
+                self.add_goals(GoalSource::Misc, obligations.into_iter().map(|o| o.into()));
             })
             .map_err(|e| {
                 debug!(?e, "failed to equate");
@@ -742,7 +738,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
             .at(&ObligationCause::dummy(), param_env)
             .sub(DefineOpaqueTypes::No, sub, sup)
             .map(|InferOk { value: (), obligations }| {
-                self.add_goals(obligations.into_iter().map(|o| o.into()));
+                self.add_goals(GoalSource::Misc, obligations.into_iter().map(|o| o.into()));
             })
             .map_err(|e| {
                 debug!(?e, "failed to subtype");
@@ -750,6 +746,26 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
             })
     }
 
+    #[instrument(level = "debug", skip(self, param_env), ret)]
+    pub(super) fn relate<T: ToTrace<'tcx>>(
+        &mut self,
+        param_env: ty::ParamEnv<'tcx>,
+        lhs: T,
+        variance: ty::Variance,
+        rhs: T,
+    ) -> Result<(), NoSolution> {
+        self.infcx
+            .at(&ObligationCause::dummy(), param_env)
+            .relate(DefineOpaqueTypes::No, lhs, variance, rhs)
+            .map(|InferOk { value: (), obligations }| {
+                self.add_goals(GoalSource::Misc, obligations.into_iter().map(|o| o.into()));
+            })
+            .map_err(|e| {
+                debug!(?e, "failed to relate");
+                NoSolution
+            })
+    }
+
     /// Equates two values returning the nested goals without adding them
     /// to the nested goals of the `EvalCtxt`.
     ///
@@ -780,7 +796,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
     ) -> T {
         self.infcx.instantiate_binder_with_fresh_vars(
             DUMMY_SP,
-            LateBoundRegionConversionTime::HigherRankedType,
+            BoundRegionConversionTime::HigherRankedType,
             value,
         )
     }
@@ -875,7 +891,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
             true,
             &mut obligations,
         )?;
-        self.add_goals(obligations.into_iter().map(|o| o.into()));
+        self.add_goals(GoalSource::Misc, obligations.into_iter().map(|o| o.into()));
         Ok(())
     }
 
@@ -895,7 +911,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
             hidden_ty,
             &mut obligations,
         );
-        self.add_goals(obligations.into_iter().map(|o| o.into()));
+        self.add_goals(GoalSource::Misc, obligations.into_iter().map(|o| o.into()));
     }
 
     // Do something for each opaque/hidden pair defined with `def_id` in the
diff --git a/compiler/rustc_trait_selection/src/solve/eval_ctxt/probe.rs b/compiler/rustc_trait_selection/src/solve/eval_ctxt/probe.rs
index 6087b9167..91fd48807 100644
--- a/compiler/rustc_trait_selection/src/solve/eval_ctxt/probe.rs
+++ b/compiler/rustc_trait_selection/src/solve/eval_ctxt/probe.rs
@@ -1,5 +1,10 @@
+use crate::solve::assembly::Candidate;
+
 use super::EvalCtxt;
-use rustc_middle::traits::solve::{inspect, CandidateSource, QueryResult};
+use rustc_middle::traits::{
+    query::NoSolution,
+    solve::{inspect, CandidateSource, QueryResult},
+};
 use std::marker::PhantomData;
 
 pub(in crate::solve) struct ProbeCtxt<'me, 'a, 'tcx, F, T> {
@@ -36,6 +41,23 @@ where
     }
 }
 
+pub(in crate::solve) struct TraitProbeCtxt<'me, 'a, 'tcx, F> {
+    cx: ProbeCtxt<'me, 'a, 'tcx, F, QueryResult<'tcx>>,
+    source: CandidateSource,
+}
+
+impl<'tcx, F> TraitProbeCtxt<'_, '_, 'tcx, F>
+where
+    F: FnOnce(&QueryResult<'tcx>) -> inspect::ProbeKind<'tcx>,
+{
+    pub(in crate::solve) fn enter(
+        self,
+        f: impl FnOnce(&mut EvalCtxt<'_, 'tcx>) -> QueryResult<'tcx>,
+    ) -> Result<Candidate<'tcx>, NoSolution> {
+        self.cx.enter(|ecx| f(ecx)).map(|result| Candidate { source: self.source, result })
+    }
+}
+
 impl<'a, 'tcx> EvalCtxt<'a, 'tcx> {
     /// `probe_kind` is only called when proof tree building is enabled so it can be
     /// as expensive as necessary to output the desired information.
@@ -69,20 +91,18 @@ impl<'a, 'tcx> EvalCtxt<'a, 'tcx> {
     pub(in crate::solve) fn probe_trait_candidate(
         &mut self,
         source: CandidateSource,
-    ) -> ProbeCtxt<
-        '_,
-        'a,
-        'tcx,
-        impl FnOnce(&QueryResult<'tcx>) -> inspect::ProbeKind<'tcx>,
-        QueryResult<'tcx>,
-    > {
-        ProbeCtxt {
-            ecx: self,
-            probe_kind: move |result: &QueryResult<'tcx>| inspect::ProbeKind::TraitCandidate {
-                source,
-                result: *result,
+    ) -> TraitProbeCtxt<'_, 'a, 'tcx, impl FnOnce(&QueryResult<'tcx>) -> inspect::ProbeKind<'tcx>>
+    {
+        TraitProbeCtxt {
+            cx: ProbeCtxt {
+                ecx: self,
+                probe_kind: move |result: &QueryResult<'tcx>| inspect::ProbeKind::TraitCandidate {
+                    source,
+                    result: *result,
+                },
+                _result: PhantomData,
             },
-            _result: PhantomData,
+            source,
         }
     }
 }
diff --git a/compiler/rustc_trait_selection/src/solve/fulfill.rs b/compiler/rustc_trait_selection/src/solve/fulfill.rs
index f1d309122..2139210b8 100644
--- a/compiler/rustc_trait_selection/src/solve/fulfill.rs
+++ b/compiler/rustc_trait_selection/src/solve/fulfill.rs
@@ -108,6 +108,11 @@ impl<'tcx> TraitEngine<'tcx> for FulfillmentCtxt<'tcx> {
                                             MismatchedProjectionTypes { err: TypeError::Mismatch },
                                         )
                                     }
+                                    ty::PredicateKind::NormalizesTo(..) => {
+                                        FulfillmentErrorCode::CodeProjectionError(
+                                            MismatchedProjectionTypes { err: TypeError::Mismatch },
+                                        )
+                                    }
                                     ty::PredicateKind::AliasRelate(_, _, _) => {
                                         FulfillmentErrorCode::CodeProjectionError(
                                             MismatchedProjectionTypes { err: TypeError::Mismatch },
@@ -135,7 +140,6 @@ impl<'tcx> TraitEngine<'tcx> for FulfillmentCtxt<'tcx> {
                                     }
                                     ty::PredicateKind::Clause(_)
                                     | ty::PredicateKind::ObjectSafe(_)
-                                    | ty::PredicateKind::ClosureKind(_, _, _)
                                     | ty::PredicateKind::Ambiguous => {
                                         FulfillmentErrorCode::CodeSelectionError(
                                             SelectionError::Unimplemented,
diff --git a/compiler/rustc_trait_selection/src/solve/inspect/analyse.rs b/compiler/rustc_trait_selection/src/solve/inspect/analyse.rs
index 69bfdd468..6db53d6dd 100644
--- a/compiler/rustc_trait_selection/src/solve/inspect/analyse.rs
+++ b/compiler/rustc_trait_selection/src/solve/inspect/analyse.rs
@@ -58,7 +58,7 @@ impl<'a, 'tcx> InspectCandidate<'a, 'tcx> {
         visitor: &mut V,
     ) -> ControlFlow<V::BreakTy> {
         // HACK: An arbitrary cutoff to avoid dealing with overflow and cycles.
-        if self.goal.depth >= 10 {
+        if self.goal.depth <= 10 {
             let infcx = self.goal.infcx;
             infcx.probe(|_| {
                 let mut instantiated_goals = vec![];
@@ -119,8 +119,7 @@ impl<'a, 'tcx> InspectGoal<'a, 'tcx> {
     ) {
         for step in &probe.steps {
             match step {
-                &inspect::ProbeStep::AddGoal(goal) => nested_goals.push(goal),
-                inspect::ProbeStep::EvaluateGoals(_) => (),
+                &inspect::ProbeStep::AddGoal(_source, goal) => nested_goals.push(goal),
                 inspect::ProbeStep::NestedProbe(ref probe) => {
                     // Nested probes have to prove goals added in their parent
                     // but do not leak them, so we truncate the added goals
@@ -129,13 +128,17 @@ impl<'a, 'tcx> InspectGoal<'a, 'tcx> {
                     self.candidates_recur(candidates, nested_goals, probe);
                     nested_goals.truncate(num_goals);
                 }
+                inspect::ProbeStep::EvaluateGoals(_)
+                | inspect::ProbeStep::CommitIfOkStart
+                | inspect::ProbeStep::CommitIfOkSuccess => (),
             }
         }
 
         match probe.kind {
             inspect::ProbeKind::NormalizedSelfTyAssembly
             | inspect::ProbeKind::UnsizeAssembly
-            | inspect::ProbeKind::UpcastProjectionCompatibility => (),
+            | inspect::ProbeKind::UpcastProjectionCompatibility
+            | inspect::ProbeKind::CommitIfOk => (),
             // We add a candidate for the root evaluation if there
             // is only one way to prove a given goal, e.g. for `WellFormed`.
             //
@@ -172,7 +175,7 @@ impl<'a, 'tcx> InspectGoal<'a, 'tcx> {
                 warn!("unexpected root evaluation: {:?}", self.evaluation);
                 return vec![];
             }
-            inspect::CanonicalGoalEvaluationKind::Evaluation { ref revisions } => {
+            inspect::CanonicalGoalEvaluationKind::Evaluation { revisions } => {
                 if let Some(last) = revisions.last() {
                     last
                 } else {
diff --git a/compiler/rustc_trait_selection/src/solve/inspect/build.rs b/compiler/rustc_trait_selection/src/solve/inspect/build.rs
index 088455b38..d8caef5b0 100644
--- a/compiler/rustc_trait_selection/src/solve/inspect/build.rs
+++ b/compiler/rustc_trait_selection/src/solve/inspect/build.rs
@@ -7,7 +7,7 @@ use std::mem;
 
 use rustc_middle::traits::query::NoSolution;
 use rustc_middle::traits::solve::{
-    CanonicalInput, Certainty, Goal, IsNormalizesToHack, QueryInput, QueryResult,
+    CanonicalInput, Certainty, Goal, GoalSource, IsNormalizesToHack, QueryInput, QueryResult,
 };
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_session::config::DumpSolverProofTree;
@@ -216,17 +216,21 @@ impl<'tcx> WipProbe<'tcx> {
 
 #[derive(Eq, PartialEq, Debug)]
 enum WipProbeStep<'tcx> {
-    AddGoal(inspect::CanonicalState<'tcx, Goal<'tcx, ty::Predicate<'tcx>>>),
+    AddGoal(GoalSource, inspect::CanonicalState<'tcx, Goal<'tcx, ty::Predicate<'tcx>>>),
     EvaluateGoals(WipAddedGoalsEvaluation<'tcx>),
     NestedProbe(WipProbe<'tcx>),
+    CommitIfOkStart,
+    CommitIfOkSuccess,
 }
 
 impl<'tcx> WipProbeStep<'tcx> {
     fn finalize(self) -> inspect::ProbeStep<'tcx> {
         match self {
-            WipProbeStep::AddGoal(goal) => inspect::ProbeStep::AddGoal(goal),
+            WipProbeStep::AddGoal(source, goal) => inspect::ProbeStep::AddGoal(source, goal),
             WipProbeStep::EvaluateGoals(eval) => inspect::ProbeStep::EvaluateGoals(eval.finalize()),
             WipProbeStep::NestedProbe(probe) => inspect::ProbeStep::NestedProbe(probe.finalize()),
+            WipProbeStep::CommitIfOkStart => inspect::ProbeStep::CommitIfOkStart,
+            WipProbeStep::CommitIfOkSuccess => inspect::ProbeStep::CommitIfOkSuccess,
         }
     }
 }
@@ -261,7 +265,7 @@ impl<'tcx> ProofTreeBuilder<'tcx> {
             GenerateProofTree::Never => ProofTreeBuilder::new_noop(),
             GenerateProofTree::IfEnabled => {
                 let opts = &tcx.sess.opts.unstable_opts;
-                match opts.dump_solver_proof_tree {
+                match opts.next_solver.map(|c| c.dump_tree).unwrap_or_default() {
                     DumpSolverProofTree::Always => ProofTreeBuilder::new_root(),
                     // `OnError` is handled by reevaluating goals in error
                     // reporting with `GenerateProofTree::Yes`.
@@ -424,7 +428,11 @@ impl<'tcx> ProofTreeBuilder<'tcx> {
         }
     }
 
-    pub fn add_goal(ecx: &mut EvalCtxt<'_, 'tcx>, goal: Goal<'tcx, ty::Predicate<'tcx>>) {
+    pub fn add_goal(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        source: GoalSource,
+        goal: Goal<'tcx, ty::Predicate<'tcx>>,
+    ) {
         // Can't use `if let Some(this) = ecx.inspect.as_mut()` here because
         // we have to immutably use the `EvalCtxt` for `make_canonical_state`.
         if ecx.inspect.is_noop() {
@@ -438,7 +446,9 @@ impl<'tcx> ProofTreeBuilder<'tcx> {
                 evaluation: WipProbe { steps, .. },
                 ..
             })
-            | DebugSolver::Probe(WipProbe { steps, .. }) => steps.push(WipProbeStep::AddGoal(goal)),
+            | DebugSolver::Probe(WipProbe { steps, .. }) => {
+                steps.push(WipProbeStep::AddGoal(source, goal))
+            }
             s => unreachable!("tried to add {goal:?} to {s:?}"),
         }
     }
@@ -459,6 +469,29 @@ impl<'tcx> ProofTreeBuilder<'tcx> {
         }
     }
 
+    /// Used by `EvalCtxt::commit_if_ok` to flatten the work done inside
+    /// of the probe into the parent.
+    pub fn integrate_snapshot(&mut self, probe: ProofTreeBuilder<'tcx>) {
+        if let Some(this) = self.as_mut() {
+            match (this, *probe.state.unwrap()) {
+                (
+                    DebugSolver::Probe(WipProbe { steps, .. })
+                    | DebugSolver::GoalEvaluationStep(WipGoalEvaluationStep {
+                        evaluation: WipProbe { steps, .. },
+                        ..
+                    }),
+                    DebugSolver::Probe(probe),
+                ) => {
+                    steps.push(WipProbeStep::CommitIfOkStart);
+                    assert_eq!(probe.kind, None);
+                    steps.extend(probe.steps);
+                    steps.push(WipProbeStep::CommitIfOkSuccess);
+                }
+                _ => unreachable!(),
+            }
+        }
+    }
+
     pub fn new_evaluate_added_goals(&mut self) -> ProofTreeBuilder<'tcx> {
         self.nested(|| WipAddedGoalsEvaluation { evaluations: vec![], result: None })
     }
diff --git a/compiler/rustc_trait_selection/src/solve/mod.rs b/compiler/rustc_trait_selection/src/solve/mod.rs
index dba5369fa..2f3111a24 100644
--- a/compiler/rustc_trait_selection/src/solve/mod.rs
+++ b/compiler/rustc_trait_selection/src/solve/mod.rs
@@ -1,7 +1,6 @@
 //! The next-generation trait solver, currently still WIP.
 //!
-//! As a user of rust, you can use `-Ztrait-solver=next` or `next-coherence`
-//! to enable the new trait solver always, or just within coherence, respectively.
+//! As a user of rust, you can use `-Znext-solver` to enable the new trait solver.
 //!
 //! As a developer of rustc, you shouldn't be using the new trait
 //! solver without asking the trait-system-refactor-initiative, but it can
@@ -16,31 +15,33 @@
 //! about it on zulip.
 use rustc_hir::def_id::DefId;
 use rustc_infer::infer::canonical::{Canonical, CanonicalVarValues};
+use rustc_infer::infer::DefineOpaqueTypes;
 use rustc_infer::traits::query::NoSolution;
 use rustc_middle::infer::canonical::CanonicalVarInfos;
 use rustc_middle::traits::solve::{
-    CanonicalResponse, Certainty, ExternalConstraintsData, Goal, IsNormalizesToHack, QueryResult,
-    Response,
+    CanonicalResponse, Certainty, ExternalConstraintsData, Goal, GoalSource, IsNormalizesToHack,
+    QueryResult, Response,
 };
-use rustc_middle::ty::{self, Ty, TyCtxt, UniverseIndex};
+use rustc_middle::ty::{self, OpaqueTypeKey, Ty, TyCtxt, UniverseIndex};
 use rustc_middle::ty::{
     CoercePredicate, RegionOutlivesPredicate, SubtypePredicate, TypeOutlivesPredicate,
 };
 
 mod alias_relate;
 mod assembly;
-mod canonicalize;
 mod eval_ctxt;
 mod fulfill;
 pub mod inspect;
 mod normalize;
+mod normalizes_to;
 mod project_goals;
 mod search_graph;
 mod trait_goals;
 
 pub use eval_ctxt::{EvalCtxt, GenerateProofTree, InferCtxtEvalExt, InferCtxtSelectExt};
 pub use fulfill::FulfillmentCtxt;
-pub(crate) use normalize::{deeply_normalize, deeply_normalize_with_skipped_universes};
+pub(crate) use normalize::deeply_normalize_for_diagnostics;
+pub use normalize::{deeply_normalize, deeply_normalize_with_skipped_universes};
 
 #[derive(Debug, Clone, Copy)]
 enum SolverMode {
@@ -63,8 +64,6 @@ enum GoalEvaluationKind {
 
 trait CanonicalResponseExt {
     fn has_no_inference_or_external_constraints(&self) -> bool;
-
-    fn has_only_region_constraints(&self) -> bool;
 }
 
 impl<'tcx> CanonicalResponseExt for Canonical<'tcx, Response<'tcx>> {
@@ -73,11 +72,6 @@ impl<'tcx> CanonicalResponseExt for Canonical<'tcx, Response<'tcx>> {
             && self.value.var_values.is_identity()
             && self.value.external_constraints.opaque_types.is_empty()
     }
-
-    fn has_only_region_constraints(&self) -> bool {
-        self.value.var_values.is_identity_modulo_regions()
-            && self.value.external_constraints.opaque_types.is_empty()
-    }
 }
 
 impl<'a, 'tcx> EvalCtxt<'a, 'tcx> {
@@ -163,7 +157,7 @@ impl<'a, 'tcx> EvalCtxt<'a, 'tcx> {
     ) -> QueryResult<'tcx> {
         match self.well_formed_goals(goal.param_env, goal.predicate) {
             Some(goals) => {
-                self.add_goals(goals);
+                self.add_goals(GoalSource::Misc, goals);
                 self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
             }
             None => self.evaluate_added_goals_and_make_canonical_response(Certainty::AMBIGUOUS),
@@ -220,7 +214,7 @@ impl<'a, 'tcx> EvalCtxt<'a, 'tcx> {
 
 impl<'tcx> EvalCtxt<'_, 'tcx> {
     #[instrument(level = "debug", skip(self))]
-    fn set_normalizes_to_hack_goal(&mut self, goal: Goal<'tcx, ty::ProjectionPredicate<'tcx>>) {
+    fn set_normalizes_to_hack_goal(&mut self, goal: Goal<'tcx, ty::NormalizesTo<'tcx>>) {
         assert!(
             self.nested_goals.normalizes_to_hack_goal.is_none(),
             "attempted to set the projection eq hack goal when one already exists"
@@ -229,15 +223,19 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
     }
 
     #[instrument(level = "debug", skip(self))]
-    fn add_goal(&mut self, goal: Goal<'tcx, ty::Predicate<'tcx>>) {
-        inspect::ProofTreeBuilder::add_goal(self, goal);
-        self.nested_goals.goals.push(goal);
+    fn add_goal(&mut self, source: GoalSource, goal: Goal<'tcx, ty::Predicate<'tcx>>) {
+        inspect::ProofTreeBuilder::add_goal(self, source, goal);
+        self.nested_goals.goals.push((source, goal));
     }
 
     #[instrument(level = "debug", skip(self, goals))]
-    fn add_goals(&mut self, goals: impl IntoIterator<Item = Goal<'tcx, ty::Predicate<'tcx>>>) {
+    fn add_goals(
+        &mut self,
+        source: GoalSource,
+        goals: impl IntoIterator<Item = Goal<'tcx, ty::Predicate<'tcx>>>,
+    ) {
         for goal in goals {
-            self.add_goal(goal);
+            self.add_goal(source, goal);
         }
     }
 
@@ -253,7 +251,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
             return None;
         }
 
-        // FIXME(-Ztrait-solver=next): We should instead try to find a `Certainty::Yes` response with
+        // FIXME(-Znext-solver): We should instead try to find a `Certainty::Yes` response with
         // a subset of the constraints that all the other responses have.
         let one = responses[0];
         if responses[1..].iter().all(|&resp| resp == one) {
@@ -294,28 +292,61 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
     /// in [`EvalCtxt::assemble_candidates_via_self_ty`] does not have to normalize
     /// the self type. It is required when structurally matching on any other
     /// arguments of a trait goal, e.g. when assembling builtin unsize candidates.
+    #[instrument(level = "debug", skip(self), ret)]
     fn try_normalize_ty(
         &mut self,
         param_env: ty::ParamEnv<'tcx>,
-        mut ty: Ty<'tcx>,
-    ) -> Result<Option<Ty<'tcx>>, NoSolution> {
-        for _ in 0..self.local_overflow_limit() {
-            let ty::Alias(_, projection_ty) = *ty.kind() else {
-                return Ok(Some(ty));
-            };
-
-            let normalized_ty = self.next_ty_infer();
+        ty: Ty<'tcx>,
+    ) -> Option<Ty<'tcx>> {
+        self.try_normalize_ty_recur(param_env, DefineOpaqueTypes::Yes, 0, ty)
+    }
+
+    fn try_normalize_ty_recur(
+        &mut self,
+        param_env: ty::ParamEnv<'tcx>,
+        define_opaque_types: DefineOpaqueTypes,
+        depth: usize,
+        ty: Ty<'tcx>,
+    ) -> Option<Ty<'tcx>> {
+        if !self.tcx().recursion_limit().value_within_limit(depth) {
+            return None;
+        }
+
+        let ty::Alias(kind, alias) = *ty.kind() else {
+            return Some(ty);
+        };
+
+        // We do no always define opaque types eagerly to allow non-defining uses in the defining scope.
+        if let (DefineOpaqueTypes::No, ty::AliasKind::Opaque) = (define_opaque_types, kind) {
+            if let Some(def_id) = alias.def_id.as_local() {
+                if self
+                    .unify_existing_opaque_tys(
+                        param_env,
+                        OpaqueTypeKey { def_id, args: alias.args },
+                        self.next_ty_infer(),
+                    )
+                    .is_empty()
+                {
+                    return Some(ty);
+                }
+            }
+        }
+
+        match self.commit_if_ok(|this| {
+            let normalized_ty = this.next_ty_infer();
             let normalizes_to_goal = Goal::new(
-                self.tcx(),
+                this.tcx(),
                 param_env,
-                ty::ProjectionPredicate { projection_ty, term: normalized_ty.into() },
+                ty::NormalizesTo { alias, term: normalized_ty.into() },
             );
-            self.add_goal(normalizes_to_goal);
-            self.try_evaluate_added_goals()?;
-            ty = self.resolve_vars_if_possible(normalized_ty);
+            this.add_goal(GoalSource::Misc, normalizes_to_goal);
+            this.try_evaluate_added_goals()?;
+            let ty = this.resolve_vars_if_possible(normalized_ty);
+            Ok(this.try_normalize_ty_recur(param_env, define_opaque_types, depth + 1, ty))
+        }) {
+            Ok(ty) => ty,
+            Err(NoSolution) => Some(ty),
         }
-
-        Ok(None)
     }
 }
 
diff --git a/compiler/rustc_trait_selection/src/solve/normalize.rs b/compiler/rustc_trait_selection/src/solve/normalize.rs
index b0a348985..55b79e6fc 100644
--- a/compiler/rustc_trait_selection/src/solve/normalize.rs
+++ b/compiler/rustc_trait_selection/src/solve/normalize.rs
@@ -4,19 +4,20 @@ use crate::traits::{needs_normalization, BoundVarReplacer, PlaceholderReplacer};
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_infer::infer::at::At;
 use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
+use rustc_infer::infer::InferCtxt;
 use rustc_infer::traits::TraitEngineExt;
 use rustc_infer::traits::{FulfillmentError, Obligation, TraitEngine};
 use rustc_middle::infer::unify_key::{ConstVariableOrigin, ConstVariableOriginKind};
-use rustc_middle::traits::Reveal;
+use rustc_middle::traits::{ObligationCause, Reveal};
 use rustc_middle::ty::{self, AliasTy, Ty, TyCtxt, UniverseIndex};
-use rustc_middle::ty::{FallibleTypeFolder, TypeSuperFoldable};
+use rustc_middle::ty::{FallibleTypeFolder, TypeFolder, TypeSuperFoldable};
 use rustc_middle::ty::{TypeFoldable, TypeVisitableExt};
 
 use super::FulfillmentCtxt;
 
 /// Deeply normalize all aliases in `value`. This does not handle inference and expects
 /// its input to be already fully resolved.
-pub(crate) fn deeply_normalize<'tcx, T: TypeFoldable<TyCtxt<'tcx>>>(
+pub fn deeply_normalize<'tcx, T: TypeFoldable<TyCtxt<'tcx>>>(
     at: At<'_, 'tcx>,
     value: T,
 ) -> Result<T, Vec<FulfillmentError<'tcx>>> {
@@ -30,7 +31,7 @@ pub(crate) fn deeply_normalize<'tcx, T: TypeFoldable<TyCtxt<'tcx>>>(
 /// Additionally takes a list of universes which represents the binders which have been
 /// entered before passing `value` to the function. This is currently needed for
 /// `normalize_erasing_regions`, which skips binders as it walks through a type.
-pub(crate) fn deeply_normalize_with_skipped_universes<'tcx, T: TypeFoldable<TyCtxt<'tcx>>>(
+pub fn deeply_normalize_with_skipped_universes<'tcx, T: TypeFoldable<TyCtxt<'tcx>>>(
     at: At<'_, 'tcx>,
     value: T,
     universes: Vec<Option<UniverseIndex>>,
@@ -75,7 +76,7 @@ impl<'tcx> NormalizationFolder<'_, 'tcx> {
             tcx,
             self.at.cause.clone(),
             self.at.param_env,
-            ty::ProjectionPredicate { projection_ty: alias, term: new_infer_ty.into() },
+            ty::NormalizesTo { alias, term: new_infer_ty.into() },
         );
 
         // Do not emit an error if normalization is known to fail but instead
@@ -128,8 +129,8 @@ impl<'tcx> NormalizationFolder<'_, 'tcx> {
             tcx,
             self.at.cause.clone(),
             self.at.param_env,
-            ty::ProjectionPredicate {
-                projection_ty: AliasTy::new(tcx, uv.def, uv.args),
+            ty::NormalizesTo {
+                alias: AliasTy::new(tcx, uv.def, uv.args),
                 term: new_infer_ct.into(),
             },
         );
@@ -230,3 +231,42 @@ impl<'tcx> FallibleTypeFolder<TyCtxt<'tcx>> for NormalizationFolder<'_, 'tcx> {
         }
     }
 }
+
+// Deeply normalize a value and return it
+pub(crate) fn deeply_normalize_for_diagnostics<'tcx, T: TypeFoldable<TyCtxt<'tcx>>>(
+    infcx: &InferCtxt<'tcx>,
+    param_env: ty::ParamEnv<'tcx>,
+    t: T,
+) -> T {
+    t.fold_with(&mut DeeplyNormalizeForDiagnosticsFolder {
+        at: infcx.at(&ObligationCause::dummy(), param_env),
+    })
+}
+
+struct DeeplyNormalizeForDiagnosticsFolder<'a, 'tcx> {
+    at: At<'a, 'tcx>,
+}
+
+impl<'tcx> TypeFolder<TyCtxt<'tcx>> for DeeplyNormalizeForDiagnosticsFolder<'_, 'tcx> {
+    fn interner(&self) -> TyCtxt<'tcx> {
+        self.at.infcx.tcx
+    }
+
+    fn fold_ty(&mut self, ty: Ty<'tcx>) -> Ty<'tcx> {
+        deeply_normalize_with_skipped_universes(
+            self.at,
+            ty,
+            vec![None; ty.outer_exclusive_binder().as_usize()],
+        )
+        .unwrap_or_else(|_| ty.super_fold_with(self))
+    }
+
+    fn fold_const(&mut self, ct: ty::Const<'tcx>) -> ty::Const<'tcx> {
+        deeply_normalize_with_skipped_universes(
+            self.at,
+            ct,
+            vec![None; ct.outer_exclusive_binder().as_usize()],
+        )
+        .unwrap_or_else(|_| ct.super_fold_with(self))
+    }
+}
diff --git a/compiler/rustc_trait_selection/src/solve/project_goals/inherent_projection.rs b/compiler/rustc_trait_selection/src/solve/normalizes_to/inherent.rs
index 28fe59b7f..b2dff9b48 100644
--- a/compiler/rustc_trait_selection/src/solve/project_goals/inherent_projection.rs
+++ b/compiler/rustc_trait_selection/src/solve/normalizes_to/inherent.rs
@@ -4,18 +4,18 @@
 //! 1. instantiate substs,
 //! 2. equate the self type, and
 //! 3. instantiate and register where clauses.
-use rustc_middle::traits::solve::{Certainty, Goal, QueryResult};
+use rustc_middle::traits::solve::{Certainty, Goal, GoalSource, QueryResult};
 use rustc_middle::ty;
 
-use super::EvalCtxt;
+use crate::solve::EvalCtxt;
 
 impl<'tcx> EvalCtxt<'_, 'tcx> {
     pub(super) fn normalize_inherent_associated_type(
         &mut self,
-        goal: Goal<'tcx, ty::ProjectionPredicate<'tcx>>,
+        goal: Goal<'tcx, ty::NormalizesTo<'tcx>>,
     ) -> QueryResult<'tcx> {
         let tcx = self.tcx();
-        let inherent = goal.predicate.projection_ty;
+        let inherent = goal.predicate.alias;
         let expected = goal.predicate.term.ty().expect("inherent consts are treated separately");
 
         let impl_def_id = tcx.parent(inherent.def_id);
@@ -38,7 +38,13 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         .expect("expected goal term to be fully unconstrained");
 
         // Check both where clauses on the impl and IAT
+        //
+        // FIXME(-Znext-solver=coinductive): I think this should be split
+        // and we tag the impl bounds with `GoalSource::ImplWhereBound`?
+        // Right not this includes both the impl and the assoc item where bounds,
+        // and I don't think the assoc item where-bounds are allowed to be coinductive.
         self.add_goals(
+            GoalSource::Misc,
             tcx.predicates_of(inherent.def_id)
                 .instantiate(tcx, inherent_substs)
                 .into_iter()
diff --git a/compiler/rustc_trait_selection/src/solve/project_goals/mod.rs b/compiler/rustc_trait_selection/src/solve/normalizes_to/mod.rs
index 240141065..0e9656a1e 100644
--- a/compiler/rustc_trait_selection/src/solve/project_goals/mod.rs
+++ b/compiler/rustc_trait_selection/src/solve/normalizes_to/mod.rs
@@ -1,7 +1,7 @@
 use crate::traits::{check_args_compatible, specialization_graph};
 
-use super::assembly::{self, structural_traits};
-use super::EvalCtxt;
+use super::assembly::{self, structural_traits, Candidate};
+use super::{EvalCtxt, GoalSource};
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::DefId;
 use rustc_hir::LangItem;
@@ -13,20 +13,20 @@ use rustc_middle::traits::solve::{
 };
 use rustc_middle::traits::BuiltinImplSource;
 use rustc_middle::ty::fast_reject::{DeepRejectCtxt, TreatParams};
-use rustc_middle::ty::ProjectionPredicate;
+use rustc_middle::ty::NormalizesTo;
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_middle::ty::{ToPredicate, TypeVisitableExt};
 use rustc_span::{sym, ErrorGuaranteed, DUMMY_SP};
 
-mod inherent_projection;
+mod inherent;
 mod opaques;
 mod weak_types;
 
 impl<'tcx> EvalCtxt<'_, 'tcx> {
     #[instrument(level = "debug", skip(self), ret)]
-    pub(super) fn compute_projection_goal(
+    pub(super) fn compute_normalizes_to_goal(
         &mut self,
-        goal: Goal<'tcx, ProjectionPredicate<'tcx>>,
+        goal: Goal<'tcx, NormalizesTo<'tcx>>,
     ) -> QueryResult<'tcx> {
         let def_id = goal.predicate.def_id();
         match self.tcx().def_kind(def_id) {
@@ -71,16 +71,13 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
     #[instrument(level = "debug", skip(self), ret)]
     fn normalize_anon_const(
         &mut self,
-        goal: Goal<'tcx, ty::ProjectionPredicate<'tcx>>,
+        goal: Goal<'tcx, ty::NormalizesTo<'tcx>>,
     ) -> QueryResult<'tcx> {
         if let Some(normalized_const) = self.try_const_eval_resolve(
             goal.param_env,
-            ty::UnevaluatedConst::new(
-                goal.predicate.projection_ty.def_id,
-                goal.predicate.projection_ty.args,
-            ),
+            ty::UnevaluatedConst::new(goal.predicate.alias.def_id, goal.predicate.alias.args),
             self.tcx()
-                .type_of(goal.predicate.projection_ty.def_id)
+                .type_of(goal.predicate.alias.def_id)
                 .no_bound_vars()
                 .expect("const ty should not rely on other generics"),
         ) {
@@ -92,17 +89,13 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
     }
 }
 
-impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
+impl<'tcx> assembly::GoalKind<'tcx> for NormalizesTo<'tcx> {
     fn self_ty(self) -> Ty<'tcx> {
         self.self_ty()
     }
 
     fn trait_ref(self, tcx: TyCtxt<'tcx>) -> ty::TraitRef<'tcx> {
-        self.projection_ty.trait_ref(tcx)
-    }
-
-    fn polarity(self) -> ty::ImplPolarity {
-        ty::ImplPolarity::Positive
+        self.alias.trait_ref(tcx)
     }
 
     fn with_self_ty(self, tcx: TyCtxt<'tcx>, self_ty: Ty<'tcx>) -> Self {
@@ -127,7 +120,7 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
                         ecx.instantiate_binder_with_infer(projection_pred);
                     ecx.eq(
                         goal.param_env,
-                        goal.predicate.projection_ty,
+                        goal.predicate.alias,
                         assumption_projection_pred.projection_ty,
                     )?;
                     ecx.eq(goal.param_env, goal.predicate.term, assumption_projection_pred.term)
@@ -135,8 +128,9 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
 
                     // Add GAT where clauses from the trait's definition
                     ecx.add_goals(
+                        GoalSource::Misc,
                         tcx.predicates_of(goal.predicate.def_id())
-                            .instantiate_own(tcx, goal.predicate.projection_ty.args)
+                            .instantiate_own(tcx, goal.predicate.alias.args)
                             .map(|(pred, _)| goal.with(tcx, pred)),
                     );
 
@@ -152,15 +146,15 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
 
     fn consider_impl_candidate(
         ecx: &mut EvalCtxt<'_, 'tcx>,
-        goal: Goal<'tcx, ProjectionPredicate<'tcx>>,
+        goal: Goal<'tcx, NormalizesTo<'tcx>>,
         impl_def_id: DefId,
-    ) -> QueryResult<'tcx> {
+    ) -> Result<Candidate<'tcx>, NoSolution> {
         let tcx = ecx.tcx();
 
-        let goal_trait_ref = goal.predicate.projection_ty.trait_ref(tcx);
+        let goal_trait_ref = goal.predicate.alias.trait_ref(tcx);
         let impl_trait_ref = tcx.impl_trait_ref(impl_def_id).unwrap();
         let drcx = DeepRejectCtxt { treat_obligation_params: TreatParams::ForLookup };
-        if !drcx.args_refs_may_unify(goal_trait_ref.args, impl_trait_ref.skip_binder().args) {
+        if !drcx.args_may_unify(goal_trait_ref.args, impl_trait_ref.skip_binder().args) {
             return Err(NoSolution);
         }
 
@@ -176,12 +170,13 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
                 .predicates
                 .into_iter()
                 .map(|pred| goal.with(tcx, pred));
-            ecx.add_goals(where_clause_bounds);
+            ecx.add_goals(GoalSource::ImplWhereBound, where_clause_bounds);
 
             // Add GAT where clauses from the trait's definition
             ecx.add_goals(
+                GoalSource::Misc,
                 tcx.predicates_of(goal.predicate.def_id())
-                    .instantiate_own(tcx, goal.predicate.projection_ty.args)
+                    .instantiate_own(tcx, goal.predicate.alias.args)
                     .map(|(pred, _)| goal.with(tcx, pred)),
             );
 
@@ -200,17 +195,21 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
             };
 
             let error_response = |ecx: &mut EvalCtxt<'_, 'tcx>, reason| {
-                let guar = tcx.sess.delay_span_bug(tcx.def_span(assoc_def.item.def_id), reason);
+                let guar = tcx.sess.span_delayed_bug(tcx.def_span(assoc_def.item.def_id), reason);
                 let error_term = match assoc_def.item.kind {
                     ty::AssocKind::Const => ty::Const::new_error(
                         tcx,
                         guar,
                         tcx.type_of(goal.predicate.def_id())
-                            .instantiate(tcx, goal.predicate.projection_ty.args),
+                            .instantiate(tcx, goal.predicate.alias.args),
                     )
                     .into(),
                     ty::AssocKind::Type => Ty::new_error(tcx, guar).into(),
-                    ty::AssocKind::Fn => unreachable!(),
+                    // This makes no sense...
+                    ty::AssocKind::Fn => span_bug!(
+                        tcx.def_span(assoc_def.item.def_id),
+                        "cannot project to an associated function"
+                    ),
                 };
                 ecx.eq(goal.param_env, goal.predicate.term, error_term)
                     .expect("expected goal term to be fully unconstrained");
@@ -231,11 +230,8 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
             //
             // And then map these args to the args of the defining impl of `Assoc`, going
             // from `[u32, u64]` to `[u32, i32, u64]`.
-            let impl_args_with_gat = goal.predicate.projection_ty.args.rebase_onto(
-                tcx,
-                goal_trait_ref.def_id,
-                impl_args,
-            );
+            let impl_args_with_gat =
+                goal.predicate.alias.args.rebase_onto(tcx, goal_trait_ref.def_id, impl_args);
             let args = ecx.translate_args(
                 goal.param_env,
                 impl_def_id,
@@ -290,7 +286,7 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
         ecx: &mut EvalCtxt<'_, 'tcx>,
         goal: Goal<'tcx, Self>,
     ) -> QueryResult<'tcx> {
-        ecx.tcx().sess.delay_span_bug(
+        ecx.tcx().sess.span_delayed_bug(
             ecx.tcx().def_span(goal.predicate.def_id()),
             "associated types not allowed on auto traits",
         );
@@ -419,7 +415,8 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
                         DUMMY_SP,
                         [ty::GenericArg::from(goal.predicate.self_ty())],
                     );
-                    ecx.add_goal(goal.with(tcx, sized_predicate));
+                    // FIXME(-Znext-solver=coinductive): Should this be `GoalSource::ImplWhereBound`?
+                    ecx.add_goal(GoalSource::Misc, goal.with(tcx, sized_predicate));
                     tcx.types.unit
                 }
 
@@ -427,7 +424,11 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
                     None => tcx.types.unit,
                     Some(field_def) => {
                         let self_ty = field_def.ty(tcx, args);
-                        ecx.add_goal(goal.with(tcx, goal.predicate.with_self_ty(tcx, self_ty)));
+                        // FIXME(-Znext-solver=coinductive): Should this be `GoalSource::ImplWhereBound`?
+                        ecx.add_goal(
+                            GoalSource::Misc,
+                            goal.with(tcx, goal.predicate.with_self_ty(tcx, self_ty)),
+                        );
                         return ecx
                             .evaluate_added_goals_and_make_canonical_response(Certainty::Yes);
                     }
@@ -437,7 +438,11 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
                 ty::Tuple(elements) => match elements.last() {
                     None => tcx.types.unit,
                     Some(&self_ty) => {
-                        ecx.add_goal(goal.with(tcx, goal.predicate.with_self_ty(tcx, self_ty)));
+                        // FIXME(-Znext-solver=coinductive): Should this be `GoalSource::ImplWhereBound`?
+                        ecx.add_goal(
+                            GoalSource::Misc,
+                            goal.with(tcx, goal.predicate.with_self_ty(tcx, self_ty)),
+                        );
                         return ecx
                             .evaluate_added_goals_and_make_canonical_response(Certainty::Yes);
                     }
@@ -520,6 +525,40 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
         )
     }
 
+    fn consider_builtin_async_iterator_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        let self_ty = goal.predicate.self_ty();
+        let ty::Coroutine(def_id, args, _) = *self_ty.kind() else {
+            return Err(NoSolution);
+        };
+
+        // Coroutines are not AsyncIterators unless they come from `gen` desugaring
+        let tcx = ecx.tcx();
+        if !tcx.coroutine_is_async_gen(def_id) {
+            return Err(NoSolution);
+        }
+
+        ecx.probe_misc_candidate("builtin AsyncIterator kind").enter(|ecx| {
+            // Take `AsyncIterator<Item = I>` and turn it into the corresponding
+            // coroutine yield ty `Poll<Option<I>>`.
+            let expected_ty = Ty::new_adt(
+                tcx,
+                tcx.adt_def(tcx.require_lang_item(LangItem::Poll, None)),
+                tcx.mk_args(&[Ty::new_adt(
+                    tcx,
+                    tcx.adt_def(tcx.require_lang_item(LangItem::Option, None)),
+                    tcx.mk_args(&[goal.predicate.term.into()]),
+                )
+                .into()]),
+            );
+            let yield_ty = args.as_coroutine().yield_ty();
+            ecx.eq(goal.param_env, expected_ty, yield_ty)?;
+            ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+        })
+    }
+
     fn consider_builtin_coroutine_candidate(
         ecx: &mut EvalCtxt<'_, 'tcx>,
         goal: Goal<'tcx, Self>,
diff --git a/compiler/rustc_trait_selection/src/solve/project_goals/opaques.rs b/compiler/rustc_trait_selection/src/solve/normalizes_to/opaques.rs
index ebd129f32..b5d1aa06e 100644
--- a/compiler/rustc_trait_selection/src/solve/project_goals/opaques.rs
+++ b/compiler/rustc_trait_selection/src/solve/normalizes_to/opaques.rs
@@ -12,10 +12,10 @@ use crate::solve::{EvalCtxt, SolverMode};
 impl<'tcx> EvalCtxt<'_, 'tcx> {
     pub(super) fn normalize_opaque_type(
         &mut self,
-        goal: Goal<'tcx, ty::ProjectionPredicate<'tcx>>,
+        goal: Goal<'tcx, ty::NormalizesTo<'tcx>>,
     ) -> QueryResult<'tcx> {
         let tcx = self.tcx();
-        let opaque_ty = goal.predicate.projection_ty;
+        let opaque_ty = goal.predicate.alias;
         let expected = goal.predicate.term.ty().expect("no such thing as an opaque const");
 
         match (goal.param_env.reveal(), self.solver_mode()) {
@@ -44,6 +44,10 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
                 // Prefer opaques registered already.
                 let opaque_type_key =
                     ty::OpaqueTypeKey { def_id: opaque_ty_def_id, args: opaque_ty.args };
+                // FIXME: This also unifies the previous hidden type with the expected.
+                //
+                // If that fails, we insert `expected` as a new hidden type instead of
+                // eagerly emitting an error.
                 let matches =
                     self.unify_existing_opaque_tys(goal.param_env, opaque_type_key, expected);
                 if !matches.is_empty() {
@@ -53,6 +57,23 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
                         return self.flounder(&matches);
                     }
                 }
+
+                let expected = match self.try_normalize_ty(goal.param_env, expected) {
+                    Some(ty) => {
+                        if ty.is_ty_var() {
+                            return self.evaluate_added_goals_and_make_canonical_response(
+                                Certainty::AMBIGUOUS,
+                            );
+                        } else {
+                            ty
+                        }
+                    }
+                    None => {
+                        return self
+                            .evaluate_added_goals_and_make_canonical_response(Certainty::OVERFLOW);
+                    }
+                };
+
                 // Otherwise, define a new opaque type
                 self.insert_hidden_type(opaque_type_key, goal.param_env, expected)?;
                 self.add_item_bounds_for_hidden_type(
diff --git a/compiler/rustc_trait_selection/src/solve/project_goals/weak_types.rs b/compiler/rustc_trait_selection/src/solve/normalizes_to/weak_types.rs
index 54de32cf6..6d5728797 100644
--- a/compiler/rustc_trait_selection/src/solve/project_goals/weak_types.rs
+++ b/compiler/rustc_trait_selection/src/solve/normalizes_to/weak_types.rs
@@ -3,18 +3,18 @@
 //!
 //! Since a weak alias is not ambiguous, this just computes the `type_of` of
 //! the alias and registers the where-clauses of the type alias.
-use rustc_middle::traits::solve::{Certainty, Goal, QueryResult};
+use rustc_middle::traits::solve::{Certainty, Goal, GoalSource, QueryResult};
 use rustc_middle::ty;
 
-use super::EvalCtxt;
+use crate::solve::EvalCtxt;
 
 impl<'tcx> EvalCtxt<'_, 'tcx> {
     pub(super) fn normalize_weak_type(
         &mut self,
-        goal: Goal<'tcx, ty::ProjectionPredicate<'tcx>>,
+        goal: Goal<'tcx, ty::NormalizesTo<'tcx>>,
     ) -> QueryResult<'tcx> {
         let tcx = self.tcx();
-        let weak_ty = goal.predicate.projection_ty;
+        let weak_ty = goal.predicate.alias;
         let expected = goal.predicate.term.ty().expect("no such thing as a const alias");
 
         let actual = tcx.type_of(weak_ty.def_id).instantiate(tcx, weak_ty.args);
@@ -22,6 +22,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
 
         // Check where clauses
         self.add_goals(
+            GoalSource::Misc,
             tcx.predicates_of(weak_ty.def_id)
                 .instantiate(tcx, weak_ty.args)
                 .predicates
diff --git a/compiler/rustc_trait_selection/src/solve/project_goals.rs b/compiler/rustc_trait_selection/src/solve/project_goals.rs
new file mode 100644
index 000000000..30ae385a8
--- /dev/null
+++ b/compiler/rustc_trait_selection/src/solve/project_goals.rs
@@ -0,0 +1,38 @@
+use crate::solve::GoalSource;
+
+use super::EvalCtxt;
+use rustc_middle::traits::solve::{Certainty, Goal, QueryResult};
+use rustc_middle::ty::{self, ProjectionPredicate};
+
+impl<'tcx> EvalCtxt<'_, 'tcx> {
+    #[instrument(level = "debug", skip(self), ret)]
+    pub(super) fn compute_projection_goal(
+        &mut self,
+        goal: Goal<'tcx, ProjectionPredicate<'tcx>>,
+    ) -> QueryResult<'tcx> {
+        let tcx = self.tcx();
+        let projection_term = match goal.predicate.term.unpack() {
+            ty::TermKind::Ty(_) => goal.predicate.projection_ty.to_ty(tcx).into(),
+            ty::TermKind::Const(_) => ty::Const::new_unevaluated(
+                tcx,
+                ty::UnevaluatedConst::new(
+                    goal.predicate.projection_ty.def_id,
+                    goal.predicate.projection_ty.args,
+                ),
+                tcx.type_of(goal.predicate.projection_ty.def_id)
+                    .instantiate(tcx, goal.predicate.projection_ty.args),
+            )
+            .into(),
+        };
+        let goal = goal.with(
+            tcx,
+            ty::PredicateKind::AliasRelate(
+                projection_term,
+                goal.predicate.term,
+                ty::AliasRelationDirection::Equate,
+            ),
+        );
+        self.add_goal(GoalSource::Misc, goal);
+        self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+    }
+}
diff --git a/compiler/rustc_trait_selection/src/solve/search_graph.rs b/compiler/rustc_trait_selection/src/solve/search_graph.rs
index 7ffa1d7d3..2a161c2d9 100644
--- a/compiler/rustc_trait_selection/src/solve/search_graph.rs
+++ b/compiler/rustc_trait_selection/src/solve/search_graph.rs
@@ -8,11 +8,13 @@ use rustc_index::IndexVec;
 use rustc_middle::dep_graph::dep_kinds;
 use rustc_middle::traits::solve::CacheData;
 use rustc_middle::traits::solve::{CanonicalInput, Certainty, EvaluationCache, QueryResult};
+use rustc_middle::ty;
 use rustc_middle::ty::TyCtxt;
 use rustc_session::Limit;
 use std::collections::hash_map::Entry;
 
 rustc_index::newtype_index! {
+    #[orderable]
     pub struct StackDepth {}
 }
 
@@ -37,7 +39,7 @@ struct StackEntry<'tcx> {
     /// If we were to use that result when later trying to prove another cycle
     /// participant, we can end up with unstable query results.
     ///
-    /// See tests/ui/new-solver/coinduction/incompleteness-unstable-result.rs for
+    /// See tests/ui/next-solver/coinduction/incompleteness-unstable-result.rs for
     /// an example of where this is needed.
     cycle_participants: FxHashSet<CanonicalInput<'tcx>>,
 }
@@ -110,37 +112,13 @@ impl<'tcx> SearchGraph<'tcx> {
         self.stack.is_empty()
     }
 
-    /// Whether we're currently in a cycle. This should only be used
-    /// for debug assertions.
-    pub(super) fn in_cycle(&self) -> bool {
-        if let Some(stack_depth) = self.stack.last_index() {
-            // Either the current goal on the stack is the root of a cycle
-            // or it depends on a goal with a lower depth.
-            self.stack[stack_depth].has_been_used
-                || self.stack[stack_depth].cycle_root_depth != stack_depth
-        } else {
-            false
-        }
-    }
-
-    /// Fetches whether the current goal encountered overflow.
-    ///
-    /// This should only be used for the check in `evaluate_goal`.
-    pub(super) fn encountered_overflow(&self) -> bool {
-        if let Some(last) = self.stack.raw.last() { last.encountered_overflow } else { false }
-    }
-
-    /// Resets `encountered_overflow` of the current goal.
-    ///
-    /// This should only be used for the check in `evaluate_goal`.
-    pub(super) fn reset_encountered_overflow(&mut self, encountered_overflow: bool) -> bool {
-        if let Some(last) = self.stack.raw.last_mut() {
-            let prev = last.encountered_overflow;
-            last.encountered_overflow = encountered_overflow;
-            prev
-        } else {
-            false
-        }
+    pub(super) fn current_goal_is_normalizes_to(&self) -> bool {
+        self.stack.raw.last().map_or(false, |e| {
+            matches!(
+                e.input.value.goal.predicate.kind().skip_binder(),
+                ty::PredicateKind::NormalizesTo(..)
+            )
+        })
     }
 
     /// Returns the remaining depth allowed for nested goals.
@@ -269,7 +247,7 @@ impl<'tcx> SearchGraph<'tcx> {
                     // in unstable results due to incompleteness.
                     //
                     // However, a test for this would be an even more complex version of
-                    // tests/ui/traits/new-solver/coinduction/incompleteness-unstable-result.rs.
+                    // tests/ui/traits/next-solver/coinduction/incompleteness-unstable-result.rs.
                     // I did not bother to write such a test and we have no regression test
                     // for this. It would be good to have such a test :)
                     #[allow(rustc::potential_query_instability)]
@@ -280,7 +258,7 @@ impl<'tcx> SearchGraph<'tcx> {
                 // until we reach a fixpoint. It is not enough to simply retry the
                 // `root` goal of this cycle.
                 //
-                // See tests/ui/traits/new-solver/cycles/fixpoint-rerun-all-cycle-heads.rs
+                // See tests/ui/traits/next-solver/cycles/fixpoint-rerun-all-cycle-heads.rs
                 // for an example.
                 self.stack[stack_depth].has_been_used = true;
                 return if let Some(result) = self.stack[stack_depth].provisional_result {
diff --git a/compiler/rustc_trait_selection/src/solve/trait_goals.rs b/compiler/rustc_trait_selection/src/solve/trait_goals.rs
index a0e2ad6e2..ac3ffd2d6 100644
--- a/compiler/rustc_trait_selection/src/solve/trait_goals.rs
+++ b/compiler/rustc_trait_selection/src/solve/trait_goals.rs
@@ -1,12 +1,14 @@
 //! Dealing with trait goals, i.e. `T: Trait<'a, U>`.
 
-use super::assembly::{self, structural_traits};
-use super::{EvalCtxt, SolverMode};
+use super::assembly::{self, structural_traits, Candidate};
+use super::{EvalCtxt, GoalSource, SolverMode};
 use rustc_hir::def_id::DefId;
 use rustc_hir::{LangItem, Movability};
 use rustc_infer::traits::query::NoSolution;
 use rustc_middle::traits::solve::inspect::ProbeKind;
-use rustc_middle::traits::solve::{CanonicalResponse, Certainty, Goal, QueryResult};
+use rustc_middle::traits::solve::{
+    CandidateSource, CanonicalResponse, Certainty, Goal, QueryResult,
+};
 use rustc_middle::traits::{BuiltinImplSource, Reveal};
 use rustc_middle::ty::fast_reject::{DeepRejectCtxt, TreatParams, TreatProjections};
 use rustc_middle::ty::{self, ToPredicate, Ty, TyCtxt};
@@ -22,10 +24,6 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
         self.trait_ref
     }
 
-    fn polarity(self) -> ty::ImplPolarity {
-        self.polarity
-    }
-
     fn with_self_ty(self, tcx: TyCtxt<'tcx>, self_ty: Ty<'tcx>) -> Self {
         self.with_self_ty(tcx, self_ty)
     }
@@ -38,14 +36,12 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
         ecx: &mut EvalCtxt<'_, 'tcx>,
         goal: Goal<'tcx, TraitPredicate<'tcx>>,
         impl_def_id: DefId,
-    ) -> QueryResult<'tcx> {
+    ) -> Result<Candidate<'tcx>, NoSolution> {
         let tcx = ecx.tcx();
 
         let impl_trait_ref = tcx.impl_trait_ref(impl_def_id).unwrap();
         let drcx = DeepRejectCtxt { treat_obligation_params: TreatParams::ForLookup };
-        if !drcx
-            .args_refs_may_unify(goal.predicate.trait_ref.args, impl_trait_ref.skip_binder().args)
-        {
+        if !drcx.args_may_unify(goal.predicate.trait_ref.args, impl_trait_ref.skip_binder().args) {
             return Err(NoSolution);
         }
 
@@ -65,7 +61,7 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
             },
         };
 
-        ecx.probe_misc_candidate("impl").enter(|ecx| {
+        ecx.probe_trait_candidate(CandidateSource::Impl(impl_def_id)).enter(|ecx| {
             let impl_args = ecx.fresh_args_for_item(impl_def_id);
             let impl_trait_ref = impl_trait_ref.instantiate(tcx, impl_args);
 
@@ -76,7 +72,7 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
                 .predicates
                 .into_iter()
                 .map(|pred| goal.with(tcx, pred));
-            ecx.add_goals(where_clause_bounds);
+            ecx.add_goals(GoalSource::ImplWhereBound, where_clause_bounds);
 
             ecx.evaluate_added_goals_and_make_canonical_response(maximal_certainty)
         })
@@ -176,7 +172,11 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
             let nested_obligations = tcx
                 .predicates_of(goal.predicate.def_id())
                 .instantiate(tcx, goal.predicate.trait_ref.args);
-            ecx.add_goals(nested_obligations.predicates.into_iter().map(|p| goal.with(tcx, p)));
+            // FIXME(-Znext-solver=coinductive): Should this be `GoalSource::ImplWhereBound`?
+            ecx.add_goals(
+                GoalSource::Misc,
+                nested_obligations.predicates.into_iter().map(|p| goal.with(tcx, p)),
+            );
             ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
         })
     }
@@ -260,7 +260,10 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
                     Err(NoSolution)
                 }
             }
-            ty::ImplPolarity::Reservation => bug!(),
+            // FIXME: Goal polarity should be split from impl polarity
+            ty::ImplPolarity::Reservation => {
+                bug!("we never expect a `Reservation` polarity in a trait goal")
+            }
         }
     }
 
@@ -374,6 +377,30 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
         ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
     }
 
+    fn consider_builtin_async_iterator_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        if goal.predicate.polarity != ty::ImplPolarity::Positive {
+            return Err(NoSolution);
+        }
+
+        let ty::Coroutine(def_id, _, _) = *goal.predicate.self_ty().kind() else {
+            return Err(NoSolution);
+        };
+
+        // Coroutines are not iterators unless they come from `gen` desugaring
+        let tcx = ecx.tcx();
+        if !tcx.coroutine_is_async_gen(def_id) {
+            return Err(NoSolution);
+        }
+
+        // Gen coroutines unconditionally implement `Iterator`
+        // Technically, we need to check that the iterator output type is Sized,
+        // but that's already proven by the coroutines being WF.
+        ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+    }
+
     fn consider_builtin_coroutine_candidate(
         ecx: &mut EvalCtxt<'_, 'tcx>,
         goal: Goal<'tcx, Self>,
@@ -425,7 +452,7 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
             return Err(NoSolution);
         }
 
-        // FIXME(-Ztrait-solver=next): Implement this when we get const working in the new solver
+        // FIXME(-Znext-solver): Implement this when we get const working in the new solver
 
         // `Destruct` is automatically implemented for every type in
         // non-const environments.
@@ -471,7 +498,7 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
             let a_ty = goal.predicate.self_ty();
             // We need to normalize the b_ty since it's destructured as a `dyn Trait`.
             let Some(b_ty) =
-                ecx.try_normalize_ty(goal.param_env, goal.predicate.trait_ref.args.type_at(1))?
+                ecx.try_normalize_ty(goal.param_env, goal.predicate.trait_ref.args.type_at(1))
             else {
                 return ecx.evaluate_added_goals_and_make_canonical_response(Certainty::OVERFLOW);
             };
@@ -489,17 +516,23 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
 
             // Check that the type implements all of the predicates of the trait object.
             // (i.e. the principal, all of the associated types match, and any auto traits)
-            ecx.add_goals(b_data.iter().map(|pred| goal.with(tcx, pred.with_self_ty(tcx, a_ty))));
+            ecx.add_goals(
+                GoalSource::ImplWhereBound,
+                b_data.iter().map(|pred| goal.with(tcx, pred.with_self_ty(tcx, a_ty))),
+            );
 
             // The type must be `Sized` to be unsized.
             if let Some(sized_def_id) = tcx.lang_items().sized_trait() {
-                ecx.add_goal(goal.with(tcx, ty::TraitRef::new(tcx, sized_def_id, [a_ty])));
+                ecx.add_goal(
+                    GoalSource::ImplWhereBound,
+                    goal.with(tcx, ty::TraitRef::new(tcx, sized_def_id, [a_ty])),
+                );
             } else {
                 return Err(NoSolution);
             }
 
             // The type must outlive the lifetime of the `dyn` we're unsizing into.
-            ecx.add_goal(goal.with(tcx, ty::OutlivesPredicate(a_ty, b_region)));
+            ecx.add_goal(GoalSource::Misc, goal.with(tcx, ty::OutlivesPredicate(a_ty, b_region)));
             ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
         })
     }
@@ -538,9 +571,8 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
             let b_ty = match ecx
                 .try_normalize_ty(goal.param_env, goal.predicate.trait_ref.args.type_at(1))
             {
-                Ok(Some(b_ty)) => b_ty,
-                Ok(None) => return vec![misc_candidate(ecx, Certainty::OVERFLOW)],
-                Err(_) => return vec![],
+                Some(b_ty) => b_ty,
+                None => return vec![misc_candidate(ecx, Certainty::OVERFLOW)],
             };
 
             let goal = goal.with(ecx.tcx(), (a_ty, b_ty));
@@ -727,11 +759,14 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         }
 
         // Also require that a_ty's lifetime outlives b_ty's lifetime.
-        self.add_goal(Goal::new(
-            self.tcx(),
-            param_env,
-            ty::Binder::dummy(ty::OutlivesPredicate(a_region, b_region)),
-        ));
+        self.add_goal(
+            GoalSource::ImplWhereBound,
+            Goal::new(
+                self.tcx(),
+                param_env,
+                ty::Binder::dummy(ty::OutlivesPredicate(a_region, b_region)),
+            ),
+        );
 
         self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
     }
@@ -804,14 +839,17 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         // Finally, we require that `TailA: Unsize<TailB>` for the tail field
         // types.
         self.eq(goal.param_env, unsized_a_ty, b_ty)?;
-        self.add_goal(goal.with(
-            tcx,
-            ty::TraitRef::new(
+        self.add_goal(
+            GoalSource::ImplWhereBound,
+            goal.with(
                 tcx,
-                tcx.lang_items().unsize_trait().unwrap(),
-                [a_tail_ty, b_tail_ty],
+                ty::TraitRef::new(
+                    tcx,
+                    tcx.lang_items().unsize_trait().unwrap(),
+                    [a_tail_ty, b_tail_ty],
+                ),
             ),
-        ));
+        );
         self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
     }
 
@@ -843,14 +881,17 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         self.eq(goal.param_env, unsized_a_ty, b_ty)?;
 
         // Similar to ADTs, require that we can unsize the tail.
-        self.add_goal(goal.with(
-            tcx,
-            ty::TraitRef::new(
+        self.add_goal(
+            GoalSource::ImplWhereBound,
+            goal.with(
                 tcx,
-                tcx.lang_items().unsize_trait().unwrap(),
-                [a_last_ty, b_last_ty],
+                ty::TraitRef::new(
+                    tcx,
+                    tcx.lang_items().unsize_trait().unwrap(),
+                    [a_last_ty, b_last_ty],
+                ),
             ),
-        ));
+        );
         self.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
     }
 
@@ -959,6 +1000,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
     ) -> QueryResult<'tcx> {
         self.probe_misc_candidate("constituent tys").enter(|ecx| {
             ecx.add_goals(
+                GoalSource::ImplWhereBound,
                 constituent_tys(ecx, goal.predicate.self_ty())?
                     .into_iter()
                     .map(|ty| goal.with(ecx.tcx(), goal.predicate.with_self_ty(ecx.tcx(), ty)))
diff --git a/compiler/rustc_trait_selection/src/traits/auto_trait.rs b/compiler/rustc_trait_selection/src/traits/auto_trait.rs
index 8096d7969..13a09917c 100644
--- a/compiler/rustc_trait_selection/src/traits/auto_trait.rs
+++ b/compiler/rustc_trait_selection/src/traits/auto_trait.rs
@@ -5,11 +5,9 @@ use super::*;
 
 use crate::errors::UnableToConstructConstantValue;
 use crate::infer::region_constraints::{Constraint, RegionConstraintData};
-use crate::infer::InferCtxt;
 use crate::traits::project::ProjectAndUnifyResult;
 use rustc_infer::infer::DefineOpaqueTypes;
 use rustc_middle::mir::interpret::ErrorHandled;
-use rustc_middle::ty::visit::TypeVisitableExt;
 use rustc_middle::ty::{ImplPolarity, Region, RegionVid};
 
 use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet};
@@ -252,7 +250,7 @@ impl<'tcx> AutoTraitFinder<'tcx> {
             fresh_preds.insert(self.clean_pred(infcx, predicate.as_predicate()));
         }
 
-        let mut select = SelectionContext::new(&infcx);
+        let mut select = SelectionContext::new(infcx);
 
         let mut already_visited = FxHashSet::default();
         let mut predicates = VecDeque::new();
@@ -410,11 +408,11 @@ impl<'tcx> AutoTraitFinder<'tcx> {
                         iter::zip(new_args.regions(), old_args.regions())
                     {
                         match (*new_region, *old_region) {
-                            // If both predicates have an `ReLateBound` (a HRTB) in the
+                            // If both predicates have an `ReBound` (a HRTB) in the
                             // same spot, we do nothing.
-                            (ty::ReLateBound(_, _), ty::ReLateBound(_, _)) => {}
+                            (ty::ReBound(_, _), ty::ReBound(_, _)) => {}
 
-                            (ty::ReLateBound(_, _), _) | (_, ty::ReVar(_)) => {
+                            (ty::ReBound(_, _), _) | (_, ty::ReVar(_)) => {
                                 // One of these is true:
                                 // The new predicate has a HRTB in a spot where the old
                                 // predicate does not (if they both had a HRTB, the previous
@@ -440,7 +438,7 @@ impl<'tcx> AutoTraitFinder<'tcx> {
                                 // `user_computed_preds`.
                                 return false;
                             }
-                            (_, ty::ReLateBound(_, _)) | (ty::ReVar(_), _) => {
+                            (_, ty::ReBound(_, _)) | (ty::ReVar(_), _) => {
                                 // This is the opposite situation as the previous arm.
                                 // One of these is true:
                                 //
@@ -820,9 +818,9 @@ impl<'tcx> AutoTraitFinder<'tcx> {
                 // the `ParamEnv`.
                 ty::PredicateKind::Clause(ty::ClauseKind::WellFormed(..))
                 | ty::PredicateKind::Clause(ty::ClauseKind::ConstArgHasType(..))
+                | ty::PredicateKind::NormalizesTo(..)
                 | ty::PredicateKind::AliasRelate(..)
                 | ty::PredicateKind::ObjectSafe(..)
-                | ty::PredicateKind::ClosureKind(..)
                 | ty::PredicateKind::Subtype(..)
                 // FIXME(generic_const_exprs): you can absolutely add this as a where clauses
                 | ty::PredicateKind::Clause(ty::ClauseKind::ConstEvaluatable(..))
diff --git a/compiler/rustc_trait_selection/src/traits/coherence.rs b/compiler/rustc_trait_selection/src/traits/coherence.rs
index dcf5fd869..533fe32f7 100644
--- a/compiler/rustc_trait_selection/src/traits/coherence.rs
+++ b/compiler/rustc_trait_selection/src/traits/coherence.rs
@@ -6,8 +6,8 @@
 
 use crate::infer::outlives::env::OutlivesEnvironment;
 use crate::infer::InferOk;
-use crate::solve::inspect;
 use crate::solve::inspect::{InspectGoal, ProofTreeInferCtxtExt, ProofTreeVisitor};
+use crate::solve::{deeply_normalize_for_diagnostics, inspect};
 use crate::traits::engine::TraitEngineExt;
 use crate::traits::query::evaluate_obligation::InferCtxtExt;
 use crate::traits::select::{IntercrateAmbiguityCause, TreatInductiveCycleAs};
@@ -20,15 +20,15 @@ use crate::traits::{
 };
 use rustc_data_structures::fx::FxIndexSet;
 use rustc_errors::Diagnostic;
+use rustc_hir::def::DefKind;
 use rustc_hir::def_id::{DefId, LOCAL_CRATE};
 use rustc_infer::infer::{DefineOpaqueTypes, InferCtxt, TyCtxtInferExt};
 use rustc_infer::traits::{util, TraitEngine};
 use rustc_middle::traits::query::NoSolution;
-use rustc_middle::traits::solve::{Certainty, Goal};
+use rustc_middle::traits::solve::{CandidateSource, Certainty, Goal};
 use rustc_middle::traits::specialization_graph::OverlapMode;
 use rustc_middle::traits::DefiningAnchor;
 use rustc_middle::ty::fast_reject::{DeepRejectCtxt, TreatParams};
-use rustc_middle::ty::print::with_no_trimmed_paths;
 use rustc_middle::ty::visit::{TypeVisitable, TypeVisitableExt};
 use rustc_middle::ty::{self, Ty, TyCtxt, TypeSuperVisitable, TypeVisitor};
 use rustc_session::lint::builtin::COINDUCTIVE_OVERLAP_IN_COHERENCE;
@@ -53,7 +53,7 @@ pub enum Conflict {
 
 pub struct OverlapResult<'tcx> {
     pub impl_header: ty::ImplHeader<'tcx>,
-    pub intercrate_ambiguity_causes: FxIndexSet<IntercrateAmbiguityCause>,
+    pub intercrate_ambiguity_causes: FxIndexSet<IntercrateAmbiguityCause<'tcx>>,
 
     /// `true` if the overlap might've been permitted before the shift
     /// to universes.
@@ -100,7 +100,7 @@ pub fn overlapping_impls(
     let impl1_ref = tcx.impl_trait_ref(impl1_def_id);
     let impl2_ref = tcx.impl_trait_ref(impl2_def_id);
     let may_overlap = match (impl1_ref, impl2_ref) {
-        (Some(a), Some(b)) => drcx.args_refs_may_unify(a.skip_binder().args, b.skip_binder().args),
+        (Some(a), Some(b)) => drcx.args_may_unify(a.skip_binder().args, b.skip_binder().args),
         (None, None) => {
             let self_ty1 = tcx.type_of(impl1_def_id).skip_binder();
             let self_ty2 = tcx.type_of(impl2_def_id).skip_binder();
@@ -245,7 +245,7 @@ fn overlap<'tcx>(
                                     let trait_ref = infcx.resolve_vars_if_possible(trait_ref);
                                     format!(
                                         "of `{}` for `{}`",
-                                        trait_ref.print_only_trait_path(),
+                                        trait_ref.print_trait_sugared(),
                                         trait_ref.self_ty()
                                     )
                                 }
@@ -273,7 +273,6 @@ fn overlap<'tcx>(
                                     causing the impls to overlap",
                                 infcx.resolve_vars_if_possible(failing_obligation.predicate)
                             ));
-                            lint
                         },
                     );
                 }
@@ -308,7 +307,13 @@ fn overlap<'tcx>(
         .iter()
         .any(|c| c.0.involves_placeholders());
 
-    let impl_header = selcx.infcx.resolve_vars_if_possible(impl1_header);
+    let mut impl_header = infcx.resolve_vars_if_possible(impl1_header);
+
+    // Deeply normalize the impl header for diagnostics, ignoring any errors if this fails.
+    if infcx.next_trait_solver() {
+        impl_header = deeply_normalize_for_diagnostics(&infcx, param_env, impl_header);
+    }
+
     Some(OverlapResult { impl_header, intercrate_ambiguity_causes, involves_placeholder })
 }
 
@@ -329,7 +334,7 @@ fn equate_impl_headers<'tcx>(
                 impl1.self_ty,
                 impl2.self_ty,
             ),
-            _ => bug!("mk_eq_impl_headers given mismatched impl kinds"),
+            _ => bug!("equate_impl_headers given mismatched impl kinds"),
         };
 
     result.map(|infer_ok| infer_ok.obligations).ok()
@@ -360,15 +365,23 @@ fn impl_intersection_has_impossible_obligation<'a, 'cx, 'tcx>(
     let infcx = selcx.infcx;
 
     obligations.iter().find(|obligation| {
-        if infcx.next_trait_solver() {
-            infcx.evaluate_obligation(obligation).map_or(false, |result| !result.may_apply())
+        let evaluation_result = if infcx.next_trait_solver() {
+            infcx.evaluate_obligation(obligation)
         } else {
             // We use `evaluate_root_obligation` to correctly track intercrate
             // ambiguity clauses. We cannot use this in the new solver.
-            selcx.evaluate_root_obligation(obligation).map_or(
-                false, // Overflow has occurred, and treat the obligation as possibly holding.
-                |result| !result.may_apply(),
-            )
+            selcx.evaluate_root_obligation(obligation)
+        };
+
+        match evaluation_result {
+            Ok(result) => !result.may_apply(),
+            // If overflow occurs, we need to conservatively treat the goal as possibly holding,
+            // since there can be instantiations of this goal that don't overflow and result in
+            // success. This isn't much of a problem in the old solver, since we treat overflow
+            // fatally (this still can be encountered: <https://github.com/rust-lang/rust/issues/105231>),
+            // but in the new solver, this is very important for correctness, since overflow
+            // *must* be treated as ambiguity for completeness.
+            Err(_overflow) => false,
         }
     })
 }
@@ -396,8 +409,11 @@ fn impl_intersection_has_negative_obligation(
 ) -> bool {
     debug!("negative_impl(impl1_def_id={:?}, impl2_def_id={:?})", impl1_def_id, impl2_def_id);
 
+    // N.B. We need to unify impl headers *with* intercrate mode, even if proving negative predicates
+    // do not need intercrate mode enabled.
     let ref infcx = tcx.infer_ctxt().intercrate(true).with_next_trait_solver(true).build();
-    let universe = infcx.universe();
+    let root_universe = infcx.universe();
+    assert_eq!(root_universe, ty::UniverseIndex::ROOT);
 
     let impl1_header = fresh_impl_header(infcx, impl1_def_id);
     let param_env =
@@ -407,13 +423,25 @@ fn impl_intersection_has_negative_obligation(
 
     // Equate the headers to find their intersection (the general type, with infer vars,
     // that may apply both impls).
-    let Some(_equate_obligations) =
+    let Some(equate_obligations) =
         equate_impl_headers(infcx, param_env, &impl1_header, &impl2_header)
     else {
         return false;
     };
 
-    plug_infer_with_placeholders(infcx, universe, (impl1_header.impl_args, impl2_header.impl_args));
+    // FIXME(with_negative_coherence): the infcx has constraints from equating
+    // the impl headers. We should use these constraints as assumptions, not as
+    // requirements, when proving the negated where clauses below.
+    drop(equate_obligations);
+    drop(infcx.take_registered_region_obligations());
+    drop(infcx.take_and_reset_region_constraints());
+
+    plug_infer_with_placeholders(
+        infcx,
+        root_universe,
+        (impl1_header.impl_args, impl2_header.impl_args),
+    );
+    let param_env = infcx.resolve_vars_if_possible(param_env);
 
     util::elaborate(tcx, tcx.predicates_of(impl2_def_id).instantiate(tcx, impl2_header.impl_args))
         .any(|(clause, _)| try_prove_negated_where_clause(infcx, clause, param_env))
@@ -458,7 +486,7 @@ fn plug_infer_with_placeholders<'tcx>(
                         ),
                     )
                 else {
-                    bug!()
+                    bug!("we always expect to be able to plug an infer var with placeholder")
                 };
                 assert_eq!(obligations, &[]);
                 ControlFlow::Continue(())
@@ -481,7 +509,7 @@ fn plug_infer_with_placeholders<'tcx>(
                         ),
                     )
                 else {
-                    bug!()
+                    bug!("we always expect to be able to plug an infer var with placeholder")
                 };
                 assert_eq!(obligations, &[]);
                 ControlFlow::Continue(())
@@ -515,7 +543,7 @@ fn plug_infer_with_placeholders<'tcx>(
                             ),
                         )
                     else {
-                        bug!()
+                        bug!("we always expect to be able to plug an infer var with placeholder")
                     };
                     assert_eq!(obligations, &[]);
                 }
@@ -540,15 +568,11 @@ fn try_prove_negated_where_clause<'tcx>(
         return false;
     };
 
-    // FIXME(with_negative_coherence): the infcx has region contraints from equating
-    // the impl headers as requirements. Given that the only region constraints we
-    // get are involving inference regions in the root, it shouldn't matter, but
-    // still sus.
-    //
-    // We probably should just throw away the region obligations registered up until
-    // now, or ideally use them as assumptions when proving the region obligations
-    // that we get from proving the negative predicate below.
-    let ref infcx = root_infcx.fork();
+    // N.B. We don't need to use intercrate mode here because we're trying to prove
+    // the *existence* of a negative goal, not the non-existence of a positive goal.
+    // Without this, we over-eagerly register coherence ambiguity candidates when
+    // impl candidates do exist.
+    let ref infcx = root_infcx.fork_with_intercrate(false);
     let ocx = ObligationCtxt::new(infcx);
 
     ocx.register_obligation(Obligation::new(
@@ -830,7 +854,7 @@ where
     }
 
     fn visit_ty(&mut self, ty: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
-        // Need to lazily normalize here in with `-Ztrait-solver=next-coherence`.
+        // Need to lazily normalize here in with `-Znext-solver=coherence`.
         let ty = match (self.lazily_normalize_ty)(ty) {
             Ok(ty) => ty,
             Err(err) => return ControlFlow::Break(OrphanCheckEarlyExit::NormalizationFailure(err)),
@@ -967,8 +991,8 @@ where
 fn compute_intercrate_ambiguity_causes<'tcx>(
     infcx: &InferCtxt<'tcx>,
     obligations: &[PredicateObligation<'tcx>],
-) -> FxIndexSet<IntercrateAmbiguityCause> {
-    let mut causes: FxIndexSet<IntercrateAmbiguityCause> = Default::default();
+) -> FxIndexSet<IntercrateAmbiguityCause<'tcx>> {
+    let mut causes: FxIndexSet<IntercrateAmbiguityCause<'tcx>> = Default::default();
 
     for obligation in obligations {
         search_ambiguity_causes(infcx, obligation.clone().into(), &mut causes);
@@ -977,11 +1001,11 @@ fn compute_intercrate_ambiguity_causes<'tcx>(
     causes
 }
 
-struct AmbiguityCausesVisitor<'a> {
-    causes: &'a mut FxIndexSet<IntercrateAmbiguityCause>,
+struct AmbiguityCausesVisitor<'a, 'tcx> {
+    causes: &'a mut FxIndexSet<IntercrateAmbiguityCause<'tcx>>,
 }
 
-impl<'a, 'tcx> ProofTreeVisitor<'tcx> for AmbiguityCausesVisitor<'a> {
+impl<'a, 'tcx> ProofTreeVisitor<'tcx> for AmbiguityCausesVisitor<'a, 'tcx> {
     type BreakTy = !;
     fn visit_goal(&mut self, goal: &InspectGoal<'_, 'tcx>) -> ControlFlow<Self::BreakTy> {
         let infcx = goal.infcx();
@@ -998,26 +1022,53 @@ impl<'a, 'tcx> ProofTreeVisitor<'tcx> for AmbiguityCausesVisitor<'a> {
 
         let Goal { param_env, predicate } = goal.goal();
 
-        // For bound predicates we simply call `infcx.replace_bound_vars_with_placeholders`
+        // For bound predicates we simply call `infcx.instantiate_binder_with_placeholders`
         // and then prove the resulting predicate as a nested goal.
         let trait_ref = match predicate.kind().no_bound_vars() {
             Some(ty::PredicateKind::Clause(ty::ClauseKind::Trait(tr))) => tr.trait_ref,
-            Some(ty::PredicateKind::Clause(ty::ClauseKind::Projection(proj))) => {
+            Some(ty::PredicateKind::Clause(ty::ClauseKind::Projection(proj)))
+                if matches!(
+                    infcx.tcx.def_kind(proj.projection_ty.def_id),
+                    DefKind::AssocTy | DefKind::AssocConst
+                ) =>
+            {
                 proj.projection_ty.trait_ref(infcx.tcx)
             }
             _ => return ControlFlow::Continue(()),
         };
 
+        // Add ambiguity causes for reservation impls.
+        for cand in goal.candidates() {
+            if let inspect::ProbeKind::TraitCandidate {
+                source: CandidateSource::Impl(def_id),
+                result: Ok(_),
+            } = cand.kind()
+            {
+                if let ty::ImplPolarity::Reservation = infcx.tcx.impl_polarity(def_id) {
+                    let message = infcx
+                        .tcx
+                        .get_attr(def_id, sym::rustc_reservation_impl)
+                        .and_then(|a| a.value_str());
+                    if let Some(message) = message {
+                        self.causes.insert(IntercrateAmbiguityCause::ReservationImpl { message });
+                    }
+                }
+            }
+        }
+
+        // Add ambiguity causes for unknowable goals.
         let mut ambiguity_cause = None;
         for cand in goal.candidates() {
             // FIXME: boiiii, using string comparisions here sure is scuffed.
-            if let inspect::ProbeKind::MiscCandidate { name: "coherence unknowable", result: _ } =
-                cand.kind()
+            if let inspect::ProbeKind::MiscCandidate {
+                name: "coherence unknowable",
+                result: Ok(_),
+            } = cand.kind()
             {
                 let lazily_normalize_ty = |ty: Ty<'tcx>| {
                     let mut fulfill_cx = <dyn TraitEngine<'tcx>>::new(infcx);
                     if matches!(ty.kind(), ty::Alias(..)) {
-                        // FIXME(-Ztrait-solver=next-coherence): we currently don't
+                        // FIXME(-Znext-solver=coherence): we currently don't
                         // normalize opaque types here, resulting in diverging behavior
                         // for TAITs.
                         match infcx
@@ -1038,25 +1089,23 @@ impl<'a, 'tcx> ProofTreeVisitor<'tcx> for AmbiguityCausesVisitor<'a> {
                         Ok(Ok(())) => warn!("expected an unknowable trait ref: {trait_ref:?}"),
                         Ok(Err(conflict)) => {
                             if !trait_ref.references_error() {
+                                // Normalize the trait ref for diagnostics, ignoring any errors if this fails.
+                                let trait_ref =
+                                    deeply_normalize_for_diagnostics(infcx, param_env, trait_ref);
+
                                 let self_ty = trait_ref.self_ty();
-                                let (trait_desc, self_desc) = with_no_trimmed_paths!({
-                                    let trait_desc = trait_ref.print_only_trait_path().to_string();
-                                    let self_desc = self_ty
-                                        .has_concrete_skeleton()
-                                        .then(|| self_ty.to_string());
-                                    (trait_desc, self_desc)
-                                });
+                                let self_ty = self_ty.has_concrete_skeleton().then(|| self_ty);
                                 ambiguity_cause = Some(match conflict {
                                     Conflict::Upstream => {
                                         IntercrateAmbiguityCause::UpstreamCrateUpdate {
-                                            trait_desc,
-                                            self_desc,
+                                            trait_ref,
+                                            self_ty,
                                         }
                                     }
                                     Conflict::Downstream => {
                                         IntercrateAmbiguityCause::DownstreamCrate {
-                                            trait_desc,
-                                            self_desc,
+                                            trait_ref,
+                                            self_ty,
                                         }
                                     }
                                 });
@@ -1093,7 +1142,7 @@ impl<'a, 'tcx> ProofTreeVisitor<'tcx> for AmbiguityCausesVisitor<'a> {
 fn search_ambiguity_causes<'tcx>(
     infcx: &InferCtxt<'tcx>,
     goal: Goal<'tcx, ty::Predicate<'tcx>>,
-    causes: &mut FxIndexSet<IntercrateAmbiguityCause>,
+    causes: &mut FxIndexSet<IntercrateAmbiguityCause<'tcx>>,
 ) {
     infcx.visit_proof_tree(goal, &mut AmbiguityCausesVisitor { causes });
 }
diff --git a/compiler/rustc_trait_selection/src/traits/const_evaluatable.rs b/compiler/rustc_trait_selection/src/traits/const_evaluatable.rs
index 62ab1e104..aee513207 100644
--- a/compiler/rustc_trait_selection/src/traits/const_evaluatable.rs
+++ b/compiler/rustc_trait_selection/src/traits/const_evaluatable.rs
@@ -65,7 +65,7 @@ pub fn is_const_evaluatable<'tcx>(
                 // FIXME(generic_const_exprs): we have a `ConstKind::Expr` which is fully concrete, but
                 // currently it is not possible to evaluate `ConstKind::Expr` so we are unable to tell if it
                 // is evaluatable or not. For now we just ICE until this is implemented.
-                Err(NotConstEvaluatable::Error(tcx.sess.delay_span_bug(
+                Err(NotConstEvaluatable::Error(tcx.sess.span_delayed_bug(
                     span,
                     "evaluating `ConstKind::Expr` is not currently supported",
                 )))
@@ -74,7 +74,7 @@ pub fn is_const_evaluatable<'tcx>(
                 let concrete = infcx.const_eval_resolve(param_env, uv, Some(span));
                 match concrete {
                     Err(ErrorHandled::TooGeneric(_)) => {
-                        Err(NotConstEvaluatable::Error(infcx.tcx.sess.delay_span_bug(
+                        Err(NotConstEvaluatable::Error(infcx.tcx.sess.span_delayed_bug(
                             span,
                             "Missing value for constant, but no error reported?",
                         )))
@@ -138,10 +138,10 @@ pub fn is_const_evaluatable<'tcx>(
                 } else if uv.has_non_region_param() {
                     NotConstEvaluatable::MentionsParam
                 } else {
-                    let guar = infcx
-                        .tcx
-                        .sess
-                        .delay_span_bug(span, "Missing value for constant, but no error reported?");
+                    let guar = infcx.tcx.sess.span_delayed_bug(
+                        span,
+                        "Missing value for constant, but no error reported?",
+                    );
                     NotConstEvaluatable::Error(guar)
                 };
 
diff --git a/compiler/rustc_trait_selection/src/traits/engine.rs b/compiler/rustc_trait_selection/src/traits/engine.rs
index d9a1a9819..013a50f9f 100644
--- a/compiler/rustc_trait_selection/src/traits/engine.rs
+++ b/compiler/rustc_trait_selection/src/traits/engine.rs
@@ -25,7 +25,6 @@ use rustc_middle::ty::ToPredicate;
 use rustc_middle::ty::TypeFoldable;
 use rustc_middle::ty::Variance;
 use rustc_middle::ty::{self, Ty, TyCtxt};
-use rustc_session::config::TraitSolver;
 
 pub trait TraitEngineExt<'tcx> {
     fn new(infcx: &InferCtxt<'tcx>) -> Box<Self>;
@@ -33,18 +32,16 @@ pub trait TraitEngineExt<'tcx> {
 
 impl<'tcx> TraitEngineExt<'tcx> for dyn TraitEngine<'tcx> {
     fn new(infcx: &InferCtxt<'tcx>) -> Box<Self> {
-        match (infcx.tcx.sess.opts.unstable_opts.trait_solver, infcx.next_trait_solver()) {
-            (TraitSolver::Classic, false) | (TraitSolver::NextCoherence, false) => {
-                Box::new(FulfillmentContext::new(infcx))
-            }
-            (TraitSolver::Classic | TraitSolver::Next | TraitSolver::NextCoherence, true) => {
-                Box::new(NextFulfillmentCtxt::new(infcx))
-            }
-            (TraitSolver::Next, false) => bug!(
-                "incompatible combination of -Ztrait-solver flag ({:?}) and InferCtxt::next_trait_solver ({:?})",
-                infcx.tcx.sess.opts.unstable_opts.trait_solver,
-                infcx.next_trait_solver()
-            ),
+        if infcx.next_trait_solver() {
+            Box::new(NextFulfillmentCtxt::new(infcx))
+        } else {
+            let new_solver_globally =
+                infcx.tcx.sess.opts.unstable_opts.next_solver.map_or(false, |c| c.globally);
+            assert!(
+                !new_solver_globally,
+                "using old solver even though new solver is enabled globally"
+            );
+            Box::new(FulfillmentContext::new(infcx))
         }
     }
 }
@@ -108,7 +105,7 @@ impl<'a, 'tcx> ObligationCtxt<'a, 'tcx> {
         param_env: ty::ParamEnv<'tcx>,
         value: T,
     ) -> T {
-        let infer_ok = self.infcx.at(&cause, param_env).normalize(value);
+        let infer_ok = self.infcx.at(cause, param_env).normalize(value);
         self.register_infer_ok_obligations(infer_ok)
     }
 
@@ -204,7 +201,7 @@ impl<'a, 'tcx> ObligationCtxt<'a, 'tcx> {
         generic_param_scope: LocalDefId,
         outlives_env: &OutlivesEnvironment<'tcx>,
     ) -> Result<(), ErrorGuaranteed> {
-        let errors = self.infcx.resolve_regions(&outlives_env);
+        let errors = self.infcx.resolve_regions(outlives_env);
         if errors.is_empty() {
             Ok(())
         } else {
diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/ambiguity.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/ambiguity.rs
index 5bc5a12a8..b246e476b 100644
--- a/compiler/rustc_trait_selection/src/traits/error_reporting/ambiguity.rs
+++ b/compiler/rustc_trait_selection/src/traits/error_reporting/ambiguity.rs
@@ -1,5 +1,5 @@
 use rustc_hir::def_id::DefId;
-use rustc_infer::infer::{InferCtxt, LateBoundRegionConversionTime};
+use rustc_infer::infer::{BoundRegionConversionTime, InferCtxt};
 use rustc_infer::traits::util::elaborate;
 use rustc_infer::traits::{Obligation, ObligationCause, PolyTraitObligation};
 use rustc_middle::ty;
@@ -53,7 +53,7 @@ pub fn recompute_applicable_impls<'tcx>(
 
         let param_env_predicate = infcx.instantiate_binder_with_fresh_vars(
             DUMMY_SP,
-            LateBoundRegionConversionTime::HigherRankedType,
+            BoundRegionConversionTime::HigherRankedType,
             poly_trait_predicate,
         );
         let param_env_trait_ref =
diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/infer_ctxt_ext.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/infer_ctxt_ext.rs
index b4835b011..0190d5ab4 100644
--- a/compiler/rustc_trait_selection/src/traits/error_reporting/infer_ctxt_ext.rs
+++ b/compiler/rustc_trait_selection/src/traits/error_reporting/infer_ctxt_ext.rs
@@ -61,8 +61,7 @@ impl<'tcx> InferCtxtExt<'tcx> for InferCtxt<'tcx> {
                     .params
                     .iter()
                     .map(|arg| {
-                        if let hir::Pat { kind: hir::PatKind::Tuple(ref args, _), span, .. } =
-                            *arg.pat
+                        if let hir::Pat { kind: hir::PatKind::Tuple(args, _), span, .. } = *arg.pat
                         {
                             Some(ArgKind::Tuple(
                                 Some(span),
@@ -92,7 +91,7 @@ impl<'tcx> InferCtxtExt<'tcx> for InferCtxt<'tcx> {
                     .inputs
                     .iter()
                     .map(|arg| match arg.kind {
-                        hir::TyKind::Tup(ref tys) => ArgKind::Tuple(
+                        hir::TyKind::Tup(tys) => ArgKind::Tuple(
                             Some(arg.span),
                             vec![("_".to_owned(), "_".to_owned()); tys.len()],
                         ),
@@ -100,7 +99,7 @@ impl<'tcx> InferCtxtExt<'tcx> for InferCtxt<'tcx> {
                     })
                     .collect::<Vec<ArgKind>>(),
             ),
-            Node::Ctor(ref variant_data) => {
+            Node::Ctor(variant_data) => {
                 let span = variant_data.ctor_hir_id().map_or(DUMMY_SP, |id| hir.span(id));
                 (span, None, vec![ArgKind::empty(); variant_data.fields().len()])
             }
diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/on_unimplemented.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/on_unimplemented.rs
index 5fbfedd3e..61e97dde5 100644
--- a/compiler/rustc_trait_selection/src/traits/error_reporting/on_unimplemented.rs
+++ b/compiler/rustc_trait_selection/src/traits/error_reporting/on_unimplemented.rs
@@ -103,7 +103,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
     /// to be the enclosing (async) block/function/closure
     fn describe_enclosure(&self, hir_id: hir::HirId) -> Option<&'static str> {
         let hir = self.tcx.hir();
-        let node = hir.find(hir_id)?;
+        let node = self.tcx.opt_hir_node(hir_id)?;
         match &node {
             hir::Node::Item(hir::Item { kind: hir::ItemKind::Fn(sig, _, body_id), .. }) => {
                 self.describe_coroutine(*body_id).or_else(|| {
@@ -184,18 +184,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             flags.push((sym::cause, Some("MainFunctionType".to_string())));
         }
 
-        if let Some(kind) = self.tcx.fn_trait_kind_from_def_id(trait_ref.def_id)
-            && let ty::Tuple(args) = trait_ref.args.type_at(1).kind()
-        {
-            let args = args
-                .iter()
-                .map(|ty| ty.to_string())
-                .collect::<Vec<_>>()
-                .join(", ");
-            flags.push((sym::Trait, Some(format!("{}({args})", kind.as_str()))));
-        } else {
-            flags.push((sym::Trait, Some(trait_ref.print_only_trait_path().to_string())));
-        }
+        flags.push((sym::Trait, Some(trait_ref.print_trait_sugared().to_string())));
 
         // Add all types without trimmed paths or visible paths, ensuring they end up with
         // their "canonical" def path.
@@ -325,7 +314,11 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
 }
 
 #[derive(Clone, Debug)]
-pub struct OnUnimplementedFormatString(Symbol);
+pub struct OnUnimplementedFormatString {
+    symbol: Symbol,
+    span: Span,
+    is_diagnostic_namespace_variant: bool,
+}
 
 #[derive(Debug)]
 pub struct OnUnimplementedDirective {
@@ -354,7 +347,7 @@ pub struct OnUnimplementedNote {
 pub enum AppendConstMessage {
     #[default]
     Default,
-    Custom(Symbol),
+    Custom(Symbol, Span),
 }
 
 #[derive(LintDiagnostic)]
@@ -376,6 +369,43 @@ impl MalformedOnUnimplementedAttrLint {
 #[help]
 pub struct MissingOptionsForOnUnimplementedAttr;
 
+#[derive(LintDiagnostic)]
+#[diag(trait_selection_ignored_diagnostic_option)]
+pub struct IgnoredDiagnosticOption {
+    pub option_name: &'static str,
+    #[label]
+    pub span: Span,
+    #[label(trait_selection_other_label)]
+    pub prev_span: Span,
+}
+
+impl IgnoredDiagnosticOption {
+    fn maybe_emit_warning<'tcx>(
+        tcx: TyCtxt<'tcx>,
+        item_def_id: DefId,
+        new: Option<Span>,
+        old: Option<Span>,
+        option_name: &'static str,
+    ) {
+        if let (Some(new_item), Some(old_item)) = (new, old) {
+            tcx.emit_spanned_lint(
+                UNKNOWN_OR_MALFORMED_DIAGNOSTIC_ATTRIBUTES,
+                tcx.local_def_id_to_hir_id(item_def_id.expect_local()),
+                new_item,
+                IgnoredDiagnosticOption { span: new_item, prev_span: old_item, option_name },
+            );
+        }
+    }
+}
+
+#[derive(LintDiagnostic)]
+#[diag(trait_selection_unknown_format_parameter_for_on_unimplemented_attr)]
+#[help]
+pub struct UnknownFormatParameterForOnUnimplementedAttr {
+    argument_name: Symbol,
+    trait_name: Symbol,
+}
+
 impl<'tcx> OnUnimplementedDirective {
     fn parse(
         tcx: TyCtxt<'tcx>,
@@ -388,8 +418,15 @@ impl<'tcx> OnUnimplementedDirective {
         let mut errored = None;
         let mut item_iter = items.iter();
 
-        let parse_value = |value_str| {
-            OnUnimplementedFormatString::try_parse(tcx, item_def_id, value_str, span).map(Some)
+        let parse_value = |value_str, value_span| {
+            OnUnimplementedFormatString::try_parse(
+                tcx,
+                item_def_id,
+                value_str,
+                value_span,
+                is_diagnostic_namespace_variant,
+            )
+            .map(Some)
         };
 
         let condition = if is_root {
@@ -402,7 +439,7 @@ impl<'tcx> OnUnimplementedDirective {
                 .ok_or_else(|| tcx.sess.emit_err(InvalidOnClauseInOnUnimplemented { span }))?;
             attr::eval_condition(cond, &tcx.sess.parse_sess, Some(tcx.features()), &mut |cfg| {
                 if let Some(value) = cfg.value
-                    && let Err(guar) = parse_value(value)
+                    && let Err(guar) = parse_value(value, cfg.span)
                 {
                     errored = Some(guar);
                 }
@@ -421,17 +458,17 @@ impl<'tcx> OnUnimplementedDirective {
         for item in item_iter {
             if item.has_name(sym::message) && message.is_none() {
                 if let Some(message_) = item.value_str() {
-                    message = parse_value(message_)?;
+                    message = parse_value(message_, item.span())?;
                     continue;
                 }
             } else if item.has_name(sym::label) && label.is_none() {
                 if let Some(label_) = item.value_str() {
-                    label = parse_value(label_)?;
+                    label = parse_value(label_, item.span())?;
                     continue;
                 }
             } else if item.has_name(sym::note) {
                 if let Some(note_) = item.value_str() {
-                    if let Some(note) = parse_value(note_)? {
+                    if let Some(note) = parse_value(note_, item.span())? {
                         notes.push(note);
                         continue;
                     }
@@ -441,7 +478,7 @@ impl<'tcx> OnUnimplementedDirective {
                 && !is_diagnostic_namespace_variant
             {
                 if let Some(parent_label_) = item.value_str() {
-                    parent_label = parse_value(parent_label_)?;
+                    parent_label = parse_value(parent_label_, item.span())?;
                     continue;
                 }
             } else if item.has_name(sym::on)
@@ -456,7 +493,7 @@ impl<'tcx> OnUnimplementedDirective {
                     match Self::parse(
                         tcx,
                         item_def_id,
-                        &items,
+                        items,
                         item.span(),
                         false,
                         is_diagnostic_namespace_variant,
@@ -474,7 +511,7 @@ impl<'tcx> OnUnimplementedDirective {
                 && !is_diagnostic_namespace_variant
             {
                 if let Some(msg) = item.value_str() {
-                    append_const_msg = Some(AppendConstMessage::Custom(msg));
+                    append_const_msg = Some(AppendConstMessage::Custom(msg, item.span()));
                     continue;
                 } else if item.is_word() {
                     append_const_msg = Some(AppendConstMessage::Default);
@@ -485,7 +522,7 @@ impl<'tcx> OnUnimplementedDirective {
             if is_diagnostic_namespace_variant {
                 tcx.emit_spanned_lint(
                     UNKNOWN_OR_MALFORMED_DIAGNOSTIC_ATTRIBUTES,
-                    tcx.hir().local_def_id_to_hir_id(item_def_id.expect_local()),
+                    tcx.local_def_id_to_hir_id(item_def_id.expect_local()),
                     vec![item.span()],
                     MalformedOnUnimplementedAttrLint::new(item.span()),
                 );
@@ -523,6 +560,54 @@ impl<'tcx> OnUnimplementedDirective {
                         subcommands.extend(directive.subcommands);
                         let mut notes = aggr.notes;
                         notes.extend(directive.notes);
+                        IgnoredDiagnosticOption::maybe_emit_warning(
+                            tcx,
+                            item_def_id,
+                            directive.message.as_ref().map(|f| f.span),
+                            aggr.message.as_ref().map(|f| f.span),
+                            "message",
+                        );
+                        IgnoredDiagnosticOption::maybe_emit_warning(
+                            tcx,
+                            item_def_id,
+                            directive.label.as_ref().map(|f| f.span),
+                            aggr.label.as_ref().map(|f| f.span),
+                            "label",
+                        );
+                        IgnoredDiagnosticOption::maybe_emit_warning(
+                            tcx,
+                            item_def_id,
+                            directive.condition.as_ref().map(|i| i.span),
+                            aggr.condition.as_ref().map(|i| i.span),
+                            "condition",
+                        );
+                        IgnoredDiagnosticOption::maybe_emit_warning(
+                            tcx,
+                            item_def_id,
+                            directive.parent_label.as_ref().map(|f| f.span),
+                            aggr.parent_label.as_ref().map(|f| f.span),
+                            "parent_label",
+                        );
+                        IgnoredDiagnosticOption::maybe_emit_warning(
+                            tcx,
+                            item_def_id,
+                            directive.append_const_msg.as_ref().and_then(|c| {
+                                if let AppendConstMessage::Custom(_, s) = c {
+                                    Some(*s)
+                                } else {
+                                    None
+                                }
+                            }),
+                            aggr.append_const_msg.as_ref().and_then(|c| {
+                                if let AppendConstMessage::Custom(_, s) = c {
+                                    Some(*s)
+                                } else {
+                                    None
+                                }
+                            }),
+                            "append_const_msg",
+                        );
+
                         Ok(Some(Self {
                             condition: aggr.condition.or(directive.condition),
                             subcommands,
@@ -560,6 +645,7 @@ impl<'tcx> OnUnimplementedDirective {
                         item_def_id,
                         value,
                         attr.span,
+                        is_diagnostic_namespace_variant,
                     )?),
                     notes: Vec::new(),
                     parent_label: None,
@@ -576,7 +662,7 @@ impl<'tcx> OnUnimplementedDirective {
 
                 tcx.emit_spanned_lint(
                     UNKNOWN_OR_MALFORMED_DIAGNOSTIC_ATTRIBUTES,
-                    tcx.hir().local_def_id_to_hir_id(item_def_id.expect_local()),
+                    tcx.local_def_id_to_hir_id(item_def_id.expect_local()),
                     report_span,
                     MalformedOnUnimplementedAttrLint::new(report_span),
                 );
@@ -587,14 +673,14 @@ impl<'tcx> OnUnimplementedDirective {
                 AttrKind::Normal(p) if !matches!(p.item.args, AttrArgs::Empty) => {
                     tcx.emit_spanned_lint(
                         UNKNOWN_OR_MALFORMED_DIAGNOSTIC_ATTRIBUTES,
-                        tcx.hir().local_def_id_to_hir_id(item_def_id.expect_local()),
+                        tcx.local_def_id_to_hir_id(item_def_id.expect_local()),
                         attr.span,
                         MalformedOnUnimplementedAttrLint::new(attr.span),
                     );
                 }
                 _ => tcx.emit_spanned_lint(
                     UNKNOWN_OR_MALFORMED_DIAGNOSTIC_ATTRIBUTES,
-                    tcx.hir().local_def_id_to_hir_id(item_def_id.expect_local()),
+                    tcx.local_def_id_to_hir_id(item_def_id.expect_local()),
                     attr.span,
                     MissingOptionsForOnUnimplementedAttr,
                 ),
@@ -602,8 +688,9 @@ impl<'tcx> OnUnimplementedDirective {
 
             Ok(None)
         } else {
-            let reported =
-                tcx.sess.delay_span_bug(DUMMY_SP, "of_item: neither meta_item_list nor value_str");
+            let reported = tcx
+                .sess
+                .span_delayed_bug(DUMMY_SP, "of_item: neither meta_item_list nor value_str");
             return Err(reported);
         };
         debug!("of_item({:?}) = {:?}", item_def_id, result);
@@ -636,7 +723,18 @@ impl<'tcx> OnUnimplementedDirective {
                         let value = cfg.value.map(|v| {
                             // `with_no_visible_paths` is also used when generating the options,
                             // so we need to match it here.
-                            ty::print::with_no_visible_paths!(OnUnimplementedFormatString(v).format(tcx, trait_ref, &options_map))
+                            ty::print::with_no_visible_paths!(
+                                OnUnimplementedFormatString {
+                                    symbol: v,
+                                    span: cfg.span,
+                                    is_diagnostic_namespace_variant: false
+                                }
+                                .format(
+                                    tcx,
+                                    trait_ref,
+                                    &options_map
+                                )
+                            )
                         });
 
                         options.contains(&(cfg.name, value))
@@ -679,19 +777,19 @@ impl<'tcx> OnUnimplementedFormatString {
         tcx: TyCtxt<'tcx>,
         item_def_id: DefId,
         from: Symbol,
-        err_sp: Span,
+        value_span: Span,
+        is_diagnostic_namespace_variant: bool,
     ) -> Result<Self, ErrorGuaranteed> {
-        let result = OnUnimplementedFormatString(from);
-        result.verify(tcx, item_def_id, err_sp)?;
+        let result = OnUnimplementedFormatString {
+            symbol: from,
+            span: value_span,
+            is_diagnostic_namespace_variant,
+        };
+        result.verify(tcx, item_def_id)?;
         Ok(result)
     }
 
-    fn verify(
-        &self,
-        tcx: TyCtxt<'tcx>,
-        item_def_id: DefId,
-        span: Span,
-    ) -> Result<(), ErrorGuaranteed> {
+    fn verify(&self, tcx: TyCtxt<'tcx>, item_def_id: DefId) -> Result<(), ErrorGuaranteed> {
         let trait_def_id = if tcx.is_trait(item_def_id) {
             item_def_id
         } else {
@@ -700,7 +798,7 @@ impl<'tcx> OnUnimplementedFormatString {
         };
         let trait_name = tcx.item_name(trait_def_id);
         let generics = tcx.generics_of(item_def_id);
-        let s = self.0.as_str();
+        let s = self.symbol.as_str();
         let parser = Parser::new(s, None, None, false, ParseMode::Format);
         let mut result = Ok(());
         for token in parser {
@@ -710,24 +808,40 @@ impl<'tcx> OnUnimplementedFormatString {
                     Position::ArgumentNamed(s) => {
                         match Symbol::intern(s) {
                             // `{ThisTraitsName}` is allowed
-                            s if s == trait_name => (),
-                            s if ALLOWED_FORMAT_SYMBOLS.contains(&s) => (),
+                            s if s == trait_name && !self.is_diagnostic_namespace_variant => (),
+                            s if ALLOWED_FORMAT_SYMBOLS.contains(&s)
+                                && !self.is_diagnostic_namespace_variant =>
+                            {
+                                ()
+                            }
                             // So is `{A}` if A is a type parameter
                             s if generics.params.iter().any(|param| param.name == s) => (),
                             s => {
-                                result = Err(struct_span_err!(
-                                    tcx.sess,
-                                    span,
-                                    E0230,
-                                    "there is no parameter `{}` on {}",
-                                    s,
-                                    if trait_def_id == item_def_id {
-                                        format!("trait `{trait_name}`")
-                                    } else {
-                                        "impl".to_string()
-                                    }
-                                )
-                                .emit());
+                                if self.is_diagnostic_namespace_variant {
+                                    tcx.emit_spanned_lint(
+                                        UNKNOWN_OR_MALFORMED_DIAGNOSTIC_ATTRIBUTES,
+                                        tcx.local_def_id_to_hir_id(item_def_id.expect_local()),
+                                        self.span,
+                                        UnknownFormatParameterForOnUnimplementedAttr {
+                                            argument_name: s,
+                                            trait_name,
+                                        },
+                                    );
+                                } else {
+                                    result = Err(struct_span_err!(
+                                        tcx.sess,
+                                        self.span,
+                                        E0230,
+                                        "there is no parameter `{}` on {}",
+                                        s,
+                                        if trait_def_id == item_def_id {
+                                            format!("trait `{trait_name}`")
+                                        } else {
+                                            "impl".to_string()
+                                        }
+                                    )
+                                    .emit());
+                                }
                             }
                         }
                     }
@@ -735,7 +849,7 @@ impl<'tcx> OnUnimplementedFormatString {
                     Position::ArgumentIs(..) | Position::ArgumentImplicitlyIs(_) => {
                         let reported = struct_span_err!(
                             tcx.sess,
-                            span,
+                            self.span,
                             E0231,
                             "only named substitution parameters are allowed"
                         )
@@ -774,37 +888,42 @@ impl<'tcx> OnUnimplementedFormatString {
             .collect::<FxHashMap<Symbol, String>>();
         let empty_string = String::new();
 
-        let s = self.0.as_str();
+        let s = self.symbol.as_str();
         let parser = Parser::new(s, None, None, false, ParseMode::Format);
         let item_context = (options.get(&sym::ItemContext)).unwrap_or(&empty_string);
         parser
             .map(|p| match p {
-                Piece::String(s) => s,
+                Piece::String(s) => s.to_owned(),
                 Piece::NextArgument(a) => match a.position {
-                    Position::ArgumentNamed(s) => {
-                        let s = Symbol::intern(s);
+                    Position::ArgumentNamed(arg) => {
+                        let s = Symbol::intern(arg);
                         match generic_map.get(&s) {
-                            Some(val) => val,
-                            None if s == name => &trait_str,
+                            Some(val) => val.to_string(),
+                            None if self.is_diagnostic_namespace_variant => {
+                                format!("{{{arg}}}")
+                            }
+                            None if s == name => trait_str.clone(),
                             None => {
                                 if let Some(val) = options.get(&s) {
-                                    val
+                                    val.clone()
                                 } else if s == sym::from_desugaring {
                                     // don't break messages using these two arguments incorrectly
-                                    &empty_string
-                                } else if s == sym::ItemContext {
-                                    &item_context
+                                    String::new()
+                                } else if s == sym::ItemContext
+                                    && !self.is_diagnostic_namespace_variant
+                                {
+                                    item_context.clone()
                                 } else if s == sym::integral {
-                                    "{integral}"
+                                    String::from("{integral}")
                                 } else if s == sym::integer_ {
-                                    "{integer}"
+                                    String::from("{integer}")
                                 } else if s == sym::float {
-                                    "{float}"
+                                    String::from("{float}")
                                 } else {
                                     bug!(
                                         "broken on_unimplemented {:?} for {:?}: \
                                       no argument matching {:?}",
-                                        self.0,
+                                        self.symbol,
                                         trait_ref,
                                         s
                                     )
@@ -812,7 +931,7 @@ impl<'tcx> OnUnimplementedFormatString {
                             }
                         }
                     }
-                    _ => bug!("broken on_unimplemented {:?} - bad format arg", self.0),
+                    _ => bug!("broken on_unimplemented {:?} - bad format arg", self.symbol),
                 },
             })
             .collect()
diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs
index 6b09bc898..a1b896d22 100644
--- a/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs
+++ b/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs
@@ -17,7 +17,7 @@ use rustc_errors::{
     ErrorGuaranteed, MultiSpan, Style, SuggestionStyle,
 };
 use rustc_hir as hir;
-use rustc_hir::def::DefKind;
+use rustc_hir::def::{DefKind, Res};
 use rustc_hir::def_id::DefId;
 use rustc_hir::intravisit::Visitor;
 use rustc_hir::is_range_literal;
@@ -26,7 +26,7 @@ use rustc_hir::{CoroutineKind, CoroutineSource, Node};
 use rustc_hir::{Expr, HirId};
 use rustc_infer::infer::error_reporting::TypeErrCtxt;
 use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
-use rustc_infer::infer::{DefineOpaqueTypes, InferOk, LateBoundRegionConversionTime};
+use rustc_infer::infer::{BoundRegionConversionTime, DefineOpaqueTypes, InferOk};
 use rustc_middle::hir::map;
 use rustc_middle::traits::IsConstable;
 use rustc_middle::ty::error::TypeError::{self, Sorts};
@@ -36,7 +36,7 @@ use rustc_middle::ty::{
     TypeSuperFoldable, TypeVisitableExt, TypeckResults,
 };
 use rustc_span::def_id::LocalDefId;
-use rustc_span::symbol::{sym, Ident, Symbol};
+use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::{BytePos, DesugaringKind, ExpnKind, MacroKind, Span, DUMMY_SP};
 use rustc_target::spec::abi;
 use std::borrow::Cow;
@@ -99,7 +99,7 @@ impl<'tcx, 'a> CoroutineData<'tcx, 'a> {
             .awaits
             .into_iter()
             .map(|id| hir.expect_expr(id))
-            .find(|await_expr| ty_matches(ty::Binder::dummy(self.0.expr_ty_adjusted(&await_expr))))
+            .find(|await_expr| ty_matches(ty::Binder::dummy(self.0.expr_ty_adjusted(await_expr))))
             .map(|expr| expr.span)
     }
 }
@@ -222,6 +222,15 @@ pub trait TypeErrCtxtExt<'tcx> {
         param_env: ty::ParamEnv<'tcx>,
     ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed>;
 
+    fn note_conflicting_fn_args(
+        &self,
+        err: &mut Diagnostic,
+        cause: &ObligationCauseCode<'tcx>,
+        expected: Ty<'tcx>,
+        found: Ty<'tcx>,
+        param_env: ty::ParamEnv<'tcx>,
+    );
+
     fn note_conflicting_closure_bounds(
         &self,
         cause: &ObligationCauseCode<'tcx>,
@@ -501,7 +510,7 @@ pub fn suggest_restriction<'tcx>(
 impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
     fn suggest_restricting_param_bound(
         &self,
-        mut err: &mut Diagnostic,
+        err: &mut Diagnostic,
         trait_pred: ty::PolyTraitPredicate<'tcx>,
         associated_ty: Option<(&'static str, Ty<'tcx>)>,
         mut body_id: LocalDefId,
@@ -521,7 +530,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
 
         // FIXME: Add check for trait bound that is already present, particularly `?Sized` so we
         //        don't suggest `T: Sized + ?Sized`.
-        while let Some(node) = self.tcx.hir().find_by_def_id(body_id) {
+        while let Some(node) = self.tcx.opt_hir_node_by_def_id(body_id) {
             match node {
                 hir::Node::Item(hir::Item {
                     ident,
@@ -533,7 +542,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                     suggest_restriction(
                         self.tcx,
                         body_id,
-                        &generics,
+                        generics,
                         "`Self`",
                         err,
                         None,
@@ -552,7 +561,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                     assert!(param_ty);
                     // Restricting `Self` for a single method.
                     suggest_restriction(
-                        self.tcx, body_id, &generics, "`Self`", err, None, projection, trait_pred,
+                        self.tcx, body_id, generics, "`Self`", err, None, projection, trait_pred,
                         None,
                     );
                     return;
@@ -575,7 +584,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                     suggest_restriction(
                         self.tcx,
                         body_id,
-                        &generics,
+                        generics,
                         "the associated type",
                         err,
                         Some(fn_sig),
@@ -595,7 +604,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                     suggest_restriction(
                         self.tcx,
                         body_id,
-                        &generics,
+                        generics,
                         "the associated type",
                         err,
                         None,
@@ -662,7 +671,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                     if suggest_constraining_type_param(
                         self.tcx,
                         generics,
-                        &mut err,
+                        err,
                         &param_name,
                         &constraint,
                         Some(trait_pred.def_id()),
@@ -690,7 +699,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                     if suggest_arbitrary_trait_bound(
                         self.tcx,
                         generics,
-                        &mut err,
+                        err,
                         trait_pred,
                         associated_ty,
                     ) {
@@ -723,7 +732,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         let Some(typeck_results) = &self.typeck_results else {
             return false;
         };
-        let hir::Node::Expr(expr) = self.tcx.hir().get(*arg_hir_id) else {
+        let hir::Node::Expr(expr) = self.tcx.hir_node(*arg_hir_id) else {
             return false;
         };
         let Some(arg_ty) = typeck_results.expr_ty_adjusted_opt(expr) else {
@@ -739,9 +748,9 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                 real_trait_pred = parent_trait_pred;
             }
 
-            // We `erase_late_bound_regions` here because `make_subregion` does not handle
-            // `ReLateBound`, and we don't particularly care about the regions.
-            let real_ty = self.tcx.erase_late_bound_regions(real_trait_pred.self_ty());
+            // We `instantiate_bound_regions_with_erased` here because `make_subregion` does not handle
+            // `ReBound`, and we don't particularly care about the regions.
+            let real_ty = self.tcx.instantiate_bound_regions_with_erased(real_trait_pred.self_ty());
             if !self.can_eq(obligation.param_env, real_ty, arg_ty) {
                 continue;
             }
@@ -776,7 +785,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                             kind:
                                 hir::ExprKind::AddrOf(hir::BorrowKind::Ref, hir::Mutability::Not, expr),
                             ..
-                        })) = self.tcx.hir().find(*arg_hir_id)
+                        })) = self.tcx.opt_hir_node(*arg_hir_id)
                         {
                             let derefs = "*".repeat(steps);
                             err.span_suggestion_verbose(
@@ -812,7 +821,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                     if self.predicate_may_hold(&obligation)
                         && self.predicate_must_hold_modulo_regions(&sized_obligation)
                     {
-                        let call_node = self.tcx.hir().get(*call_hir_id);
+                        let call_node = self.tcx.hir_node(*call_hir_id);
                         let msg = "consider dereferencing here";
                         let is_receiver = matches!(
                             call_node,
@@ -871,7 +880,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         };
 
         let hir = self.tcx.hir();
-        let hir_id = hir.local_def_id_to_hir_id(def_id.as_local()?);
+        let hir_id = self.tcx.local_def_id_to_hir_id(def_id.as_local()?);
         match hir.find_parent(hir_id) {
             Some(hir::Node::Stmt(hir::Stmt { kind: hir::StmtKind::Local(local), .. })) => {
                 get_name(err, &local.pat.kind)
@@ -908,7 +917,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
 
         let self_ty = self.instantiate_binder_with_fresh_vars(
             DUMMY_SP,
-            LateBoundRegionConversionTime::FnCall,
+            BoundRegionConversionTime::FnCall,
             trait_pred.self_ty(),
         );
 
@@ -1034,10 +1043,10 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         let hir::ExprKind::Path(hir::QPath::Resolved(None, path)) = expr.kind else {
             return;
         };
-        let hir::def::Res::Local(hir_id) = path.res else {
+        let Res::Local(hir_id) = path.res else {
             return;
         };
-        let Some(hir::Node::Pat(pat)) = self.tcx.hir().find(hir_id) else {
+        let Some(hir::Node::Pat(pat)) = self.tcx.opt_hir_node(hir_id) else {
             return;
         };
         let Some(hir::Node::Local(hir::Local { ty: None, init: Some(init), .. })) =
@@ -1097,7 +1106,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         else {
             return false;
         };
-        let arg_node = self.tcx.hir().get(*arg_hir_id);
+        let arg_node = self.tcx.hir_node(*arg_hir_id);
         let Node::Expr(Expr { kind: hir::ExprKind::Path(_), .. }) = arg_node else { return false };
 
         let clone_trait = self.tcx.require_lang_item(LangItem::Clone, None);
@@ -1237,7 +1246,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
 
         let output = self.instantiate_binder_with_fresh_vars(
             DUMMY_SP,
-            LateBoundRegionConversionTime::FnCall,
+            BoundRegionConversionTime::FnCall,
             output,
         );
         let inputs = inputs
@@ -1246,7 +1255,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             .map(|ty| {
                 self.instantiate_binder_with_fresh_vars(
                     DUMMY_SP,
-                    LateBoundRegionConversionTime::FnCall,
+                    BoundRegionConversionTime::FnCall,
                     inputs.rebind(*ty),
                 )
             })
@@ -1273,7 +1282,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         let code = if let ObligationCauseCode::FunctionArgumentObligation { parent_code, .. } =
             obligation.cause.code()
         {
-            &parent_code
+            parent_code
         } else if let ObligationCauseCode::ItemObligation(_)
         | ObligationCauseCode::ExprItemObligation(..) = obligation.cause.code()
         {
@@ -1378,14 +1387,21 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                         err.message =
                             vec![(rustc_errors::DiagnosticMessage::from(msg), Style::NoStyle)];
                     }
+                    let mut file = None;
                     err.span_label(
                         span,
                         format!(
                             "the trait `{}` is not implemented for `{}`",
                             old_pred.print_modifiers_and_trait_path(),
-                            old_pred.self_ty().skip_binder(),
+                            self.tcx.short_ty_string(old_pred.self_ty().skip_binder(), &mut file),
                         ),
                     );
+                    if let Some(file) = file {
+                        err.note(format!(
+                            "the full type name has been written to '{}'",
+                            file.display()
+                        ));
+                    }
 
                     if imm_ref_self_ty_satisfies_pred && mut_ref_self_ty_satisfies_pred {
                         err.span_suggestions(
@@ -1618,8 +1634,8 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                 }
             }
             if let hir::ExprKind::Path(hir::QPath::Resolved(None, path)) = expr.kind
-                && let hir::def::Res::Local(hir_id) = path.res
-                && let Some(hir::Node::Pat(binding)) = self.tcx.hir().find(hir_id)
+                && let Res::Local(hir_id) = path.res
+                && let Some(hir::Node::Pat(binding)) = self.tcx.opt_hir_node(hir_id)
                 && let Some(hir::Node::Local(local)) = self.tcx.hir().find_parent(binding.hir_id)
                 && let None = local.ty
                 && let Some(binding_expr) = local.init
@@ -1634,9 +1650,8 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
 
     fn suggest_remove_await(&self, obligation: &PredicateObligation<'tcx>, err: &mut Diagnostic) {
         let hir = self.tcx.hir();
-        if let ObligationCauseCode::AwaitableExpr(Some(hir_id)) =
-            obligation.cause.code().peel_derives()
-            && let hir::Node::Expr(expr) = hir.get(*hir_id)
+        if let ObligationCauseCode::AwaitableExpr(hir_id) = obligation.cause.code().peel_derives()
+            && let hir::Node::Expr(expr) = self.tcx.hir_node(*hir_id)
         {
             // FIXME: use `obligation.predicate.kind()...trait_ref.self_ty()` to see if we have `()`
             // and if not maybe suggest doing something else? If we kept the expression around we
@@ -1786,7 +1801,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         trait_pred: ty::PolyTraitPredicate<'tcx>,
     ) -> bool {
         let hir = self.tcx.hir();
-        let node = hir.find_by_def_id(obligation.cause.body_id);
+        let node = self.tcx.opt_hir_node_by_def_id(obligation.cause.body_id);
         if let Some(hir::Node::Item(hir::Item { kind: hir::ItemKind::Fn(sig, _, body_id), .. })) = node
             && let hir::ExprKind::Block(blk, _) = &hir.body(*body_id).value.kind
             && sig.decl.output.span().overlaps(span)
@@ -1821,9 +1836,8 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
     }
 
     fn return_type_span(&self, obligation: &PredicateObligation<'tcx>) -> Option<Span> {
-        let hir = self.tcx.hir();
         let Some(hir::Node::Item(hir::Item { kind: hir::ItemKind::Fn(sig, ..), .. })) =
-            hir.find_by_def_id(obligation.cause.body_id)
+            self.tcx.opt_hir_node_by_def_id(obligation.cause.body_id)
         else {
             return None;
         };
@@ -1867,7 +1881,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         let body = self.tcx.hir().body(self.tcx.hir().body_owned_by(obligation.cause.body_id));
 
         let mut visitor = ReturnsVisitor::default();
-        visitor.visit_body(&body);
+        visitor.visit_body(body);
 
         let mut sugg =
             vec![(span.shrink_to_lo(), "Box<".to_string()), (span.shrink_to_hi(), ">".to_string())];
@@ -1915,14 +1929,14 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         }
 
         let hir = self.tcx.hir();
-        let node = hir.find_by_def_id(obligation.cause.body_id);
+        let node = self.tcx.opt_hir_node_by_def_id(obligation.cause.body_id);
         if let Some(hir::Node::Item(hir::Item { kind: hir::ItemKind::Fn(_, _, body_id), .. })) =
             node
         {
             let body = hir.body(*body_id);
             // Point at all the `return`s in the function as they have failed trait bounds.
             let mut visitor = ReturnsVisitor::default();
-            visitor.visit_body(&body);
+            visitor.visit_body(body);
             let typeck_results = self.typeck_results.as_ref().unwrap();
             for expr in &visitor.returns {
                 if let Some(returned_ty) = typeck_results.node_type_opt(expr.hir_id) {
@@ -2006,6 +2020,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         let signature_kind = format!("{argument_kind} signature");
         err.note_expected_found(&signature_kind, expected_str, &signature_kind, found_str);
 
+        self.note_conflicting_fn_args(&mut err, cause, expected, found, param_env);
         self.note_conflicting_closure_bounds(cause, &mut err);
 
         if let Some(found_node) = found_node {
@@ -2015,6 +2030,158 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         err
     }
 
+    fn note_conflicting_fn_args(
+        &self,
+        err: &mut Diagnostic,
+        cause: &ObligationCauseCode<'tcx>,
+        expected: Ty<'tcx>,
+        found: Ty<'tcx>,
+        param_env: ty::ParamEnv<'tcx>,
+    ) {
+        let ObligationCauseCode::FunctionArgumentObligation { arg_hir_id, .. } = cause else {
+            return;
+        };
+        let ty::FnPtr(expected) = expected.kind() else {
+            return;
+        };
+        let ty::FnPtr(found) = found.kind() else {
+            return;
+        };
+        let Some(Node::Expr(arg)) = self.tcx.opt_hir_node(*arg_hir_id) else {
+            return;
+        };
+        let hir::ExprKind::Path(path) = arg.kind else {
+            return;
+        };
+        let expected_inputs = self.tcx.instantiate_bound_regions_with_erased(*expected).inputs();
+        let found_inputs = self.tcx.instantiate_bound_regions_with_erased(*found).inputs();
+        let both_tys = expected_inputs.iter().copied().zip(found_inputs.iter().copied());
+
+        let arg_expr = |infcx: &InferCtxt<'tcx>, name, expected: Ty<'tcx>, found: Ty<'tcx>| {
+            let (expected_ty, expected_refs) = get_deref_type_and_refs(expected);
+            let (found_ty, found_refs) = get_deref_type_and_refs(found);
+
+            if infcx.can_eq(param_env, found_ty, expected_ty) {
+                if found_refs.len() == expected_refs.len()
+                    && found_refs.iter().eq(expected_refs.iter())
+                {
+                    name
+                } else if found_refs.len() > expected_refs.len() {
+                    let refs = &found_refs[..found_refs.len() - expected_refs.len()];
+                    if found_refs[..expected_refs.len()].iter().eq(expected_refs.iter()) {
+                        format!(
+                            "{}{name}",
+                            refs.iter()
+                                .map(|mutbl| format!("&{}", mutbl.prefix_str()))
+                                .collect::<Vec<_>>()
+                                .join(""),
+                        )
+                    } else {
+                        // The refs have different mutability.
+                        format!(
+                            "{}*{name}",
+                            refs.iter()
+                                .map(|mutbl| format!("&{}", mutbl.prefix_str()))
+                                .collect::<Vec<_>>()
+                                .join(""),
+                        )
+                    }
+                } else if expected_refs.len() > found_refs.len() {
+                    format!(
+                        "{}{name}",
+                        (0..(expected_refs.len() - found_refs.len()))
+                            .map(|_| "*")
+                            .collect::<Vec<_>>()
+                            .join(""),
+                    )
+                } else {
+                    format!(
+                        "{}{name}",
+                        found_refs
+                            .iter()
+                            .map(|mutbl| format!("&{}", mutbl.prefix_str()))
+                            .chain(found_refs.iter().map(|_| "*".to_string()))
+                            .collect::<Vec<_>>()
+                            .join(""),
+                    )
+                }
+            } else {
+                format!("/* {found} */")
+            }
+        };
+        let args_have_same_underlying_type = both_tys.clone().all(|(expected, found)| {
+            let (expected_ty, _) = get_deref_type_and_refs(expected);
+            let (found_ty, _) = get_deref_type_and_refs(found);
+            self.can_eq(param_env, found_ty, expected_ty)
+        });
+        let (closure_names, call_names): (Vec<_>, Vec<_>) = if args_have_same_underlying_type
+            && !expected_inputs.is_empty()
+            && expected_inputs.len() == found_inputs.len()
+            && let Some(typeck) = &self.typeck_results
+            && let Res::Def(res_kind, fn_def_id) = typeck.qpath_res(&path, *arg_hir_id)
+            && res_kind.is_fn_like()
+        {
+            let closure: Vec<_> = self
+                .tcx
+                .fn_arg_names(fn_def_id)
+                .iter()
+                .enumerate()
+                .map(|(i, ident)| {
+                    if ident.name.is_empty() || ident.name == kw::SelfLower {
+                        format!("arg{i}")
+                    } else {
+                        format!("{ident}")
+                    }
+                })
+                .collect();
+            let args = closure
+                .iter()
+                .zip(both_tys)
+                .map(|(name, (expected, found))| {
+                    arg_expr(self.infcx, name.to_owned(), expected, found)
+                })
+                .collect();
+            (closure, args)
+        } else {
+            let closure_args = expected_inputs
+                .iter()
+                .enumerate()
+                .map(|(i, _)| format!("arg{i}"))
+                .collect::<Vec<_>>();
+            let call_args = both_tys
+                .enumerate()
+                .map(|(i, (expected, found))| {
+                    arg_expr(self.infcx, format!("arg{i}"), expected, found)
+                })
+                .collect::<Vec<_>>();
+            (closure_args, call_args)
+        };
+        let closure_names: Vec<_> = closure_names
+            .into_iter()
+            .zip(expected_inputs.iter())
+            .map(|(name, ty)| {
+                format!(
+                    "{name}{}",
+                    if ty.has_infer_types() {
+                        String::new()
+                    } else if ty.references_error() {
+                        ": /* type */".to_string()
+                    } else {
+                        format!(": {ty}")
+                    }
+                )
+            })
+            .collect();
+        err.multipart_suggestion(
+            "consider wrapping the function in a closure",
+            vec![
+                (arg.span.shrink_to_lo(), format!("|{}| ", closure_names.join(", "))),
+                (arg.span.shrink_to_hi(), format!("({})", call_names.join(", "))),
+            ],
+            Applicability::MaybeIncorrect,
+        );
+    }
+
     // Add a note if there are two `Fn`-family bounds that have conflicting argument
     // requirements, which will always cause a closure to have a type error.
     fn note_conflicting_closure_bounds(
@@ -2287,8 +2454,8 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             // represent regions that are part of the suspended
             // coroutine frame. Bound regions are preserved by
             // `erase_regions` and so we must also call
-            // `erase_late_bound_regions`.
-            let ty_erased = self.tcx.erase_late_bound_regions(ty);
+            // `instantiate_bound_regions_with_erased`.
+            let ty_erased = self.tcx.instantiate_bound_regions_with_erased(ty);
             let ty_erased = self.tcx.erase_regions(ty_erased);
             let eq = ty_erased == target_ty_erased;
             debug!(?ty_erased, ?target_ty_erased, ?eq);
@@ -2300,7 +2467,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         // cycles. If we can't use resolved types because the coroutine comes from another crate,
         // we still provide a targeted error but without all the relevant spans.
         let coroutine_data = match &self.typeck_results {
-            Some(t) if t.hir_owner.to_def_id() == coroutine_did_root => CoroutineData(&t),
+            Some(t) if t.hir_owner.to_def_id() == coroutine_did_root => CoroutineData(t),
             _ if coroutine_did.is_local() => {
                 CoroutineData(self.tcx.typeck(coroutine_did.expect_local()))
             }
@@ -2344,7 +2511,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
 
         if interior_or_upvar_span.is_none() {
             interior_or_upvar_span =
-                coroutine_data.try_get_upvar_span(&self, coroutine_did, ty_matches);
+                coroutine_data.try_get_upvar_span(self, coroutine_did, ty_matches);
         }
 
         if interior_or_upvar_span.is_none() && !coroutine_did.is_local() {
@@ -2415,7 +2582,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                             .tcx
                             .parent(coroutine_did)
                             .as_local()
-                            .map(|parent_did| hir.local_def_id_to_hir_id(parent_did))
+                            .map(|parent_did| self.tcx.local_def_id_to_hir_id(parent_did))
                             .and_then(|parent_hir_id| hir.opt_name(parent_hir_id))
                             .map(|name| {
                                 format!("future returned by `{name}` is not {trait_name}")
@@ -2426,11 +2593,28 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                         CoroutineKind::Async(CoroutineSource::Closure) => {
                             format!("future created by async closure is not {trait_name}")
                         }
+                        CoroutineKind::AsyncGen(CoroutineSource::Fn) => self
+                            .tcx
+                            .parent(coroutine_did)
+                            .as_local()
+                            .map(|parent_did| self.tcx.local_def_id_to_hir_id(parent_did))
+                            .and_then(|parent_hir_id| hir.opt_name(parent_hir_id))
+                            .map(|name| {
+                                format!("async iterator returned by `{name}` is not {trait_name}")
+                            })?,
+                        CoroutineKind::AsyncGen(CoroutineSource::Block) => {
+                            format!("async iterator created by async gen block is not {trait_name}")
+                        }
+                        CoroutineKind::AsyncGen(CoroutineSource::Closure) => {
+                            format!(
+                                "async iterator created by async gen closure is not {trait_name}"
+                            )
+                        }
                         CoroutineKind::Gen(CoroutineSource::Fn) => self
                             .tcx
                             .parent(coroutine_did)
                             .as_local()
-                            .map(|parent_did| hir.local_def_id_to_hir_id(parent_did))
+                            .map(|parent_did| self.tcx.local_def_id_to_hir_id(parent_did))
                             .and_then(|parent_hir_id| hir.opt_name(parent_hir_id))
                             .map(|name| {
                                 format!("iterator returned by `{name}` is not {trait_name}")
@@ -2517,7 +2701,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             CoroutineInteriorOrUpvar::Upvar(upvar_span) => {
                 // `Some((ref_ty, is_mut))` if `target_ty` is `&T` or `&mut T` and fails to impl `Send`
                 let non_send = match target_ty.kind() {
-                    ty::Ref(_, ref_ty, mutability) => match self.evaluate_obligation(&obligation) {
+                    ty::Ref(_, ref_ty, mutability) => match self.evaluate_obligation(obligation) {
                         Ok(eval) if !eval.may_apply() => Some((ref_ty, mutability.is_mut())),
                         _ => None,
                     },
@@ -2593,11 +2777,9 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             | ObligationCauseCode::MethodReceiver
             | ObligationCauseCode::ReturnNoExpression
             | ObligationCauseCode::UnifyReceiver(..)
-            | ObligationCauseCode::OpaqueType
             | ObligationCauseCode::MiscObligation
             | ObligationCauseCode::WellFormed(..)
             | ObligationCauseCode::MatchImpl(..)
-            | ObligationCauseCode::ReturnType
             | ObligationCauseCode::ReturnValue(_)
             | ObligationCauseCode::BlockTailExpression(..)
             | ObligationCauseCode::AwaitableExpr(_)
@@ -2608,7 +2790,9 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             | ObligationCauseCode::BinOp { .. }
             | ObligationCauseCode::AscribeUserTypeProvePredicate(..)
             | ObligationCauseCode::DropImpl
-            | ObligationCauseCode::ConstParam(_) => {}
+            | ObligationCauseCode::ConstParam(_)
+            | ObligationCauseCode::ReferenceOutlivesReferent(..)
+            | ObligationCauseCode::ObjectTypeBound(..) => {}
             ObligationCauseCode::RustCall => {
                 if let Some(pred) = predicate.to_opt_poly_trait_pred()
                     && Some(pred.def_id()) == self.tcx.lang_items().sized_trait()
@@ -2622,19 +2806,6 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             ObligationCauseCode::TupleElem => {
                 err.note("only the last element of a tuple may have a dynamically sized type");
             }
-            ObligationCauseCode::ProjectionWf(data) => {
-                err.note(format!("required so that the projection `{data}` is well-formed"));
-            }
-            ObligationCauseCode::ReferenceOutlivesReferent(ref_ty) => {
-                err.note(format!(
-                    "required so that reference `{ref_ty}` does not outlive its referent"
-                ));
-            }
-            ObligationCauseCode::ObjectTypeBound(object_ty, region) => {
-                err.note(format!(
-                    "required so that the lifetime bound of `{region}` for `{object_ty}` is satisfied",
-                ));
-            }
             ObligationCauseCode::ItemObligation(_)
             | ObligationCauseCode::ExprItemObligation(..) => {
                 // We hold the `DefId` of the item introducing the obligation, but displaying it
@@ -2723,11 +2894,14 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                                 .collect::<Vec<_>>();
                             if !impls.is_empty() {
                                 let len = impls.len();
-                                let mut types = impls.iter()
-                                    .map(|t| with_no_trimmed_paths!(format!(
-                                        "  {}",
-                                        tcx.type_of(*t).instantiate_identity(),
-                                    )))
+                                let mut types = impls
+                                    .iter()
+                                    .map(|t| {
+                                        with_no_trimmed_paths!(format!(
+                                            "  {}",
+                                            tcx.type_of(*t).instantiate_identity(),
+                                        ))
+                                    })
                                     .collect::<Vec<_>>();
                                 let post = if types.len() > 9 {
                                     types.truncate(8);
@@ -2749,50 +2923,62 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                 }
             }
             ObligationCauseCode::Coercion { source, target } => {
-                let (source, source_file) =
-                    self.tcx.short_ty_string(self.resolve_vars_if_possible(source));
-                let (target, target_file) =
-                    self.tcx.short_ty_string(self.resolve_vars_if_possible(target));
+                let mut file = None;
+                let source =
+                    self.tcx.short_ty_string(self.resolve_vars_if_possible(source), &mut file);
+                let target =
+                    self.tcx.short_ty_string(self.resolve_vars_if_possible(target), &mut file);
                 err.note(with_forced_trimmed_paths!(format!(
                     "required for the cast from `{source}` to `{target}`",
                 )));
-                if let Some(file) = source_file {
-                    err.note(format!(
-                        "the full name for the source type has been written to '{}'",
-                        file.display(),
-                    ));
-                }
-                if let Some(file) = target_file {
+                if let Some(file) = file {
                     err.note(format!(
-                        "the full name for the target type has been written to '{}'",
+                        "the full name for the type has been written to '{}'",
                         file.display(),
                     ));
                 }
             }
-            ObligationCauseCode::RepeatElementCopy { is_constable, elt_type, elt_span, elt_stmt_span } => {
+            ObligationCauseCode::RepeatElementCopy {
+                is_constable,
+                elt_type,
+                elt_span,
+                elt_stmt_span,
+            } => {
                 err.note(
                     "the `Copy` trait is required because this value will be copied for each element of the array",
                 );
                 let value_kind = match is_constable {
                     IsConstable::Fn => Some("the result of the function call"),
                     IsConstable::Ctor => Some("the result of the constructor"),
-                    _ => None
+                    _ => None,
                 };
                 let sm = tcx.sess.source_map();
-                if let Some(value_kind) = value_kind &&
-                    let Ok(snip) = sm.span_to_snippet(elt_span)
+                if let Some(value_kind) = value_kind
+                    && let Ok(snip) = sm.span_to_snippet(elt_span)
                 {
                     let help_msg = format!(
                         "consider creating a new `const` item and initializing it with {value_kind} \
-                        to be used in the repeat position");
+                        to be used in the repeat position"
+                    );
                     let indentation = sm.indentation_before(elt_stmt_span).unwrap_or_default();
-                    err.multipart_suggestion(help_msg, vec![
-                        (elt_stmt_span.shrink_to_lo(), format!("const ARRAY_REPEAT_VALUE: {elt_type} = {snip};\n{indentation}")),
-                        (elt_span, "ARRAY_REPEAT_VALUE".to_string())
-                    ], Applicability::MachineApplicable);
+                    err.multipart_suggestion(
+                        help_msg,
+                        vec![
+                            (
+                                elt_stmt_span.shrink_to_lo(),
+                                format!(
+                                    "const ARRAY_REPEAT_VALUE: {elt_type} = {snip};\n{indentation}"
+                                ),
+                            ),
+                            (elt_span, "ARRAY_REPEAT_VALUE".to_string()),
+                        ],
+                        Applicability::MachineApplicable,
+                    );
                 }
 
-                if self.tcx.sess.is_nightly_build() && matches!(is_constable, IsConstable::Fn|IsConstable::Ctor) {
+                if self.tcx.sess.is_nightly_build()
+                    && matches!(is_constable, IsConstable::Fn | IsConstable::Ctor)
+                {
                     err.help(
                         "create an inline `const` block, see RFC #2920 \
                          <https://github.com/rust-lang/rfcs/pull/2920> for more information",
@@ -2801,7 +2987,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             }
             ObligationCauseCode::VariableType(hir_id) => {
                 let parent_node = self.tcx.hir().parent_id(hir_id);
-                match self.tcx.hir().find(parent_node) {
+                match self.tcx.opt_hir_node(parent_node) {
                     Some(Node::Local(hir::Local { ty: Some(ty), .. })) => {
                         err.span_suggestion_verbose(
                             ty.span.shrink_to_lo(),
@@ -2945,7 +3131,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                     "all values captured by value by a closure must have a statically known size",
                 );
                 let hir::ExprKind::Closure(closure) =
-                    self.tcx.hir().get_by_def_id(closure_def_id).expect_expr().kind
+                    self.tcx.hir_node_by_def_id(closure_def_id).expect_expr().kind
                 else {
                     bug!("expected closure in SizedClosureCapture obligation");
                 };
@@ -2957,16 +3143,16 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             }
             ObligationCauseCode::SizedCoroutineInterior(coroutine_def_id) => {
                 let what = match self.tcx.coroutine_kind(coroutine_def_id) {
-                    None | Some(hir::CoroutineKind::Coroutine) | Some(hir::CoroutineKind::Gen(_)) => "yield",
+                    None
+                    | Some(hir::CoroutineKind::Coroutine)
+                    | Some(hir::CoroutineKind::Gen(_)) => "yield",
                     Some(hir::CoroutineKind::Async(..)) => "await",
+                    Some(hir::CoroutineKind::AsyncGen(_)) => "yield`/`await",
                 };
                 err.note(format!(
                     "all values live across `{what}` must have a statically known size"
                 ));
             }
-            ObligationCauseCode::ConstPatternStructural => {
-                err.note("constants used for pattern-matching must derive `PartialEq` and `Eq`");
-            }
             ObligationCauseCode::SharedStatic => {
                 err.note("shared static variables must have a type that implements `Sync`");
             }
@@ -3000,9 +3186,9 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                 // Don't print the tuple of capture types
                 'print: {
                     if !is_upvar_tys_infer_tuple {
-                        let msg = with_forced_trimmed_paths!(format!(
-                            "required because it appears within the type `{ty}`",
-                        ));
+                        let mut file = None;
+                        let ty_str = self.tcx.short_ty_string(ty, &mut file);
+                        let msg = format!("required because it appears within the type `{ty_str}`");
                         match ty.kind() {
                             ty::Adt(def, _) => match self.tcx.opt_item_ident(def.did()) {
                                 Some(ident) => err.span_note(ident.span, msg),
@@ -3099,8 +3285,9 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                 let mut parent_trait_pred =
                     self.resolve_vars_if_possible(data.derived.parent_trait_pred);
                 let parent_def_id = parent_trait_pred.def_id();
-                let (self_ty, file) =
-                    self.tcx.short_ty_string(parent_trait_pred.skip_binder().self_ty());
+                let mut file = None;
+                let self_ty =
+                    self.tcx.short_ty_string(parent_trait_pred.skip_binder().self_ty(), &mut file);
                 let msg = format!(
                     "required for `{self_ty}` to implement `{}`",
                     parent_trait_pred.print_modifiers_and_trait_path()
@@ -3197,8 +3384,10 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                         count,
                         pluralize!(count)
                     ));
-                    let (self_ty, file) =
-                        self.tcx.short_ty_string(parent_trait_pred.skip_binder().self_ty());
+                    let mut file = None;
+                    let self_ty = self
+                        .tcx
+                        .short_ty_string(parent_trait_pred.skip_binder().self_ty(), &mut file);
                     err.note(format!(
                         "required for `{self_ty}` to implement `{}`",
                         parent_trait_pred.print_modifiers_and_trait_path()
@@ -3283,7 +3472,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                         err,
                         predicate,
                         param_env,
-                        &parent_code,
+                        parent_code,
                         obligated_types,
                         seen_requirements,
                     )
@@ -3352,7 +3541,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                 let self_ty = self.resolve_vars_if_possible(trait_pred.self_ty());
                 let impls_future = self.type_implements_trait(
                     future_trait,
-                    [self.tcx.erase_late_bound_regions(self_ty)],
+                    [self.tcx.instantiate_bound_regions_with_erased(self_ty)],
                     obligation.param_env,
                 );
                 if !impls_future.must_apply_modulo_regions() {
@@ -3443,17 +3632,19 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             is_derivable_trait &&
                 // Ensure all fields impl the trait.
                 adt.all_fields().all(|field| {
-                    let field_ty = field.ty(self.tcx, args);
+                    let field_ty = ty::GenericArg::from(field.ty(self.tcx, args));
                     let trait_args = match diagnostic_name {
                         sym::PartialEq | sym::PartialOrd => {
                             Some(field_ty)
                         }
                         _ => None,
                     };
+                    // Also add host param, if present
+                    let host = self.tcx.generics_of(trait_pred.def_id()).host_effect_index.map(|idx| trait_pred.skip_binder().trait_ref.args[idx]);
                     let trait_pred = trait_pred.map_bound_ref(|tr| ty::TraitPredicate {
                         trait_ref: ty::TraitRef::new(self.tcx,
                             trait_pred.def_id(),
-                            [field_ty].into_iter().chain(trait_args),
+                            [field_ty].into_iter().chain(trait_args).chain(host),
                         ),
                         ..*tr
                     });
@@ -3474,6 +3665,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                     trait_pred.skip_binder().self_ty(),
                     diagnostic_name,
                 ),
+                // FIXME(effects, const_trait_impl) derive_const as suggestion?
                 format!("#[derive({diagnostic_name})]\n"),
                 Applicability::MaybeIncorrect,
             );
@@ -3513,13 +3705,13 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         call_hir_id: HirId,
     ) {
         let tcx = self.tcx;
-        let hir = tcx.hir();
-        if let Some(Node::Expr(expr)) = hir.find(arg_hir_id)
+        if let Some(Node::Expr(expr)) = tcx.opt_hir_node(arg_hir_id)
             && let Some(typeck_results) = &self.typeck_results
         {
             if let hir::Expr { kind: hir::ExprKind::Block(block, _), .. } = expr {
                 let inner_expr = expr.peel_blocks();
-                let ty = typeck_results.expr_ty_adjusted_opt(inner_expr)
+                let ty = typeck_results
+                    .expr_ty_adjusted_opt(inner_expr)
                     .unwrap_or(Ty::new_misc_error(tcx));
                 let span = inner_expr.span;
                 if Some(span) != err.span.primary_span() {
@@ -3538,16 +3730,15 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                             tcx.lang_items().fn_once_trait(),
                             tcx.lang_items().fn_mut_trait(),
                             tcx.lang_items().fn_trait(),
-                        ].contains(&Some(pred.def_id()))
+                        ]
+                        .contains(&Some(pred.def_id()))
                     {
                         if let [stmt, ..] = block.stmts
                             && let hir::StmtKind::Semi(value) = stmt.kind
                             && let hir::ExprKind::Closure(hir::Closure {
-                                body,
-                                fn_decl_span,
-                                ..
+                                body, fn_decl_span, ..
                             }) = value.kind
-                            && let body = hir.body(*body)
+                            && let body = tcx.hir().body(*body)
                             && !matches!(body.value.kind, hir::ExprKind::Block(..))
                         {
                             // Check if the failed predicate was an expectation of a closure type
@@ -3568,9 +3759,11 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                                 "you might have meant to create the closure instead of a block",
                                 format!(
                                     "|{}| ",
-                                    (0..pred.trait_ref.args.len() - 1).map(|_| "_")
+                                    (0..pred.trait_ref.args.len() - 1)
+                                        .map(|_| "_")
                                         .collect::<Vec<_>>()
-                                        .join(", ")),
+                                        .join(", ")
+                                ),
                                 Applicability::MaybeIncorrect,
                             );
                         }
@@ -3595,7 +3788,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                     let where_pred = self.instantiate_binder_with_placeholders(where_pred);
                     let failed_pred = self.instantiate_binder_with_fresh_vars(
                         expr.span,
-                        LateBoundRegionConversionTime::FnCall,
+                        BoundRegionConversionTime::FnCall,
                         failed_pred,
                     );
 
@@ -3627,21 +3820,21 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                 }
             }
             if let hir::ExprKind::Path(hir::QPath::Resolved(None, path)) = expr.kind
-                && let hir::Path { res: hir::def::Res::Local(hir_id), .. } = path
-                && let Some(hir::Node::Pat(binding)) = self.tcx.hir().find(*hir_id)
+                && let hir::Path { res: Res::Local(hir_id), .. } = path
+                && let Some(hir::Node::Pat(binding)) = self.tcx.opt_hir_node(*hir_id)
                 && let parent_hir_id = self.tcx.hir().parent_id(binding.hir_id)
-                && let Some(hir::Node::Local(local)) = self.tcx.hir().find(parent_hir_id)
+                && let Some(hir::Node::Local(local)) = self.tcx.opt_hir_node(parent_hir_id)
                 && let Some(binding_expr) = local.init
             {
                 // If the expression we're calling on is a binding, we want to point at the
                 // `let` when talking about the type. Otherwise we'll point at every part
                 // of the method chain with the type.
-                self.point_at_chain(binding_expr, &typeck_results, type_diffs, param_env, err);
+                self.point_at_chain(binding_expr, typeck_results, type_diffs, param_env, err);
             } else {
-                self.point_at_chain(expr, &typeck_results, type_diffs, param_env, err);
+                self.point_at_chain(expr, typeck_results, type_diffs, param_env, err);
             }
         }
-        let call_node = hir.find(call_hir_id);
+        let call_node = tcx.opt_hir_node(call_hir_id);
         if let Some(Node::Expr(hir::Expr {
             kind: hir::ExprKind::MethodCall(path, rcvr, ..), ..
         })) = call_node
@@ -3651,7 +3844,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             }
         }
 
-        if let Some(Node::Expr(expr)) = hir.find(call_hir_id) {
+        if let Some(Node::Expr(expr)) = tcx.opt_hir_node(call_hir_id) {
             if let hir::ExprKind::Call(hir::Expr { span, .. }, _)
             | hir::ExprKind::MethodCall(
                 hir::PathSegment { ident: Ident { span, .. }, .. },
@@ -3812,7 +4005,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                             continue;
                         };
                         let hir = tcx.hir();
-                        let node = hir.get_by_def_id(hir.get_parent_item(expr.hir_id).def_id);
+                        let node = tcx.hir_node_by_def_id(hir.get_parent_item(expr.hir_id).def_id);
 
                         let pred = ty::Binder::dummy(ty::TraitPredicate {
                             trait_ref: ty::TraitRef::from_lang_item(
@@ -3832,7 +4025,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                         suggest_restriction(
                             tcx,
                             hir.body_owner_def_id(body_id),
-                            &generics,
+                            generics,
                             &format!("type parameter `{ty}`"),
                             err,
                             node.fn_sig(),
@@ -3887,8 +4080,8 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             );
 
             if let hir::ExprKind::Path(hir::QPath::Resolved(None, path)) = expr.kind
-                && let hir::Path { res: hir::def::Res::Local(hir_id), .. } = path
-                && let Some(hir::Node::Pat(binding)) = self.tcx.hir().find(*hir_id)
+                && let hir::Path { res: Res::Local(hir_id), .. } = path
+                && let Some(hir::Node::Pat(binding)) = self.tcx.opt_hir_node(*hir_id)
                 && let Some(parent) = self.tcx.hir().find_parent(binding.hir_id)
             {
                 // We've reached the root of the method call chain...
@@ -4267,7 +4460,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             }
             return;
         };
-        let Some(hir::Node::TraitItem(item)) = self.tcx.hir().find_by_def_id(fn_def_id) else {
+        let Some(hir::Node::TraitItem(item)) = self.tcx.opt_hir_node_by_def_id(fn_def_id) else {
             return;
         };
 
@@ -4342,17 +4535,6 @@ fn hint_missing_borrow<'tcx>(
 
     let args = fn_decl.inputs.iter();
 
-    fn get_deref_type_and_refs(mut ty: Ty<'_>) -> (Ty<'_>, Vec<hir::Mutability>) {
-        let mut refs = vec![];
-
-        while let ty::Ref(_, new_ty, mutbl) = ty.kind() {
-            ty = *new_ty;
-            refs.push(*mutbl);
-        }
-
-        (ty, refs)
-    }
-
     let mut to_borrow = Vec::new();
     let mut remove_borrow = Vec::new();
 
@@ -4483,7 +4665,7 @@ pub trait NextTypeParamName {
 impl NextTypeParamName for &[hir::GenericParam<'_>] {
     fn next_type_param_name(&self, name: Option<&str>) -> String {
         // This is the list of possible parameter names that we might suggest.
-        let name = name.and_then(|n| n.chars().next()).map(|c| c.to_string().to_uppercase());
+        let name = name.and_then(|n| n.chars().next()).map(|c| c.to_uppercase().to_string());
         let name = name.as_deref();
         let possible_names = [name.unwrap_or("T"), "T", "U", "V", "X", "Y", "Z", "A", "B", "C"];
         let used_names = self
@@ -4512,7 +4694,7 @@ impl<'a, 'hir> hir::intravisit::Visitor<'hir> for ReplaceImplTraitVisitor<'a> {
     fn visit_ty(&mut self, t: &'hir hir::Ty<'hir>) {
         if let hir::TyKind::Path(hir::QPath::Resolved(
             None,
-            hir::Path { res: hir::def::Res::Def(_, segment_did), .. },
+            hir::Path { res: Res::Def(_, segment_did), .. },
         )) = t.kind
         {
             if self.param_did == *segment_did {
@@ -4530,6 +4712,7 @@ impl<'a, 'hir> hir::intravisit::Visitor<'hir> for ReplaceImplTraitVisitor<'a> {
 }
 
 pub(super) fn get_explanation_based_on_obligation<'tcx>(
+    tcx: TyCtxt<'tcx>,
     obligation: &PredicateObligation<'tcx>,
     trait_ref: ty::PolyTraitRef<'tcx>,
     trait_predicate: &ty::PolyTraitPredicate<'tcx>,
@@ -4550,13 +4733,13 @@ pub(super) fn get_explanation_based_on_obligation<'tcx>(
                 pre_message,
                 trait_predicate.print_modifiers_and_trait_path(),
                 desc,
-                trait_ref.skip_binder().self_ty(),
+                tcx.short_ty_string(trait_ref.skip_binder().self_ty(), &mut None),
             ),
             None => format!(
                 "{}the trait `{}` is not implemented for `{}`",
                 pre_message,
                 trait_predicate.print_modifiers_and_trait_path(),
-                trait_ref.skip_binder().self_ty(),
+                tcx.short_ty_string(trait_ref.skip_binder().self_ty(), &mut None),
             ),
         }
     }
@@ -4599,9 +4782,15 @@ pub fn suggest_desugaring_async_fn_to_impl_future_in_trait<'tcx>(
         return None;
     };
 
-    let future = tcx.hir().get_by_def_id(opaque_def_id).expect_item().expect_opaque_ty();
-    let Some(hir::GenericBound::LangItemTrait(_, _, _, generics)) = future.bounds.get(0) else {
-        // `async fn` should always lower to a lang item bound... but don't ICE.
+    let future = tcx.hir_node_by_def_id(opaque_def_id).expect_item().expect_opaque_ty();
+    let [hir::GenericBound::Trait(trait_ref, _)] = future.bounds else {
+        // `async fn` should always lower to a single bound... but don't ICE.
+        return None;
+    };
+    let Some(hir::PathSegment { args: Some(generics), .. }) =
+        trait_ref.trait_ref.path.segments.last()
+    else {
+        // desugaring to a single path segment for `Future<...>`.
         return None;
     };
     let Some(hir::TypeBindingKind::Equality { term: hir::Term::Ty(future_output_ty) }) =
@@ -4645,3 +4834,14 @@ pub fn suggest_desugaring_async_fn_to_impl_future_in_trait<'tcx>(
 
     Some(sugg)
 }
+
+fn get_deref_type_and_refs(mut ty: Ty<'_>) -> (Ty<'_>, Vec<hir::Mutability>) {
+    let mut refs = vec![];
+
+    while let ty::Ref(_, new_ty, mutbl) = ty.kind() {
+        ty = *new_ty;
+        refs.push(*mutbl);
+    }
+
+    (ty, refs)
+}
diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs
index ba2e3d1ae..9ee091bbd 100644
--- a/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs
+++ b/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs
@@ -3,6 +3,7 @@ use super::suggestions::{get_explanation_based_on_obligation, TypeErrCtxtExt as
 use crate::errors::{ClosureFnMutLabel, ClosureFnOnceLabel, ClosureKindMismatch};
 use crate::infer::error_reporting::{TyCategory, TypeAnnotationNeeded as ErrorCode};
 use crate::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
+use crate::infer::InferCtxtExt as _;
 use crate::infer::{self, InferCtxt};
 use crate::traits::error_reporting::infer_ctxt_ext::InferCtxtExt;
 use crate::traits::error_reporting::{ambiguity, ambiguity::Ambiguity::*};
@@ -36,11 +37,11 @@ use rustc_middle::ty::{
     self, SubtypePredicate, ToPolyTraitRef, ToPredicate, TraitRef, Ty, TyCtxt, TypeFoldable,
     TypeVisitable, TypeVisitableExt,
 };
-use rustc_session::config::{DumpSolverProofTree, TraitSolver};
+use rustc_session::config::DumpSolverProofTree;
 use rustc_session::Limit;
 use rustc_span::def_id::LOCAL_CRATE;
 use rustc_span::symbol::sym;
-use rustc_span::{ExpnKind, Span, DUMMY_SP};
+use rustc_span::{BytePos, ExpnKind, Span, Symbol, DUMMY_SP};
 use std::borrow::Cow;
 use std::fmt;
 use std::iter;
@@ -98,8 +99,21 @@ pub trait TypeErrCtxtExt<'tcx> {
         error: &SelectionError<'tcx>,
     );
 
+    fn emit_specialized_closure_kind_error(
+        &self,
+        obligation: &PredicateObligation<'tcx>,
+        trait_ref: ty::PolyTraitRef<'tcx>,
+    ) -> Option<ErrorGuaranteed>;
+
     fn fn_arg_obligation(&self, obligation: &PredicateObligation<'tcx>) -> bool;
 
+    fn try_conversion_context(
+        &self,
+        obligation: &PredicateObligation<'tcx>,
+        trait_ref: ty::TraitRef<'tcx>,
+        err: &mut Diagnostic,
+    ) -> bool;
+
     fn report_const_param_not_wf(
         &self,
         ty: Ty<'tcx>,
@@ -212,7 +226,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             }
         }
 
-        self.tcx.sess.delay_span_bug(DUMMY_SP, "expected fulfillment errors")
+        self.tcx.sess.span_delayed_bug(DUMMY_SP, "expected fulfillment errors")
     }
 
     /// Reports that an overflow has occurred and halts compilation. We
@@ -236,7 +250,12 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         err.emit();
 
         self.tcx.sess.abort_if_errors();
-        bug!();
+        // FIXME: this should be something like `build_overflow_error_fatal`, which returns
+        // `DiagnosticBuilder<', !>`. Then we don't even need anything after that `emit()`.
+        unreachable!(
+            "did not expect compilation to continue after `abort_if_errors`, \
+            since an error was definitely emitted!"
+        );
     }
 
     fn build_overflow_error<T>(
@@ -356,14 +375,16 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
     ) {
         let tcx = self.tcx;
 
-        if tcx.sess.opts.unstable_opts.dump_solver_proof_tree == DumpSolverProofTree::OnError {
+        if tcx.sess.opts.unstable_opts.next_solver.map(|c| c.dump_tree).unwrap_or_default()
+            == DumpSolverProofTree::OnError
+        {
             dump_proof_tree(root_obligation, self.infcx);
         }
 
         let mut span = obligation.cause.span;
         // FIXME: statically guarantee this by tainting after the diagnostic is emitted
         self.set_tainted_by_errors(
-            tcx.sess.delay_span_bug(span, "`report_selection_error` did not emit an error"),
+            tcx.sess.span_delayed_bug(span, "`report_selection_error` did not emit an error"),
         );
 
         let mut err = match *error {
@@ -411,6 +432,11 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                     ty::PredicateKind::Clause(ty::ClauseKind::Trait(trait_predicate)) => {
                         let trait_predicate = bound_predicate.rebind(trait_predicate);
                         let trait_predicate = self.resolve_vars_if_possible(trait_predicate);
+                        let trait_ref = trait_predicate.to_poly_trait_ref();
+
+                        if let Some(_guar) = self.emit_specialized_closure_kind_error(&obligation, trait_ref) {
+                            return;
+                        }
 
                         // FIXME(effects)
                         let predicate_is_const = false;
@@ -425,22 +451,22 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                             // reported on the binding definition (#56607).
                             return;
                         }
-                        let trait_ref = trait_predicate.to_poly_trait_ref();
-                        let (post_message, pre_message, type_def, file_note) = self
+                        let mut file = None;
+                        let (post_message, pre_message, type_def) = self
                             .get_parent_trait_ref(obligation.cause.code())
                             .map(|(t, s)| {
-                                let (t, file) = self.tcx.short_ty_string(t);
+                                let t = self.tcx.short_ty_string(t, &mut file);
                                 (
                                     format!(" in `{t}`"),
                                     format!("within `{t}`, "),
                                     s.map(|s| (format!("within this `{t}`"), s)),
-                                    file.map(|file| format!(
-                                        "the full trait has been written to '{}'",
-                                        file.display(),
-                                    ))
                                 )
                             })
                             .unwrap_or_default();
+                        let file_note = file.map(|file| format!(
+                            "the full trait has been written to '{}'",
+                            file.display(),
+                        ));
 
                         let OnUnimplementedNote {
                             message,
@@ -499,6 +525,11 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
 
                         let mut err = struct_span_err!(self.tcx.sess, span, E0277, "{}", err_msg);
 
+                        let mut suggested = false;
+                        if is_try_conversion {
+                            suggested = self.try_conversion_context(&obligation, trait_ref.skip_binder(), &mut err);
+                        }
+
                         if is_try_conversion && let Some(ret_span) = self.return_type_span(&obligation) {
                             err.span_label(
                                 ret_span,
@@ -511,7 +542,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
 
                         if Some(trait_ref.def_id()) == tcx.lang_items().tuple_trait() {
                             self.add_tuple_trait_message(
-                                &obligation.cause.code().peel_derives(),
+                                obligation.cause.code().peel_derives(),
                                 &mut err,
                             );
                         }
@@ -524,6 +555,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                         }
 
                         let explanation = get_explanation_based_on_obligation(
+                            self.tcx,
                             &obligation,
                             trait_ref,
                             &trait_predicate,
@@ -569,7 +601,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                             if Some(trait_ref.def_id()) == self.tcx.lang_items().sized_trait() {
                                 self.suggest_borrowing_for_object_cast(
                                     &mut err,
-                                    &root_obligation,
+                                    root_obligation,
                                     source,
                                     target,
                                 );
@@ -599,8 +631,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
 
                         self.suggest_floating_point_literal(&obligation, &mut err, &trait_ref);
                         self.suggest_dereferencing_index(&obligation, &mut err, trait_predicate);
-                        let mut suggested =
-                            self.suggest_dereferences(&obligation, &mut err, trait_predicate);
+                        suggested |= self.suggest_dereferences(&obligation, &mut err, trait_predicate);
                         suggested |= self.suggest_fn_call(&obligation, &mut err, trait_predicate);
                         let impl_candidates = self.find_similar_impl_candidates(trait_predicate);
                         suggested = if let &[cand] = &impl_candidates[..] {
@@ -612,7 +643,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                                     span.shrink_to_hi(),
                                     format!(
                                         "the trait `{}` is implemented for fn pointer `{}`, try casting using `as`",
-                                        cand.print_only_trait_path(),
+                                        cand.print_trait_sugared(),
                                         cand.self_ty(),
                                     ),
                                     format!(" as {}", cand.self_ty()),
@@ -786,30 +817,22 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                         report_object_safety_error(self.tcx, span, trait_def_id, violations)
                     }
 
-                    ty::PredicateKind::ClosureKind(closure_def_id, closure_args, kind) => {
-                        let found_kind = self.closure_kind(closure_args).unwrap();
-                        self.report_closure_error(&obligation, closure_def_id, found_kind, kind)
-                    }
-
                     ty::PredicateKind::Clause(ty::ClauseKind::WellFormed(ty)) => {
                         let ty = self.resolve_vars_if_possible(ty);
-                        match self.tcx.sess.opts.unstable_opts.trait_solver {
-                            TraitSolver::Classic => {
-                                // WF predicates cannot themselves make
-                                // errors. They can only block due to
-                                // ambiguity; otherwise, they always
-                                // degenerate into other obligations
-                                // (which may fail).
-                                span_bug!(span, "WF predicate not satisfied for {:?}", ty);
-                            }
-                            TraitSolver::Next | TraitSolver::NextCoherence => {
-                                // FIXME: we'll need a better message which takes into account
-                                // which bounds actually failed to hold.
-                                self.tcx.sess.struct_span_err(
-                                    span,
-                                    format!("the type `{ty}` is not well-formed"),
-                                )
-                            }
+                        if self.next_trait_solver() {
+                            // FIXME: we'll need a better message which takes into account
+                            // which bounds actually failed to hold.
+                            self.tcx.sess.struct_span_err(
+                                span,
+                                format!("the type `{ty}` is not well-formed"),
+                            )
+                        } else {
+                            // WF predicates cannot themselves make
+                            // errors. They can only block due to
+                            // ambiguity; otherwise, they always
+                            // degenerate into other obligations
+                            // (which may fail).
+                            span_bug!(span, "WF predicate not satisfied for {:?}", ty);
                         }
                     }
 
@@ -837,6 +860,11 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
 
                     ty::PredicateKind::Ambiguous => span_bug!(span, "ambiguous"),
 
+                    ty::PredicateKind::NormalizesTo(..) => span_bug!(
+                        span,
+                        "NormalizesTo predicate should never be the predicate cause of a SelectionError"
+                    ),
+
                     ty::PredicateKind::AliasRelate(..) => span_bug!(
                         span,
                         "AliasRelate predicate should never be the predicate cause of a SelectionError"
@@ -927,18 +955,48 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         err.emit();
     }
 
+    fn emit_specialized_closure_kind_error(
+        &self,
+        obligation: &PredicateObligation<'tcx>,
+        trait_ref: ty::PolyTraitRef<'tcx>,
+    ) -> Option<ErrorGuaranteed> {
+        if let ty::Closure(closure_def_id, closure_args) = *trait_ref.self_ty().skip_binder().kind()
+            && let Some(expected_kind) = self.tcx.fn_trait_kind_from_def_id(trait_ref.def_id())
+            && let Some(found_kind) = self.closure_kind(closure_args)
+            && !found_kind.extends(expected_kind)
+            && let sig = closure_args.as_closure().sig()
+            && self.can_sub(
+                obligation.param_env,
+                trait_ref,
+                sig.map_bound(|sig| {
+                    ty::TraitRef::new(
+                        self.tcx,
+                        trait_ref.def_id(),
+                        [trait_ref.self_ty().skip_binder(), sig.inputs()[0]],
+                    )
+                }),
+            )
+        {
+            let mut err =
+                self.report_closure_error(&obligation, closure_def_id, found_kind, expected_kind);
+            self.note_obligation_cause(&mut err, &obligation);
+            self.point_at_returns_when_relevant(&mut err, &obligation);
+            Some(err.emit())
+        } else {
+            None
+        }
+    }
+
     fn fn_arg_obligation(&self, obligation: &PredicateObligation<'tcx>) -> bool {
-        if let ObligationCauseCode::FunctionArgumentObligation {
-            arg_hir_id,
-            ..
-        } = obligation.cause.code()
-            && let Some(Node::Expr(arg)) = self.tcx.hir().find(*arg_hir_id)
+        if let ObligationCauseCode::FunctionArgumentObligation { arg_hir_id, .. } =
+            obligation.cause.code()
+            && let Some(Node::Expr(arg)) = self.tcx.opt_hir_node(*arg_hir_id)
             && let arg = arg.peel_borrows()
             && let hir::ExprKind::Path(hir::QPath::Resolved(
                 None,
                 hir::Path { res: hir::def::Res::Local(hir_id), .. },
             )) = arg.kind
-            && let Some(Node::Pat(pat)) = self.tcx.hir().find(*hir_id)
+            && let Some(Node::Pat(pat)) = self.tcx.opt_hir_node(*hir_id)
             && let Some(preds) = self.reported_trait_errors.borrow().get(&pat.span)
             && preds.contains(&obligation.predicate)
         {
@@ -947,6 +1005,223 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         false
     }
 
+    /// When the `E` of the resulting `Result<T, E>` in an expression `foo().bar().baz()?`,
+    /// identify thoe method chain sub-expressions that could or could not have been annotated
+    /// with `?`.
+    fn try_conversion_context(
+        &self,
+        obligation: &PredicateObligation<'tcx>,
+        trait_ref: ty::TraitRef<'tcx>,
+        err: &mut Diagnostic,
+    ) -> bool {
+        let span = obligation.cause.span;
+        struct V<'v> {
+            search_span: Span,
+            found: Option<&'v hir::Expr<'v>>,
+        }
+        impl<'v> Visitor<'v> for V<'v> {
+            fn visit_expr(&mut self, ex: &'v hir::Expr<'v>) {
+                if let hir::ExprKind::Match(expr, _arms, hir::MatchSource::TryDesugar(_)) = ex.kind
+                {
+                    if ex.span.with_lo(ex.span.hi() - BytePos(1)).source_equal(self.search_span) {
+                        if let hir::ExprKind::Call(_, [expr, ..]) = expr.kind {
+                            self.found = Some(expr);
+                            return;
+                        }
+                    }
+                }
+                hir::intravisit::walk_expr(self, ex);
+            }
+        }
+        let hir_id = self.tcx.local_def_id_to_hir_id(obligation.cause.body_id);
+        let body_id = match self.tcx.opt_hir_node(hir_id) {
+            Some(hir::Node::Item(hir::Item { kind: hir::ItemKind::Fn(_, _, body_id), .. })) => {
+                body_id
+            }
+            _ => return false,
+        };
+        let mut v = V { search_span: span, found: None };
+        v.visit_body(self.tcx.hir().body(*body_id));
+        let Some(expr) = v.found else {
+            return false;
+        };
+        let Some(typeck) = &self.typeck_results else {
+            return false;
+        };
+        let Some((ObligationCauseCode::QuestionMark, Some(y))) = obligation.cause.code().parent()
+        else {
+            return false;
+        };
+        if !self.tcx.is_diagnostic_item(sym::FromResidual, y.def_id()) {
+            return false;
+        }
+        let self_ty = trait_ref.self_ty();
+        let found_ty = trait_ref.args.get(1).and_then(|a| a.as_type());
+
+        let mut prev_ty = self.resolve_vars_if_possible(
+            typeck.expr_ty_adjusted_opt(expr).unwrap_or(Ty::new_misc_error(self.tcx)),
+        );
+
+        // We always look at the `E` type, because that's the only one affected by `?`. If the
+        // incorrect `Result<T, E>` is because of the `T`, we'll get an E0308 on the whole
+        // expression, after the `?` has "unwrapped" the `T`.
+        let get_e_type = |prev_ty: Ty<'tcx>| -> Option<Ty<'tcx>> {
+            let ty::Adt(def, args) = prev_ty.kind() else {
+                return None;
+            };
+            let Some(arg) = args.get(1) else {
+                return None;
+            };
+            if !self.tcx.is_diagnostic_item(sym::Result, def.did()) {
+                return None;
+            }
+            arg.as_type()
+        };
+
+        let mut suggested = false;
+        let mut chain = vec![];
+
+        // The following logic is simlar to `point_at_chain`, but that's focused on associated types
+        let mut expr = expr;
+        while let hir::ExprKind::MethodCall(path_segment, rcvr_expr, args, span) = expr.kind {
+            // Point at every method call in the chain with the `Result` type.
+            // let foo = bar.iter().map(mapper)?;
+            //               ------ -----------
+            expr = rcvr_expr;
+            chain.push((span, prev_ty));
+
+            let next_ty = self.resolve_vars_if_possible(
+                typeck.expr_ty_adjusted_opt(expr).unwrap_or(Ty::new_misc_error(self.tcx)),
+            );
+
+            let is_diagnostic_item = |symbol: Symbol, ty: Ty<'tcx>| {
+                let ty::Adt(def, _) = ty.kind() else {
+                    return false;
+                };
+                self.tcx.is_diagnostic_item(symbol, def.did())
+            };
+            // For each method in the chain, see if this is `Result::map_err` or
+            // `Option::ok_or_else` and if it is, see if the closure passed to it has an incorrect
+            // trailing `;`.
+            if let Some(ty) = get_e_type(prev_ty)
+                && let Some(found_ty) = found_ty
+                // Ideally we would instead use `FnCtxt::lookup_method_for_diagnostic` for 100%
+                // accurate check, but we are in the wrong stage to do that and looking for
+                // `Result::map_err` by checking the Self type and the path segment is enough.
+                // sym::ok_or_else
+                && (
+                    ( // Result::map_err
+                        path_segment.ident.name == sym::map_err
+                            && is_diagnostic_item(sym::Result, next_ty)
+                    ) || ( // Option::ok_or_else
+                        path_segment.ident.name == sym::ok_or_else
+                            && is_diagnostic_item(sym::Option, next_ty)
+                    )
+                )
+                // Found `Result<_, ()>?`
+                && let ty::Tuple(tys) = found_ty.kind()
+                && tys.is_empty()
+                // The current method call returns `Result<_, ()>`
+                && self.can_eq(obligation.param_env, ty, found_ty)
+                // There's a single argument in the method call and it is a closure
+                && args.len() == 1
+                && let Some(arg) = args.get(0)
+                && let hir::ExprKind::Closure(closure) = arg.kind
+                // The closure has a block for its body with no tail expression
+                && let body = self.tcx.hir().body(closure.body)
+                && let hir::ExprKind::Block(block, _) = body.value.kind
+                && let None = block.expr
+                // The last statement is of a type that can be converted to the return error type
+                && let [.., stmt] = block.stmts
+                && let hir::StmtKind::Semi(expr) = stmt.kind
+                && let expr_ty = self.resolve_vars_if_possible(
+                    typeck.expr_ty_adjusted_opt(expr)
+                        .unwrap_or(Ty::new_misc_error(self.tcx)),
+                )
+                && self
+                    .infcx
+                    .type_implements_trait(
+                        self.tcx.get_diagnostic_item(sym::From).unwrap(),
+                        [self_ty, expr_ty],
+                        obligation.param_env,
+                    )
+                    .must_apply_modulo_regions()
+            {
+                suggested = true;
+                err.span_suggestion_short(
+                    stmt.span.with_lo(expr.span.hi()),
+                    "remove this semicolon",
+                    String::new(),
+                    Applicability::MachineApplicable,
+                );
+            }
+
+            prev_ty = next_ty;
+
+            if let hir::ExprKind::Path(hir::QPath::Resolved(None, path)) = expr.kind
+                && let hir::Path { res: hir::def::Res::Local(hir_id), .. } = path
+                && let Some(hir::Node::Pat(binding)) = self.tcx.opt_hir_node(*hir_id)
+                && let Some(parent) = self.tcx.hir().find_parent(binding.hir_id)
+            {
+                // We've reached the root of the method call chain...
+                if let hir::Node::Local(local) = parent
+                    && let Some(binding_expr) = local.init
+                {
+                    // ...and it is a binding. Get the binding creation and continue the chain.
+                    expr = binding_expr;
+                }
+                if let hir::Node::Param(_param) = parent {
+                    // ...and it is a an fn argument.
+                    break;
+                }
+            }
+        }
+        // `expr` is now the "root" expression of the method call chain, which can be any
+        // expression kind, like a method call or a path. If this expression is `Result<T, E>` as
+        // well, then we also point at it.
+        prev_ty = self.resolve_vars_if_possible(
+            typeck.expr_ty_adjusted_opt(expr).unwrap_or(Ty::new_misc_error(self.tcx)),
+        );
+        chain.push((expr.span, prev_ty));
+
+        let mut prev = None;
+        for (span, err_ty) in chain.into_iter().rev() {
+            let err_ty = get_e_type(err_ty);
+            let err_ty = match (err_ty, prev) {
+                (Some(err_ty), Some(prev)) if !self.can_eq(obligation.param_env, err_ty, prev) => {
+                    err_ty
+                }
+                (Some(err_ty), None) => err_ty,
+                _ => {
+                    prev = err_ty;
+                    continue;
+                }
+            };
+            if self
+                .infcx
+                .type_implements_trait(
+                    self.tcx.get_diagnostic_item(sym::From).unwrap(),
+                    [self_ty, err_ty],
+                    obligation.param_env,
+                )
+                .must_apply_modulo_regions()
+            {
+                if !suggested {
+                    err.span_label(span, format!("this has type `Result<_, {err_ty}>`"));
+                }
+            } else {
+                err.span_label(
+                    span,
+                    format!(
+                        "this can't be annotated with `?` because it has type `Result<_, {err_ty}>`",
+                    ),
+                );
+            }
+            prev = Some(err_ty);
+        }
+        suggested
+    }
+
     fn report_const_param_not_wf(
         &self,
         ty: Ty<'tcx>,
@@ -1291,7 +1566,9 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
 
     #[instrument(skip(self), level = "debug")]
     fn report_fulfillment_error(&self, error: &FulfillmentError<'tcx>) {
-        if self.tcx.sess.opts.unstable_opts.dump_solver_proof_tree == DumpSolverProofTree::OnError {
+        if self.tcx.sess.opts.unstable_opts.next_solver.map(|c| c.dump_tree).unwrap_or_default()
+            == DumpSolverProofTree::OnError
+        {
             dump_proof_tree(&error.root_obligation, self.infcx);
         }
 
@@ -1377,7 +1654,7 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             {
                 let data = self.instantiate_binder_with_fresh_vars(
                     obligation.cause.span,
-                    infer::LateBoundRegionConversionTime::HigherRankedType,
+                    infer::BoundRegionConversionTime::HigherRankedType,
                     bound_predicate.rebind(data),
                 );
                 let unnormalized_term = match data.term.unpack() {
@@ -1397,7 +1674,7 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                     )
                     .into(),
                 };
-                // FIXME(-Ztrait-solver=next): For diagnostic purposes, it would be nice
+                // FIXME(-Znext-solver): For diagnostic purposes, it would be nice
                 // to deeply normalize this type.
                 let normalized_term =
                     ocx.normalize(&obligation.cause, obligation.param_env, unnormalized_term);
@@ -1413,7 +1690,6 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                         | ObligationCauseCode::ExprItemObligation(..)
                         | ObligationCauseCode::ExprBindingObligation(..)
                         | ObligationCauseCode::Coercion { .. }
-                        | ObligationCauseCode::OpaqueType
                 );
 
                 // constrain inference variables a bit more to nested obligations from normalize so
@@ -1653,6 +1929,9 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             hir::CoroutineKind::Async(hir::CoroutineSource::Block) => "an async block",
             hir::CoroutineKind::Async(hir::CoroutineSource::Fn) => "an async function",
             hir::CoroutineKind::Async(hir::CoroutineSource::Closure) => "an async closure",
+            hir::CoroutineKind::AsyncGen(hir::CoroutineSource::Block) => "an async gen block",
+            hir::CoroutineKind::AsyncGen(hir::CoroutineSource::Fn) => "an async gen function",
+            hir::CoroutineKind::AsyncGen(hir::CoroutineSource::Closure) => "an async gen closure",
             hir::CoroutineKind::Gen(hir::CoroutineSource::Block) => "a gen block",
             hir::CoroutineKind::Gen(hir::CoroutineSource::Fn) => "a gen function",
             hir::CoroutineKind::Gen(hir::CoroutineSource::Closure) => "a gen closure",
@@ -1751,7 +2030,7 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                         ct_op: |ct| ct.normalize(self.tcx, ty::ParamEnv::empty()),
                     });
                 err.highlighted_help(vec![
-                    (format!("the trait `{}` ", cand.print_only_trait_path()), Style::NoStyle),
+                    (format!("the trait `{}` ", cand.print_trait_sugared()), Style::NoStyle),
                     ("is".to_string(), Style::Highlight),
                     (" implemented for `".to_string(), Style::NoStyle),
                     (cand.self_ty().to_string(), Style::Highlight),
@@ -1787,7 +2066,7 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                         _ => (" implemented for `", ""),
                     };
                 err.highlighted_help(vec![
-                    (format!("the trait `{}` ", cand.print_only_trait_path()), Style::NoStyle),
+                    (format!("the trait `{}` ", cand.print_trait_sugared()), Style::NoStyle),
                     ("is".to_string(), Style::Highlight),
                     (desc.to_string(), Style::NoStyle),
                     (cand.self_ty().to_string(), Style::Highlight),
@@ -1820,7 +2099,7 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             let end = if candidates.len() <= 9 { candidates.len() } else { 8 };
             err.help(format!(
                 "the following {other}types implement trait `{}`:{}{}",
-                trait_ref.print_only_trait_path(),
+                trait_ref.print_trait_sugared(),
                 candidates[..end].join(""),
                 if candidates.len() > 9 {
                     format!("\nand {} others", candidates.len() - 8)
@@ -1967,7 +2246,7 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                 }
             }
             ObligationCauseCode::FunctionArgumentObligation { parent_code, .. } => {
-                self.get_parent_trait_ref(&parent_code)
+                self.get_parent_trait_ref(parent_code)
             }
             _ => None,
         }
@@ -2182,7 +2461,7 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                         self.tcx.hir().maybe_body_owned_by(obligation.cause.body_id)
                 {
                     let mut expr_finder = FindExprBySpan::new(span);
-                    expr_finder.visit_expr(&self.tcx.hir().body(body_id).value);
+                    expr_finder.visit_expr(self.tcx.hir().body(body_id).value);
 
                     if let Some(hir::Expr {
                         kind: hir::ExprKind::Path(hir::QPath::Resolved(None, path)),
@@ -2228,7 +2507,7 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                                 ident: trait_name,
                                 kind: hir::ItemKind::Trait(_, _, _, _, trait_item_refs),
                                 ..
-                            })) = self.tcx.hir().find_by_def_id(local_def_id)
+                            })) = self.tcx.opt_hir_node_by_def_id(local_def_id)
                             && let Some(method_ref) = trait_item_refs
                                 .iter()
                                 .find(|item_ref| item_ref.ident == *assoc_item_name)
@@ -2731,7 +3010,7 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                         Some(format!("{cannot_do_this} in const contexts"))
                     }
                     // overridden post message
-                    (true, Some(AppendConstMessage::Custom(custom_msg))) => {
+                    (true, Some(AppendConstMessage::Custom(custom_msg, _))) => {
                         Some(format!("{cannot_do_this}{custom_msg}"))
                     }
                     // fallback to generic message
@@ -2752,7 +3031,8 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         use rustc_transmute::Answer;
 
         // Erase regions because layout code doesn't particularly care about regions.
-        let trait_ref = self.tcx.erase_regions(self.tcx.erase_late_bound_regions(trait_ref));
+        let trait_ref =
+            self.tcx.erase_regions(self.tcx.instantiate_bound_regions_with_erased(trait_ref));
 
         let src_and_dst = rustc_transmute::Types {
             dst: trait_ref.args.type_at(0),
@@ -2933,7 +3213,7 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                 obligation.param_env,
             ) {
                 self.report_similar_impl_candidates_for_root_obligation(
-                    &obligation,
+                    obligation,
                     *trait_predicate,
                     body_def_id,
                     err,
@@ -3040,18 +3320,18 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         // Additional context information explaining why the closure only implements
         // a particular trait.
         if let Some(typeck_results) = &self.typeck_results {
-            let hir_id = self.tcx.hir().local_def_id_to_hir_id(closure_def_id.expect_local());
+            let hir_id = self.tcx.local_def_id_to_hir_id(closure_def_id.expect_local());
             match (found_kind, typeck_results.closure_kind_origins().get(hir_id)) {
                 (ty::ClosureKind::FnOnce, Some((span, place))) => {
                     err.fn_once_label = Some(ClosureFnOnceLabel {
                         span: *span,
-                        place: ty::place_to_string_for_capture(self.tcx, &place),
+                        place: ty::place_to_string_for_capture(self.tcx, place),
                     })
                 }
                 (ty::ClosureKind::FnMut, Some((span, place))) => {
                     err.fn_mut_label = Some(ClosureFnMutLabel {
                         span: *span,
-                        place: ty::place_to_string_for_capture(self.tcx, &place),
+                        place: ty::place_to_string_for_capture(self.tcx, place),
                     })
                 }
                 _ => {}
@@ -3115,7 +3395,7 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         };
 
         if let Some(diag) =
-            self.tcx.sess.diagnostic().steal_diagnostic(self.tcx.def_span(def_id), StashKey::Cycle)
+            self.tcx.sess.dcx().steal_diagnostic(self.tcx.def_span(def_id), StashKey::Cycle)
         {
             diag.cancel();
         }
@@ -3163,7 +3443,7 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         let mut not_tupled = false;
 
         let found = match found_trait_ref.skip_binder().args.type_at(1).kind() {
-            ty::Tuple(ref tys) => vec![ArgKind::empty(); tys.len()],
+            ty::Tuple(tys) => vec![ArgKind::empty(); tys.len()],
             _ => {
                 not_tupled = true;
                 vec![ArgKind::empty()]
@@ -3172,7 +3452,7 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
 
         let expected_ty = expected_trait_ref.skip_binder().args.type_at(1);
         let expected = match expected_ty.kind() {
-            ty::Tuple(ref tys) => {
+            ty::Tuple(tys) => {
                 tys.iter().map(|t| ArgKind::from_expected_ty(t, Some(span))).collect()
             }
             _ => {
@@ -3253,20 +3533,30 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         }
 
         match obligation.predicate.kind().skip_binder() {
-            ty::PredicateKind::Clause(ty::ClauseKind::ConstEvaluatable(ct)) => {
-                let ty::ConstKind::Unevaluated(uv) = ct.kind() else {
+            ty::PredicateKind::Clause(ty::ClauseKind::ConstEvaluatable(ct)) => match ct.kind() {
+                ty::ConstKind::Unevaluated(uv) => {
+                    let mut err =
+                        self.tcx.sess.struct_span_err(span, "unconstrained generic constant");
+                    let const_span = self.tcx.def_span(uv.def);
+                    match self.tcx.sess.source_map().span_to_snippet(const_span) {
+                            Ok(snippet) => err.help(format!(
+                                "try adding a `where` bound using this expression: `where [(); {snippet}]:`"
+                            )),
+                            _ => err.help("consider adding a `where` bound using this expression"),
+                        };
+                    Some(err)
+                }
+                ty::ConstKind::Expr(_) => {
+                    let err = self
+                        .tcx
+                        .sess
+                        .struct_span_err(span, format!("unconstrained generic constant `{ct}`"));
+                    Some(err)
+                }
+                _ => {
                     bug!("const evaluatable failed for non-unevaluated const `{ct:?}`");
-                };
-                let mut err = self.tcx.sess.struct_span_err(span, "unconstrained generic constant");
-                let const_span = self.tcx.def_span(uv.def);
-                match self.tcx.sess.source_map().span_to_snippet(const_span) {
-                    Ok(snippet) => err.help(format!(
-                        "try adding a `where` bound using this expression: `where [(); {snippet}]:`"
-                    )),
-                    _ => err.help("consider adding a `where` bound using this expression"),
-                };
-                Some(err)
-            }
+                }
+            },
             _ => {
                 span_bug!(
                     span,
diff --git a/compiler/rustc_trait_selection/src/traits/fulfill.rs b/compiler/rustc_trait_selection/src/traits/fulfill.rs
index fb9cf51b5..045d7e444 100644
--- a/compiler/rustc_trait_selection/src/traits/fulfill.rs
+++ b/compiler/rustc_trait_selection/src/traits/fulfill.rs
@@ -1,4 +1,5 @@
 use crate::infer::{InferCtxt, TyOrConstInferVar};
+use crate::traits::error_reporting::TypeErrCtxtExt;
 use rustc_data_structures::captures::Captures;
 use rustc_data_structures::obligation_forest::ProcessResult;
 use rustc_data_structures::obligation_forest::{Error, ForestObligation, Outcome};
@@ -115,12 +116,13 @@ impl<'tcx> TraitEngine<'tcx> for FulfillmentContext<'tcx> {
     fn register_predicate_obligation(
         &mut self,
         infcx: &InferCtxt<'tcx>,
-        obligation: PredicateObligation<'tcx>,
+        mut obligation: PredicateObligation<'tcx>,
     ) {
         assert_eq!(self.usable_in_snapshot, infcx.num_open_snapshots());
         // this helps to reduce duplicate errors, as well as making
         // debug output much nicer to read and so on.
-        let obligation = infcx.resolve_vars_if_possible(obligation);
+        debug_assert!(!obligation.param_env.has_non_region_infer());
+        obligation.predicate = infcx.resolve_vars_if_possible(obligation.predicate);
 
         debug!(?obligation, "register_predicate_obligation");
 
@@ -350,7 +352,6 @@ impl<'a, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'tcx> {
                 | ty::PredicateKind::Clause(ty::ClauseKind::ConstArgHasType(..))
                 | ty::PredicateKind::Clause(ty::ClauseKind::WellFormed(_))
                 | ty::PredicateKind::ObjectSafe(_)
-                | ty::PredicateKind::ClosureKind(..)
                 | ty::PredicateKind::Subtype(_)
                 | ty::PredicateKind::Coerce(_)
                 | ty::PredicateKind::Clause(ty::ClauseKind::ConstEvaluatable(..))
@@ -360,8 +361,11 @@ impl<'a, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'tcx> {
                     ProcessResult::Changed(mk_pending(vec![obligation.with(infcx.tcx, pred)]))
                 }
                 ty::PredicateKind::Ambiguous => ProcessResult::Unchanged,
+                ty::PredicateKind::NormalizesTo(..) => {
+                    bug!("NormalizesTo is only used by the new solver")
+                }
                 ty::PredicateKind::AliasRelate(..) => {
-                    bug!("AliasRelate is only used for new solver")
+                    bug!("AliasRelate is only used by the new solver")
                 }
             },
             Some(pred) => match pred {
@@ -411,17 +415,30 @@ impl<'a, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'tcx> {
                     }
                 }
 
-                ty::PredicateKind::ClosureKind(_, closure_args, kind) => {
-                    match self.selcx.infcx.closure_kind(closure_args) {
-                        Some(closure_kind) => {
-                            if closure_kind.extends(kind) {
-                                ProcessResult::Changed(vec![])
-                            } else {
-                                ProcessResult::Error(CodeSelectionError(Unimplemented))
-                            }
-                        }
-                        None => ProcessResult::Unchanged,
-                    }
+                ty::PredicateKind::Ambiguous => ProcessResult::Unchanged,
+                ty::PredicateKind::NormalizesTo(..) => {
+                    bug!("NormalizesTo is only used by the new solver")
+                }
+                ty::PredicateKind::AliasRelate(..) => {
+                    bug!("AliasRelate is only used by the new solver")
+                }
+
+                // General case overflow check. Allow `process_trait_obligation`
+                // and `process_projection_obligation` to handle checking for
+                // the recursion limit themselves. Also don't check some
+                // predicate kinds that don't give further obligations.
+                _ if !self
+                    .selcx
+                    .tcx()
+                    .recursion_limit()
+                    .value_within_limit(obligation.recursion_depth) =>
+                {
+                    self.selcx.infcx.err_ctxt().report_overflow_error(
+                        &obligation.predicate,
+                        obligation.cause.span,
+                        false,
+                        |_| {},
+                    );
                 }
 
                 ty::PredicateKind::Clause(ty::ClauseKind::WellFormed(arg)) => {
@@ -454,7 +471,12 @@ impl<'a, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'tcx> {
                                 vec![TyOrConstInferVar::Ty(a), TyOrConstInferVar::Ty(b)];
                             ProcessResult::Unchanged
                         }
-                        Ok(Ok(ok)) => ProcessResult::Changed(mk_pending(ok.obligations)),
+                        Ok(Ok(mut ok)) => {
+                            for subobligation in &mut ok.obligations {
+                                subobligation.set_depth_from_parent(obligation.recursion_depth);
+                            }
+                            ProcessResult::Changed(mk_pending(ok.obligations))
+                        }
                         Ok(Err(err)) => {
                             let expected_found =
                                 ExpectedFound::new(subtype.a_is_expected, subtype.a, subtype.b);
@@ -625,10 +647,6 @@ impl<'a, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'tcx> {
                         }
                     }
                 }
-                ty::PredicateKind::Ambiguous => ProcessResult::Unchanged,
-                ty::PredicateKind::AliasRelate(..) => {
-                    bug!("AliasRelate is only used for new solver")
-                }
                 ty::PredicateKind::Clause(ty::ClauseKind::ConstArgHasType(ct, ty)) => {
                     match self.selcx.infcx.at(&obligation.cause, obligation.param_env).eq(
                         DefineOpaqueTypes::No,
diff --git a/compiler/rustc_trait_selection/src/traits/misc.rs b/compiler/rustc_trait_selection/src/traits/misc.rs
index ab07b10c6..cf4fa2337 100644
--- a/compiler/rustc_trait_selection/src/traits/misc.rs
+++ b/compiler/rustc_trait_selection/src/traits/misc.rs
@@ -9,7 +9,7 @@ use rustc_infer::infer::canonical::Canonical;
 use rustc_infer::infer::{RegionResolutionError, TyCtxtInferExt};
 use rustc_infer::traits::query::NoSolution;
 use rustc_infer::{infer::outlives::env::OutlivesEnvironment, traits::FulfillmentError};
-use rustc_middle::ty::{self, AdtDef, GenericArg, List, ParamEnv, Ty, TyCtxt, TypeVisitableExt};
+use rustc_middle::ty::{self, AdtDef, GenericArg, List, Ty, TyCtxt, TypeVisitableExt};
 use rustc_span::DUMMY_SP;
 
 use super::outlives_bounds::InferCtxtExt;
@@ -173,7 +173,7 @@ pub fn all_fields_implement_trait<'tcx>(
             // between expected and found const-generic types. Don't report an
             // additional copy error here, since it's not typically useful.
             if !normalization_errors.is_empty() || ty.references_error() {
-                tcx.sess.delay_span_bug(field_span, format!("couldn't normalize struct field `{unnormalized_ty}` when checking {tr} implementation", tr = tcx.def_path_str(trait_def_id)));
+                tcx.sess.span_delayed_bug(field_span, format!("couldn't normalize struct field `{unnormalized_ty}` when checking {tr} implementation", tr = tcx.def_path_str(trait_def_id)));
                 continue;
             }
 
@@ -209,10 +209,10 @@ pub fn all_fields_implement_trait<'tcx>(
 
 pub fn check_tys_might_be_eq<'tcx>(
     tcx: TyCtxt<'tcx>,
-    canonical: Canonical<'tcx, (ParamEnv<'tcx>, Ty<'tcx>, Ty<'tcx>)>,
+    canonical: Canonical<'tcx, ty::ParamEnvAnd<'tcx, (Ty<'tcx>, Ty<'tcx>)>>,
 ) -> Result<(), NoSolution> {
-    let (infcx, (param_env, ty_a, ty_b), _) =
-        tcx.infer_ctxt().build_with_canonical(DUMMY_SP, &canonical);
+    let (infcx, key, _) = tcx.infer_ctxt().build_with_canonical(DUMMY_SP, &canonical);
+    let (param_env, (ty_a, ty_b)) = key.into_parts();
     let ocx = ObligationCtxt::new(&infcx);
 
     let result = ocx.eq(&ObligationCause::dummy(), param_env, ty_a, ty_b);
diff --git a/compiler/rustc_trait_selection/src/traits/mod.rs b/compiler/rustc_trait_selection/src/traits/mod.rs
index fff5510bb..8c5f1e907 100644
--- a/compiler/rustc_trait_selection/src/traits/mod.rs
+++ b/compiler/rustc_trait_selection/src/traits/mod.rs
@@ -215,7 +215,7 @@ fn do_normalize_predicates<'tcx>(
     // the normalized predicates.
     let errors = infcx.resolve_regions(&outlives_env);
     if !errors.is_empty() {
-        tcx.sess.delay_span_bug(
+        tcx.sess.span_delayed_bug(
             span,
             format!("failed region resolution while normalizing {elaborated_env:?}: {errors:?}"),
         );
@@ -315,7 +315,7 @@ pub fn normalize_param_env_or_error<'tcx>(
             // We do not normalize types here as there is no backwards compatibility requirement
             // for us to do so.
             //
-            // FIXME(-Ztrait-solver=next): remove this hack since we have deferred projection equality
+            // FIXME(-Znext-solver): remove this hack since we have deferred projection equality
             predicate.fold_with(&mut ConstNormalizer(tcx))
         }),
     )
@@ -386,7 +386,7 @@ pub fn normalize_param_env_or_error<'tcx>(
 
 /// Normalize a type and process all resulting obligations, returning any errors.
 ///
-/// FIXME(-Ztrait-solver=next): This should be replaced by `At::deeply_normalize`
+/// FIXME(-Znext-solver): This should be replaced by `At::deeply_normalize`
 /// which has the same behavior with the new solver. Because using a separate
 /// fulfillment context worsens caching in the old solver, `At::deeply_normalize`
 /// is still lazy with the old solver as it otherwise negatively impacts perf.
@@ -484,7 +484,7 @@ fn is_impossible_associated_item(
             t.super_visit_with(self)
         }
         fn visit_region(&mut self, r: ty::Region<'tcx>) -> ControlFlow<Self::BreakTy> {
-            if let ty::ReEarlyBound(param) = r.kind()
+            if let ty::ReEarlyParam(param) = r.kind()
                 && let param_def_id = self.generics.region_param(&param, self.tcx).def_id
                 && self.tcx.parent(param_def_id) == self.trait_item_def_id
             {
diff --git a/compiler/rustc_trait_selection/src/traits/object_safety.rs b/compiler/rustc_trait_selection/src/traits/object_safety.rs
index 17c7f94ee..7ac37315f 100644
--- a/compiler/rustc_trait_selection/src/traits/object_safety.rs
+++ b/compiler/rustc_trait_selection/src/traits/object_safety.rs
@@ -84,7 +84,7 @@ fn check_is_object_safe(tcx: TyCtxt<'_>, trait_def_id: DefId) -> bool {
                 span,
             ) = violation
             {
-                lint_object_unsafe_trait(tcx, *span, trait_def_id, &violation);
+                lint_object_unsafe_trait(tcx, *span, trait_def_id, violation);
             }
         }
         return true;
@@ -192,9 +192,8 @@ fn lint_object_unsafe_trait(
             );
             if node.is_some() {
                 // Only provide the help if its a local trait, otherwise it's not
-                violation.solution(err);
+                violation.solution().add_to(err);
             }
-            err
         },
     );
 }
@@ -329,7 +328,7 @@ fn super_predicates_have_non_lifetime_binders(
     tcx.super_predicates_of(trait_def_id)
         .predicates
         .iter()
-        .filter_map(|(pred, span)| pred.has_non_region_late_bound().then_some(*span))
+        .filter_map(|(pred, span)| pred.has_non_region_bound_vars().then_some(*span))
         .collect()
 }
 
@@ -345,7 +344,7 @@ fn generics_require_sized_self(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
     // Search for a predicate like `Self : Sized` amongst the trait bounds.
     let predicates = tcx.predicates_of(def_id);
     let predicates = predicates.instantiate_identity(tcx).predicates;
-    elaborate(tcx, predicates.into_iter()).any(|pred| match pred.kind().skip_binder() {
+    elaborate(tcx, predicates).any(|pred| match pred.kind().skip_binder() {
         ty::ClauseKind::Trait(ref trait_pred) => {
             trait_pred.def_id() == sized_def_id && trait_pred.self_ty().is_param(0)
         }
@@ -509,7 +508,7 @@ fn virtual_call_violations_for_method<'tcx>(
                     Ok(layout) => Some(layout.abi),
                     Err(err) => {
                         // #78372
-                        tcx.sess.delay_span_bug(
+                        tcx.sess.span_delayed_bug(
                             tcx.def_span(method.def_id),
                             format!("error: {err}\n while computing layout for type {ty:?}"),
                         );
@@ -525,7 +524,7 @@ fn virtual_call_violations_for_method<'tcx>(
             match abi_of_ty(unit_receiver_ty) {
                 Some(Abi::Scalar(..)) => (),
                 abi => {
-                    tcx.sess.delay_span_bug(
+                    tcx.sess.span_delayed_bug(
                         tcx.def_span(method.def_id),
                         format!(
                             "receiver when `Self = ()` should have a Scalar ABI; found {abi:?}"
@@ -543,7 +542,7 @@ fn virtual_call_violations_for_method<'tcx>(
             match abi_of_ty(trait_object_receiver) {
                 Some(Abi::ScalarPair(..)) => (),
                 abi => {
-                    tcx.sess.delay_span_bug(
+                    tcx.sess.span_delayed_bug(
                         tcx.def_span(method.def_id),
                         format!(
                             "receiver when `Self = {trait_object_ty}` should have a ScalarPair ABI; found {abi:?}"
@@ -595,9 +594,7 @@ fn virtual_call_violations_for_method<'tcx>(
             // would already have reported an error at the definition of the
             // auto trait.
             if pred_trait_ref.args.len() != 1 {
-                tcx.sess
-                    .diagnostic()
-                    .delay_span_bug(span, "auto traits cannot have generic parameters");
+                tcx.sess.dcx().span_delayed_bug(span, "auto traits cannot have generic parameters");
             }
             return false;
         }
diff --git a/compiler/rustc_trait_selection/src/traits/outlives_bounds.rs b/compiler/rustc_trait_selection/src/traits/outlives_bounds.rs
index 32bbd626d..7513f88cf 100644
--- a/compiler/rustc_trait_selection/src/traits/outlives_bounds.rs
+++ b/compiler/rustc_trait_selection/src/traits/outlives_bounds.rs
@@ -72,7 +72,7 @@ impl<'a, 'tcx: 'a> InferCtxtExt<'a, 'tcx> for InferCtxt<'tcx> {
         };
 
         let mut constraints = QueryRegionConstraints::default();
-        let Ok(InferOk { value, obligations }) = self
+        let Ok(InferOk { value: mut bounds, obligations }) = self
             .instantiate_nll_query_response_and_region_obligations(
                 &ObligationCause::dummy(),
                 param_env,
@@ -85,6 +85,10 @@ impl<'a, 'tcx: 'a> InferCtxtExt<'a, 'tcx> for InferCtxt<'tcx> {
         };
         assert_eq!(&obligations, &[]);
 
+        // Because of #109628, we may have unexpected placeholders. Ignore them!
+        // FIXME(#109628): panic in this case once the issue is fixed.
+        bounds.retain(|bound| !bound.has_placeholders());
+
         if !constraints.is_empty() {
             let span = self.tcx.def_span(body_id);
 
@@ -107,14 +111,14 @@ impl<'a, 'tcx: 'a> InferCtxtExt<'a, 'tcx> for InferCtxt<'tcx> {
 
             let errors = ocx.select_all_or_error();
             if !errors.is_empty() {
-                self.tcx.sess.delay_span_bug(
+                self.tcx.sess.span_delayed_bug(
                     span,
                     "implied_outlives_bounds failed to solve obligations from instantiation",
                 );
             }
         };
 
-        value
+        bounds
     }
 
     fn implied_bounds_tys(
diff --git a/compiler/rustc_trait_selection/src/traits/project.rs b/compiler/rustc_trait_selection/src/traits/project.rs
index 471d10dbd..a1b0ada0e 100644
--- a/compiler/rustc_trait_selection/src/traits/project.rs
+++ b/compiler/rustc_trait_selection/src/traits/project.rs
@@ -18,7 +18,7 @@ use rustc_middle::traits::ImplSourceUserDefinedData;
 
 use crate::errors::InherentProjectionNormalizationOverflow;
 use crate::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKind};
-use crate::infer::{InferCtxt, InferOk, LateBoundRegionConversionTime};
+use crate::infer::{BoundRegionConversionTime, InferCtxt, InferOk};
 use crate::traits::error_reporting::TypeErrCtxtExt as _;
 use crate::traits::query::evaluate_obligation::InferCtxtExt as _;
 use crate::traits::select::ProjectionMatchesProjection;
@@ -67,7 +67,7 @@ pub trait NormalizeExt<'tcx> {
     /// same goals in both a temporary and the shared context which negatively impacts
     /// performance as these don't share caching.
     ///
-    /// FIXME(-Ztrait-solver=next): This has the same behavior as `traits::fully_normalize`
+    /// FIXME(-Znext-solver): This has the same behavior as `traits::fully_normalize`
     /// in the new solver, but because of performance reasons, we currently reuse an
     /// existing fulfillment context in the old solver. Once we also eagerly prove goals with
     /// the old solver or have removed the old solver, remove `traits::fully_normalize` and
@@ -191,7 +191,9 @@ impl<'tcx> ProjectionCandidateSet<'tcx> {
                 match (current, candidate) {
                     (ParamEnv(..), ParamEnv(..)) => convert_to_ambiguous = (),
                     (ParamEnv(..), _) => return false,
-                    (_, ParamEnv(..)) => unreachable!(),
+                    (_, ParamEnv(..)) => bug!(
+                        "should never prefer non-param-env candidates over param-env candidates"
+                    ),
                     (_, _) => convert_to_ambiguous = (),
                 }
             }
@@ -584,7 +586,7 @@ impl<'a, 'b, 'tcx> TypeFolder<TyCtxt<'tcx>> for AssocTypeNormalizer<'a, 'b, 'tcx
                         data,
                         self.cause.clone(),
                         self.depth,
-                        &mut self.obligations,
+                        self.obligations,
                     )
                 } else {
                     opt_normalize_projection_type(
@@ -593,7 +595,7 @@ impl<'a, 'b, 'tcx> TypeFolder<TyCtxt<'tcx>> for AssocTypeNormalizer<'a, 'b, 'tcx
                         data,
                         self.cause.clone(),
                         self.depth,
-                        &mut self.obligations,
+                        self.obligations,
                     )
                     .ok()
                     .flatten()
@@ -632,7 +634,7 @@ impl<'a, 'b, 'tcx> TypeFolder<TyCtxt<'tcx>> for AssocTypeNormalizer<'a, 'b, 'tcx
                     data,
                     self.cause.clone(),
                     self.depth,
-                    &mut self.obligations,
+                    self.obligations,
                 )
                 .ok()
                 .flatten()
@@ -717,7 +719,7 @@ impl<'a, 'b, 'tcx> TypeFolder<TyCtxt<'tcx>> for AssocTypeNormalizer<'a, 'b, 'tcx
                     data,
                     self.cause.clone(),
                     self.depth,
-                    &mut self.obligations,
+                    self.obligations,
                 )
             }
 
@@ -732,7 +734,7 @@ impl<'a, 'b, 'tcx> TypeFolder<TyCtxt<'tcx>> for AssocTypeNormalizer<'a, 'b, 'tcx
                     data,
                     self.cause.clone(),
                     self.depth,
-                    &mut self.obligations,
+                    self.obligations,
                 );
 
                 PlaceholderReplacer::replace_placeholders(
@@ -894,16 +896,16 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for BoundVarReplacer<'_, 'tcx> {
 
     fn fold_region(&mut self, r: ty::Region<'tcx>) -> ty::Region<'tcx> {
         match *r {
-            ty::ReLateBound(debruijn, _)
-                if debruijn.as_usize() + 1
-                    > self.current_index.as_usize() + self.universe_indices.len() =>
+            ty::ReBound(debruijn, _)
+                if debruijn.as_usize()
+                    >= self.current_index.as_usize() + self.universe_indices.len() =>
             {
                 bug!(
                     "Bound vars {r:#?} outside of `self.universe_indices`: {:#?}",
                     self.universe_indices
                 );
             }
-            ty::ReLateBound(debruijn, br) if debruijn >= self.current_index => {
+            ty::ReBound(debruijn, br) if debruijn >= self.current_index => {
                 let universe = self.universe_for(debruijn);
                 let p = ty::PlaceholderRegion { universe, bound: br };
                 self.mapped_regions.insert(p, br);
@@ -1001,7 +1003,7 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for PlaceholderReplacer<'_, 'tcx> {
         &mut self,
         t: ty::Binder<'tcx, T>,
     ) -> ty::Binder<'tcx, T> {
-        if !t.has_placeholders() && !t.has_infer_regions() {
+        if !t.has_placeholders() && !t.has_infer() {
             return t;
         }
         self.current_index.shift_in(1);
@@ -1034,7 +1036,7 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for PlaceholderReplacer<'_, 'tcx> {
                         let db = ty::DebruijnIndex::from_usize(
                             self.universe_indices.len() - index + self.current_index.as_usize() - 1,
                         );
-                        ty::Region::new_late_bound(self.interner(), db, *replace_var)
+                        ty::Region::new_bound(self.interner(), db, *replace_var)
                     }
                     None => r1,
                 }
@@ -1048,6 +1050,7 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for PlaceholderReplacer<'_, 'tcx> {
     }
 
     fn fold_ty(&mut self, ty: Ty<'tcx>) -> Ty<'tcx> {
+        let ty = self.infcx.shallow_resolve(ty);
         match *ty.kind() {
             ty::Placeholder(p) => {
                 let replace_var = self.mapped_types.get(&p);
@@ -1063,16 +1066,23 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for PlaceholderReplacer<'_, 'tcx> {
                         );
                         Ty::new_bound(self.infcx.tcx, db, *replace_var)
                     }
-                    None => ty,
+                    None => {
+                        if ty.has_infer() {
+                            ty.super_fold_with(self)
+                        } else {
+                            ty
+                        }
+                    }
                 }
             }
 
-            _ if ty.has_placeholders() || ty.has_infer_regions() => ty.super_fold_with(self),
+            _ if ty.has_placeholders() || ty.has_infer() => ty.super_fold_with(self),
             _ => ty,
         }
     }
 
     fn fold_const(&mut self, ct: ty::Const<'tcx>) -> ty::Const<'tcx> {
+        let ct = self.infcx.shallow_resolve(ct);
         if let ty::ConstKind::Placeholder(p) = ct.kind() {
             let replace_var = self.mapped_consts.get(&p);
             match replace_var {
@@ -1087,7 +1097,13 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for PlaceholderReplacer<'_, 'tcx> {
                     );
                     ty::Const::new_bound(self.infcx.tcx, db, *replace_var, ct.ty())
                 }
-                None => ct,
+                None => {
+                    if ct.has_infer() {
+                        ct.super_fold_with(self)
+                    } else {
+                        ct
+                    }
+                }
             }
         } else {
             ct.super_fold_with(self)
@@ -1440,11 +1456,22 @@ pub fn compute_inherent_assoc_ty_args<'a, 'b, 'tcx>(
 
     // Infer the generic parameters of the impl by unifying the
     // impl type with the self type of the projection.
-    let self_ty = alias_ty.self_ty();
+    let mut self_ty = alias_ty.self_ty();
+    if !selcx.infcx.next_trait_solver() {
+        self_ty = normalize_with_depth_to(
+            selcx,
+            param_env,
+            cause.clone(),
+            depth + 1,
+            self_ty,
+            obligations,
+        );
+    }
+
     match selcx.infcx.at(&cause, param_env).eq(DefineOpaqueTypes::No, impl_ty, self_ty) {
         Ok(mut ok) => obligations.append(&mut ok.obligations),
         Err(_) => {
-            tcx.sess.delay_span_bug(
+            tcx.sess.span_delayed_bug(
                 cause.span,
                 format!(
                     "{self_ty:?} was a subtype of {impl_ty:?} during selection but now it is not"
@@ -1798,11 +1825,18 @@ fn assemble_candidates_from_impls<'cx, 'tcx>(
                 let self_ty = selcx.infcx.shallow_resolve(obligation.predicate.self_ty());
 
                 let lang_items = selcx.tcx().lang_items();
-                if [lang_items.coroutine_trait(), lang_items.future_trait(), lang_items.iterator_trait()].contains(&Some(trait_ref.def_id))
-                    || selcx.tcx().fn_trait_kind_from_def_id(trait_ref.def_id).is_some()
+                if [
+                    lang_items.coroutine_trait(),
+                    lang_items.future_trait(),
+                    lang_items.iterator_trait(),
+                    lang_items.async_iterator_trait(),
+                    lang_items.fn_trait(),
+                    lang_items.fn_mut_trait(),
+                    lang_items.fn_once_trait(),
+                ].contains(&Some(trait_ref.def_id))
                 {
                     true
-                } else if lang_items.discriminant_kind_trait() == Some(trait_ref.def_id) {
+                }else if lang_items.discriminant_kind_trait() == Some(trait_ref.def_id) {
                     match self_ty.kind() {
                         ty::Bool
                         | ty::Char
@@ -1949,7 +1983,7 @@ fn assemble_candidates_from_impls<'cx, 'tcx>(
             ImplSource::Builtin(BuiltinImplSource::TraitUpcasting { .. }, _)
             | ImplSource::Builtin(BuiltinImplSource::TupleUnsizing, _) => {
                 // These traits have no associated types.
-                selcx.tcx().sess.delay_span_bug(
+                selcx.tcx().sess.span_delayed_bug(
                     obligation.cause.span,
                     format!("Cannot project an associated type from `{impl_source:?}`"),
                 );
@@ -2017,6 +2051,8 @@ fn confirm_select_candidate<'cx, 'tcx>(
                 confirm_future_candidate(selcx, obligation, data)
             } else if lang_items.iterator_trait() == Some(trait_def_id) {
                 confirm_iterator_candidate(selcx, obligation, data)
+            } else if lang_items.async_iterator_trait() == Some(trait_def_id) {
+                confirm_async_iterator_candidate(selcx, obligation, data)
             } else if selcx.tcx().fn_trait_kind_from_def_id(trait_def_id).is_some() {
                 if obligation.predicate.self_ty().is_closure() {
                     confirm_closure_candidate(selcx, obligation, data)
@@ -2046,12 +2082,13 @@ fn confirm_coroutine_candidate<'cx, 'tcx>(
     obligation: &ProjectionTyObligation<'tcx>,
     nested: Vec<PredicateObligation<'tcx>>,
 ) -> Progress<'tcx> {
-    let ty::Coroutine(_, args, _) =
-        selcx.infcx.shallow_resolve(obligation.predicate.self_ty()).kind()
-    else {
-        unreachable!()
+    let self_ty = selcx.infcx.shallow_resolve(obligation.predicate.self_ty());
+    let ty::Coroutine(_, args, _) = self_ty.kind() else {
+        unreachable!(
+            "expected coroutine self type for built-in coroutine candidate, found {self_ty}"
+        )
     };
-    let coroutine_sig = args.as_coroutine().poly_sig();
+    let coroutine_sig = args.as_coroutine().sig();
     let Normalized { value: coroutine_sig, obligations } = normalize_with_depth(
         selcx,
         obligation.param_env,
@@ -2066,29 +2103,31 @@ fn confirm_coroutine_candidate<'cx, 'tcx>(
 
     let coroutine_def_id = tcx.require_lang_item(LangItem::Coroutine, None);
 
-    let predicate = super::util::coroutine_trait_ref_and_outputs(
+    let (trait_ref, yield_ty, return_ty) = super::util::coroutine_trait_ref_and_outputs(
         tcx,
         coroutine_def_id,
         obligation.predicate.self_ty(),
         coroutine_sig,
-    )
-    .map_bound(|(trait_ref, yield_ty, return_ty)| {
-        let name = tcx.associated_item(obligation.predicate.def_id).name;
-        let ty = if name == sym::Return {
-            return_ty
-        } else if name == sym::Yield {
-            yield_ty
-        } else {
-            bug!()
-        };
+    );
 
-        ty::ProjectionPredicate {
-            projection_ty: ty::AliasTy::new(tcx, obligation.predicate.def_id, trait_ref.args),
-            term: ty.into(),
-        }
-    });
+    let name = tcx.associated_item(obligation.predicate.def_id).name;
+    let ty = if name == sym::Return {
+        return_ty
+    } else if name == sym::Yield {
+        yield_ty
+    } else {
+        span_bug!(
+            tcx.def_span(obligation.predicate.def_id),
+            "unexpected associated type: `Coroutine::{name}`"
+        );
+    };
 
-    confirm_param_env_candidate(selcx, obligation, predicate, false)
+    let predicate = ty::ProjectionPredicate {
+        projection_ty: ty::AliasTy::new(tcx, obligation.predicate.def_id, trait_ref.args),
+        term: ty.into(),
+    };
+
+    confirm_param_env_candidate(selcx, obligation, ty::Binder::dummy(predicate), false)
         .with_addl_obligations(nested)
         .with_addl_obligations(obligations)
 }
@@ -2098,12 +2137,13 @@ fn confirm_future_candidate<'cx, 'tcx>(
     obligation: &ProjectionTyObligation<'tcx>,
     nested: Vec<PredicateObligation<'tcx>>,
 ) -> Progress<'tcx> {
-    let ty::Coroutine(_, args, _) =
-        selcx.infcx.shallow_resolve(obligation.predicate.self_ty()).kind()
-    else {
-        unreachable!()
+    let self_ty = selcx.infcx.shallow_resolve(obligation.predicate.self_ty());
+    let ty::Coroutine(_, args, _) = self_ty.kind() else {
+        unreachable!(
+            "expected coroutine self type for built-in async future candidate, found {self_ty}"
+        )
     };
-    let coroutine_sig = args.as_coroutine().poly_sig();
+    let coroutine_sig = args.as_coroutine().sig();
     let Normalized { value: coroutine_sig, obligations } = normalize_with_depth(
         selcx,
         obligation.param_env,
@@ -2117,22 +2157,21 @@ fn confirm_future_candidate<'cx, 'tcx>(
     let tcx = selcx.tcx();
     let fut_def_id = tcx.require_lang_item(LangItem::Future, None);
 
-    let predicate = super::util::future_trait_ref_and_outputs(
+    let (trait_ref, return_ty) = super::util::future_trait_ref_and_outputs(
         tcx,
         fut_def_id,
         obligation.predicate.self_ty(),
         coroutine_sig,
-    )
-    .map_bound(|(trait_ref, return_ty)| {
-        debug_assert_eq!(tcx.associated_item(obligation.predicate.def_id).name, sym::Output);
+    );
 
-        ty::ProjectionPredicate {
-            projection_ty: ty::AliasTy::new(tcx, obligation.predicate.def_id, trait_ref.args),
-            term: return_ty.into(),
-        }
-    });
+    debug_assert_eq!(tcx.associated_item(obligation.predicate.def_id).name, sym::Output);
+
+    let predicate = ty::ProjectionPredicate {
+        projection_ty: ty::AliasTy::new(tcx, obligation.predicate.def_id, trait_ref.args),
+        term: return_ty.into(),
+    };
 
-    confirm_param_env_candidate(selcx, obligation, predicate, false)
+    confirm_param_env_candidate(selcx, obligation, ty::Binder::dummy(predicate), false)
         .with_addl_obligations(nested)
         .with_addl_obligations(obligations)
 }
@@ -2142,12 +2181,54 @@ fn confirm_iterator_candidate<'cx, 'tcx>(
     obligation: &ProjectionTyObligation<'tcx>,
     nested: Vec<PredicateObligation<'tcx>>,
 ) -> Progress<'tcx> {
+    let self_ty = selcx.infcx.shallow_resolve(obligation.predicate.self_ty());
+    let ty::Coroutine(_, args, _) = self_ty.kind() else {
+        unreachable!("expected coroutine self type for built-in gen candidate, found {self_ty}")
+    };
+    let gen_sig = args.as_coroutine().sig();
+    let Normalized { value: gen_sig, obligations } = normalize_with_depth(
+        selcx,
+        obligation.param_env,
+        obligation.cause.clone(),
+        obligation.recursion_depth + 1,
+        gen_sig,
+    );
+
+    debug!(?obligation, ?gen_sig, ?obligations, "confirm_iterator_candidate");
+
+    let tcx = selcx.tcx();
+    let iter_def_id = tcx.require_lang_item(LangItem::Iterator, None);
+
+    let (trait_ref, yield_ty) = super::util::iterator_trait_ref_and_outputs(
+        tcx,
+        iter_def_id,
+        obligation.predicate.self_ty(),
+        gen_sig,
+    );
+
+    debug_assert_eq!(tcx.associated_item(obligation.predicate.def_id).name, sym::Item);
+
+    let predicate = ty::ProjectionPredicate {
+        projection_ty: ty::AliasTy::new(tcx, obligation.predicate.def_id, trait_ref.args),
+        term: yield_ty.into(),
+    };
+
+    confirm_param_env_candidate(selcx, obligation, ty::Binder::dummy(predicate), false)
+        .with_addl_obligations(nested)
+        .with_addl_obligations(obligations)
+}
+
+fn confirm_async_iterator_candidate<'cx, 'tcx>(
+    selcx: &mut SelectionContext<'cx, 'tcx>,
+    obligation: &ProjectionTyObligation<'tcx>,
+    nested: Vec<PredicateObligation<'tcx>>,
+) -> Progress<'tcx> {
     let ty::Coroutine(_, args, _) =
         selcx.infcx.shallow_resolve(obligation.predicate.self_ty()).kind()
     else {
         unreachable!()
     };
-    let gen_sig = args.as_coroutine().poly_sig();
+    let gen_sig = args.as_coroutine().sig();
     let Normalized { value: gen_sig, obligations } = normalize_with_depth(
         selcx,
         obligation.param_env,
@@ -2156,27 +2237,34 @@ fn confirm_iterator_candidate<'cx, 'tcx>(
         gen_sig,
     );
 
-    debug!(?obligation, ?gen_sig, ?obligations, "confirm_iterator_candidate");
+    debug!(?obligation, ?gen_sig, ?obligations, "confirm_async_iterator_candidate");
 
     let tcx = selcx.tcx();
-    let iter_def_id = tcx.require_lang_item(LangItem::Iterator, None);
+    let iter_def_id = tcx.require_lang_item(LangItem::AsyncIterator, None);
 
-    let predicate = super::util::iterator_trait_ref_and_outputs(
+    let (trait_ref, yield_ty) = super::util::async_iterator_trait_ref_and_outputs(
         tcx,
         iter_def_id,
         obligation.predicate.self_ty(),
         gen_sig,
-    )
-    .map_bound(|(trait_ref, yield_ty)| {
-        debug_assert_eq!(tcx.associated_item(obligation.predicate.def_id).name, sym::Item);
+    );
 
-        ty::ProjectionPredicate {
-            projection_ty: ty::AliasTy::new(tcx, obligation.predicate.def_id, trait_ref.args),
-            term: yield_ty.into(),
-        }
-    });
+    debug_assert_eq!(tcx.associated_item(obligation.predicate.def_id).name, sym::Item);
 
-    confirm_param_env_candidate(selcx, obligation, predicate, false)
+    let ty::Adt(_poll_adt, args) = *yield_ty.kind() else {
+        bug!();
+    };
+    let ty::Adt(_option_adt, args) = *args.type_at(0).kind() else {
+        bug!();
+    };
+    let item_ty = args.type_at(0);
+
+    let predicate = ty::ProjectionPredicate {
+        projection_ty: ty::AliasTy::new(tcx, obligation.predicate.def_id, trait_ref.args),
+        term: item_ty.into(),
+    };
+
+    confirm_param_env_candidate(selcx, obligation, ty::Binder::dummy(predicate), false)
         .with_addl_obligations(nested)
         .with_addl_obligations(obligations)
 }
@@ -2239,8 +2327,9 @@ fn confirm_fn_pointer_candidate<'cx, 'tcx>(
     obligation: &ProjectionTyObligation<'tcx>,
     nested: Vec<PredicateObligation<'tcx>>,
 ) -> Progress<'tcx> {
+    let tcx = selcx.tcx();
     let fn_type = selcx.infcx.shallow_resolve(obligation.predicate.self_ty());
-    let sig = fn_type.fn_sig(selcx.tcx());
+    let sig = fn_type.fn_sig(tcx);
     let Normalized { value: sig, obligations } = normalize_with_depth(
         selcx,
         obligation.param_env,
@@ -2249,9 +2338,24 @@ fn confirm_fn_pointer_candidate<'cx, 'tcx>(
         sig,
     );
 
-    confirm_callable_candidate(selcx, obligation, sig, util::TupleArgumentsFlag::Yes)
-        .with_addl_obligations(nested)
-        .with_addl_obligations(obligations)
+    let host_effect_param = match *fn_type.kind() {
+        ty::FnDef(def_id, args) => tcx
+            .generics_of(def_id)
+            .host_effect_index
+            .map_or(tcx.consts.true_, |idx| args.const_at(idx)),
+        ty::FnPtr(_) => tcx.consts.true_,
+        _ => unreachable!("only expected FnPtr or FnDef in `confirm_fn_pointer_candidate`"),
+    };
+
+    confirm_callable_candidate(
+        selcx,
+        obligation,
+        sig,
+        util::TupleArgumentsFlag::Yes,
+        host_effect_param,
+    )
+    .with_addl_obligations(nested)
+    .with_addl_obligations(obligations)
 }
 
 fn confirm_closure_candidate<'cx, 'tcx>(
@@ -2259,9 +2363,9 @@ fn confirm_closure_candidate<'cx, 'tcx>(
     obligation: &ProjectionTyObligation<'tcx>,
     nested: Vec<PredicateObligation<'tcx>>,
 ) -> Progress<'tcx> {
-    let ty::Closure(_, args) = selcx.infcx.shallow_resolve(obligation.predicate.self_ty()).kind()
-    else {
-        unreachable!()
+    let self_ty = selcx.infcx.shallow_resolve(obligation.predicate.self_ty());
+    let ty::Closure(_, args) = self_ty.kind() else {
+        unreachable!("expected closure self type for closure candidate, found {self_ty}")
     };
     let closure_sig = args.as_closure().sig();
     let Normalized { value: closure_sig, obligations } = normalize_with_depth(
@@ -2274,9 +2378,16 @@ fn confirm_closure_candidate<'cx, 'tcx>(
 
     debug!(?obligation, ?closure_sig, ?obligations, "confirm_closure_candidate");
 
-    confirm_callable_candidate(selcx, obligation, closure_sig, util::TupleArgumentsFlag::No)
-        .with_addl_obligations(nested)
-        .with_addl_obligations(obligations)
+    confirm_callable_candidate(
+        selcx,
+        obligation,
+        closure_sig,
+        util::TupleArgumentsFlag::No,
+        // FIXME(effects): This doesn't handle const closures correctly!
+        selcx.tcx().consts.true_,
+    )
+    .with_addl_obligations(nested)
+    .with_addl_obligations(obligations)
 }
 
 fn confirm_callable_candidate<'cx, 'tcx>(
@@ -2284,6 +2395,7 @@ fn confirm_callable_candidate<'cx, 'tcx>(
     obligation: &ProjectionTyObligation<'tcx>,
     fn_sig: ty::PolyFnSig<'tcx>,
     flag: util::TupleArgumentsFlag,
+    fn_host_effect: ty::Const<'tcx>,
 ) -> Progress<'tcx> {
     let tcx = selcx.tcx();
 
@@ -2298,6 +2410,7 @@ fn confirm_callable_candidate<'cx, 'tcx>(
         obligation.predicate.self_ty(),
         fn_sig,
         flag,
+        fn_host_effect,
     )
     .map_bound(|(trait_ref, ret_type)| ty::ProjectionPredicate {
         projection_ty: ty::AliasTy::new(tcx, fn_once_output_def_id, trait_ref.args),
@@ -2319,7 +2432,7 @@ fn confirm_param_env_candidate<'cx, 'tcx>(
 
     let cache_entry = infcx.instantiate_binder_with_fresh_vars(
         cause.span,
-        LateBoundRegionConversionTime::HigherRankedType,
+        BoundRegionConversionTime::HigherRankedType,
         poly_cache_entry,
     );
 
diff --git a/compiler/rustc_trait_selection/src/traits/query/dropck_outlives.rs b/compiler/rustc_trait_selection/src/traits/query/dropck_outlives.rs
index f8efa6a1f..ec80df1d6 100644
--- a/compiler/rustc_trait_selection/src/traits/query/dropck_outlives.rs
+++ b/compiler/rustc_trait_selection/src/traits/query/dropck_outlives.rs
@@ -48,9 +48,7 @@ pub fn trivial_dropck_outlives<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> bool {
         // (T1..Tn) and closures have same properties as T1..Tn --
         // check if *all* of them are trivial.
         ty::Tuple(tys) => tys.iter().all(|t| trivial_dropck_outlives(tcx, t)),
-        ty::Closure(_, ref args) => {
-            trivial_dropck_outlives(tcx, args.as_closure().tupled_upvars_ty())
-        }
+        ty::Closure(_, args) => trivial_dropck_outlives(tcx, args.as_closure().tupled_upvars_ty()),
 
         ty::Adt(def, _) => {
             if Some(def.did()) == tcx.lang_items().manually_drop() {
@@ -239,7 +237,7 @@ pub fn dtorck_constraint_for_ty_inner<'tcx>(
                 // By the time this code runs, all type variables ought to
                 // be fully resolved.
 
-                tcx.sess.delay_span_bug(
+                tcx.sess.span_delayed_bug(
                     span,
                     format!("upvar_tys for closure not found. Expected capture information for closure {ty}",),
                 );
@@ -288,7 +286,7 @@ pub fn dtorck_constraint_for_ty_inner<'tcx>(
             if !args.is_valid() {
                 // By the time this code runs, all type variables ought to
                 // be fully resolved.
-                tcx.sess.delay_span_bug(
+                tcx.sess.span_delayed_bug(
                     span,
                     format!("upvar_tys for coroutine not found. Expected capture information for coroutine {ty}",),
                 );
diff --git a/compiler/rustc_trait_selection/src/traits/query/evaluate_obligation.rs b/compiler/rustc_trait_selection/src/traits/query/evaluate_obligation.rs
index 65f32b1c4..d812d537d 100644
--- a/compiler/rustc_trait_selection/src/traits/query/evaluate_obligation.rs
+++ b/compiler/rustc_trait_selection/src/traits/query/evaluate_obligation.rs
@@ -72,7 +72,7 @@ impl<'tcx> InferCtxtExt<'tcx> for InferCtxt<'tcx> {
                 let mut fulfill_cx = crate::solve::FulfillmentCtxt::new(self);
                 fulfill_cx.register_predicate_obligation(self, obligation.clone());
                 // True errors
-                // FIXME(-Ztrait-solver=next): Overflows are reported as ambig here, is that OK?
+                // FIXME(-Znext-solver): Overflows are reported as ambig here, is that OK?
                 if !fulfill_cx.select_where_possible(self).is_empty() {
                     Ok(EvaluationResult::EvaluatedToErr)
                 } else if !fulfill_cx.select_all_or_error(self).is_empty() {
@@ -108,7 +108,7 @@ impl<'tcx> InferCtxtExt<'tcx> for InferCtxt<'tcx> {
         match self.evaluate_obligation(obligation) {
             Ok(result) => result,
             Err(OverflowError::Canonical) => {
-                let mut selcx = SelectionContext::new(&self);
+                let mut selcx = SelectionContext::new(self);
                 selcx.evaluate_root_obligation(obligation).unwrap_or_else(|r| match r {
                     OverflowError::Canonical => {
                         span_bug!(
diff --git a/compiler/rustc_trait_selection/src/traits/query/normalize.rs b/compiler/rustc_trait_selection/src/traits/query/normalize.rs
index 2e31b560b..ed55533bc 100644
--- a/compiler/rustc_trait_selection/src/traits/query/normalize.rs
+++ b/compiler/rustc_trait_selection/src/traits/query/normalize.rs
@@ -150,7 +150,7 @@ impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for MaxEscapingBoundVarVisitor {
     #[inline]
     fn visit_region(&mut self, r: ty::Region<'tcx>) -> ControlFlow<Self::BreakTy> {
         match *r {
-            ty::ReLateBound(debruijn, _) if debruijn > self.outer_index => {
+            ty::ReBound(debruijn, _) if debruijn > self.outer_index => {
                 self.escaping =
                     self.escaping.max(debruijn.as_usize() - self.outer_index.as_usize());
             }
@@ -160,14 +160,12 @@ impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for MaxEscapingBoundVarVisitor {
     }
 
     fn visit_const(&mut self, ct: ty::Const<'tcx>) -> ControlFlow<Self::BreakTy> {
-        match ct.kind() {
-            ty::ConstKind::Bound(debruijn, _) if debruijn >= self.outer_index => {
-                self.escaping =
-                    self.escaping.max(debruijn.as_usize() - self.outer_index.as_usize());
-                ControlFlow::Continue(())
-            }
-            _ => ct.super_visit_with(self),
+        if ct.outer_exclusive_binder() > self.outer_index {
+            self.escaping = self
+                .escaping
+                .max(ct.outer_exclusive_binder().as_usize() - self.outer_index.as_usize());
         }
+        ControlFlow::Continue(())
     }
 }
 
@@ -287,14 +285,14 @@ impl<'cx, 'tcx> FallibleTypeFolder<TyCtxt<'tcx>> for QueryNormalizer<'cx, 'tcx>
                     ty::Projection => tcx.normalize_projection_ty(c_data),
                     ty::Weak => tcx.normalize_weak_ty(c_data),
                     ty::Inherent => tcx.normalize_inherent_projection_ty(c_data),
-                    _ => unreachable!(),
+                    kind => unreachable!("did not expect {kind:?} due to match arm above"),
                 }?;
                 // We don't expect ambiguity.
                 if !result.value.is_proven() {
                     // Rustdoc normalizes possibly not well-formed types, so only
                     // treat this as a bug if we're not in rustdoc.
                     if !tcx.sess.opts.actually_rustdoc {
-                        tcx.sess.delay_span_bug(
+                        tcx.sess.span_delayed_bug(
                             DUMMY_SP,
                             format!("unexpected ambiguity: {c_data:?} {result:?}"),
                         );
diff --git a/compiler/rustc_trait_selection/src/traits/query/type_op/ascribe_user_type.rs b/compiler/rustc_trait_selection/src/traits/query/type_op/ascribe_user_type.rs
index 302b6016e..152ceeee8 100644
--- a/compiler/rustc_trait_selection/src/traits/query/type_op/ascribe_user_type.rs
+++ b/compiler/rustc_trait_selection/src/traits/query/type_op/ascribe_user_type.rs
@@ -26,7 +26,7 @@ impl<'tcx> super::QueryTypeOp<'tcx> for AscribeUserType<'tcx> {
         tcx.type_op_ascribe_user_type(canonicalized)
     }
 
-    fn perform_locally_in_new_solver(
+    fn perform_locally_with_next_solver(
         ocx: &ObligationCtxt<'_, 'tcx>,
         key: ParamEnvAnd<'tcx, Self>,
     ) -> Result<Self::QueryResponse, NoSolution> {
diff --git a/compiler/rustc_trait_selection/src/traits/query/type_op/custom.rs b/compiler/rustc_trait_selection/src/traits/query/type_op/custom.rs
index c81bc5790..18bb56ba4 100644
--- a/compiler/rustc_trait_selection/src/traits/query/type_op/custom.rs
+++ b/compiler/rustc_trait_selection/src/traits/query/type_op/custom.rs
@@ -82,13 +82,13 @@ where
     let value = infcx.commit_if_ok(|_| {
         let ocx = ObligationCtxt::new(infcx);
         let value = op(&ocx).map_err(|_| {
-            infcx.tcx.sess.delay_span_bug(span, format!("error performing operation: {name}"))
+            infcx.tcx.sess.span_delayed_bug(span, format!("error performing operation: {name}"))
         })?;
         let errors = ocx.select_all_or_error();
         if errors.is_empty() {
             Ok(value)
         } else {
-            Err(infcx.tcx.sess.delay_span_bug(
+            Err(infcx.tcx.sess.span_delayed_bug(
                 DUMMY_SP,
                 format!("errors selecting obligation during MIR typeck: {errors:?}"),
             ))
diff --git a/compiler/rustc_trait_selection/src/traits/query/type_op/eq.rs b/compiler/rustc_trait_selection/src/traits/query/type_op/eq.rs
index f65893088..57e649f3e 100644
--- a/compiler/rustc_trait_selection/src/traits/query/type_op/eq.rs
+++ b/compiler/rustc_trait_selection/src/traits/query/type_op/eq.rs
@@ -23,7 +23,7 @@ impl<'tcx> super::QueryTypeOp<'tcx> for Eq<'tcx> {
         tcx.type_op_eq(canonicalized)
     }
 
-    fn perform_locally_in_new_solver(
+    fn perform_locally_with_next_solver(
         ocx: &ObligationCtxt<'_, 'tcx>,
         key: ParamEnvAnd<'tcx, Self>,
     ) -> Result<Self::QueryResponse, NoSolution> {
diff --git a/compiler/rustc_trait_selection/src/traits/query/type_op/implied_outlives_bounds.rs b/compiler/rustc_trait_selection/src/traits/query/type_op/implied_outlives_bounds.rs
index e345fc39e..ba6ed2987 100644
--- a/compiler/rustc_trait_selection/src/traits/query/type_op/implied_outlives_bounds.rs
+++ b/compiler/rustc_trait_selection/src/traits/query/type_op/implied_outlives_bounds.rs
@@ -50,7 +50,7 @@ impl<'tcx> super::QueryTypeOp<'tcx> for ImpliedOutlivesBounds<'tcx> {
         tcx.implied_outlives_bounds(canonicalized)
     }
 
-    fn perform_locally_in_new_solver(
+    fn perform_locally_with_next_solver(
         ocx: &ObligationCtxt<'_, 'tcx>,
         key: ParamEnvAnd<'tcx, Self>,
     ) -> Result<Self::QueryResponse, NoSolution> {
@@ -123,20 +123,19 @@ pub fn compute_implied_outlives_bounds_inner<'tcx>(
                 Some(pred) => pred,
             };
             match pred {
-                ty::PredicateKind::Clause(ty::ClauseKind::Trait(..))
                 // FIXME(const_generics): Make sure that `<'a, 'b, const N: &'a &'b u32>` is sound
                 // if we ever support that
+                ty::PredicateKind::Clause(ty::ClauseKind::Trait(..))
                 | ty::PredicateKind::Clause(ty::ClauseKind::ConstArgHasType(..))
                 | ty::PredicateKind::Subtype(..)
                 | ty::PredicateKind::Coerce(..)
                 | ty::PredicateKind::Clause(ty::ClauseKind::Projection(..))
-                | ty::PredicateKind::ClosureKind(..)
                 | ty::PredicateKind::ObjectSafe(..)
                 | ty::PredicateKind::Clause(ty::ClauseKind::ConstEvaluatable(..))
                 | ty::PredicateKind::ConstEquate(..)
                 | ty::PredicateKind::Ambiguous
-                | ty::PredicateKind::AliasRelate(..)
-                 => {}
+                | ty::PredicateKind::NormalizesTo(..)
+                | ty::PredicateKind::AliasRelate(..) => {}
 
                 // We need to search through *all* WellFormed predicates
                 ty::PredicateKind::Clause(ty::ClauseKind::WellFormed(arg)) => {
@@ -144,10 +143,9 @@ pub fn compute_implied_outlives_bounds_inner<'tcx>(
                 }
 
                 // We need to register region relationships
-                ty::PredicateKind::Clause(ty::ClauseKind::RegionOutlives(ty::OutlivesPredicate(
-                    r_a,
-                    r_b,
-                ))) => outlives_bounds.push(ty::OutlivesPredicate(r_a.into(), r_b)),
+                ty::PredicateKind::Clause(ty::ClauseKind::RegionOutlives(
+                    ty::OutlivesPredicate(r_a, r_b),
+                )) => outlives_bounds.push(ty::OutlivesPredicate(r_a.into(), r_b)),
 
                 ty::PredicateKind::Clause(ty::ClauseKind::TypeOutlives(ty::OutlivesPredicate(
                     ty_a,
@@ -186,7 +184,9 @@ pub fn compute_implied_outlives_bounds_inner<'tcx>(
                 push_outlives_components(tcx, ty_a, &mut components);
                 implied_bounds.extend(implied_bounds_from_components(r_b, components))
             }
-            ty::GenericArgKind::Const(_) => unreachable!(),
+            ty::GenericArgKind::Const(_) => {
+                unreachable!("consts do not participate in outlives bounds")
+            }
         }
     }
 
@@ -208,6 +208,11 @@ fn implied_bounds_from_components<'tcx>(
                 Component::Region(r) => Some(OutlivesBound::RegionSubRegion(sub_region, r)),
                 Component::Param(p) => Some(OutlivesBound::RegionSubParam(sub_region, p)),
                 Component::Alias(p) => Some(OutlivesBound::RegionSubAlias(sub_region, p)),
+                Component::Placeholder(_p) => {
+                    // FIXME(non_lifetime_binders): Placeholders don't currently
+                    // imply anything for outlives, though they could easily.
+                    None
+                }
                 Component::EscapingAlias(_) =>
                 // If the projection has escaping regions, don't
                 // try to infer any implied bounds even for its
diff --git a/compiler/rustc_trait_selection/src/traits/query/type_op/mod.rs b/compiler/rustc_trait_selection/src/traits/query/type_op/mod.rs
index 9d7933e23..272f1a54f 100644
--- a/compiler/rustc_trait_selection/src/traits/query/type_op/mod.rs
+++ b/compiler/rustc_trait_selection/src/traits/query/type_op/mod.rs
@@ -89,7 +89,7 @@ pub trait QueryTypeOp<'tcx>: fmt::Debug + Copy + TypeFoldable<TyCtxt<'tcx>> + 't
     /// make sure to feed it predefined opaque types and the defining anchor
     /// and that would require duplicating all of the tcx queries. Instead,
     /// just perform these ops locally.
-    fn perform_locally_in_new_solver(
+    fn perform_locally_with_next_solver(
         ocx: &ObligationCtxt<'_, 'tcx>,
         key: ParamEnvAnd<'tcx, Self>,
     ) -> Result<Self::QueryResponse, NoSolution>;
@@ -149,7 +149,7 @@ where
         if infcx.next_trait_solver() {
             return Ok(scrape_region_constraints(
                 infcx,
-                |ocx| QueryTypeOp::perform_locally_in_new_solver(ocx, self),
+                |ocx| QueryTypeOp::perform_locally_with_next_solver(ocx, self),
                 "query type op",
                 span,
             )?
@@ -159,7 +159,7 @@ where
         let mut region_constraints = QueryRegionConstraints::default();
         let (output, error_info, mut obligations, _) =
             Q::fully_perform_into(self, infcx, &mut region_constraints).map_err(|_| {
-                infcx.tcx.sess.delay_span_bug(span, format!("error performing {self:?}"))
+                infcx.tcx.sess.span_delayed_bug(span, format!("error performing {self:?}"))
             })?;
 
         // Typically, instantiating NLL query results does not
@@ -188,7 +188,7 @@ where
                 }
             }
             if !progress {
-                return Err(infcx.tcx.sess.delay_span_bug(
+                return Err(infcx.tcx.sess.span_delayed_bug(
                     span,
                     format!("ambiguity processing {obligations:?} from {self:?}"),
                 ));
diff --git a/compiler/rustc_trait_selection/src/traits/query/type_op/normalize.rs b/compiler/rustc_trait_selection/src/traits/query/type_op/normalize.rs
index 9559f5002..3b33f6e61 100644
--- a/compiler/rustc_trait_selection/src/traits/query/type_op/normalize.rs
+++ b/compiler/rustc_trait_selection/src/traits/query/type_op/normalize.rs
@@ -25,11 +25,11 @@ where
         T::type_op_method(tcx, canonicalized)
     }
 
-    fn perform_locally_in_new_solver(
+    fn perform_locally_with_next_solver(
         ocx: &ObligationCtxt<'_, 'tcx>,
         key: ParamEnvAnd<'tcx, Self>,
     ) -> Result<Self::QueryResponse, NoSolution> {
-        // FIXME(-Ztrait-solver=next): shouldn't be using old normalizer
+        // FIXME(-Znext-solver): shouldn't be using old normalizer
         Ok(ocx.normalize(&ObligationCause::dummy(), key.param_env, key.value.value))
     }
 }
diff --git a/compiler/rustc_trait_selection/src/traits/query/type_op/outlives.rs b/compiler/rustc_trait_selection/src/traits/query/type_op/outlives.rs
index f2c1243f9..07587e374 100644
--- a/compiler/rustc_trait_selection/src/traits/query/type_op/outlives.rs
+++ b/compiler/rustc_trait_selection/src/traits/query/type_op/outlives.rs
@@ -42,7 +42,7 @@ impl<'tcx> super::QueryTypeOp<'tcx> for DropckOutlives<'tcx> {
         tcx.dropck_outlives(canonicalized)
     }
 
-    fn perform_locally_in_new_solver(
+    fn perform_locally_with_next_solver(
         ocx: &ObligationCtxt<'_, 'tcx>,
         key: ParamEnvAnd<'tcx, Self>,
     ) -> Result<Self::QueryResponse, NoSolution> {
diff --git a/compiler/rustc_trait_selection/src/traits/query/type_op/prove_predicate.rs b/compiler/rustc_trait_selection/src/traits/query/type_op/prove_predicate.rs
index 789ef6472..e21ede47f 100644
--- a/compiler/rustc_trait_selection/src/traits/query/type_op/prove_predicate.rs
+++ b/compiler/rustc_trait_selection/src/traits/query/type_op/prove_predicate.rs
@@ -40,7 +40,7 @@ impl<'tcx> super::QueryTypeOp<'tcx> for ProvePredicate<'tcx> {
         tcx.type_op_prove_predicate(canonicalized)
     }
 
-    fn perform_locally_in_new_solver(
+    fn perform_locally_with_next_solver(
         ocx: &ObligationCtxt<'_, 'tcx>,
         key: ParamEnvAnd<'tcx, Self>,
     ) -> Result<Self::QueryResponse, NoSolution> {
diff --git a/compiler/rustc_trait_selection/src/traits/query/type_op/subtype.rs b/compiler/rustc_trait_selection/src/traits/query/type_op/subtype.rs
index 10976d5cd..ae11b0825 100644
--- a/compiler/rustc_trait_selection/src/traits/query/type_op/subtype.rs
+++ b/compiler/rustc_trait_selection/src/traits/query/type_op/subtype.rs
@@ -20,7 +20,7 @@ impl<'tcx> super::QueryTypeOp<'tcx> for Subtype<'tcx> {
         tcx.type_op_subtype(canonicalized)
     }
 
-    fn perform_locally_in_new_solver(
+    fn perform_locally_with_next_solver(
         ocx: &ObligationCtxt<'_, 'tcx>,
         key: ParamEnvAnd<'tcx, Self>,
     ) -> Result<Self::QueryResponse, NoSolution> {
diff --git a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
index f4b6d3bcf..38b1046ac 100644
--- a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
+++ b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
@@ -115,6 +115,8 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                     self.assemble_future_candidates(obligation, &mut candidates);
                 } else if lang_items.iterator_trait() == Some(def_id) {
                     self.assemble_iterator_candidates(obligation, &mut candidates);
+                } else if lang_items.async_iterator_trait() == Some(def_id) {
+                    self.assemble_async_iterator_candidates(obligation, &mut candidates);
                 }
 
                 self.assemble_closure_candidates(obligation, &mut candidates);
@@ -155,10 +157,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             .infcx
             .probe(|_| self.match_projection_obligation_against_definition_bounds(obligation));
 
-        // FIXME(effects) proper constness needed?
-        candidates.vec.extend(
-            result.into_iter().map(|idx| ProjectionCandidate(idx, ty::BoundConstness::NotConst)),
-        );
+        candidates.vec.extend(result.into_iter().map(|idx| ProjectionCandidate(idx)));
     }
 
     /// Given an obligation like `<SomeTrait for T>`, searches the obligations that the caller
@@ -261,6 +260,34 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         }
     }
 
+    fn assemble_async_iterator_candidates(
+        &mut self,
+        obligation: &PolyTraitObligation<'tcx>,
+        candidates: &mut SelectionCandidateSet<'tcx>,
+    ) {
+        let self_ty = obligation.self_ty().skip_binder();
+        if let ty::Coroutine(did, args, _) = *self_ty.kind() {
+            // gen constructs get lowered to a special kind of coroutine that
+            // should directly `impl AsyncIterator`.
+            if self.tcx().coroutine_is_async_gen(did) {
+                debug!(?self_ty, ?obligation, "assemble_iterator_candidates",);
+
+                // Can only confirm this candidate if we have constrained
+                // the `Yield` type to at least `Poll<Option<?0>>`..
+                let ty::Adt(_poll_def, args) = *args.as_coroutine().yield_ty().kind() else {
+                    candidates.ambiguous = true;
+                    return;
+                };
+                let ty::Adt(_option_def, _) = *args.type_at(0).kind() else {
+                    candidates.ambiguous = true;
+                    return;
+                };
+
+                candidates.vec.push(AsyncIteratorCandidate);
+            }
+        }
+    }
+
     /// Checks for the artificial impl that the compiler will create for an obligation like `X :
     /// FnMut<..>` where `X` is a closure type.
     ///
@@ -291,8 +318,11 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                         }
                     }
                     None => {
-                        debug!("assemble_unboxed_candidates: closure_kind not yet known");
-                        candidates.vec.push(ClosureCandidate { is_const });
+                        if kind == ty::ClosureKind::FnOnce {
+                            candidates.vec.push(ClosureCandidate { is_const });
+                        } else {
+                            candidates.ambiguous = true;
+                        }
                     }
                 }
             }
@@ -328,17 +358,23 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             // Provide an impl, but only for suitable `fn` pointers.
             ty::FnPtr(sig) => {
                 if sig.is_fn_trait_compatible() {
-                    candidates.vec.push(FnPointerCandidate { is_const: false });
+                    candidates
+                        .vec
+                        .push(FnPointerCandidate { fn_host_effect: self.tcx().consts.true_ });
                 }
             }
             // Provide an impl for suitable functions, rejecting `#[target_feature]` functions (RFC 2396).
-            ty::FnDef(def_id, _) => {
-                if self.tcx().fn_sig(def_id).skip_binder().is_fn_trait_compatible()
-                    && self.tcx().codegen_fn_attrs(def_id).target_features.is_empty()
+            ty::FnDef(def_id, args) => {
+                let tcx = self.tcx();
+                if tcx.fn_sig(def_id).skip_binder().is_fn_trait_compatible()
+                    && tcx.codegen_fn_attrs(def_id).target_features.is_empty()
                 {
-                    candidates
-                        .vec
-                        .push(FnPointerCandidate { is_const: self.tcx().is_const_fn(def_id) });
+                    candidates.vec.push(FnPointerCandidate {
+                        fn_host_effect: tcx
+                            .generics_of(def_id)
+                            .host_effect_index
+                            .map_or(tcx.consts.true_, |idx| args.const_at(idx)),
+                    });
                 }
             }
             _ => {}
@@ -375,7 +411,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 // consider a "quick reject". This avoids creating more types
                 // and so forth that we need to.
                 let impl_trait_ref = self.tcx().impl_trait_ref(impl_def_id).unwrap();
-                if !drcx.args_refs_may_unify(obligation_args, impl_trait_ref.skip_binder().args) {
+                if !drcx.args_may_unify(obligation_args, impl_trait_ref.skip_binder().args) {
                     return;
                 }
                 if self.reject_fn_ptr_impls(
@@ -555,7 +591,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 }
 
                 ty::Alias(ty::Opaque, _) => {
-                    if candidates.vec.iter().any(|c| matches!(c, ProjectionCandidate(..))) {
+                    if candidates.vec.iter().any(|c| matches!(c, ProjectionCandidate(_))) {
                         // We do not generate an auto impl candidate for `impl Trait`s which already
                         // reference our auto trait.
                         //
@@ -634,7 +670,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
 
             let self_ty = placeholder_trait_predicate.self_ty();
             let principal_trait_ref = match self_ty.kind() {
-                ty::Dynamic(ref data, ..) => {
+                ty::Dynamic(data, ..) => {
                     if data.auto_traits().any(|did| did == obligation.predicate.def_id()) {
                         debug!(
                             "assemble_candidates_from_object_ty: matched builtin bound, \
@@ -759,10 +795,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
 
         match (source.kind(), target.kind()) {
             // Trait+Kx+'a -> Trait+Ky+'b (upcasts).
-            (
-                &ty::Dynamic(ref a_data, a_region, ty::Dyn),
-                &ty::Dynamic(ref b_data, b_region, ty::Dyn),
-            ) => {
+            (&ty::Dynamic(a_data, a_region, ty::Dyn), &ty::Dynamic(b_data, b_region, ty::Dyn)) => {
                 // Upcast coercions permit several things:
                 //
                 // 1. Dropping auto traits, e.g., `Foo + Send` to `Foo`
@@ -923,9 +956,18 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
     ) {
         // If the predicate is `~const Destruct` in a non-const environment, we don't actually need
         // to check anything. We'll short-circuit checking any obligations in confirmation, too.
-        // FIXME(effects)
-        if true {
-            candidates.vec.push(ConstDestructCandidate(None));
+        let Some(host_effect_index) =
+            self.tcx().generics_of(obligation.predicate.def_id()).host_effect_index
+        else {
+            candidates.vec.push(BuiltinCandidate { has_nested: false });
+            return;
+        };
+        // If the obligation has `host = true`, then the obligation is non-const and it's always
+        // trivially implemented.
+        if obligation.predicate.skip_binder().trait_ref.args.const_at(host_effect_index)
+            == self.tcx().consts.true_
+        {
+            candidates.vec.push(BuiltinCandidate { has_nested: false });
             return;
         }
 
@@ -1051,7 +1093,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
     ) {
         // The regions of a type don't affect the size of the type
         let tcx = self.tcx();
-        let self_ty = tcx.erase_late_bound_regions(obligation.predicate.self_ty());
+        let self_ty = tcx.instantiate_bound_regions_with_erased(obligation.predicate.self_ty());
         // We should erase regions from both the param-env and type, since both
         // may have infer regions. Specifically, after canonicalizing and instantiating,
         // early bound regions turn into region vars in both the new and old solver.
diff --git a/compiler/rustc_trait_selection/src/traits/select/confirmation.rs b/compiler/rustc_trait_selection/src/traits/select/confirmation.rs
index 4bfa341e3..ce3fc2185 100644
--- a/compiler/rustc_trait_selection/src/traits/select/confirmation.rs
+++ b/compiler/rustc_trait_selection/src/traits/select/confirmation.rs
@@ -9,7 +9,7 @@
 use rustc_ast::Mutability;
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_hir::lang_items::LangItem;
-use rustc_infer::infer::LateBoundRegionConversionTime::HigherRankedType;
+use rustc_infer::infer::BoundRegionConversionTime::HigherRankedType;
 use rustc_infer::infer::{DefineOpaqueTypes, InferOk};
 use rustc_middle::traits::{BuiltinImplSource, SelectionOutputTypeParameterMismatch};
 use rustc_middle::ty::{
@@ -71,7 +71,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 ImplSource::Builtin(BuiltinImplSource::Misc, data)
             }
 
-            ProjectionCandidate(idx, _) => {
+            ProjectionCandidate(idx) => {
                 let obligations = self.confirm_projection_candidate(obligation, idx)?;
                 ImplSource::Param(obligations)
             }
@@ -98,8 +98,13 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 ImplSource::Builtin(BuiltinImplSource::Misc, vtable_iterator)
             }
 
-            FnPointerCandidate { is_const } => {
-                let data = self.confirm_fn_pointer_candidate(obligation, is_const)?;
+            AsyncIteratorCandidate => {
+                let vtable_iterator = self.confirm_async_iterator_candidate(obligation)?;
+                ImplSource::Builtin(BuiltinImplSource::Misc, vtable_iterator)
+            }
+
+            FnPointerCandidate { fn_host_effect } => {
+                let data = self.confirm_fn_pointer_candidate(obligation, fn_host_effect)?;
                 ImplSource::Builtin(BuiltinImplSource::Misc, data)
             }
 
@@ -327,8 +332,9 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         // care about other regions. Erasing late-bound regions is equivalent
         // to instantiating the binder with placeholders then erasing those
         // placeholder regions.
-        let predicate =
-            self.tcx().erase_regions(self.tcx().erase_late_bound_regions(obligation.predicate));
+        let predicate = self
+            .tcx()
+            .erase_regions(self.tcx().instantiate_bound_regions_with_erased(obligation.predicate));
 
         let Some(assume) = rustc_transmute::Assume::from_const(
             self.infcx.tcx,
@@ -393,7 +399,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 obligation.recursion_depth + 1,
                 obligation.param_env,
                 trait_def_id,
-                &trait_ref.args,
+                trait_ref.args,
                 obligation.predicate,
             );
 
@@ -454,7 +460,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             recursion_depth,
             param_env,
             impl_def_id,
-            &args.value,
+            args.value,
             parent_trait_pred,
         );
 
@@ -552,7 +558,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             let defs: &ty::Generics = tcx.generics_of(assoc_type);
 
             if !defs.params.is_empty() && !tcx.features().generic_associated_types_extended {
-                tcx.sess.delay_span_bug(
+                tcx.sess.span_delayed_bug(
                     obligation.cause.span,
                     "GATs in trait object shouldn't have been considered",
                 );
@@ -592,7 +598,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                             let kind = ty::BoundRegionKind::BrNamed(param.def_id, param.name);
                             let bound_var = ty::BoundVariableKind::Region(kind);
                             bound_vars.push(bound_var);
-                            ty::Region::new_late_bound(
+                            ty::Region::new_bound(
                                 tcx,
                                 ty::INNERMOST,
                                 ty::BoundRegion {
@@ -647,8 +653,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
     fn confirm_fn_pointer_candidate(
         &mut self,
         obligation: &PolyTraitObligation<'tcx>,
-        // FIXME(effects)
-        _is_const: bool,
+        fn_host_effect: ty::Const<'tcx>,
     ) -> Result<Vec<PredicateObligation<'tcx>>, SelectionError<'tcx>> {
         debug!(?obligation, "confirm_fn_pointer_candidate");
 
@@ -669,6 +674,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             self_ty,
             sig,
             util::TupleArgumentsFlag::Yes,
+            fn_host_effect,
         )
         .map_bound(|(trait_ref, _)| trait_ref);
 
@@ -707,7 +713,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             obligation.recursion_depth,
             obligation.param_env,
             trait_def_id,
-            &args,
+            args,
             obligation.predicate,
         );
 
@@ -730,7 +736,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
 
         debug!(?obligation, ?coroutine_def_id, ?args, "confirm_coroutine_candidate");
 
-        let coroutine_sig = args.as_coroutine().poly_sig();
+        let coroutine_sig = args.as_coroutine().sig();
 
         // NOTE: The self-type is a coroutine type and hence is
         // in fact unparameterized (or at least does not reference any
@@ -741,15 +747,14 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             .no_bound_vars()
             .expect("unboxed closure type should not capture bound vars from the predicate");
 
-        let trait_ref = super::util::coroutine_trait_ref_and_outputs(
+        let (trait_ref, _, _) = super::util::coroutine_trait_ref_and_outputs(
             self.tcx(),
             obligation.predicate.def_id(),
             self_ty,
             coroutine_sig,
-        )
-        .map_bound(|(trait_ref, ..)| trait_ref);
+        );
 
-        let nested = self.confirm_poly_trait_refs(obligation, trait_ref)?;
+        let nested = self.confirm_poly_trait_refs(obligation, ty::Binder::dummy(trait_ref))?;
         debug!(?trait_ref, ?nested, "coroutine candidate obligations");
 
         Ok(nested)
@@ -769,17 +774,16 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
 
         debug!(?obligation, ?coroutine_def_id, ?args, "confirm_future_candidate");
 
-        let coroutine_sig = args.as_coroutine().poly_sig();
+        let coroutine_sig = args.as_coroutine().sig();
 
-        let trait_ref = super::util::future_trait_ref_and_outputs(
+        let (trait_ref, _) = super::util::future_trait_ref_and_outputs(
             self.tcx(),
             obligation.predicate.def_id(),
             obligation.predicate.no_bound_vars().expect("future has no bound vars").self_ty(),
             coroutine_sig,
-        )
-        .map_bound(|(trait_ref, ..)| trait_ref);
+        );
 
-        let nested = self.confirm_poly_trait_refs(obligation, trait_ref)?;
+        let nested = self.confirm_poly_trait_refs(obligation, ty::Binder::dummy(trait_ref))?;
         debug!(?trait_ref, ?nested, "future candidate obligations");
 
         Ok(nested)
@@ -799,17 +803,45 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
 
         debug!(?obligation, ?coroutine_def_id, ?args, "confirm_iterator_candidate");
 
-        let gen_sig = args.as_coroutine().poly_sig();
+        let gen_sig = args.as_coroutine().sig();
 
-        let trait_ref = super::util::iterator_trait_ref_and_outputs(
+        let (trait_ref, _) = super::util::iterator_trait_ref_and_outputs(
             self.tcx(),
             obligation.predicate.def_id(),
             obligation.predicate.no_bound_vars().expect("iterator has no bound vars").self_ty(),
             gen_sig,
-        )
-        .map_bound(|(trait_ref, ..)| trait_ref);
+        );
 
-        let nested = self.confirm_poly_trait_refs(obligation, trait_ref)?;
+        let nested = self.confirm_poly_trait_refs(obligation, ty::Binder::dummy(trait_ref))?;
+        debug!(?trait_ref, ?nested, "iterator candidate obligations");
+
+        Ok(nested)
+    }
+
+    fn confirm_async_iterator_candidate(
+        &mut self,
+        obligation: &PolyTraitObligation<'tcx>,
+    ) -> Result<Vec<PredicateObligation<'tcx>>, SelectionError<'tcx>> {
+        // Okay to skip binder because the args on coroutine types never
+        // touch bound regions, they just capture the in-scope
+        // type/region parameters.
+        let self_ty = self.infcx.shallow_resolve(obligation.self_ty().skip_binder());
+        let ty::Coroutine(coroutine_def_id, args, _) = *self_ty.kind() else {
+            bug!("closure candidate for non-closure {:?}", obligation);
+        };
+
+        debug!(?obligation, ?coroutine_def_id, ?args, "confirm_async_iterator_candidate");
+
+        let gen_sig = args.as_coroutine().sig();
+
+        let (trait_ref, _) = super::util::async_iterator_trait_ref_and_outputs(
+            self.tcx(),
+            obligation.predicate.def_id(),
+            obligation.predicate.no_bound_vars().expect("iterator has no bound vars").self_ty(),
+            gen_sig,
+        );
+
+        let nested = self.confirm_poly_trait_refs(obligation, ty::Binder::dummy(trait_ref))?;
         debug!(?trait_ref, ?nested, "iterator candidate obligations");
 
         Ok(nested)
@@ -820,11 +852,6 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         &mut self,
         obligation: &PolyTraitObligation<'tcx>,
     ) -> Result<Vec<PredicateObligation<'tcx>>, SelectionError<'tcx>> {
-        let kind = self
-            .tcx()
-            .fn_trait_kind_from_def_id(obligation.predicate.def_id())
-            .unwrap_or_else(|| bug!("closure candidate for non-fn trait {:?}", obligation));
-
         // Okay to skip binder because the args on closure types never
         // touch bound regions, they just capture the in-scope
         // type/region parameters.
@@ -833,16 +860,12 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             bug!("closure candidate for non-closure {:?}", obligation);
         };
 
-        let trait_ref = self.closure_trait_ref_unnormalized(obligation, args);
-        let mut nested = self.confirm_poly_trait_refs(obligation, trait_ref)?;
+        let trait_ref =
+            self.closure_trait_ref_unnormalized(obligation, args, self.tcx().consts.true_);
+        let nested = self.confirm_poly_trait_refs(obligation, trait_ref)?;
 
         debug!(?closure_def_id, ?trait_ref, ?nested, "confirm closure candidate obligations");
 
-        nested.push(obligation.with(
-            self.tcx(),
-            ty::Binder::dummy(ty::PredicateKind::ClosureKind(closure_def_id, args, kind)),
-        ));
-
         Ok(nested)
     }
 
@@ -920,8 +943,12 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         let a_ty = self.infcx.shallow_resolve(predicate.self_ty());
         let b_ty = self.infcx.shallow_resolve(predicate.trait_ref.args.type_at(1));
 
-        let ty::Dynamic(a_data, a_region, ty::Dyn) = *a_ty.kind() else { bug!() };
-        let ty::Dynamic(b_data, b_region, ty::Dyn) = *b_ty.kind() else { bug!() };
+        let ty::Dynamic(a_data, a_region, ty::Dyn) = *a_ty.kind() else {
+            bug!("expected `dyn` type in `confirm_trait_upcasting_unsize_candidate`")
+        };
+        let ty::Dynamic(b_data, b_region, ty::Dyn) = *b_ty.kind() else {
+            bug!("expected `dyn` type in `confirm_trait_upcasting_unsize_candidate`")
+        };
 
         let source_principal = a_data.principal().unwrap().with_self_ty(tcx, a_ty);
         let unnormalized_upcast_principal =
@@ -985,7 +1012,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
 
         Ok(match (source.kind(), target.kind()) {
             // Trait+Kx+'a -> Trait+Ky+'b (auto traits and lifetime subtyping).
-            (&ty::Dynamic(ref data_a, r_a, dyn_a), &ty::Dynamic(ref data_b, r_b, dyn_b))
+            (&ty::Dynamic(data_a, r_a, dyn_a), &ty::Dynamic(data_b, r_b, dyn_b))
                 if dyn_a == dyn_b =>
             {
                 // See `assemble_candidates_for_unsizing` for more info.
@@ -1030,7 +1057,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             }
 
             // `T` -> `Trait`
-            (_, &ty::Dynamic(ref data, r, ty::Dyn)) => {
+            (_, &ty::Dynamic(data, r, ty::Dyn)) => {
                 let mut object_dids = data.auto_traits().chain(data.principal_def_id());
                 if let Some(did) = object_dids.find(|did| !tcx.check_is_object_safe(*did)) {
                     return Err(TraitNotObjectSafe(did));
@@ -1184,11 +1211,13 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         obligation: &PolyTraitObligation<'tcx>,
         impl_def_id: Option<DefId>,
     ) -> Result<Vec<PredicateObligation<'tcx>>, SelectionError<'tcx>> {
-        // `~const Destruct` in a non-const environment is always trivially true, since our type is `Drop`
-        // FIXME(effects)
-        if true {
-            return Ok(vec![]);
-        }
+        let Some(host_effect_index) =
+            self.tcx().generics_of(obligation.predicate.def_id()).host_effect_index
+        else {
+            bug!()
+        };
+        let host_effect_param: ty::GenericArg<'tcx> =
+            obligation.predicate.skip_binder().trait_ref.args.const_at(host_effect_index).into();
 
         let drop_trait = self.tcx().require_lang_item(LangItem::Drop, None);
 
@@ -1277,7 +1306,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                     let tcx = self.tcx();
                     stack.extend(tcx.coroutine_hidden_types(def_id).map(|bty| {
                         let ty = bty.instantiate(tcx, args);
-                        debug_assert!(!ty.has_late_bound_regions());
+                        debug_assert!(!ty.has_bound_regions());
                         ty
                     }))
                 }
@@ -1285,7 +1314,6 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 // If we have a projection type, make sure to normalize it so we replace it
                 // with a fresh infer variable
                 ty::Alias(ty::Projection | ty::Inherent, ..) => {
-                    // FIXME(effects) this needs constness
                     let predicate = normalize_with_depth_to(
                         self,
                         obligation.param_env,
@@ -1296,7 +1324,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                                 self.tcx(),
                                 LangItem::Destruct,
                                 cause.span,
-                                [nested_ty],
+                                [nested_ty.into(), host_effect_param],
                             ),
                             polarity: ty::ImplPolarity::Positive,
                         }),
@@ -1316,13 +1344,12 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 // since it's either not `const Drop` (and we raise an error during selection),
                 // or it's an ADT (and we need to check for a custom impl during selection)
                 _ => {
-                    // FIXME(effects) this needs constness
                     let predicate = self_ty.rebind(ty::TraitPredicate {
                         trait_ref: ty::TraitRef::from_lang_item(
                             self.tcx(),
                             LangItem::Destruct,
                             cause.span,
-                            [nested_ty],
+                            [nested_ty.into(), host_effect_param],
                         ),
                         polarity: ty::ImplPolarity::Positive,
                     });
diff --git a/compiler/rustc_trait_selection/src/traits/select/mod.rs b/compiler/rustc_trait_selection/src/traits/select/mod.rs
index 08208cc60..23f7bdd15 100644
--- a/compiler/rustc_trait_selection/src/traits/select/mod.rs
+++ b/compiler/rustc_trait_selection/src/traits/select/mod.rs
@@ -32,8 +32,8 @@ use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_errors::Diagnostic;
 use rustc_hir as hir;
 use rustc_hir::def_id::DefId;
+use rustc_infer::infer::BoundRegionConversionTime;
 use rustc_infer::infer::DefineOpaqueTypes;
-use rustc_infer::infer::LateBoundRegionConversionTime;
 use rustc_infer::traits::TraitObligation;
 use rustc_middle::dep_graph::dep_kinds;
 use rustc_middle::dep_graph::DepNodeIndex;
@@ -46,6 +46,7 @@ use rustc_middle::ty::GenericArgsRef;
 use rustc_middle::ty::{self, EarlyBinder, PolyProjectionPredicate, ToPolyTraitRef, ToPredicate};
 use rustc_middle::ty::{Ty, TyCtxt, TypeFoldable, TypeVisitableExt};
 use rustc_span::symbol::sym;
+use rustc_span::Symbol;
 
 use std::cell::{Cell, RefCell};
 use std::cmp;
@@ -59,13 +60,13 @@ mod candidate_assembly;
 mod confirmation;
 
 #[derive(Clone, Debug, Eq, PartialEq, Hash)]
-pub enum IntercrateAmbiguityCause {
-    DownstreamCrate { trait_desc: String, self_desc: Option<String> },
-    UpstreamCrateUpdate { trait_desc: String, self_desc: Option<String> },
-    ReservationImpl { message: String },
+pub enum IntercrateAmbiguityCause<'tcx> {
+    DownstreamCrate { trait_ref: ty::TraitRef<'tcx>, self_ty: Option<Ty<'tcx>> },
+    UpstreamCrateUpdate { trait_ref: ty::TraitRef<'tcx>, self_ty: Option<Ty<'tcx>> },
+    ReservationImpl { message: Symbol },
 }
 
-impl IntercrateAmbiguityCause {
+impl<'tcx> IntercrateAmbiguityCause<'tcx> {
     /// Emits notes when the overlap is caused by complex intercrate ambiguities.
     /// See #23980 for details.
     pub fn add_intercrate_ambiguity_hint(&self, err: &mut Diagnostic) {
@@ -73,28 +74,32 @@ impl IntercrateAmbiguityCause {
     }
 
     pub fn intercrate_ambiguity_hint(&self) -> String {
-        match self {
-            IntercrateAmbiguityCause::DownstreamCrate { trait_desc, self_desc } => {
-                let self_desc = if let Some(ty) = self_desc {
-                    format!(" for type `{ty}`")
-                } else {
-                    String::new()
-                };
-                format!("downstream crates may implement trait `{trait_desc}`{self_desc}")
+        with_no_trimmed_paths!(match self {
+            IntercrateAmbiguityCause::DownstreamCrate { trait_ref, self_ty } => {
+                format!(
+                    "downstream crates may implement trait `{trait_desc}`{self_desc}",
+                    trait_desc = trait_ref.print_trait_sugared(),
+                    self_desc = if let Some(self_ty) = self_ty {
+                        format!(" for type `{self_ty}`")
+                    } else {
+                        String::new()
+                    }
+                )
             }
-            IntercrateAmbiguityCause::UpstreamCrateUpdate { trait_desc, self_desc } => {
-                let self_desc = if let Some(ty) = self_desc {
-                    format!(" for type `{ty}`")
-                } else {
-                    String::new()
-                };
+            IntercrateAmbiguityCause::UpstreamCrateUpdate { trait_ref, self_ty } => {
                 format!(
                     "upstream crates may add a new impl of trait `{trait_desc}`{self_desc} \
-                     in future versions"
+                in future versions",
+                    trait_desc = trait_ref.print_trait_sugared(),
+                    self_desc = if let Some(self_ty) = self_ty {
+                        format!(" for type `{self_ty}`")
+                    } else {
+                        String::new()
+                    }
                 )
             }
-            IntercrateAmbiguityCause::ReservationImpl { message } => message.clone(),
-        }
+            IntercrateAmbiguityCause::ReservationImpl { message } => message.to_string(),
+        })
     }
 }
 
@@ -114,7 +119,7 @@ pub struct SelectionContext<'cx, 'tcx> {
     /// We don't do his until we detect a coherence error because it can
     /// lead to false overflow results (#47139) and because always
     /// computing it may negatively impact performance.
-    intercrate_ambiguity_causes: Option<FxIndexSet<IntercrateAmbiguityCause>>,
+    intercrate_ambiguity_causes: Option<FxIndexSet<IntercrateAmbiguityCause<'tcx>>>,
 
     /// The mode that trait queries run in, which informs our error handling
     /// policy. In essence, canonicalized queries need their errors propagated
@@ -217,8 +222,8 @@ pub enum TreatInductiveCycleAs {
 impl From<TreatInductiveCycleAs> for EvaluationResult {
     fn from(treat: TreatInductiveCycleAs) -> EvaluationResult {
         match treat {
-            TreatInductiveCycleAs::Ambig => EvaluatedToUnknown,
-            TreatInductiveCycleAs::Recur => EvaluatedToRecur,
+            TreatInductiveCycleAs::Ambig => EvaluatedToAmbigStackDependent,
+            TreatInductiveCycleAs::Recur => EvaluatedToErrStackDependent,
         }
     }
 }
@@ -270,7 +275,9 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
     /// Gets the intercrate ambiguity causes collected since tracking
     /// was enabled and disables tracking at the same time. If
     /// tracking is not enabled, just returns an empty vector.
-    pub fn take_intercrate_ambiguity_causes(&mut self) -> FxIndexSet<IntercrateAmbiguityCause> {
+    pub fn take_intercrate_ambiguity_causes(
+        &mut self,
+    ) -> FxIndexSet<IntercrateAmbiguityCause<'tcx>> {
         assert!(self.is_intercrate());
         self.intercrate_ambiguity_causes.take().unwrap_or_default()
     }
@@ -367,7 +374,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         debug_assert!(!self.infcx.next_trait_solver());
         // Watch out for overflow. This intentionally bypasses (and does
         // not update) the cache.
-        self.check_recursion_limit(&stack.obligation, &stack.obligation)?;
+        self.check_recursion_limit(stack.obligation, stack.obligation)?;
 
         // Check the cache. Note that we freshen the trait-ref
         // separately rather than using `stack.fresh_trait_ref` --
@@ -416,7 +423,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                     let mut no_candidates_apply = true;
 
                     for c in candidate_set.vec.iter() {
-                        if self.evaluate_candidate(stack, &c)?.may_apply() {
+                        if self.evaluate_candidate(stack, c)?.may_apply() {
                             no_candidates_apply = false;
                             break;
                         }
@@ -428,19 +435,11 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                         );
                         if !trait_ref.references_error() {
                             let self_ty = trait_ref.self_ty();
-                            let (trait_desc, self_desc) = with_no_trimmed_paths!({
-                                let trait_desc = trait_ref.print_only_trait_path().to_string();
-                                let self_desc =
-                                    self_ty.has_concrete_skeleton().then(|| self_ty.to_string());
-                                (trait_desc, self_desc)
-                            });
+                            let self_ty = self_ty.has_concrete_skeleton().then(|| self_ty);
                             let cause = if let Conflict::Upstream = conflict {
-                                IntercrateAmbiguityCause::UpstreamCrateUpdate {
-                                    trait_desc,
-                                    self_desc,
-                                }
+                                IntercrateAmbiguityCause::UpstreamCrateUpdate { trait_ref, self_ty }
                             } else {
-                                IntercrateAmbiguityCause::DownstreamCrate { trait_desc, self_desc }
+                                IntercrateAmbiguityCause::DownstreamCrate { trait_ref, self_ty }
                             };
                             debug!(?cause, "evaluate_stack: pushing cause");
                             self.intercrate_ambiguity_causes.as_mut().unwrap().insert(cause);
@@ -799,7 +798,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                     // A global type with no free lifetimes or generic parameters
                     // outlives anything.
                     if pred.0.has_free_regions()
-                        || pred.0.has_late_bound_regions()
+                        || pred.0.has_bound_regions()
                         || pred.0.has_non_region_infer()
                         || pred.0.has_non_region_infer()
                     {
@@ -885,19 +884,6 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                     }
                 }
 
-                ty::PredicateKind::ClosureKind(_, closure_args, kind) => {
-                    match self.infcx.closure_kind(closure_args) {
-                        Some(closure_kind) => {
-                            if closure_kind.extends(kind) {
-                                Ok(EvaluatedToOk)
-                            } else {
-                                Ok(EvaluatedToErr)
-                            }
-                        }
-                        None => Ok(EvaluatedToAmbig),
-                    }
-                }
-
                 ty::PredicateKind::Clause(ty::ClauseKind::ConstEvaluatable(uv)) => {
                     match const_evaluatable::is_const_evaluatable(
                         self.infcx,
@@ -1004,8 +990,11 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                         }
                     }
                 }
+                ty::PredicateKind::NormalizesTo(..) => {
+                    bug!("NormalizesTo is only used by the new solver")
+                }
                 ty::PredicateKind::AliasRelate(..) => {
-                    bug!("AliasRelate is only used for new solver")
+                    bug!("AliasRelate is only used by the new solver")
                 }
                 ty::PredicateKind::Ambiguous => Ok(EvaluatedToAmbig),
                 ty::PredicateKind::Clause(ty::ClauseKind::ConstArgHasType(ct, ty)) => {
@@ -1237,15 +1226,11 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         if unbound_input_types
             && stack.iter().skip(1).any(|prev| {
                 stack.obligation.param_env == prev.obligation.param_env
-                    && self.match_fresh_trait_refs(
-                        stack.fresh_trait_pred,
-                        prev.fresh_trait_pred,
-                        prev.obligation.param_env,
-                    )
+                    && self.match_fresh_trait_refs(stack.fresh_trait_pred, prev.fresh_trait_pred)
             })
         {
             debug!("evaluate_stack --> unbound argument, recursive --> giving up",);
-            return Ok(EvaluatedToUnknown);
+            return Ok(EvaluatedToAmbigStackDependent);
         }
 
         match self.candidate_from_obligation(stack) {
@@ -1464,20 +1449,17 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         if let ImplCandidate(def_id) = candidate {
             if let ty::ImplPolarity::Reservation = tcx.impl_polarity(def_id) {
                 if let Some(intercrate_ambiguity_clauses) = &mut self.intercrate_ambiguity_causes {
-                    let value = tcx
+                    let message = tcx
                         .get_attr(def_id, sym::rustc_reservation_impl)
                         .and_then(|a| a.value_str());
-                    if let Some(value) = value {
+                    if let Some(message) = message {
                         debug!(
                             "filter_reservation_impls: \
                                  reservation impl ambiguity on {:?}",
                             def_id
                         );
-                        intercrate_ambiguity_clauses.insert(
-                            IntercrateAmbiguityCause::ReservationImpl {
-                                message: value.to_string(),
-                            },
-                        );
+                        intercrate_ambiguity_clauses
+                            .insert(IntercrateAmbiguityCause::ReservationImpl { message });
                     }
                 }
                 return Ok(None);
@@ -1751,7 +1733,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         let mut nested_obligations = Vec::new();
         let infer_predicate = self.infcx.instantiate_binder_with_fresh_vars(
             obligation.cause.span,
-            LateBoundRegionConversionTime::HigherRankedType,
+            BoundRegionConversionTime::HigherRankedType,
             env_predicate,
         );
         let infer_projection = if potentially_unnormalized_candidates {
@@ -1841,7 +1823,7 @@ impl<'tcx> SelectionContext<'_, 'tcx> {
         // the param_env so that it can be given the lowest priority. See
         // #50825 for the motivation for this.
         let is_global =
-            |cand: &ty::PolyTraitPredicate<'tcx>| cand.is_global() && !cand.has_late_bound_vars();
+            |cand: &ty::PolyTraitPredicate<'tcx>| cand.is_global() && !cand.has_bound_vars();
 
         // (*) Prefer `BuiltinCandidate { has_nested: false }`, `PointeeCandidate`,
         // `DiscriminantKindCandidate`, `ConstDestructCandidate`
@@ -1879,7 +1861,9 @@ impl<'tcx> SelectionContext<'_, 'tcx> {
             }
 
             // Drop otherwise equivalent non-const fn pointer candidates
-            (FnPointerCandidate { .. }, FnPointerCandidate { is_const: false }) => DropVictim::Yes,
+            (FnPointerCandidate { .. }, FnPointerCandidate { fn_host_effect }) => {
+                DropVictim::drop_if(*fn_host_effect == self.tcx().consts.true_)
+            }
 
             (
                 ParamCandidate(ref other_cand),
@@ -1889,6 +1873,7 @@ impl<'tcx> SelectionContext<'_, 'tcx> {
                 | CoroutineCandidate
                 | FutureCandidate
                 | IteratorCandidate
+                | AsyncIteratorCandidate
                 | FnPointerCandidate { .. }
                 | BuiltinObjectCandidate
                 | BuiltinUnsizeCandidate
@@ -1896,7 +1881,7 @@ impl<'tcx> SelectionContext<'_, 'tcx> {
                 | BuiltinCandidate { .. }
                 | TraitAliasCandidate
                 | ObjectCandidate(_)
-                | ProjectionCandidate(..),
+                | ProjectionCandidate(_),
             ) => {
                 // We have a where clause so don't go around looking
                 // for impls. Arbitrarily give param candidates priority
@@ -1906,7 +1891,7 @@ impl<'tcx> SelectionContext<'_, 'tcx> {
                 // here (see issue #50825).
                 DropVictim::drop_if(!is_global(other_cand))
             }
-            (ObjectCandidate(_) | ProjectionCandidate(..), ParamCandidate(ref victim_cand)) => {
+            (ObjectCandidate(_) | ProjectionCandidate(_), ParamCandidate(ref victim_cand)) => {
                 // Prefer these to a global where-clause bound
                 // (see issue #50825).
                 if is_global(victim_cand) { DropVictim::Yes } else { DropVictim::No }
@@ -1918,6 +1903,7 @@ impl<'tcx> SelectionContext<'_, 'tcx> {
                 | CoroutineCandidate
                 | FutureCandidate
                 | IteratorCandidate
+                | AsyncIteratorCandidate
                 | FnPointerCandidate { .. }
                 | BuiltinObjectCandidate
                 | BuiltinUnsizeCandidate
@@ -1933,26 +1919,27 @@ impl<'tcx> SelectionContext<'_, 'tcx> {
                 )
             }
 
-            (ProjectionCandidate(i, _), ProjectionCandidate(j, _))
+            (ProjectionCandidate(i), ProjectionCandidate(j))
             | (ObjectCandidate(i), ObjectCandidate(j)) => {
                 // Arbitrarily pick the lower numbered candidate for backwards
                 // compatibility reasons. Don't let this affect inference.
                 DropVictim::drop_if(i < j && !has_non_region_infer)
             }
-            (ObjectCandidate(_), ProjectionCandidate(..))
-            | (ProjectionCandidate(..), ObjectCandidate(_)) => {
+            (ObjectCandidate(_), ProjectionCandidate(_))
+            | (ProjectionCandidate(_), ObjectCandidate(_)) => {
                 bug!("Have both object and projection candidate")
             }
 
             // Arbitrarily give projection and object candidates priority.
             (
-                ObjectCandidate(_) | ProjectionCandidate(..),
+                ObjectCandidate(_) | ProjectionCandidate(_),
                 ImplCandidate(..)
                 | AutoImplCandidate
                 | ClosureCandidate { .. }
                 | CoroutineCandidate
                 | FutureCandidate
                 | IteratorCandidate
+                | AsyncIteratorCandidate
                 | FnPointerCandidate { .. }
                 | BuiltinObjectCandidate
                 | BuiltinUnsizeCandidate
@@ -1968,13 +1955,14 @@ impl<'tcx> SelectionContext<'_, 'tcx> {
                 | CoroutineCandidate
                 | FutureCandidate
                 | IteratorCandidate
+                | AsyncIteratorCandidate
                 | FnPointerCandidate { .. }
                 | BuiltinObjectCandidate
                 | BuiltinUnsizeCandidate
                 | TraitUpcastingUnsizeCandidate(_)
                 | BuiltinCandidate { .. }
                 | TraitAliasCandidate,
-                ObjectCandidate(_) | ProjectionCandidate(..),
+                ObjectCandidate(_) | ProjectionCandidate(_),
             ) => DropVictim::No,
 
             (&ImplCandidate(other_def), &ImplCandidate(victim_def)) => {
@@ -2075,6 +2063,7 @@ impl<'tcx> SelectionContext<'_, 'tcx> {
                 | CoroutineCandidate
                 | FutureCandidate
                 | IteratorCandidate
+                | AsyncIteratorCandidate
                 | FnPointerCandidate { .. }
                 | BuiltinObjectCandidate
                 | BuiltinUnsizeCandidate
@@ -2086,6 +2075,7 @@ impl<'tcx> SelectionContext<'_, 'tcx> {
                 | CoroutineCandidate
                 | FutureCandidate
                 | IteratorCandidate
+                | AsyncIteratorCandidate
                 | FnPointerCandidate { .. }
                 | BuiltinObjectCandidate
                 | BuiltinUnsizeCandidate
@@ -2218,7 +2208,7 @@ impl<'tcx> SelectionContext<'_, 'tcx> {
                 }
             }
 
-            ty::CoroutineWitness(def_id, ref args) => {
+            ty::CoroutineWitness(def_id, args) => {
                 let hidden_types = bind_coroutine_hidden_types_above(
                     self.infcx,
                     def_id,
@@ -2307,23 +2297,23 @@ impl<'tcx> SelectionContext<'_, 'tcx> {
 
             ty::Array(element_ty, _) | ty::Slice(element_ty) => t.rebind(vec![element_ty]),
 
-            ty::Tuple(ref tys) => {
+            ty::Tuple(tys) => {
                 // (T1, ..., Tn) -- meets any bound that all of T1...Tn meet
                 t.rebind(tys.iter().collect())
             }
 
-            ty::Closure(_, ref args) => {
+            ty::Closure(_, args) => {
                 let ty = self.infcx.shallow_resolve(args.as_closure().tupled_upvars_ty());
                 t.rebind(vec![ty])
             }
 
-            ty::Coroutine(_, ref args, _) => {
+            ty::Coroutine(_, args, _) => {
                 let ty = self.infcx.shallow_resolve(args.as_coroutine().tupled_upvars_ty());
                 let witness = args.as_coroutine().witness();
                 t.rebind([ty].into_iter().chain(iter::once(witness)).collect())
             }
 
-            ty::CoroutineWitness(def_id, ref args) => {
+            ty::CoroutineWitness(def_id, args) => {
                 bind_coroutine_hidden_types_above(self.infcx, def_id, args, t.bound_vars())
             }
 
@@ -2436,7 +2426,7 @@ impl<'tcx> SelectionContext<'_, 'tcx> {
                 // the placeholder trait ref may fail due the Generalizer relation
                 // raising a CyclicalTy error due to a sub_root_var relation
                 // for a variable being generalized...
-                let guar = self.infcx.tcx.sess.delay_span_bug(
+                let guar = self.infcx.tcx.sess.span_delayed_bug(
                     obligation.cause.span,
                     format!(
                         "Impl {impl_def_id:?} was matchable against {obligation:?} but now is not"
@@ -2638,9 +2628,8 @@ impl<'tcx> SelectionContext<'_, 'tcx> {
         &self,
         previous: ty::PolyTraitPredicate<'tcx>,
         current: ty::PolyTraitPredicate<'tcx>,
-        param_env: ty::ParamEnv<'tcx>,
     ) -> bool {
-        let mut matcher = MatchAgainstFreshVars::new(self.tcx(), param_env);
+        let mut matcher = MatchAgainstFreshVars::new(self.tcx());
         matcher.relate(previous, current).is_ok()
     }
 
@@ -2668,6 +2657,7 @@ impl<'tcx> SelectionContext<'_, 'tcx> {
         &mut self,
         obligation: &PolyTraitObligation<'tcx>,
         args: GenericArgsRef<'tcx>,
+        fn_host_effect: ty::Const<'tcx>,
     ) -> ty::PolyTraitRef<'tcx> {
         let closure_sig = args.as_closure().sig();
 
@@ -2688,6 +2678,7 @@ impl<'tcx> SelectionContext<'_, 'tcx> {
             self_ty,
             closure_sig,
             util::TupleArgumentsFlag::No,
+            fn_host_effect,
         )
         .map_bound(|(trait_ref, _)| trait_ref)
     }
@@ -3103,7 +3094,7 @@ fn bind_coroutine_hidden_types_above<'tcx>(
                                 kind: ty::BrAnon,
                             };
                             counter += 1;
-                            ty::Region::new_late_bound(tcx, current_depth, br)
+                            ty::Region::new_bound(tcx, current_depth, br)
                         }
                         r => bug!("unexpected region: {r:?}"),
                     })
diff --git a/compiler/rustc_trait_selection/src/traits/specialize/mod.rs b/compiler/rustc_trait_selection/src/traits/specialize/mod.rs
index efab29743..71a88f5f0 100644
--- a/compiler/rustc_trait_selection/src/traits/specialize/mod.rs
+++ b/compiler/rustc_trait_selection/src/traits/specialize/mod.rs
@@ -37,7 +37,7 @@ pub struct OverlapError<'tcx> {
     pub with_impl: DefId,
     pub trait_ref: ty::TraitRef<'tcx>,
     pub self_ty: Option<Ty<'tcx>>,
-    pub intercrate_ambiguity_causes: FxIndexSet<IntercrateAmbiguityCause>,
+    pub intercrate_ambiguity_causes: FxIndexSet<IntercrateAmbiguityCause<'tcx>>,
     pub involves_placeholder: bool,
 }
 
@@ -107,7 +107,7 @@ pub fn translate_args_with_cause<'tcx>(
         param_env, source_impl, source_args, target_node
     );
     let source_trait_ref =
-        infcx.tcx.impl_trait_ref(source_impl).unwrap().instantiate(infcx.tcx, &source_args);
+        infcx.tcx.impl_trait_ref(source_impl).unwrap().instantiate(infcx.tcx, source_args);
 
     // translate the Self and Param parts of the substitution, since those
     // vary across impls
@@ -197,25 +197,22 @@ fn fulfill_implication<'tcx>(
         param_env, source_trait_ref, target_impl
     );
 
-    let source_trait_ref = match traits::fully_normalize(
-        &infcx,
-        ObligationCause::dummy(),
-        param_env,
-        source_trait_ref,
-    ) {
-        Ok(source_trait_ref) => source_trait_ref,
-        Err(_errors) => {
-            infcx.tcx.sess.delay_span_bug(
-                infcx.tcx.def_span(source_impl),
-                format!("failed to fully normalize {source_trait_ref}"),
-            );
-            source_trait_ref
-        }
-    };
+    let source_trait_ref =
+        match traits::fully_normalize(infcx, ObligationCause::dummy(), param_env, source_trait_ref)
+        {
+            Ok(source_trait_ref) => source_trait_ref,
+            Err(_errors) => {
+                infcx.tcx.sess.span_delayed_bug(
+                    infcx.tcx.def_span(source_impl),
+                    format!("failed to fully normalize {source_trait_ref}"),
+                );
+                source_trait_ref
+            }
+        };
 
     let source_trait = ImplSubject::Trait(source_trait_ref);
 
-    let selcx = &mut SelectionContext::new(&infcx);
+    let selcx = &mut SelectionContext::new(infcx);
     let target_args = infcx.fresh_args_for_item(DUMMY_SP, target_impl);
     let (target_trait, obligations) =
         util::impl_subject_and_oblig(selcx, param_env, target_impl, target_args, error_cause);
@@ -415,7 +412,7 @@ fn report_conflicting_impls<'tcx>(
     let msg = DelayDm(|| {
         format!(
             "conflicting implementations of trait `{}`{}{}",
-            overlap.trait_ref.print_only_trait_path(),
+            overlap.trait_ref.print_trait_sugared(),
             overlap.self_ty.map_or_else(String::new, |ty| format!(" for type `{ty}`")),
             match used_to_be_allowed {
                 Some(FutureCompatOverlapErrorKind::Issue33140) => ": (E0119)",
@@ -434,7 +431,10 @@ fn report_conflicting_impls<'tcx>(
                 decorate(tcx, &overlap, impl_span, &mut err);
                 Some(err.emit())
             } else {
-                Some(tcx.sess.delay_span_bug(impl_span, "impl should have failed the orphan check"))
+                Some(
+                    tcx.sess
+                        .span_delayed_bug(impl_span, "impl should have failed the orphan check"),
+                )
             };
             sg.has_errored = reported;
         }
@@ -445,12 +445,11 @@ fn report_conflicting_impls<'tcx>(
             };
             tcx.struct_span_lint_hir(
                 lint,
-                tcx.hir().local_def_id_to_hir_id(impl_def_id),
+                tcx.local_def_id_to_hir_id(impl_def_id),
                 impl_span,
                 msg,
                 |err| {
                     decorate(tcx, &overlap, impl_span, err);
-                    err
                 },
             );
         }
diff --git a/compiler/rustc_trait_selection/src/traits/structural_match.rs b/compiler/rustc_trait_selection/src/traits/structural_match.rs
index 5960415a8..f8e47cacc 100644
--- a/compiler/rustc_trait_selection/src/traits/structural_match.rs
+++ b/compiler/rustc_trait_selection/src/traits/structural_match.rs
@@ -135,7 +135,7 @@ impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for Search<'tcx> {
                 bug!("unexpected type during structural-match checking: {:?}", ty);
             }
             ty::Error(_) => {
-                self.tcx.sess.delay_span_bug(self.span, "ty::Error in structural-match check");
+                self.tcx.sess.span_delayed_bug(self.span, "ty::Error in structural-match check");
                 // We still want to check other types after encountering an error,
                 // as this may still emit relevant errors.
                 return ControlFlow::Continue(());
diff --git a/compiler/rustc_trait_selection/src/traits/structural_normalize.rs b/compiler/rustc_trait_selection/src/traits/structural_normalize.rs
index 9d6be7689..e0f9fdc38 100644
--- a/compiler/rustc_trait_selection/src/traits/structural_normalize.rs
+++ b/compiler/rustc_trait_selection/src/traits/structural_normalize.rs
@@ -3,7 +3,7 @@ use rustc_infer::infer::type_variable::{TypeVariableOrigin, TypeVariableOriginKi
 use rustc_infer::traits::{FulfillmentError, TraitEngine};
 use rustc_middle::ty::{self, Ty};
 
-use crate::traits::{query::evaluate_obligation::InferCtxtExt, NormalizeExt, Obligation};
+use crate::traits::{NormalizeExt, Obligation};
 
 pub trait StructurallyNormalizeExt<'tcx> {
     fn structurally_normalize(
@@ -16,46 +16,43 @@ pub trait StructurallyNormalizeExt<'tcx> {
 impl<'tcx> StructurallyNormalizeExt<'tcx> for At<'_, 'tcx> {
     fn structurally_normalize(
         &self,
-        mut ty: Ty<'tcx>,
+        ty: Ty<'tcx>,
         fulfill_cx: &mut dyn TraitEngine<'tcx>,
     ) -> Result<Ty<'tcx>, Vec<FulfillmentError<'tcx>>> {
         assert!(!ty.is_ty_var(), "should have resolved vars before calling");
 
         if self.infcx.next_trait_solver() {
-            // FIXME(-Ztrait-solver=next): correctly handle
-            // overflow here.
-            for _ in 0..256 {
-                let ty::Alias(ty::Projection | ty::Inherent | ty::Weak, projection_ty) = *ty.kind()
-                else {
-                    break;
-                };
-
-                let new_infer_ty = self.infcx.next_ty_var(TypeVariableOrigin {
-                    kind: TypeVariableOriginKind::NormalizeProjectionType,
-                    span: self.cause.span,
-                });
-                let obligation = Obligation::new(
-                    self.infcx.tcx,
-                    self.cause.clone(),
-                    self.param_env,
-                    ty::Binder::dummy(ty::ProjectionPredicate {
-                        projection_ty,
-                        term: new_infer_ty.into(),
-                    }),
-                );
-                if self.infcx.predicate_may_hold(&obligation) {
-                    fulfill_cx.register_predicate_obligation(self.infcx, obligation);
-                    let errors = fulfill_cx.select_where_possible(self.infcx);
-                    if !errors.is_empty() {
-                        return Err(errors);
-                    }
-                    ty = self.infcx.resolve_vars_if_possible(new_infer_ty);
-                } else {
-                    break;
-                }
+            // FIXME(-Znext-solver): Should we resolve opaques here?
+            let ty::Alias(ty::Projection | ty::Inherent | ty::Weak, _) = *ty.kind() else {
+                return Ok(ty);
+            };
+
+            let new_infer_ty = self.infcx.next_ty_var(TypeVariableOrigin {
+                kind: TypeVariableOriginKind::NormalizeProjectionType,
+                span: self.cause.span,
+            });
+
+            // We simply emit an `alias-eq` goal here, since that will take care of
+            // normalizing the LHS of the projection until it is a rigid projection
+            // (or a not-yet-defined opaque in scope).
+            let obligation = Obligation::new(
+                self.infcx.tcx,
+                self.cause.clone(),
+                self.param_env,
+                ty::PredicateKind::AliasRelate(
+                    ty.into(),
+                    new_infer_ty.into(),
+                    ty::AliasRelationDirection::Equate,
+                ),
+            );
+
+            fulfill_cx.register_predicate_obligation(self.infcx, obligation);
+            let errors = fulfill_cx.select_where_possible(self.infcx);
+            if !errors.is_empty() {
+                return Err(errors);
             }
 
-            Ok(ty)
+            Ok(self.infcx.resolve_vars_if_possible(new_infer_ty))
         } else {
             Ok(self.normalize(ty).into_value_registering_obligations(self.infcx, fulfill_cx))
         }
diff --git a/compiler/rustc_trait_selection/src/traits/util.rs b/compiler/rustc_trait_selection/src/traits/util.rs
index bbde0c827..19eae93df 100644
--- a/compiler/rustc_trait_selection/src/traits/util.rs
+++ b/compiler/rustc_trait_selection/src/traits/util.rs
@@ -220,9 +220,8 @@ pub fn impl_subject_and_oblig<'a, 'tcx>(
         selcx.infcx.at(&ObligationCause::dummy(), param_env).normalize(predicates);
     let impl_obligations = super::predicates_for_generics(cause, param_env, predicates);
 
-    let impl_obligations = impl_obligations
-        .chain(normalization_obligations1.into_iter())
-        .chain(normalization_obligations2.into_iter());
+    let impl_obligations =
+        impl_obligations.chain(normalization_obligations1).chain(normalization_obligations2);
 
     (subject, impl_obligations)
 }
@@ -265,13 +264,26 @@ pub fn closure_trait_ref_and_return_type<'tcx>(
     self_ty: Ty<'tcx>,
     sig: ty::PolyFnSig<'tcx>,
     tuple_arguments: TupleArgumentsFlag,
+    fn_host_effect: ty::Const<'tcx>,
 ) -> ty::Binder<'tcx, (ty::TraitRef<'tcx>, Ty<'tcx>)> {
     assert!(!self_ty.has_escaping_bound_vars());
     let arguments_tuple = match tuple_arguments {
         TupleArgumentsFlag::No => sig.skip_binder().inputs()[0],
         TupleArgumentsFlag::Yes => Ty::new_tup(tcx, sig.skip_binder().inputs()),
     };
-    let trait_ref = ty::TraitRef::new(tcx, fn_trait_def_id, [self_ty, arguments_tuple]);
+    let trait_ref = if tcx.generics_of(fn_trait_def_id).host_effect_index.is_some() {
+        ty::TraitRef::new(
+            tcx,
+            fn_trait_def_id,
+            [
+                ty::GenericArg::from(self_ty),
+                ty::GenericArg::from(arguments_tuple),
+                ty::GenericArg::from(fn_host_effect),
+            ],
+        )
+    } else {
+        ty::TraitRef::new(tcx, fn_trait_def_id, [self_ty, arguments_tuple])
+    };
     sig.map_bound(|sig| (trait_ref, sig.output()))
 }
 
@@ -279,33 +291,44 @@ pub fn coroutine_trait_ref_and_outputs<'tcx>(
     tcx: TyCtxt<'tcx>,
     fn_trait_def_id: DefId,
     self_ty: Ty<'tcx>,
-    sig: ty::PolyGenSig<'tcx>,
-) -> ty::Binder<'tcx, (ty::TraitRef<'tcx>, Ty<'tcx>, Ty<'tcx>)> {
+    sig: ty::GenSig<'tcx>,
+) -> (ty::TraitRef<'tcx>, Ty<'tcx>, Ty<'tcx>) {
     assert!(!self_ty.has_escaping_bound_vars());
-    let trait_ref = ty::TraitRef::new(tcx, fn_trait_def_id, [self_ty, sig.skip_binder().resume_ty]);
-    sig.map_bound(|sig| (trait_ref, sig.yield_ty, sig.return_ty))
+    let trait_ref = ty::TraitRef::new(tcx, fn_trait_def_id, [self_ty, sig.resume_ty]);
+    (trait_ref, sig.yield_ty, sig.return_ty)
 }
 
 pub fn future_trait_ref_and_outputs<'tcx>(
     tcx: TyCtxt<'tcx>,
     fn_trait_def_id: DefId,
     self_ty: Ty<'tcx>,
-    sig: ty::PolyGenSig<'tcx>,
-) -> ty::Binder<'tcx, (ty::TraitRef<'tcx>, Ty<'tcx>)> {
+    sig: ty::GenSig<'tcx>,
+) -> (ty::TraitRef<'tcx>, Ty<'tcx>) {
     assert!(!self_ty.has_escaping_bound_vars());
     let trait_ref = ty::TraitRef::new(tcx, fn_trait_def_id, [self_ty]);
-    sig.map_bound(|sig| (trait_ref, sig.return_ty))
+    (trait_ref, sig.return_ty)
 }
 
 pub fn iterator_trait_ref_and_outputs<'tcx>(
     tcx: TyCtxt<'tcx>,
     iterator_def_id: DefId,
     self_ty: Ty<'tcx>,
-    sig: ty::PolyGenSig<'tcx>,
-) -> ty::Binder<'tcx, (ty::TraitRef<'tcx>, Ty<'tcx>)> {
+    sig: ty::GenSig<'tcx>,
+) -> (ty::TraitRef<'tcx>, Ty<'tcx>) {
     assert!(!self_ty.has_escaping_bound_vars());
     let trait_ref = ty::TraitRef::new(tcx, iterator_def_id, [self_ty]);
-    sig.map_bound(|sig| (trait_ref, sig.yield_ty))
+    (trait_ref, sig.yield_ty)
+}
+
+pub fn async_iterator_trait_ref_and_outputs<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    async_iterator_def_id: DefId,
+    self_ty: Ty<'tcx>,
+    sig: ty::GenSig<'tcx>,
+) -> (ty::TraitRef<'tcx>, Ty<'tcx>) {
+    assert!(!self_ty.has_escaping_bound_vars());
+    let trait_ref = ty::TraitRef::new(tcx, async_iterator_def_id, [self_ty]);
+    (trait_ref, sig.yield_ty)
 }
 
 pub fn impl_item_is_final(tcx: TyCtxt<'_>, assoc_item: &ty::AssocItem) -> bool {
diff --git a/compiler/rustc_trait_selection/src/traits/wf.rs b/compiler/rustc_trait_selection/src/traits/wf.rs
index fe5b625e4..0f8d9c6bf 100644
--- a/compiler/rustc_trait_selection/src/traits/wf.rs
+++ b/compiler/rustc_trait_selection/src/traits/wf.rs
@@ -624,7 +624,7 @@ impl<'a, 'tcx> WfPredicates<'a, 'tcx> {
                     // Note that we handle the len is implicitly checked while walking `arg`.
                 }
 
-                ty::Tuple(ref tys) => {
+                ty::Tuple(tys) => {
                     if let Some((_last, rest)) = tys.split_last() {
                         for &elem in rest {
                             self.require_sized(elem, traits::TupleElem);
@@ -761,18 +761,15 @@ impl<'a, 'tcx> WfPredicates<'a, 'tcx> {
                     let defer_to_coercion = self.tcx().features().object_safe_for_dispatch;
 
                     if !defer_to_coercion {
-                        let cause = self.cause(traits::WellFormed(None));
-                        let component_traits = data.auto_traits().chain(data.principal_def_id());
-                        let tcx = self.tcx();
-                        self.out.extend(component_traits.map(|did| {
-                            traits::Obligation::with_depth(
-                                tcx,
-                                cause.clone(),
+                        if let Some(principal) = data.principal_def_id() {
+                            self.out.push(traits::Obligation::with_depth(
+                                self.tcx(),
+                                self.cause(traits::WellFormed(None)),
                                 depth,
                                 param_env,
-                                ty::Binder::dummy(ty::PredicateKind::ObjectSafe(did)),
-                            )
-                        }));
+                                ty::Binder::dummy(ty::PredicateKind::ObjectSafe(principal)),
+                            ));
+                        }
                     }
                 }
 
@@ -913,20 +910,15 @@ pub fn object_region_bounds<'tcx>(
     tcx: TyCtxt<'tcx>,
     existential_predicates: &'tcx ty::List<ty::PolyExistentialPredicate<'tcx>>,
 ) -> Vec<ty::Region<'tcx>> {
-    // Since we don't actually *know* the self type for an object,
-    // this "open(err)" serves as a kind of dummy standin -- basically
-    // a placeholder type.
-    let open_ty = Ty::new_fresh(tcx, 0);
-
     let predicates = existential_predicates.iter().filter_map(|predicate| {
         if let ty::ExistentialPredicate::Projection(_) = predicate.skip_binder() {
             None
         } else {
-            Some(predicate.with_self_ty(tcx, open_ty))
+            Some(predicate.with_self_ty(tcx, tcx.types.trait_object_dummy_self))
         }
     });
 
-    required_region_bounds(tcx, open_ty, predicates)
+    required_region_bounds(tcx, tcx.types.trait_object_dummy_self, predicates)
 }
 
 /// Given a set of predicates that apply to an object type, returns
diff --git a/compiler/rustc_traits/src/evaluate_obligation.rs b/compiler/rustc_traits/src/evaluate_obligation.rs
index 9fd550495..ce22da23f 100644
--- a/compiler/rustc_traits/src/evaluate_obligation.rs
+++ b/compiler/rustc_traits/src/evaluate_obligation.rs
@@ -22,7 +22,7 @@ fn evaluate_obligation<'tcx>(
     debug!("evaluate_obligation: goal={:#?}", goal);
     let ParamEnvAnd { param_env, value: predicate } = goal;
 
-    let mut selcx = SelectionContext::with_query_mode(&infcx, TraitQueryMode::Canonical);
+    let mut selcx = SelectionContext::with_query_mode(infcx, TraitQueryMode::Canonical);
     let obligation = Obligation::new(tcx, ObligationCause::dummy(), param_env, predicate);
 
     selcx.evaluate_root_obligation(&obligation)
diff --git a/compiler/rustc_traits/src/normalize_erasing_regions.rs b/compiler/rustc_traits/src/normalize_erasing_regions.rs
index 2563e3ed1..0576fe010 100644
--- a/compiler/rustc_traits/src/normalize_erasing_regions.rs
+++ b/compiler/rustc_traits/src/normalize_erasing_regions.rs
@@ -4,18 +4,12 @@ use rustc_middle::traits::query::NoSolution;
 use rustc_middle::ty::{self, ParamEnvAnd, TyCtxt, TypeFoldable, TypeVisitableExt};
 use rustc_trait_selection::traits::query::normalize::QueryNormalizeExt;
 use rustc_trait_selection::traits::{Normalized, ObligationCause};
-use std::sync::atomic::Ordering;
 
 pub(crate) fn provide(p: &mut Providers) {
     *p = Providers {
         try_normalize_generic_arg_after_erasing_regions: |tcx, goal| {
             debug!("try_normalize_generic_arg_after_erasing_regions(goal={:#?}", goal);
 
-            tcx.sess
-                .perf_stats
-                .normalize_generic_arg_after_erasing_regions
-                .fetch_add(1, Ordering::Relaxed);
-
             try_normalize_after_erasing_regions(tcx, goal)
         },
         ..*p
@@ -61,10 +55,10 @@ fn not_outlives_predicate(p: ty::Predicate<'_>) -> bool {
         ty::PredicateKind::Clause(ty::ClauseKind::Trait(..))
         | ty::PredicateKind::Clause(ty::ClauseKind::Projection(..))
         | ty::PredicateKind::Clause(ty::ClauseKind::ConstArgHasType(..))
+        | ty::PredicateKind::NormalizesTo(..)
         | ty::PredicateKind::AliasRelate(..)
         | ty::PredicateKind::Clause(ty::ClauseKind::WellFormed(..))
         | ty::PredicateKind::ObjectSafe(..)
-        | ty::PredicateKind::ClosureKind(..)
         | ty::PredicateKind::Subtype(..)
         | ty::PredicateKind::Coerce(..)
         | ty::PredicateKind::Clause(ty::ClauseKind::ConstEvaluatable(..))
diff --git a/compiler/rustc_traits/src/normalize_projection_ty.rs b/compiler/rustc_traits/src/normalize_projection_ty.rs
index 01bb1ca70..b8c71bc96 100644
--- a/compiler/rustc_traits/src/normalize_projection_ty.rs
+++ b/compiler/rustc_traits/src/normalize_projection_ty.rs
@@ -10,7 +10,6 @@ use rustc_trait_selection::traits::query::{
 use rustc_trait_selection::traits::{
     self, FulfillmentErrorCode, ObligationCause, SelectionContext,
 };
-use std::sync::atomic::Ordering;
 
 pub(crate) fn provide(p: &mut Providers) {
     *p = Providers {
@@ -27,7 +26,6 @@ fn normalize_projection_ty<'tcx>(
 ) -> Result<&'tcx Canonical<'tcx, QueryResponse<'tcx, NormalizationResult<'tcx>>>, NoSolution> {
     debug!("normalize_provider(goal={:#?})", goal);
 
-    tcx.sess.perf_stats.normalize_projection_ty.fetch_add(1, Ordering::Relaxed);
     tcx.infer_ctxt().enter_canonical_trait_query(
         &goal,
         |ocx, ParamEnvAnd { param_env, value: goal }| {
@@ -78,7 +76,6 @@ fn normalize_weak_ty<'tcx>(
 ) -> Result<&'tcx Canonical<'tcx, QueryResponse<'tcx, NormalizationResult<'tcx>>>, NoSolution> {
     debug!("normalize_provider(goal={:#?})", goal);
 
-    tcx.sess.perf_stats.normalize_projection_ty.fetch_add(1, Ordering::Relaxed);
     tcx.infer_ctxt().enter_canonical_trait_query(
         &goal,
         |ocx, ParamEnvAnd { param_env, value: goal }| {
diff --git a/compiler/rustc_transmute/Cargo.toml b/compiler/rustc_transmute/Cargo.toml
index 07420985a..066016231 100644
--- a/compiler/rustc_transmute/Cargo.toml
+++ b/compiler/rustc_transmute/Cargo.toml
@@ -27,5 +27,5 @@ rustc = [
 
 [dev-dependencies]
 # tidy-alphabetical-start
-itertools = "0.10.1"
+itertools = "0.11"
 # tidy-alphabetical-end
diff --git a/compiler/rustc_ty_utils/Cargo.toml b/compiler/rustc_ty_utils/Cargo.toml
index 7e00f1e0c..f8eb82da8 100644
--- a/compiler/rustc_ty_utils/Cargo.toml
+++ b/compiler/rustc_ty_utils/Cargo.toml
@@ -5,7 +5,7 @@ edition = "2021"
 
 [dependencies]
 # tidy-alphabetical-start
-itertools = "0.10.1"
+itertools = "0.11"
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
 rustc_fluent_macro = { path = "../rustc_fluent_macro" }
diff --git a/compiler/rustc_ty_utils/src/abi.rs b/compiler/rustc_ty_utils/src/abi.rs
index fcf6626bb..a5f11ca23 100644
--- a/compiler/rustc_ty_utils/src/abi.rs
+++ b/compiler/rustc_ty_utils/src/abi.rs
@@ -16,7 +16,7 @@ use rustc_target::spec::abi::Abi as SpecAbi;
 
 use std::iter;
 
-pub fn provide(providers: &mut Providers) {
+pub(crate) fn provide(providers: &mut Providers) {
     *providers = Providers { fn_abi_of_fn_ptr, fn_abi_of_instance, ..*providers };
 }
 
@@ -82,7 +82,7 @@ fn fn_sig_for_fn_abi<'tcx>(
                 var: ty::BoundVar::from_usize(bound_vars.len() - 1),
                 kind: ty::BoundRegionKind::BrEnv,
             };
-            let env_region = ty::Region::new_late_bound(tcx, ty::INNERMOST, br);
+            let env_region = ty::Region::new_bound(tcx, ty::INNERMOST, br);
             let env_ty = tcx.closure_env_ty(def_id, args, env_region).unwrap();
 
             let sig = sig.skip_binder();
@@ -98,70 +98,130 @@ fn fn_sig_for_fn_abi<'tcx>(
             )
         }
         ty::Coroutine(did, args, _) => {
-            let sig = args.as_coroutine().poly_sig();
+            let coroutine_kind = tcx.coroutine_kind(did).unwrap();
+            let sig = args.as_coroutine().sig();
 
-            let bound_vars = tcx.mk_bound_variable_kinds_from_iter(
-                sig.bound_vars().iter().chain(iter::once(ty::BoundVariableKind::Region(ty::BrEnv))),
-            );
+            let bound_vars = tcx.mk_bound_variable_kinds_from_iter(iter::once(
+                ty::BoundVariableKind::Region(ty::BrEnv),
+            ));
             let br = ty::BoundRegion {
                 var: ty::BoundVar::from_usize(bound_vars.len() - 1),
                 kind: ty::BoundRegionKind::BrEnv,
             };
-            let env_ty =
-                Ty::new_mut_ref(tcx, ty::Region::new_late_bound(tcx, ty::INNERMOST, br), ty);
+            let env_ty = Ty::new_mut_ref(tcx, ty::Region::new_bound(tcx, ty::INNERMOST, br), ty);
 
             let pin_did = tcx.require_lang_item(LangItem::Pin, None);
             let pin_adt_ref = tcx.adt_def(pin_did);
             let pin_args = tcx.mk_args(&[env_ty.into()]);
-            let env_ty = Ty::new_adt(tcx, pin_adt_ref, pin_args);
+            let env_ty = match coroutine_kind {
+                hir::CoroutineKind::Gen(_) => {
+                    // Iterator::next doesn't accept a pinned argument,
+                    // unlike for all other coroutine kinds.
+                    env_ty
+                }
+                hir::CoroutineKind::Async(_)
+                | hir::CoroutineKind::AsyncGen(_)
+                | hir::CoroutineKind::Coroutine => Ty::new_adt(tcx, pin_adt_ref, pin_args),
+            };
 
-            let sig = sig.skip_binder();
             // The `FnSig` and the `ret_ty` here is for a coroutines main
             // `Coroutine::resume(...) -> CoroutineState` function in case we
-            // have an ordinary coroutine, or the `Future::poll(...) -> Poll`
-            // function in case this is a special coroutine backing an async construct.
-            let (resume_ty, ret_ty) = if tcx.coroutine_is_async(did) {
-                // The signature should be `Future::poll(_, &mut Context<'_>) -> Poll<Output>`
-                let poll_did = tcx.require_lang_item(LangItem::Poll, None);
-                let poll_adt_ref = tcx.adt_def(poll_did);
-                let poll_args = tcx.mk_args(&[sig.return_ty.into()]);
-                let ret_ty = Ty::new_adt(tcx, poll_adt_ref, poll_args);
-
-                // We have to replace the `ResumeTy` that is used for type and borrow checking
-                // with `&mut Context<'_>` which is used in codegen.
-                #[cfg(debug_assertions)]
-                {
-                    if let ty::Adt(resume_ty_adt, _) = sig.resume_ty.kind() {
-                        let expected_adt =
-                            tcx.adt_def(tcx.require_lang_item(LangItem::ResumeTy, None));
-                        assert_eq!(*resume_ty_adt, expected_adt);
-                    } else {
-                        panic!("expected `ResumeTy`, found `{:?}`", sig.resume_ty);
-                    };
+            // have an ordinary coroutine, the `Future::poll(...) -> Poll`
+            // function in case this is a special coroutine backing an async construct
+            // or the `Iterator::next(...) -> Option` function in case this is a
+            // special coroutine backing a gen construct.
+            let (resume_ty, ret_ty) = match coroutine_kind {
+                hir::CoroutineKind::Async(_) => {
+                    // The signature should be `Future::poll(_, &mut Context<'_>) -> Poll<Output>`
+                    assert_eq!(sig.yield_ty, tcx.types.unit);
+
+                    let poll_did = tcx.require_lang_item(LangItem::Poll, None);
+                    let poll_adt_ref = tcx.adt_def(poll_did);
+                    let poll_args = tcx.mk_args(&[sig.return_ty.into()]);
+                    let ret_ty = Ty::new_adt(tcx, poll_adt_ref, poll_args);
+
+                    // We have to replace the `ResumeTy` that is used for type and borrow checking
+                    // with `&mut Context<'_>` which is used in codegen.
+                    #[cfg(debug_assertions)]
+                    {
+                        if let ty::Adt(resume_ty_adt, _) = sig.resume_ty.kind() {
+                            let expected_adt =
+                                tcx.adt_def(tcx.require_lang_item(LangItem::ResumeTy, None));
+                            assert_eq!(*resume_ty_adt, expected_adt);
+                        } else {
+                            panic!("expected `ResumeTy`, found `{:?}`", sig.resume_ty);
+                        };
+                    }
+                    let context_mut_ref = Ty::new_task_context(tcx);
+
+                    (Some(context_mut_ref), ret_ty)
                 }
-                let context_mut_ref = Ty::new_task_context(tcx);
+                hir::CoroutineKind::Gen(_) => {
+                    // The signature should be `Iterator::next(_) -> Option<Yield>`
+                    let option_did = tcx.require_lang_item(LangItem::Option, None);
+                    let option_adt_ref = tcx.adt_def(option_did);
+                    let option_args = tcx.mk_args(&[sig.yield_ty.into()]);
+                    let ret_ty = Ty::new_adt(tcx, option_adt_ref, option_args);
 
-                (context_mut_ref, ret_ty)
-            } else {
-                // The signature should be `Coroutine::resume(_, Resume) -> CoroutineState<Yield, Return>`
-                let state_did = tcx.require_lang_item(LangItem::CoroutineState, None);
-                let state_adt_ref = tcx.adt_def(state_did);
-                let state_args = tcx.mk_args(&[sig.yield_ty.into(), sig.return_ty.into()]);
-                let ret_ty = Ty::new_adt(tcx, state_adt_ref, state_args);
+                    assert_eq!(sig.return_ty, tcx.types.unit);
+                    assert_eq!(sig.resume_ty, tcx.types.unit);
+
+                    (None, ret_ty)
+                }
+                hir::CoroutineKind::AsyncGen(_) => {
+                    // The signature should be
+                    // `AsyncIterator::poll_next(_, &mut Context<'_>) -> Poll<Option<Output>>`
+                    assert_eq!(sig.return_ty, tcx.types.unit);
+
+                    // Yield type is already `Poll<Option<yield_ty>>`
+                    let ret_ty = sig.yield_ty;
+
+                    // We have to replace the `ResumeTy` that is used for type and borrow checking
+                    // with `&mut Context<'_>` which is used in codegen.
+                    #[cfg(debug_assertions)]
+                    {
+                        if let ty::Adt(resume_ty_adt, _) = sig.resume_ty.kind() {
+                            let expected_adt =
+                                tcx.adt_def(tcx.require_lang_item(LangItem::ResumeTy, None));
+                            assert_eq!(*resume_ty_adt, expected_adt);
+                        } else {
+                            panic!("expected `ResumeTy`, found `{:?}`", sig.resume_ty);
+                        };
+                    }
+                    let context_mut_ref = Ty::new_task_context(tcx);
 
-                (sig.resume_ty, ret_ty)
+                    (Some(context_mut_ref), ret_ty)
+                }
+                hir::CoroutineKind::Coroutine => {
+                    // The signature should be `Coroutine::resume(_, Resume) -> CoroutineState<Yield, Return>`
+                    let state_did = tcx.require_lang_item(LangItem::CoroutineState, None);
+                    let state_adt_ref = tcx.adt_def(state_did);
+                    let state_args = tcx.mk_args(&[sig.yield_ty.into(), sig.return_ty.into()]);
+                    let ret_ty = Ty::new_adt(tcx, state_adt_ref, state_args);
+
+                    (Some(sig.resume_ty), ret_ty)
+                }
             };
 
-            ty::Binder::bind_with_vars(
+            let fn_sig = if let Some(resume_ty) = resume_ty {
                 tcx.mk_fn_sig(
                     [env_ty, resume_ty],
                     ret_ty,
                     false,
                     hir::Unsafety::Normal,
                     rustc_target::spec::abi::Abi::Rust,
-                ),
-                bound_vars,
-            )
+                )
+            } else {
+                // `Iterator::next` doesn't have a `resume` argument.
+                tcx.mk_fn_sig(
+                    [env_ty],
+                    ret_ty,
+                    false,
+                    hir::Unsafety::Normal,
+                    rustc_target::spec::abi::Abi::Rust,
+                )
+            };
+            ty::Binder::bind_with_vars(fn_sig, bound_vars)
         }
         _ => bug!("unexpected type {:?} in Instance::fn_sig", ty),
     }
@@ -327,6 +387,93 @@ fn adjust_for_rust_scalar<'tcx>(
     }
 }
 
+/// Ensure that the ABI makes basic sense.
+fn fn_abi_sanity_check<'tcx>(
+    cx: &LayoutCx<'tcx, TyCtxt<'tcx>>,
+    fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
+    spec_abi: SpecAbi,
+) {
+    fn fn_arg_sanity_check<'tcx>(
+        cx: &LayoutCx<'tcx, TyCtxt<'tcx>>,
+        fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
+        spec_abi: SpecAbi,
+        arg: &ArgAbi<'tcx, Ty<'tcx>>,
+    ) {
+        match &arg.mode {
+            PassMode::Ignore => {}
+            PassMode::Direct(_) => {
+                // Here the Rust type is used to determine the actual ABI, so we have to be very
+                // careful. Scalar/ScalarPair is fine, since backends will generally use
+                // `layout.abi` and ignore everything else. We should just reject `Aggregate`
+                // entirely here, but some targets need to be fixed first.
+                if matches!(arg.layout.abi, Abi::Aggregate { .. }) {
+                    // For an unsized type we'd only pass the sized prefix, so there is no universe
+                    // in which we ever want to allow this.
+                    assert!(
+                        arg.layout.is_sized(),
+                        "`PassMode::Direct` for unsized type in ABI: {:#?}",
+                        fn_abi
+                    );
+                    // This really shouldn't happen even for sized aggregates, since
+                    // `immediate_llvm_type` will use `layout.fields` to turn this Rust type into an
+                    // LLVM type. This means all sorts of Rust type details leak into the ABI.
+                    // However wasm sadly *does* currently use this mode so we have to allow it --
+                    // but we absolutely shouldn't let any more targets do that.
+                    // (Also see <https://github.com/rust-lang/rust/issues/115666>.)
+                    //
+                    // The unstable abi `PtxKernel` also uses Direct for now.
+                    // It needs to switch to something else before stabilization can happen.
+                    // (See issue: https://github.com/rust-lang/rust/issues/117271)
+                    assert!(
+                        matches!(&*cx.tcx.sess.target.arch, "wasm32" | "wasm64")
+                            || matches!(spec_abi, SpecAbi::PtxKernel | SpecAbi::Unadjusted),
+                        r#"`PassMode::Direct` for aggregates only allowed for "unadjusted" and "ptx-kernel" functions and on wasm\nProblematic type: {:#?}"#,
+                        arg.layout,
+                    );
+                }
+            }
+            PassMode::Pair(_, _) => {
+                // Similar to `Direct`, we need to make sure that backends use `layout.abi` and
+                // ignore the rest of the layout.
+                assert!(
+                    matches!(arg.layout.abi, Abi::ScalarPair(..)),
+                    "PassMode::Pair for type {}",
+                    arg.layout.ty
+                );
+            }
+            PassMode::Cast { .. } => {
+                // `Cast` means "transmute to `CastType`"; that only makes sense for sized types.
+                assert!(arg.layout.is_sized());
+            }
+            PassMode::Indirect { meta_attrs: None, .. } => {
+                // No metadata, must be sized.
+                // Conceptually, unsized arguments must be copied around, which requires dynamically
+                // determining their size, which we cannot do without metadata. Consult
+                // t-opsem before removing this check.
+                assert!(arg.layout.is_sized());
+            }
+            PassMode::Indirect { meta_attrs: Some(_), on_stack, .. } => {
+                // With metadata. Must be unsized and not on the stack.
+                assert!(arg.layout.is_unsized() && !on_stack);
+                // Also, must not be `extern` type.
+                let tail = cx.tcx.struct_tail_with_normalize(arg.layout.ty, |ty| ty, || {});
+                if matches!(tail.kind(), ty::Foreign(..)) {
+                    // These types do not have metadata, so having `meta_attrs` is bogus.
+                    // Conceptually, unsized arguments must be copied around, which requires dynamically
+                    // determining their size. Therefore, we cannot allow `extern` types here. Consult
+                    // t-opsem before removing this check.
+                    panic!("unsized arguments must not be `extern` types");
+                }
+            }
+        }
+    }
+
+    for arg in fn_abi.args.iter() {
+        fn_arg_sanity_check(cx, fn_abi, spec_abi, arg);
+    }
+    fn_arg_sanity_check(cx, fn_abi, spec_abi, &fn_abi.ret);
+}
+
 // FIXME(eddyb) perhaps group the signature/type-containing (or all of them?)
 // arguments of this method, into a separate `struct`.
 #[tracing::instrument(level = "debug", skip(cx, caller_location, fn_def_id, force_thin_self_ptr))]
@@ -453,6 +600,7 @@ fn fn_abi_new_uncached<'tcx>(
     };
     fn_abi_adjust_for_abi(cx, &mut fn_abi, sig.abi, fn_def_id)?;
     debug!("fn_abi_new_uncached = {:?}", fn_abi);
+    fn_abi_sanity_check(cx, &fn_abi, sig.abi);
     Ok(cx.tcx.arena.alloc(fn_abi))
 }
 
@@ -464,6 +612,24 @@ fn fn_abi_adjust_for_abi<'tcx>(
     fn_def_id: Option<DefId>,
 ) -> Result<(), &'tcx FnAbiError<'tcx>> {
     if abi == SpecAbi::Unadjusted {
+        // The "unadjusted" ABI passes aggregates in "direct" mode. That's fragile but needed for
+        // some LLVM intrinsics.
+        fn unadjust<'tcx>(arg: &mut ArgAbi<'tcx, Ty<'tcx>>) {
+            // This still uses `PassMode::Pair` for ScalarPair types. That's unlikely to be intended,
+            // but who knows what breaks if we change this now.
+            if matches!(arg.layout.abi, Abi::Aggregate { .. }) {
+                assert!(
+                    arg.layout.abi.is_sized(),
+                    "'unadjusted' ABI does not support unsized arguments"
+                );
+            }
+            arg.make_direct_deprecated();
+        }
+
+        unadjust(&mut fn_abi.ret);
+        for arg in fn_abi.args.iter_mut() {
+            unadjust(arg);
+        }
         return Ok(());
     }
 
@@ -520,13 +686,14 @@ fn fn_abi_adjust_for_abi<'tcx>(
 
                 _ => return,
             }
-            // `Aggregate` ABI must be adjusted to ensure that ABI-compatible Rust types are passed
-            // the same way.
+            // Compute `Aggregate` ABI.
+
+            let is_indirect_not_on_stack =
+                matches!(arg.mode, PassMode::Indirect { on_stack: false, .. });
+            assert!(is_indirect_not_on_stack, "{:?}", arg);
 
             let size = arg.layout.size;
-            if arg.layout.is_unsized() || size > Pointer(AddressSpace::DATA).size(cx) {
-                arg.make_indirect();
-            } else {
+            if !arg.layout.is_unsized() && size <= Pointer(AddressSpace::DATA).size(cx) {
                 // We want to pass small aggregates as immediates, but using
                 // an LLVM aggregate type for this leads to bad optimizations,
                 // so we pick an appropriately sized integer type instead.
diff --git a/compiler/rustc_ty_utils/src/assoc.rs b/compiler/rustc_ty_utils/src/assoc.rs
index 3e140793b..db37bec4b 100644
--- a/compiler/rustc_ty_utils/src/assoc.rs
+++ b/compiler/rustc_ty_utils/src/assoc.rs
@@ -2,13 +2,12 @@ use rustc_data_structures::fx::FxIndexSet;
 use rustc_hir as hir;
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::{DefId, DefIdMap, LocalDefId};
-use rustc_hir::definitions::DefPathData;
 use rustc_hir::intravisit::{self, Visitor};
 use rustc_middle::query::Providers;
 use rustc_middle::ty::{self, GenericArgs, ImplTraitInTraitData, Ty, TyCtxt};
 use rustc_span::symbol::kw;
 
-pub fn provide(providers: &mut Providers) {
+pub(crate) fn provide(providers: &mut Providers) {
     *providers = Providers {
         associated_item,
         associated_item_def_ids,
@@ -23,7 +22,7 @@ pub fn provide(providers: &mut Providers) {
 fn associated_item_def_ids(tcx: TyCtxt<'_>, def_id: LocalDefId) -> &[DefId] {
     let item = tcx.hir().expect_item(def_id);
     match item.kind {
-        hir::ItemKind::Trait(.., ref trait_item_refs) => {
+        hir::ItemKind::Trait(.., trait_item_refs) => {
             // We collect RPITITs for each trait method's return type and create a
             // corresponding associated item using associated_types_for_impl_traits_in_associated_fn
             // query.
@@ -43,11 +42,11 @@ fn associated_item_def_ids(tcx: TyCtxt<'_>, def_id: LocalDefId) -> &[DefId] {
                                     trait_fn_def_id,
                                 )
                             })
-                            .map(|def_id| *def_id),
+                            .copied(),
                     ),
             )
         }
-        hir::ItemKind::Impl(ref impl_) => {
+        hir::ItemKind::Impl(impl_) => {
             // We collect RPITITs for each trait method's return type, on the impl side too and
             // create a corresponding associated item using
             // associated_types_for_impl_traits_in_associated_fn query.
@@ -69,7 +68,7 @@ fn associated_item_def_ids(tcx: TyCtxt<'_>, def_id: LocalDefId) -> &[DefId] {
                                     impl_fn_def_id,
                                 )
                             })
-                            .map(|def_id| *def_id)
+                            .copied()
                     })),
             )
         }
@@ -94,11 +93,11 @@ fn impl_item_implementor_ids(tcx: TyCtxt<'_>, impl_id: DefId) -> DefIdMap<DefId>
 }
 
 fn associated_item(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::AssocItem {
-    let id = tcx.hir().local_def_id_to_hir_id(def_id);
+    let id = tcx.local_def_id_to_hir_id(def_id);
     let parent_def_id = tcx.hir().get_parent_item(id);
     let parent_item = tcx.hir().expect_item(parent_def_id.def_id);
     match parent_item.kind {
-        hir::ItemKind::Impl(ref impl_) => {
+        hir::ItemKind::Impl(impl_) => {
             if let Some(impl_item_ref) = impl_.items.iter().find(|i| i.id.owner_id.def_id == def_id)
             {
                 let assoc_item = associated_item_from_impl_item_ref(impl_item_ref);
@@ -107,7 +106,7 @@ fn associated_item(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::AssocItem {
             }
         }
 
-        hir::ItemKind::Trait(.., ref trait_item_refs) => {
+        hir::ItemKind::Trait(.., trait_item_refs) => {
             if let Some(trait_item_ref) =
                 trait_item_refs.iter().find(|i| i.id.owner_id.def_id == def_id)
             {
@@ -254,13 +253,11 @@ fn associated_type_for_impl_trait_in_trait(
     assert_eq!(tcx.def_kind(trait_def_id), DefKind::Trait);
 
     let span = tcx.def_span(opaque_ty_def_id);
-    let trait_assoc_ty = tcx.at(span).create_def(trait_def_id, DefPathData::ImplTraitAssocTy);
+    let trait_assoc_ty = tcx.at(span).create_def(trait_def_id, kw::Empty, DefKind::AssocTy);
 
     let local_def_id = trait_assoc_ty.def_id();
     let def_id = local_def_id.to_def_id();
 
-    trait_assoc_ty.opt_def_kind(Some(DefKind::AssocTy));
-
     // There's no HIR associated with this new synthesized `def_id`, so feed
     // `opt_local_def_id_to_hir_id` with `None`.
     trait_assoc_ty.opt_local_def_id_to_hir_id(None);
@@ -348,8 +345,7 @@ fn associated_type_for_impl_trait_in_impl(
     let impl_local_def_id = tcx.local_parent(impl_fn_def_id);
 
     let decl = tcx
-        .hir()
-        .find_by_def_id(impl_fn_def_id)
+        .opt_hir_node_by_def_id(impl_fn_def_id)
         .expect("expected item")
         .fn_decl()
         .expect("expected decl");
@@ -357,13 +353,11 @@ fn associated_type_for_impl_trait_in_impl(
         hir::FnRetTy::DefaultReturn(_) => tcx.def_span(impl_fn_def_id),
         hir::FnRetTy::Return(ty) => ty.span,
     };
-    let impl_assoc_ty = tcx.at(span).create_def(impl_local_def_id, DefPathData::ImplTraitAssocTy);
+    let impl_assoc_ty = tcx.at(span).create_def(impl_local_def_id, kw::Empty, DefKind::AssocTy);
 
     let local_def_id = impl_assoc_ty.def_id();
     let def_id = local_def_id.to_def_id();
 
-    impl_assoc_ty.opt_def_kind(Some(DefKind::AssocTy));
-
     // There's no HIR associated with this new synthesized `def_id`, so feed
     // `opt_local_def_id_to_hir_id` with `None`.
     impl_assoc_ty.opt_local_def_id_to_hir_id(None);
diff --git a/compiler/rustc_ty_utils/src/consts.rs b/compiler/rustc_ty_utils/src/consts.rs
index 35487d3b6..b521a5c11 100644
--- a/compiler/rustc_ty_utils/src/consts.rs
+++ b/compiler/rustc_ty_utils/src/consts.rs
@@ -17,7 +17,7 @@ use crate::errors::{GenericConstantTooComplex, GenericConstantTooComplexSub};
 
 /// Destructures array, ADT or tuple constants into the constants
 /// of their fields.
-pub(crate) fn destructure_const<'tcx>(
+fn destructure_const<'tcx>(
     tcx: TyCtxt<'tcx>,
     const_: ty::Const<'tcx>,
 ) -> ty::DestructuredConst<'tcx> {
@@ -375,7 +375,7 @@ impl<'a, 'tcx> IsThirPolymorphic<'a, 'tcx> {
 
 impl<'a, 'tcx> visit::Visitor<'a, 'tcx> for IsThirPolymorphic<'a, 'tcx> {
     fn thir(&self) -> &'a thir::Thir<'tcx> {
-        &self.thir
+        self.thir
     }
 
     #[instrument(skip(self), level = "debug")]
@@ -396,7 +396,7 @@ impl<'a, 'tcx> visit::Visitor<'a, 'tcx> for IsThirPolymorphic<'a, 'tcx> {
 }
 
 /// Builds an abstract const, do not use this directly, but use `AbstractConst::new` instead.
-pub fn thir_abstract_const(
+fn thir_abstract_const(
     tcx: TyCtxt<'_>,
     def: LocalDefId,
 ) -> Result<Option<ty::EarlyBinder<ty::Const<'_>>>, ErrorGuaranteed> {
@@ -428,6 +428,6 @@ pub fn thir_abstract_const(
     Ok(Some(ty::EarlyBinder::bind(recurse_build(tcx, body, body_id, root_span)?)))
 }
 
-pub fn provide(providers: &mut Providers) {
+pub(crate) fn provide(providers: &mut Providers) {
     *providers = Providers { destructure_const, thir_abstract_const, ..*providers };
 }
diff --git a/compiler/rustc_ty_utils/src/implied_bounds.rs b/compiler/rustc_ty_utils/src/implied_bounds.rs
index 5c34df1ed..3f9bd509b 100644
--- a/compiler/rustc_ty_utils/src/implied_bounds.rs
+++ b/compiler/rustc_ty_utils/src/implied_bounds.rs
@@ -7,7 +7,7 @@ use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_span::Span;
 use std::iter;
 
-pub fn provide(providers: &mut Providers) {
+pub(crate) fn provide(providers: &mut Providers) {
     *providers = Providers {
         assumed_wf_types,
         assumed_wf_types_for_rpitit: |tcx, def_id| {
@@ -53,9 +53,9 @@ fn assumed_wf_types<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalDefId) -> &'tcx [(Ty<'
         DefKind::AssocTy if let Some(data) = tcx.opt_rpitit_info(def_id.to_def_id()) => {
             match data {
                 ty::ImplTraitInTraitData::Trait { fn_def_id, .. } => {
-                    // We need to remap all of the late-bound lifetimes in theassumed wf types
-                    // of the fn (which are represented as ReFree) to the early-bound lifetimes
-                    // of the RPITIT (which are represented by ReEarlyBound owned by the opaque).
+                    // We need to remap all of the late-bound lifetimes in the assumed wf types
+                    // of the fn (which are represented as ReLateParam) to the early-bound lifetimes
+                    // of the RPITIT (which are represented by ReEarlyParam owned by the opaque).
                     // Luckily, this is very easy to do because we already have that mapping
                     // stored in the HIR of this RPITIT.
                     //
@@ -65,19 +65,19 @@ fn assumed_wf_types<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalDefId) -> &'tcx [(Ty<'
                     let mut mapping = FxHashMap::default();
                     let generics = tcx.generics_of(def_id);
 
-                    // For each captured opaque lifetime, if it's late-bound (`ReFree` in this case,
-                    // since it has been liberated), map it back to the early-bound lifetime of
+                    // For each captured opaque lifetime, if it's late-bound (`ReLateParam` in this
+                    // case, since it has been liberated), map it back to the early-bound lifetime of
                     // the GAT. Since RPITITs also have all of the fn's generics, we slice only
                     // the end of the list corresponding to the opaque's generics.
                     for param in &generics.params[tcx.generics_of(fn_def_id).params.len()..] {
                         let orig_lt =
                             tcx.map_rpit_lifetime_to_fn_lifetime(param.def_id.expect_local());
-                        if matches!(*orig_lt, ty::ReFree(..)) {
+                        if matches!(*orig_lt, ty::ReLateParam(..)) {
                             mapping.insert(
                                 orig_lt,
-                                ty::Region::new_early_bound(
+                                ty::Region::new_early_param(
                                     tcx,
-                                    ty::EarlyBoundRegion {
+                                    ty::EarlyParamRegion {
                                         def_id: param.def_id,
                                         index: param.index,
                                         name: param.name,
@@ -90,7 +90,7 @@ fn assumed_wf_types<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalDefId) -> &'tcx [(Ty<'
                     let remapped_wf_tys = tcx.fold_regions(
                         tcx.assumed_wf_types(fn_def_id.expect_local()).to_vec(),
                         |region, _| {
-                            // If `region` is a `ReFree` that is captured by the
+                            // If `region` is a `ReLateParam` that is captured by the
                             // opaque, remap it to its corresponding the early-
                             // bound region.
                             if let Some(remapped_region) = mapping.get(&region) {
@@ -156,13 +156,12 @@ fn assumed_wf_types<'tcx>(tcx: TyCtxt<'tcx>, def_id: LocalDefId) -> &'tcx [(Ty<'
         | DefKind::Field
         | DefKind::LifetimeParam
         | DefKind::GlobalAsm
-        | DefKind::Closure
-        | DefKind::Coroutine => ty::List::empty(),
+        | DefKind::Closure => ty::List::empty(),
     }
 }
 
 fn fn_sig_spans(tcx: TyCtxt<'_>, def_id: LocalDefId) -> impl Iterator<Item = Span> + '_ {
-    let node = tcx.hir().get(tcx.local_def_id_to_hir_id(def_id));
+    let node = tcx.hir_node_by_def_id(def_id);
     if let Some(decl) = node.fn_decl() {
         decl.inputs.iter().map(|ty| ty.span).chain(iter::once(decl.output.span()))
     } else {
diff --git a/compiler/rustc_ty_utils/src/instance.rs b/compiler/rustc_ty_utils/src/instance.rs
index 1487f40fd..f1c9bb23e 100644
--- a/compiler/rustc_ty_utils/src/instance.rs
+++ b/compiler/rustc_ty_utils/src/instance.rs
@@ -61,7 +61,7 @@ fn resolve_instance<'tcx>(
 
         Ok(Some(Instance { def, args }))
     };
-    debug!("inner_resolve_instance: result={:?}", result);
+    debug!("resolve_instance: result={:?}", result);
     result
 }
 
@@ -79,7 +79,7 @@ fn resolve_associated_item<'tcx>(
     let vtbl = match tcx.codegen_select_candidate((param_env, trait_ref)) {
         Ok(vtbl) => vtbl,
         Err(CodegenObligationError::Ambiguity) => {
-            let reported = tcx.sess.delay_span_bug(
+            let reported = tcx.sess.span_delayed_bug(
                 tcx.def_span(trait_item_id),
                 format!(
                     "encountered ambiguity selecting `{trait_ref:?}` during codegen, presuming due to \
@@ -171,7 +171,7 @@ fn resolve_associated_item<'tcx>(
 
             // Any final impl is required to define all associated items.
             if !leaf_def.item.defaultness(tcx).has_value() {
-                let guard = tcx.sess.delay_span_bug(
+                let guard = tcx.sess.span_delayed_bug(
                     tcx.def_span(leaf_def.item.def_id),
                     "missing value for assoc item in impl",
                 );
@@ -271,6 +271,21 @@ fn resolve_associated_item<'tcx>(
                     debug_assert!(tcx.defaultness(trait_item_id).has_value());
                     Some(Instance::new(trait_item_id, rcvr_args))
                 }
+            } else if Some(trait_ref.def_id) == lang_items.async_iterator_trait() {
+                let ty::Coroutine(coroutine_def_id, args, _) = *rcvr_args.type_at(0).kind() else {
+                    bug!()
+                };
+
+                if cfg!(debug_assertions) && tcx.item_name(trait_item_id) != sym::poll_next {
+                    span_bug!(
+                        tcx.def_span(coroutine_def_id),
+                        "no definition for `{trait_ref}::{}` for built-in coroutine type",
+                        tcx.item_name(trait_item_id)
+                    )
+                }
+
+                // `AsyncIterator::poll_next` is generated by the compiler.
+                Some(Instance { def: ty::InstanceDef::Item(coroutine_def_id), args })
             } else if Some(trait_ref.def_id) == lang_items.coroutine_trait() {
                 let ty::Coroutine(coroutine_def_id, args, _) = *rcvr_args.type_at(0).kind() else {
                     bug!()
@@ -328,6 +343,6 @@ fn resolve_associated_item<'tcx>(
     })
 }
 
-pub fn provide(providers: &mut Providers) {
+pub(crate) fn provide(providers: &mut Providers) {
     *providers = Providers { resolve_instance, ..*providers };
 }
diff --git a/compiler/rustc_ty_utils/src/layout.rs b/compiler/rustc_ty_utils/src/layout.rs
index 283862b5e..7918965e0 100644
--- a/compiler/rustc_ty_utils/src/layout.rs
+++ b/compiler/rustc_ty_utils/src/layout.rs
@@ -8,9 +8,7 @@ use rustc_middle::ty::layout::{
     IntegerExt, LayoutCx, LayoutError, LayoutOf, TyAndLayout, MAX_SIMD_LANES,
 };
 use rustc_middle::ty::print::with_no_trimmed_paths;
-use rustc_middle::ty::{
-    self, AdtDef, EarlyBinder, GenericArgsRef, ReprOptions, Ty, TyCtxt, TypeVisitableExt,
-};
+use rustc_middle::ty::{self, AdtDef, EarlyBinder, GenericArgsRef, Ty, TyCtxt, TypeVisitableExt};
 use rustc_session::{DataTypeKind, FieldInfo, FieldKind, SizeKind, VariantInfo};
 use rustc_span::symbol::Symbol;
 use rustc_span::DUMMY_SP;
@@ -24,7 +22,7 @@ use crate::errors::{
 };
 use crate::layout_sanity_check::sanity_check_layout;
 
-pub fn provide(providers: &mut Providers) {
+pub(crate) fn provide(providers: &mut Providers) {
     *providers = Providers { layout_of, ..*providers };
 }
 
@@ -65,7 +63,11 @@ fn layout_of<'tcx>(
     let layout = layout_of_uncached(&cx, ty)?;
     let layout = TyAndLayout { ty, layout };
 
-    record_layout_for_printing(&cx, layout);
+    // If we are running with `-Zprint-type-sizes`, maybe record layouts
+    // for dumping later.
+    if cx.tcx.sess.opts.unstable_opts.print_type_sizes {
+        record_layout_for_printing(&cx, layout);
+    }
 
     sanity_check_layout(&cx, &layout);
 
@@ -89,7 +91,7 @@ fn univariant_uninterned<'tcx>(
     let dl = cx.data_layout();
     let pack = repr.pack;
     if pack.is_some() && repr.align.is_some() {
-        cx.tcx.sess.delay_span_bug(DUMMY_SP, "struct cannot be packed and aligned");
+        cx.tcx.sess.span_delayed_bug(DUMMY_SP, "struct cannot be packed and aligned");
         return Err(cx.tcx.arena.alloc(LayoutError::Unknown(ty)));
     }
 
@@ -316,7 +318,7 @@ fn layout_of_uncached<'tcx>(
 
         ty::Coroutine(def_id, args, _) => coroutine_layout(cx, ty, def_id, args)?,
 
-        ty::Closure(_, ref args) => {
+        ty::Closure(_, args) => {
             let tys = args.as_closure().upvar_tys();
             univariant(
                 &tys.iter()
@@ -342,7 +344,7 @@ fn layout_of_uncached<'tcx>(
         ty::Adt(def, args) if def.repr().simd() => {
             if !def.is_struct() {
                 // Should have yielded E0517 by now.
-                tcx.sess.delay_span_bug(
+                tcx.sess.span_delayed_bug(
                     DUMMY_SP,
                     "#[repr(simd)] was applied to an ADT that is not a struct",
                 );
@@ -372,7 +374,7 @@ fn layout_of_uncached<'tcx>(
             // (should be caught by typeck)
             for fi in fields {
                 if fi.ty(tcx, args) != f0_ty {
-                    tcx.sess.delay_span_bug(
+                    tcx.sess.span_delayed_bug(
                         DUMMY_SP,
                         "#[repr(simd)] was applied to an ADT with heterogeneous field type",
                     );
@@ -431,7 +433,21 @@ fn layout_of_uncached<'tcx>(
                 .size
                 .checked_mul(e_len, dl)
                 .ok_or_else(|| error(cx, LayoutError::SizeOverflow(ty)))?;
-            let align = dl.vector_align(size);
+
+            let (abi, align) = if def.repr().packed() && !e_len.is_power_of_two() {
+                // Non-power-of-two vectors have padding up to the next power-of-two.
+                // If we're a packed repr, remove the padding while keeping the alignment as close
+                // to a vector as possible.
+                (
+                    Abi::Aggregate { sized: true },
+                    AbiAndPrefAlign {
+                        abi: Align::max_for_offset(size),
+                        pref: dl.vector_align(size).pref,
+                    },
+                )
+            } else {
+                (Abi::Vector { element: e_abi, count: e_len }, dl.vector_align(size))
+            };
             let size = size.align_to(align.abi);
 
             // Compute the placement of the vector fields:
@@ -444,7 +460,7 @@ fn layout_of_uncached<'tcx>(
             tcx.mk_layout(LayoutS {
                 variants: Variants::Single { index: FIRST_VARIANT },
                 fields,
-                abi: Abi::Vector { element: e_abi, count: e_len },
+                abi,
                 largest_niche: e_ly.largest_niche,
                 size,
                 align,
@@ -469,7 +485,7 @@ fn layout_of_uncached<'tcx>(
 
             if def.is_union() {
                 if def.repr().pack.is_some() && def.repr().align.is_some() {
-                    cx.tcx.sess.delay_span_bug(
+                    cx.tcx.sess.span_delayed_bug(
                         tcx.def_span(def.did()),
                         "union cannot be packed and aligned",
                     );
@@ -724,7 +740,7 @@ fn coroutine_layout<'tcx>(
     let Some(info) = tcx.coroutine_layout(def_id) else {
         return Err(error(cx, LayoutError::Unknown(ty)));
     };
-    let (ineligible_locals, assignments) = coroutine_saved_local_eligibility(&info);
+    let (ineligible_locals, assignments) = coroutine_saved_local_eligibility(info);
 
     // Build a prefix layout, including "promoting" all ineligible
     // locals as part of the prefix. We compute the layout of all of
@@ -740,11 +756,11 @@ fn coroutine_layout<'tcx>(
     };
     let tag_layout = cx.tcx.mk_layout(LayoutS::scalar(cx, tag));
 
-    let promoted_layouts = ineligible_locals
-        .iter()
-        .map(|local| subst_field(info.field_tys[local].ty))
-        .map(|ty| Ty::new_maybe_uninit(tcx, ty))
-        .map(|ty| Ok(cx.layout_of(ty)?.layout));
+    let promoted_layouts = ineligible_locals.iter().map(|local| {
+        let field_ty = subst_field(info.field_tys[local].ty);
+        let uninit_ty = Ty::new_maybe_uninit(tcx, field_ty);
+        Ok(cx.spanned_layout_of(uninit_ty, info.field_tys[local].source_info.span)?.layout)
+    });
     let prefix_layouts = args
         .as_coroutine()
         .prefix_tys()
@@ -815,7 +831,10 @@ fn coroutine_layout<'tcx>(
                     Assigned(_) => bug!("assignment does not match variant"),
                     Ineligible(_) => false,
                 })
-                .map(|local| subst_field(info.field_tys[*local].ty));
+                .map(|local| {
+                    let field_ty = subst_field(info.field_tys[*local].ty);
+                    Ty::new_maybe_uninit(tcx, field_ty)
+                });
 
             let mut variant = univariant_uninterned(
                 cx,
@@ -911,21 +930,7 @@ fn coroutine_layout<'tcx>(
     Ok(layout)
 }
 
-/// This is invoked by the `layout_of` query to record the final
-/// layout of each type.
-#[inline(always)]
 fn record_layout_for_printing<'tcx>(cx: &LayoutCx<'tcx, TyCtxt<'tcx>>, layout: TyAndLayout<'tcx>) {
-    // If we are running with `-Zprint-type-sizes`, maybe record layouts
-    // for dumping later.
-    if cx.tcx.sess.opts.unstable_opts.print_type_sizes {
-        record_layout_for_printing_outlined(cx, layout)
-    }
-}
-
-fn record_layout_for_printing_outlined<'tcx>(
-    cx: &LayoutCx<'tcx, TyCtxt<'tcx>>,
-    layout: TyAndLayout<'tcx>,
-) {
     // Ignore layouts that are done with non-empty environments or
     // non-monomorphic layouts, as the user only wants to see the stuff
     // resulting from the final codegen session.
diff --git a/compiler/rustc_ty_utils/src/lib.rs b/compiler/rustc_ty_utils/src/lib.rs
index dabe25589..fa1f94e8b 100644
--- a/compiler/rustc_ty_utils/src/lib.rs
+++ b/compiler/rustc_ty_utils/src/lib.rs
@@ -5,16 +5,16 @@
 //! This API is completely unstable and subject to change.
 
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
-#![cfg_attr(not(bootstrap), doc(rust_logo))]
-#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
-#![cfg_attr(not(bootstrap), allow(internal_features))]
+#![doc(rust_logo)]
+#![feature(rustdoc_internals)]
+#![allow(internal_features)]
 #![feature(assert_matches)]
 #![feature(associated_type_defaults)]
+#![feature(box_patterns)]
+#![feature(if_let_guard)]
 #![feature(iterator_try_collect)]
 #![feature(let_chains)]
-#![feature(if_let_guard)]
 #![feature(never_type)]
-#![feature(box_patterns)]
 #![recursion_limit = "256"]
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
@@ -24,8 +24,6 @@ extern crate rustc_middle;
 #[macro_use]
 extern crate tracing;
 
-use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
-use rustc_fluent_macro::fluent_messages;
 use rustc_middle::query::Providers;
 
 mod abi;
@@ -34,17 +32,17 @@ mod common_traits;
 mod consts;
 mod errors;
 mod implied_bounds;
-pub mod instance;
+mod instance;
 mod layout;
 mod layout_sanity_check;
 mod needs_drop;
 mod opaque_types;
-pub mod representability;
-pub mod sig_types;
+mod representability;
+mod sig_types;
 mod structural_match;
 mod ty;
 
-fluent_messages! { "../messages.ftl" }
+rustc_fluent_macro::fluent_messages! { "../messages.ftl" }
 
 pub fn provide(providers: &mut Providers) {
     abi::provide(providers);
diff --git a/compiler/rustc_ty_utils/src/opaque_types.rs b/compiler/rustc_ty_utils/src/opaque_types.rs
index 9242a1a75..db43c31cc 100644
--- a/compiler/rustc_ty_utils/src/opaque_types.rs
+++ b/compiler/rustc_ty_utils/src/opaque_types.rs
@@ -81,8 +81,8 @@ impl<'tcx> OpaqueTypeCollector<'tcx> {
     /// For the above example, this function returns `true` for `f1` and `false` for `f2`.
     #[instrument(level = "trace", skip(self), ret)]
     fn check_tait_defining_scope(&self, opaque_def_id: LocalDefId) -> bool {
-        let mut hir_id = self.tcx.hir().local_def_id_to_hir_id(self.item);
-        let opaque_hir_id = self.tcx.hir().local_def_id_to_hir_id(opaque_def_id);
+        let mut hir_id = self.tcx.local_def_id_to_hir_id(self.item);
+        let opaque_hir_id = self.tcx.local_def_id_to_hir_id(opaque_def_id);
 
         // Named opaque types can be defined by any siblings or children of siblings.
         let scope = self.tcx.hir().get_defining_scope(opaque_hir_id);
@@ -159,10 +159,10 @@ impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for OpaqueTypeCollector<'tcx> {
                 // Only check that the parent generics of the TAIT/RPIT are unique.
                 // the args owned by the opaque are going to always be duplicate
                 // lifetime params for RPITs, and empty for TAITs.
-                match self
-                    .tcx
-                    .uses_unique_generic_params(&alias_ty.args[..parent_count], CheckRegions::Bound)
-                {
+                match self.tcx.uses_unique_generic_params(
+                    &alias_ty.args[..parent_count],
+                    CheckRegions::FromFunction,
+                ) {
                     Ok(()) => {
                         // FIXME: implement higher kinded lifetime bounds on nested opaque types. They are not
                         // supported at all, so this is sound to do, but once we want to support them, you'll
@@ -238,7 +238,7 @@ impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for OpaqueTypeCollector<'tcx> {
                                     .instantiate(self.tcx, impl_args)
                                     .visit_with(self);
                             } else {
-                                self.tcx.sess.delay_span_bug(
+                                self.tcx.sess.span_delayed_bug(
                                     self.tcx.def_span(assoc.def_id),
                                     "item had incorrect args",
                                 );
@@ -313,7 +313,7 @@ fn opaque_types_defined_by<'tcx>(
         | DefKind::Impl { .. } => {}
         // Closures and coroutines are type checked with their parent, so we need to allow all
         // opaques from the closure signature *and* from the parent body.
-        DefKind::Closure | DefKind::Coroutine | DefKind::InlineConst => {
+        DefKind::Closure | DefKind::InlineConst => {
             collector.opaques.extend(tcx.opaque_types_defined_by(tcx.local_parent(item)));
         }
     }
diff --git a/compiler/rustc_ty_utils/src/representability.rs b/compiler/rustc_ty_utils/src/representability.rs
index f34e0df2c..3aaa2e73b 100644
--- a/compiler/rustc_ty_utils/src/representability.rs
+++ b/compiler/rustc_ty_utils/src/representability.rs
@@ -6,7 +6,7 @@ use rustc_middle::query::Providers;
 use rustc_middle::ty::{self, Representability, Ty, TyCtxt};
 use rustc_span::def_id::LocalDefId;
 
-pub fn provide(providers: &mut Providers) {
+pub(crate) fn provide(providers: &mut Providers) {
     *providers =
         Providers { representability, representability_adt_ty, params_in_repr, ..*providers };
 }
diff --git a/compiler/rustc_ty_utils/src/sig_types.rs b/compiler/rustc_ty_utils/src/sig_types.rs
index ccdc61201..b155a4ac8 100644
--- a/compiler/rustc_ty_utils/src/sig_types.rs
+++ b/compiler/rustc_ty_utils/src/sig_types.rs
@@ -8,7 +8,7 @@ use rustc_middle::ty::TyCtxt;
 use rustc_span::Span;
 use rustc_type_ir::visit::TypeVisitable;
 
-pub trait SpannedTypeVisitor<'tcx> {
+pub(crate) trait SpannedTypeVisitor<'tcx> {
     type BreakTy = !;
     fn visit(
         &mut self,
@@ -17,7 +17,7 @@ pub trait SpannedTypeVisitor<'tcx> {
     ) -> ControlFlow<Self::BreakTy>;
 }
 
-pub fn walk_types<'tcx, V: SpannedTypeVisitor<'tcx>>(
+pub(crate) fn walk_types<'tcx, V: SpannedTypeVisitor<'tcx>>(
     tcx: TyCtxt<'tcx>,
     item: LocalDefId,
     visitor: &mut V,
@@ -28,7 +28,7 @@ pub fn walk_types<'tcx, V: SpannedTypeVisitor<'tcx>>(
         // Walk over the signature of the function
         DefKind::AssocFn | DefKind::Fn => {
             let ty_sig = tcx.fn_sig(item).instantiate_identity();
-            let hir_sig = tcx.hir().get_by_def_id(item).fn_decl().unwrap();
+            let hir_sig = tcx.hir_node_by_def_id(item).fn_decl().unwrap();
             // Walk over the inputs and outputs manually in order to get good spans for them.
             visitor.visit(hir_sig.output.span(), ty_sig.output());
             for (hir, ty) in hir_sig.inputs.iter().zip(ty_sig.inputs().iter()) {
@@ -42,7 +42,7 @@ pub fn walk_types<'tcx, V: SpannedTypeVisitor<'tcx>>(
         DefKind::TyAlias {..} | DefKind::AssocTy |
         // Walk over the type of the item
         DefKind::Static(_) | DefKind::Const | DefKind::AssocConst | DefKind::AnonConst => {
-            let span = match tcx.hir().get_by_def_id(item).ty() {
+            let span = match tcx.hir_node_by_def_id(item).ty() {
                 Some(ty) => ty.span,
                 _ => tcx.def_span(item),
             };
@@ -67,14 +67,14 @@ pub fn walk_types<'tcx, V: SpannedTypeVisitor<'tcx>>(
         // These are not part of a public API, they can only appear as hidden types, and there
         // the interesting parts are solely in the signature of the containing item's opaque type
         // or dyn type.
-        DefKind::InlineConst | DefKind::Closure | DefKind::Coroutine => {}
+        DefKind::InlineConst | DefKind::Closure => {}
         DefKind::Impl { of_trait } => {
             if of_trait {
-                let span = tcx.hir().get_by_def_id(item).expect_item().expect_impl().of_trait.unwrap().path.span;
+                let span = tcx.hir_node_by_def_id(item).expect_item().expect_impl().of_trait.unwrap().path.span;
                 let args = &tcx.impl_trait_ref(item).unwrap().instantiate_identity().args[1..];
                 visitor.visit(span, args)?;
             }
-            let span = match tcx.hir().get_by_def_id(item).ty() {
+            let span = match tcx.hir_node_by_def_id(item).ty() {
                 Some(ty) => ty.span,
                 _ => tcx.def_span(item),
             };
diff --git a/compiler/rustc_ty_utils/src/structural_match.rs b/compiler/rustc_ty_utils/src/structural_match.rs
index 215acbe2c..6e7a98877 100644
--- a/compiler/rustc_ty_utils/src/structural_match.rs
+++ b/compiler/rustc_ty_utils/src/structural_match.rs
@@ -39,6 +39,6 @@ fn has_structural_eq_impls<'tcx>(tcx: TyCtxt<'tcx>, adt_ty: Ty<'tcx>) -> bool {
     ocx.select_all_or_error().is_empty()
 }
 
-pub fn provide(providers: &mut Providers) {
+pub(crate) fn provide(providers: &mut Providers) {
     providers.has_structural_eq_impls = has_structural_eq_impls;
 }
diff --git a/compiler/rustc_ty_utils/src/ty.rs b/compiler/rustc_ty_utils/src/ty.rs
index abf3e108e..2158aacab 100644
--- a/compiler/rustc_ty_utils/src/ty.rs
+++ b/compiler/rustc_ty_utils/src/ty.rs
@@ -25,7 +25,7 @@ fn sized_constraint_for_ty<'tcx>(
             vec![ty]
         }
 
-        Tuple(ref tys) => match tys.last() {
+        Tuple(tys) => match tys.last() {
             None => vec![],
             Some(&ty) => sized_constraint_for_ty(tcx, adtdef, ty),
         },
@@ -74,7 +74,7 @@ fn sized_constraint_for_ty<'tcx>(
 }
 
 fn defaultness(tcx: TyCtxt<'_>, def_id: LocalDefId) -> hir::Defaultness {
-    match tcx.hir().get_by_def_id(def_id) {
+    match tcx.hir_node_by_def_id(def_id) {
         hir::Node::Item(hir::Item { kind: hir::ItemKind::Impl(impl_), .. }) => impl_.defaultness,
         hir::Node::ImplItem(hir::ImplItem { defaultness, .. })
         | hir::Node::TraitItem(hir::TraitItem { defaultness, .. }) => *defaultness,
@@ -195,7 +195,7 @@ impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for ImplTraitInTraitFinder<'_, 'tcx> {
             // bounds of the RPITIT. Shift these binders back out when
             // constructing the top-level projection predicate.
             let shifted_alias_ty = self.tcx.fold_regions(unshifted_alias_ty, |re, depth| {
-                if let ty::ReLateBound(index, bv) = re.kind() {
+                if let ty::ReBound(index, bv) = re.kind() {
                     if depth != ty::INNERMOST {
                         return ty::Region::new_error_with_message(
                             self.tcx,
@@ -203,11 +203,7 @@ impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for ImplTraitInTraitFinder<'_, 'tcx> {
                             "we shouldn't walk non-predicate binders with `impl Trait`...",
                         );
                     }
-                    ty::Region::new_late_bound(
-                        self.tcx,
-                        index.shifted_out_to_binder(self.depth),
-                        bv,
-                    )
+                    ty::Region::new_bound(self.tcx, index.shifted_out_to_binder(self.depth), bv)
                 } else {
                     re
                 }
@@ -289,7 +285,7 @@ fn issue33140_self_ty(tcx: TyCtxt<'_>, def_id: DefId) -> Option<EarlyBinder<Ty<'
 
     let self_ty = trait_ref.self_ty();
     let self_ty_matches = match self_ty.kind() {
-        ty::Dynamic(ref data, re, _) if re.is_static() => data.principal().is_none(),
+        ty::Dynamic(data, re, _) if re.is_static() => data.principal().is_none(),
         _ => false,
     };
 
@@ -304,7 +300,7 @@ fn issue33140_self_ty(tcx: TyCtxt<'_>, def_id: DefId) -> Option<EarlyBinder<Ty<'
 
 /// Check if a function is async.
 fn asyncness(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::Asyncness {
-    let node = tcx.hir().get_by_def_id(def_id);
+    let node = tcx.hir_node_by_def_id(def_id);
     node.fn_sig().map_or(ty::Asyncness::No, |sig| match sig.header.asyncness {
         hir::IsAsync::Async(_) => ty::Asyncness::Yes,
         hir::IsAsync::NotAsync => ty::Asyncness::No,
@@ -355,7 +351,7 @@ fn unsizing_params_for_adt<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> BitSet<u32
     unsizing_params
 }
 
-pub fn provide(providers: &mut Providers) {
+pub(crate) fn provide(providers: &mut Providers) {
     *providers = Providers {
         asyncness,
         adt_sized_constraint,
diff --git a/compiler/rustc_type_ir/Cargo.toml b/compiler/rustc_type_ir/Cargo.toml
index b39ba3059..3a08d89cc 100644
--- a/compiler/rustc_type_ir/Cargo.toml
+++ b/compiler/rustc_type_ir/Cargo.toml
@@ -7,9 +7,20 @@ edition = "2021"
 # tidy-alphabetical-start
 bitflags = "1.2.1"
 derivative = "2.2.0"
-rustc_data_structures = { path = "../rustc_data_structures" }
-rustc_index = { path = "../rustc_index" }
-rustc_macros = { path = "../rustc_macros" }
-rustc_serialize = { path = "../rustc_serialize" }
-smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
+rustc_data_structures = { path = "../rustc_data_structures", optional = true }
+rustc_index = { path = "../rustc_index", default-features = false }
+rustc_macros = { path = "../rustc_macros", optional = true }
+rustc_serialize = { path = "../rustc_serialize", optional = true }
+smallvec = { version = "1.8.1" }
 # tidy-alphabetical-end
+
+[features]
+default = ["nightly"]
+nightly = [
+    "smallvec/may_dangle",
+    "smallvec/union",
+    "rustc_index/nightly",
+    "rustc_serialize",
+    "rustc_data_structures",
+    "rustc_macros",
+]
diff --git a/compiler/rustc_type_ir/src/canonical.rs b/compiler/rustc_type_ir/src/canonical.rs
index ace9eade7..5bc2bfe28 100644
--- a/compiler/rustc_type_ir/src/canonical.rs
+++ b/compiler/rustc_type_ir/src/canonical.rs
@@ -2,18 +2,16 @@ use std::fmt;
 use std::hash::Hash;
 use std::ops::ControlFlow;
 
-use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
-
 use crate::fold::{FallibleTypeFolder, TypeFoldable};
 use crate::visit::{TypeVisitable, TypeVisitor};
-use crate::{HashStableContext, Interner, UniverseIndex};
+use crate::{Interner, PlaceholderLike, UniverseIndex};
 
 /// A "canonicalized" type `V` is one where all free inference
 /// variables have been rewritten to "canonical vars". These are
 /// numbered starting from 0 in order of first appearance.
 #[derive(derivative::Derivative)]
 #[derivative(Clone(bound = "V: Clone"), Hash(bound = "V: Hash"))]
-#[derive(TyEncodable, TyDecodable)]
+#[cfg_attr(feature = "nightly", derive(TyEncodable, TyDecodable, HashStable_NoContext))]
 pub struct Canonical<I: Interner, V> {
     pub value: V,
     pub max_universe: UniverseIndex,
@@ -60,17 +58,6 @@ impl<I: Interner, V> Canonical<I, V> {
     }
 }
 
-impl<CTX: HashStableContext, I: Interner, V: HashStable<CTX>> HashStable<CTX> for Canonical<I, V>
-where
-    I::CanonicalVars: HashStable<CTX>,
-{
-    fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) {
-        self.value.hash_stable(hcx, hasher);
-        self.max_universe.hash_stable(hcx, hasher);
-        self.variables.hash_stable(hcx, hasher);
-    }
-}
-
 impl<I: Interner, V: Eq> Eq for Canonical<I, V> {}
 
 impl<I: Interner, V: PartialEq> PartialEq for Canonical<I, V> {
@@ -126,3 +113,243 @@ where
         self.variables.visit_with(folder)
     }
 }
+
+/// Information about a canonical variable that is included with the
+/// canonical value. This is sufficient information for code to create
+/// a copy of the canonical value in some other inference context,
+/// with fresh inference variables replacing the canonical values.
+#[derive(derivative::Derivative)]
+#[derivative(Clone(bound = ""), Copy(bound = ""), Hash(bound = ""), Debug(bound = ""))]
+#[cfg_attr(feature = "nightly", derive(TyDecodable, TyEncodable, HashStable_NoContext))]
+pub struct CanonicalVarInfo<I: Interner> {
+    pub kind: CanonicalVarKind<I>,
+}
+
+impl<I: Interner> PartialEq for CanonicalVarInfo<I> {
+    fn eq(&self, other: &Self) -> bool {
+        self.kind == other.kind
+    }
+}
+
+impl<I: Interner> Eq for CanonicalVarInfo<I> {}
+
+impl<I: Interner> TypeVisitable<I> for CanonicalVarInfo<I>
+where
+    I::Ty: TypeVisitable<I>,
+{
+    fn visit_with<V: TypeVisitor<I>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
+        self.kind.visit_with(visitor)
+    }
+}
+
+impl<I: Interner> TypeFoldable<I> for CanonicalVarInfo<I>
+where
+    I::Ty: TypeFoldable<I>,
+{
+    fn try_fold_with<F: FallibleTypeFolder<I>>(self, folder: &mut F) -> Result<Self, F::Error> {
+        Ok(CanonicalVarInfo { kind: self.kind.try_fold_with(folder)? })
+    }
+}
+
+impl<I: Interner> CanonicalVarInfo<I> {
+    pub fn universe(self) -> UniverseIndex {
+        self.kind.universe()
+    }
+
+    #[must_use]
+    pub fn with_updated_universe(self, ui: UniverseIndex) -> CanonicalVarInfo<I> {
+        CanonicalVarInfo { kind: self.kind.with_updated_universe(ui) }
+    }
+
+    pub fn is_existential(&self) -> bool {
+        match self.kind {
+            CanonicalVarKind::Ty(_) => true,
+            CanonicalVarKind::PlaceholderTy(_) => false,
+            CanonicalVarKind::Region(_) => true,
+            CanonicalVarKind::PlaceholderRegion(..) => false,
+            CanonicalVarKind::Const(..) => true,
+            CanonicalVarKind::PlaceholderConst(_, _) => false,
+            CanonicalVarKind::Effect => true,
+        }
+    }
+
+    pub fn is_region(&self) -> bool {
+        match self.kind {
+            CanonicalVarKind::Region(_) | CanonicalVarKind::PlaceholderRegion(_) => true,
+            CanonicalVarKind::Ty(_)
+            | CanonicalVarKind::PlaceholderTy(_)
+            | CanonicalVarKind::Const(_, _)
+            | CanonicalVarKind::PlaceholderConst(_, _)
+            | CanonicalVarKind::Effect => false,
+        }
+    }
+
+    pub fn expect_placeholder_index(self) -> usize {
+        match self.kind {
+            CanonicalVarKind::Ty(_)
+            | CanonicalVarKind::Region(_)
+            | CanonicalVarKind::Const(_, _)
+            | CanonicalVarKind::Effect => panic!("expected placeholder: {self:?}"),
+
+            CanonicalVarKind::PlaceholderRegion(placeholder) => placeholder.var().as_usize(),
+            CanonicalVarKind::PlaceholderTy(placeholder) => placeholder.var().as_usize(),
+            CanonicalVarKind::PlaceholderConst(placeholder, _) => placeholder.var().as_usize(),
+        }
+    }
+}
+
+/// Describes the "kind" of the canonical variable. This is a "kind"
+/// in the type-theory sense of the term -- i.e., a "meta" type system
+/// that analyzes type-like values.
+#[derive(derivative::Derivative)]
+#[derivative(Clone(bound = ""), Copy(bound = ""), Hash(bound = ""), Debug(bound = ""))]
+#[cfg_attr(feature = "nightly", derive(TyDecodable, TyEncodable, HashStable_NoContext))]
+pub enum CanonicalVarKind<I: Interner> {
+    /// Some kind of type inference variable.
+    Ty(CanonicalTyVarKind),
+
+    /// A "placeholder" that represents "any type".
+    PlaceholderTy(I::PlaceholderTy),
+
+    /// Region variable `'?R`.
+    Region(UniverseIndex),
+
+    /// A "placeholder" that represents "any region". Created when you
+    /// are solving a goal like `for<'a> T: Foo<'a>` to represent the
+    /// bound region `'a`.
+    PlaceholderRegion(I::PlaceholderRegion),
+
+    /// Some kind of const inference variable.
+    Const(UniverseIndex, I::Ty),
+
+    /// Effect variable `'?E`.
+    Effect,
+
+    /// A "placeholder" that represents "any const".
+    PlaceholderConst(I::PlaceholderConst, I::Ty),
+}
+
+impl<I: Interner> PartialEq for CanonicalVarKind<I> {
+    fn eq(&self, other: &Self) -> bool {
+        match (self, other) {
+            (Self::Ty(l0), Self::Ty(r0)) => l0 == r0,
+            (Self::PlaceholderTy(l0), Self::PlaceholderTy(r0)) => l0 == r0,
+            (Self::Region(l0), Self::Region(r0)) => l0 == r0,
+            (Self::PlaceholderRegion(l0), Self::PlaceholderRegion(r0)) => l0 == r0,
+            (Self::Const(l0, l1), Self::Const(r0, r1)) => l0 == r0 && l1 == r1,
+            (Self::PlaceholderConst(l0, l1), Self::PlaceholderConst(r0, r1)) => {
+                l0 == r0 && l1 == r1
+            }
+            _ => std::mem::discriminant(self) == std::mem::discriminant(other),
+        }
+    }
+}
+
+impl<I: Interner> Eq for CanonicalVarKind<I> {}
+
+impl<I: Interner> TypeVisitable<I> for CanonicalVarKind<I>
+where
+    I::Ty: TypeVisitable<I>,
+{
+    fn visit_with<V: TypeVisitor<I>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
+        match self {
+            CanonicalVarKind::Ty(_)
+            | CanonicalVarKind::PlaceholderTy(_)
+            | CanonicalVarKind::Region(_)
+            | CanonicalVarKind::PlaceholderRegion(_)
+            | CanonicalVarKind::Effect => ControlFlow::Continue(()),
+            CanonicalVarKind::Const(_, ty) | CanonicalVarKind::PlaceholderConst(_, ty) => {
+                ty.visit_with(visitor)
+            }
+        }
+    }
+}
+
+impl<I: Interner> TypeFoldable<I> for CanonicalVarKind<I>
+where
+    I::Ty: TypeFoldable<I>,
+{
+    fn try_fold_with<F: FallibleTypeFolder<I>>(self, folder: &mut F) -> Result<Self, F::Error> {
+        Ok(match self {
+            CanonicalVarKind::Ty(kind) => CanonicalVarKind::Ty(kind),
+            CanonicalVarKind::Region(kind) => CanonicalVarKind::Region(kind),
+            CanonicalVarKind::Const(kind, ty) => {
+                CanonicalVarKind::Const(kind, ty.try_fold_with(folder)?)
+            }
+            CanonicalVarKind::PlaceholderTy(placeholder) => {
+                CanonicalVarKind::PlaceholderTy(placeholder)
+            }
+            CanonicalVarKind::PlaceholderRegion(placeholder) => {
+                CanonicalVarKind::PlaceholderRegion(placeholder)
+            }
+            CanonicalVarKind::PlaceholderConst(placeholder, ty) => {
+                CanonicalVarKind::PlaceholderConst(placeholder, ty.try_fold_with(folder)?)
+            }
+            CanonicalVarKind::Effect => CanonicalVarKind::Effect,
+        })
+    }
+}
+
+impl<I: Interner> CanonicalVarKind<I> {
+    pub fn universe(self) -> UniverseIndex {
+        match self {
+            CanonicalVarKind::Ty(CanonicalTyVarKind::General(ui)) => ui,
+            CanonicalVarKind::Region(ui) => ui,
+            CanonicalVarKind::Const(ui, _) => ui,
+            CanonicalVarKind::PlaceholderTy(placeholder) => placeholder.universe(),
+            CanonicalVarKind::PlaceholderRegion(placeholder) => placeholder.universe(),
+            CanonicalVarKind::PlaceholderConst(placeholder, _) => placeholder.universe(),
+            CanonicalVarKind::Ty(CanonicalTyVarKind::Float | CanonicalTyVarKind::Int) => {
+                UniverseIndex::ROOT
+            }
+            CanonicalVarKind::Effect => UniverseIndex::ROOT,
+        }
+    }
+
+    /// Replaces the universe of this canonical variable with `ui`.
+    ///
+    /// In case this is a float or int variable, this causes an ICE if
+    /// the updated universe is not the root.
+    pub fn with_updated_universe(self, ui: UniverseIndex) -> CanonicalVarKind<I> {
+        match self {
+            CanonicalVarKind::Ty(CanonicalTyVarKind::General(_)) => {
+                CanonicalVarKind::Ty(CanonicalTyVarKind::General(ui))
+            }
+            CanonicalVarKind::Region(_) => CanonicalVarKind::Region(ui),
+            CanonicalVarKind::Const(_, ty) => CanonicalVarKind::Const(ui, ty),
+
+            CanonicalVarKind::PlaceholderTy(placeholder) => {
+                CanonicalVarKind::PlaceholderTy(placeholder.with_updated_universe(ui))
+            }
+            CanonicalVarKind::PlaceholderRegion(placeholder) => {
+                CanonicalVarKind::PlaceholderRegion(placeholder.with_updated_universe(ui))
+            }
+            CanonicalVarKind::PlaceholderConst(placeholder, ty) => {
+                CanonicalVarKind::PlaceholderConst(placeholder.with_updated_universe(ui), ty)
+            }
+            CanonicalVarKind::Ty(CanonicalTyVarKind::Int | CanonicalTyVarKind::Float)
+            | CanonicalVarKind::Effect => {
+                assert_eq!(ui, UniverseIndex::ROOT);
+                self
+            }
+        }
+    }
+}
+
+/// Rust actually has more than one category of type variables;
+/// notably, the type variables we create for literals (e.g., 22 or
+/// 22.) can only be instantiated with integral/float types (e.g.,
+/// usize or f32). In order to faithfully reproduce a type, we need to
+/// know what set of types a given type variable can be unified with.
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+#[cfg_attr(feature = "nightly", derive(TyDecodable, TyEncodable, HashStable_NoContext))]
+pub enum CanonicalTyVarKind {
+    /// General type variable `?T` that can be unified with arbitrary types.
+    General(UniverseIndex),
+
+    /// Integral type variable `?I` (that can only be unified with integral types).
+    Int,
+
+    /// Floating-point type variable `?F` (that can only be unified with float types).
+    Float,
+}
diff --git a/compiler/rustc_type_ir/src/const_kind.rs b/compiler/rustc_type_ir/src/const_kind.rs
index 33782b13c..0aaaad5af 100644
--- a/compiler/rustc_type_ir/src/const_kind.rs
+++ b/compiler/rustc_type_ir/src/const_kind.rs
@@ -1,8 +1,8 @@
-use rustc_data_structures::stable_hasher::HashStable;
-use rustc_data_structures::stable_hasher::StableHasher;
+#[cfg(feature = "nightly")]
+use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use std::fmt;
 
-use crate::{DebruijnIndex, DebugWithInfcx, HashStableContext, InferCtxtLike, Interner, WithInfcx};
+use crate::{DebruijnIndex, DebugWithInfcx, InferCtxtLike, Interner, WithInfcx};
 
 use self::ConstKind::*;
 
@@ -10,19 +10,20 @@ use self::ConstKind::*;
 #[derive(derivative::Derivative)]
 #[derivative(
     Clone(bound = ""),
+    Copy(bound = ""),
     PartialOrd(bound = ""),
     PartialOrd = "feature_allow_slow_enum",
     Ord(bound = ""),
     Ord = "feature_allow_slow_enum",
     Hash(bound = "")
 )]
-#[derive(TyEncodable, TyDecodable)]
+#[cfg_attr(feature = "nightly", derive(TyEncodable, TyDecodable, HashStable_NoContext))]
 pub enum ConstKind<I: Interner> {
     /// A const generic parameter.
     Param(I::ParamConst),
 
     /// Infer the value of the const.
-    Infer(I::InferConst),
+    Infer(InferConst),
 
     /// Bound const variable, used only when preparing a trait query.
     Bound(DebruijnIndex, I::BoundConst),
@@ -47,48 +48,6 @@ pub enum ConstKind<I: Interner> {
     Expr(I::ExprConst),
 }
 
-const fn const_kind_discriminant<I: Interner>(value: &ConstKind<I>) -> usize {
-    match value {
-        Param(_) => 0,
-        Infer(_) => 1,
-        Bound(_, _) => 2,
-        Placeholder(_) => 3,
-        Unevaluated(_) => 4,
-        Value(_) => 5,
-        Error(_) => 6,
-        Expr(_) => 7,
-    }
-}
-
-impl<CTX: HashStableContext, I: Interner> HashStable<CTX> for ConstKind<I>
-where
-    I::ParamConst: HashStable<CTX>,
-    I::InferConst: HashStable<CTX>,
-    I::BoundConst: HashStable<CTX>,
-    I::PlaceholderConst: HashStable<CTX>,
-    I::AliasConst: HashStable<CTX>,
-    I::ValueConst: HashStable<CTX>,
-    I::ErrorGuaranteed: HashStable<CTX>,
-    I::ExprConst: HashStable<CTX>,
-{
-    fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) {
-        const_kind_discriminant(self).hash_stable(hcx, hasher);
-        match self {
-            Param(p) => p.hash_stable(hcx, hasher),
-            Infer(i) => i.hash_stable(hcx, hasher),
-            Bound(d, b) => {
-                d.hash_stable(hcx, hasher);
-                b.hash_stable(hcx, hasher);
-            }
-            Placeholder(p) => p.hash_stable(hcx, hasher),
-            Unevaluated(u) => u.hash_stable(hcx, hasher),
-            Value(v) => v.hash_stable(hcx, hasher),
-            Error(e) => e.hash_stable(hcx, hasher),
-            Expr(e) => e.hash_stable(hcx, hasher),
-        }
-    }
-}
-
 impl<I: Interner> PartialEq for ConstKind<I> {
     fn eq(&self, other: &Self) -> bool {
         match (self, other) {
@@ -123,7 +82,7 @@ impl<I: Interner> DebugWithInfcx<I> for ConstKind<I> {
         match this.data {
             Param(param) => write!(f, "{param:?}"),
             Infer(var) => write!(f, "{:?}", &this.wrap(var)),
-            Bound(debruijn, var) => crate::debug_bound_var(f, *debruijn, var.clone()),
+            Bound(debruijn, var) => crate::debug_bound_var(f, *debruijn, var),
             Placeholder(placeholder) => write!(f, "{placeholder:?}"),
             Unevaluated(uv) => {
                 write!(f, "{:?}", &this.wrap(uv))
@@ -134,3 +93,81 @@ impl<I: Interner> DebugWithInfcx<I> for ConstKind<I> {
         }
     }
 }
+
+rustc_index::newtype_index! {
+    /// A **`const`** **v**ariable **ID**.
+    #[encodable]
+    #[orderable]
+    #[debug_format = "?{}c"]
+    #[gate_rustc_only]
+    pub struct ConstVid {}
+}
+
+rustc_index::newtype_index! {
+    /// An **effect** **v**ariable **ID**.
+    ///
+    /// Handling effect infer variables happens separately from const infer variables
+    /// because we do not want to reuse any of the const infer machinery. If we try to
+    /// relate an effect variable with a normal one, we would ICE, which can catch bugs
+    /// where we are not correctly using the effect var for an effect param. Fallback
+    /// is also implemented on top of having separate effect and normal const variables.
+    #[encodable]
+    #[orderable]
+    #[debug_format = "?{}e"]
+    #[gate_rustc_only]
+    pub struct EffectVid {}
+}
+
+/// An inference variable for a const, for use in const generics.
+#[derive(Copy, Clone, Eq, PartialEq, PartialOrd, Ord, Hash)]
+#[cfg_attr(feature = "nightly", derive(TyEncodable, TyDecodable))]
+pub enum InferConst {
+    /// Infer the value of the const.
+    Var(ConstVid),
+    /// Infer the value of the effect.
+    ///
+    /// For why this is separate from the `Var` variant above, see the
+    /// documentation on `EffectVid`.
+    EffectVar(EffectVid),
+    /// A fresh const variable. See `infer::freshen` for more details.
+    Fresh(u32),
+}
+
+impl fmt::Debug for InferConst {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            InferConst::Var(var) => write!(f, "{var:?}"),
+            InferConst::EffectVar(var) => write!(f, "{var:?}"),
+            InferConst::Fresh(var) => write!(f, "Fresh({var:?})"),
+        }
+    }
+}
+impl<I: Interner> DebugWithInfcx<I> for InferConst {
+    fn fmt<Infcx: InferCtxtLike<Interner = I>>(
+        this: WithInfcx<'_, Infcx, &Self>,
+        f: &mut core::fmt::Formatter<'_>,
+    ) -> core::fmt::Result {
+        match *this.data {
+            InferConst::Var(vid) => match this.infcx.universe_of_ct(vid) {
+                None => write!(f, "{:?}", this.data),
+                Some(universe) => write!(f, "?{}_{}c", vid.index(), universe.index()),
+            },
+            InferConst::EffectVar(vid) => write!(f, "?{}e", vid.index()),
+            InferConst::Fresh(_) => {
+                unreachable!()
+            }
+        }
+    }
+}
+
+#[cfg(feature = "nightly")]
+impl<CTX> HashStable<CTX> for InferConst {
+    fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) {
+        match self {
+            InferConst::Var(_) | InferConst::EffectVar(_) => {
+                panic!("const variables should not be hashed: {self:?}")
+            }
+            InferConst::Fresh(i) => i.hash_stable(hcx, hasher),
+        }
+    }
+}
diff --git a/compiler/rustc_type_ir/src/debug.rs b/compiler/rustc_type_ir/src/debug.rs
index 4ea3eb3e8..9c8e45b43 100644
--- a/compiler/rustc_type_ir/src/debug.rs
+++ b/compiler/rustc_type_ir/src/debug.rs
@@ -1,36 +1,46 @@
-use crate::{Interner, UniverseIndex};
+use crate::{ConstVid, InferCtxtLike, Interner, TyVid, UniverseIndex};
 
 use core::fmt;
 use std::marker::PhantomData;
 
-pub trait InferCtxtLike {
-    type Interner: Interner;
+pub struct NoInfcx<I>(PhantomData<I>);
 
-    fn universe_of_ty(&self, ty: <Self::Interner as Interner>::InferTy) -> Option<UniverseIndex>;
+impl<I: Interner> InferCtxtLike for NoInfcx<I> {
+    type Interner = I;
 
-    fn universe_of_lt(
-        &self,
-        lt: <Self::Interner as Interner>::InferRegion,
-    ) -> Option<UniverseIndex>;
+    fn interner(&self) -> Self::Interner {
+        unreachable!()
+    }
 
-    fn universe_of_ct(&self, ct: <Self::Interner as Interner>::InferConst)
-    -> Option<UniverseIndex>;
-}
+    fn universe_of_ty(&self, _ty: TyVid) -> Option<UniverseIndex> {
+        None
+    }
 
-pub struct NoInfcx<I>(PhantomData<I>);
+    fn universe_of_lt(&self, _lt: I::InferRegion) -> Option<UniverseIndex> {
+        None
+    }
 
-impl<I: Interner> InferCtxtLike for NoInfcx<I> {
-    type Interner = I;
+    fn universe_of_ct(&self, _ct: ConstVid) -> Option<UniverseIndex> {
+        None
+    }
 
-    fn universe_of_ty(&self, _ty: <I as Interner>::InferTy) -> Option<UniverseIndex> {
+    fn root_ty_var(&self, vid: TyVid) -> TyVid {
+        vid
+    }
+
+    fn probe_ty_var(&self, _vid: TyVid) -> Option<I::Ty> {
         None
     }
 
-    fn universe_of_ct(&self, _ct: <I as Interner>::InferConst) -> Option<UniverseIndex> {
+    fn opportunistic_resolve_lt_var(&self, _vid: I::InferRegion) -> Option<I::Region> {
         None
     }
 
-    fn universe_of_lt(&self, _lt: <I as Interner>::InferRegion) -> Option<UniverseIndex> {
+    fn root_ct_var(&self, vid: ConstVid) -> ConstVid {
+        vid
+    }
+
+    fn probe_ct_var(&self, _vid: ConstVid) -> Option<I::Const> {
         None
     }
 }
diff --git a/compiler/rustc_type_ir/src/flags.rs b/compiler/rustc_type_ir/src/flags.rs
index 8472a0845..af741a0a3 100644
--- a/compiler/rustc_type_ir/src/flags.rs
+++ b/compiler/rustc_type_ir/src/flags.rs
@@ -8,7 +8,7 @@ bitflags! {
         // required.
         /// Does this have `Param`?
         const HAS_TY_PARAM                = 1 << 0;
-        /// Does this have `ReEarlyBound`?
+        /// Does this have `ReEarlyParam`?
         const HAS_RE_PARAM                = 1 << 1;
         /// Does this have `ConstKind::Param`?
         const HAS_CT_PARAM                = 1 << 2;
@@ -85,20 +85,20 @@ bitflags! {
         const HAS_ERROR                   = 1 << 14;
 
         /// Does this have any region that "appears free" in the type?
-        /// Basically anything but `ReLateBound` and `ReErased`.
+        /// Basically anything but `ReBound` and `ReErased`.
         const HAS_FREE_REGIONS            = 1 << 15;
 
-        /// Does this have any `ReLateBound` regions?
-        const HAS_RE_LATE_BOUND           = 1 << 16;
+        /// Does this have any `ReBound` regions?
+        const HAS_RE_BOUND                = 1 << 16;
         /// Does this have any `Bound` types?
-        const HAS_TY_LATE_BOUND           = 1 << 17;
+        const HAS_TY_BOUND                = 1 << 17;
         /// Does this have any `ConstKind::Bound` consts?
-        const HAS_CT_LATE_BOUND           = 1 << 18;
+        const HAS_CT_BOUND                = 1 << 18;
         /// Does this have any bound variables?
         /// Used to check if a global bound is safe to evaluate.
-        const HAS_LATE_BOUND              = TypeFlags::HAS_RE_LATE_BOUND.bits
-                                          | TypeFlags::HAS_TY_LATE_BOUND.bits
-                                          | TypeFlags::HAS_CT_LATE_BOUND.bits;
+        const HAS_BOUND_VARS              = TypeFlags::HAS_RE_BOUND.bits
+                                          | TypeFlags::HAS_TY_BOUND.bits
+                                          | TypeFlags::HAS_CT_BOUND.bits;
 
         /// Does this have any `ReErased` regions?
         const HAS_RE_ERASED               = 1 << 19;
diff --git a/compiler/rustc_type_ir/src/fold.rs b/compiler/rustc_type_ir/src/fold.rs
index fc56400df..8d4025883 100644
--- a/compiler/rustc_type_ir/src/fold.rs
+++ b/compiler/rustc_type_ir/src/fold.rs
@@ -45,12 +45,18 @@
 //! - u.fold_with(folder)
 //! ```
 
-use rustc_data_structures::sync::Lrc;
 use rustc_index::{Idx, IndexVec};
 use std::mem;
 
+use crate::Lrc;
 use crate::{visit::TypeVisitable, Interner};
 
+#[cfg(feature = "nightly")]
+type Never = !;
+
+#[cfg(not(feature = "nightly"))]
+type Never = std::convert::Infallible;
+
 /// This trait is implemented for every type that can be folded,
 /// providing the skeleton of the traversal.
 ///
@@ -79,7 +85,10 @@ pub trait TypeFoldable<I: Interner>: TypeVisitable<I> {
     /// folders. Do not override this method, to ensure coherence with
     /// `try_fold_with`.
     fn fold_with<F: TypeFolder<I>>(self, folder: &mut F) -> Self {
-        self.try_fold_with(folder).into_ok()
+        match self.try_fold_with(folder) {
+            Ok(t) => t,
+            Err(e) => match e {},
+        }
     }
 }
 
@@ -100,7 +109,10 @@ pub trait TypeSuperFoldable<I: Interner>: TypeFoldable<I> {
     /// infallible folders. Do not override this method, to ensure coherence
     /// with `try_super_fold_with`.
     fn super_fold_with<F: TypeFolder<I>>(self, folder: &mut F) -> Self {
-        self.try_super_fold_with(folder).into_ok()
+        match self.try_super_fold_with(folder) {
+            Ok(t) => t,
+            Err(e) => match e {},
+        }
     }
 }
 
@@ -113,7 +125,7 @@ pub trait TypeSuperFoldable<I: Interner>: TypeFoldable<I> {
 /// A blanket implementation of [`FallibleTypeFolder`] will defer to
 /// the infallible methods of this trait to ensure that the two APIs
 /// are coherent.
-pub trait TypeFolder<I: Interner>: FallibleTypeFolder<I, Error = !> {
+pub trait TypeFolder<I: Interner>: FallibleTypeFolder<I, Error = Never> {
     fn interner(&self) -> I;
 
     fn fold_binder<T>(&mut self, t: I::Binder<T>) -> I::Binder<T>
@@ -208,13 +220,13 @@ impl<I: Interner, F> FallibleTypeFolder<I> for F
 where
     F: TypeFolder<I>,
 {
-    type Error = !;
+    type Error = Never;
 
     fn interner(&self) -> I {
         TypeFolder::interner(self)
     }
 
-    fn try_fold_binder<T>(&mut self, t: I::Binder<T>) -> Result<I::Binder<T>, !>
+    fn try_fold_binder<T>(&mut self, t: I::Binder<T>) -> Result<I::Binder<T>, Never>
     where
         T: TypeFoldable<I>,
         I::Binder<T>: TypeSuperFoldable<I>,
@@ -222,25 +234,25 @@ where
         Ok(self.fold_binder(t))
     }
 
-    fn try_fold_ty(&mut self, t: I::Ty) -> Result<I::Ty, !>
+    fn try_fold_ty(&mut self, t: I::Ty) -> Result<I::Ty, Never>
     where
         I::Ty: TypeSuperFoldable<I>,
     {
         Ok(self.fold_ty(t))
     }
 
-    fn try_fold_region(&mut self, r: I::Region) -> Result<I::Region, !> {
+    fn try_fold_region(&mut self, r: I::Region) -> Result<I::Region, Never> {
         Ok(self.fold_region(r))
     }
 
-    fn try_fold_const(&mut self, c: I::Const) -> Result<I::Const, !>
+    fn try_fold_const(&mut self, c: I::Const) -> Result<I::Const, Never>
     where
         I::Const: TypeSuperFoldable<I>,
     {
         Ok(self.fold_const(c))
     }
 
-    fn try_fold_predicate(&mut self, p: I::Predicate) -> Result<I::Predicate, !>
+    fn try_fold_predicate(&mut self, p: I::Predicate) -> Result<I::Predicate, Never>
     where
         I::Predicate: TypeSuperFoldable<I>,
     {
@@ -311,7 +323,7 @@ impl<I: Interner, T: TypeFoldable<I>> TypeFoldable<I> for Lrc<T> {
             // Call to `Lrc::make_mut` above guarantees that `unique` is the
             // sole reference to the contained value, so we can avoid doing
             // a checked `get_mut` here.
-            let slot = Lrc::get_mut_unchecked(&mut unique);
+            let slot = Lrc::get_mut(&mut unique).unwrap_unchecked();
 
             // Semantically move the contained type out from `unique`, fold
             // it, then move the folded value back into `unique`. Should
diff --git a/compiler/rustc_type_ir/src/infcx.rs b/compiler/rustc_type_ir/src/infcx.rs
new file mode 100644
index 000000000..28b71f0ea
--- /dev/null
+++ b/compiler/rustc_type_ir/src/infcx.rs
@@ -0,0 +1,40 @@
+use crate::{ConstVid, Interner, TyVid, UniverseIndex};
+
+pub trait InferCtxtLike {
+    type Interner: Interner;
+
+    fn interner(&self) -> Self::Interner;
+
+    fn universe_of_ty(&self, ty: TyVid) -> Option<UniverseIndex>;
+
+    /// Resolve `TyVid` to its root `TyVid`.
+    fn root_ty_var(&self, vid: TyVid) -> TyVid;
+
+    /// Resolve `TyVid` to its inferred type, if it has been equated with a non-infer type.
+    fn probe_ty_var(&self, vid: TyVid) -> Option<<Self::Interner as Interner>::Ty>;
+
+    fn universe_of_lt(
+        &self,
+        lt: <Self::Interner as Interner>::InferRegion,
+    ) -> Option<UniverseIndex>;
+
+    /// Resolve `InferRegion` to its inferred region, if it has been equated with
+    /// a non-infer region.
+    ///
+    /// FIXME: This has slightly different semantics than `{probe,resolve}_{ty,ct}_var`,
+    /// that has to do with the fact unlike `Ty` or `Const` vars, in rustc, we may
+    /// not always be able to *name* the root region var from the universe of the
+    /// var we're trying to resolve. That's why it's called *opportunistic*.
+    fn opportunistic_resolve_lt_var(
+        &self,
+        vid: <Self::Interner as Interner>::InferRegion,
+    ) -> Option<<Self::Interner as Interner>::Region>;
+
+    fn universe_of_ct(&self, ct: ConstVid) -> Option<UniverseIndex>;
+
+    /// Resolve `ConstVid` to its root `ConstVid`.
+    fn root_ct_var(&self, vid: ConstVid) -> ConstVid;
+
+    /// Resolve `ConstVid` to its inferred type, if it has been equated with a non-infer type.
+    fn probe_ct_var(&self, vid: ConstVid) -> Option<<Self::Interner as Interner>::Const>;
+}
diff --git a/compiler/rustc_type_ir/src/interner.rs b/compiler/rustc_type_ir/src/interner.rs
index 7f75e5b35..188910ecc 100644
--- a/compiler/rustc_type_ir/src/interner.rs
+++ b/compiler/rustc_type_ir/src/interner.rs
@@ -2,68 +2,110 @@ use smallvec::SmallVec;
 use std::fmt::Debug;
 use std::hash::Hash;
 
-use crate::{DebugWithInfcx, Mutability};
+use crate::{
+    BoundVar, CanonicalVarInfo, ConstKind, DebruijnIndex, DebugWithInfcx, RegionKind, TyKind,
+    UniverseIndex,
+};
 
 pub trait Interner: Sized {
-    type DefId: Clone + Debug + Hash + Ord;
-    type AdtDef: Clone + Debug + Hash + Ord;
+    type DefId: Copy + Debug + Hash + Ord;
+    type AdtDef: Copy + Debug + Hash + Ord;
 
-    type GenericArgs: Clone
+    type GenericArgs: Copy
         + DebugWithInfcx<Self>
         + Hash
         + Ord
         + IntoIterator<Item = Self::GenericArg>;
-    type GenericArg: Clone + DebugWithInfcx<Self> + Hash + Ord;
-    type Term: Clone + Debug + Hash + Ord;
+    type GenericArg: Copy + DebugWithInfcx<Self> + Hash + Ord;
+    type Term: Copy + Debug + Hash + Ord;
 
     type Binder<T>;
-    type TypeAndMut: Clone + Debug + Hash + Ord;
-    type CanonicalVars: Clone + Debug + Hash + Eq;
+    type CanonicalVars: Copy + Debug + Hash + Eq + IntoIterator<Item = CanonicalVarInfo<Self>>;
 
     // Kinds of tys
-    type Ty: Clone + DebugWithInfcx<Self> + Hash + Ord;
-    type Tys: Clone + Debug + Hash + Ord + IntoIterator<Item = Self::Ty>;
-    type AliasTy: Clone + DebugWithInfcx<Self> + Hash + Ord;
-    type ParamTy: Clone + Debug + Hash + Ord;
-    type BoundTy: Clone + Debug + Hash + Ord;
-    type PlaceholderTy: Clone + Debug + Hash + Ord;
-    type InferTy: Clone + DebugWithInfcx<Self> + Hash + Ord;
+    type Ty: Copy
+        + DebugWithInfcx<Self>
+        + Hash
+        + Ord
+        + Into<Self::GenericArg>
+        + IntoKind<Kind = TyKind<Self>>;
+    type Tys: Copy + Debug + Hash + Ord + IntoIterator<Item = Self::Ty>;
+    type AliasTy: Copy + DebugWithInfcx<Self> + Hash + Ord;
+    type ParamTy: Copy + Debug + Hash + Ord;
+    type BoundTy: Copy + Debug + Hash + Ord;
+    type PlaceholderTy: Copy + Debug + Hash + Ord + PlaceholderLike;
 
     // Things stored inside of tys
-    type ErrorGuaranteed: Clone + Debug + Hash + Ord;
-    type BoundExistentialPredicates: Clone + DebugWithInfcx<Self> + Hash + Ord;
-    type PolyFnSig: Clone + DebugWithInfcx<Self> + Hash + Ord;
-    type AllocId: Clone + Debug + Hash + Ord;
+    type ErrorGuaranteed: Copy + Debug + Hash + Ord;
+    type BoundExistentialPredicates: Copy + DebugWithInfcx<Self> + Hash + Ord;
+    type PolyFnSig: Copy + DebugWithInfcx<Self> + Hash + Ord;
+    type AllocId: Copy + Debug + Hash + Ord;
 
     // Kinds of consts
-    type Const: Clone + DebugWithInfcx<Self> + Hash + Ord;
-    type InferConst: Clone + DebugWithInfcx<Self> + Hash + Ord;
-    type AliasConst: Clone + DebugWithInfcx<Self> + Hash + Ord;
-    type PlaceholderConst: Clone + Debug + Hash + Ord;
-    type ParamConst: Clone + Debug + Hash + Ord;
-    type BoundConst: Clone + Debug + Hash + Ord;
-    type ValueConst: Clone + Debug + Hash + Ord;
-    type ExprConst: Clone + DebugWithInfcx<Self> + Hash + Ord;
+    type Const: Copy
+        + DebugWithInfcx<Self>
+        + Hash
+        + Ord
+        + Into<Self::GenericArg>
+        + IntoKind<Kind = ConstKind<Self>>
+        + ConstTy<Self>;
+    type AliasConst: Copy + DebugWithInfcx<Self> + Hash + Ord;
+    type PlaceholderConst: Copy + Debug + Hash + Ord + PlaceholderLike;
+    type ParamConst: Copy + Debug + Hash + Ord;
+    type BoundConst: Copy + Debug + Hash + Ord;
+    type ValueConst: Copy + Debug + Hash + Ord;
+    type ExprConst: Copy + DebugWithInfcx<Self> + Hash + Ord;
 
     // Kinds of regions
-    type Region: Clone + DebugWithInfcx<Self> + Hash + Ord;
-    type EarlyBoundRegion: Clone + Debug + Hash + Ord;
-    type BoundRegion: Clone + Debug + Hash + Ord;
-    type FreeRegion: Clone + Debug + Hash + Ord;
-    type InferRegion: Clone + DebugWithInfcx<Self> + Hash + Ord;
-    type PlaceholderRegion: Clone + Debug + Hash + Ord;
+    type Region: Copy
+        + DebugWithInfcx<Self>
+        + Hash
+        + Ord
+        + Into<Self::GenericArg>
+        + IntoKind<Kind = RegionKind<Self>>;
+    type EarlyParamRegion: Copy + Debug + Hash + Ord;
+    type LateParamRegion: Copy + Debug + Hash + Ord;
+    type BoundRegion: Copy + Debug + Hash + Ord;
+    type InferRegion: Copy + DebugWithInfcx<Self> + Hash + Ord;
+    type PlaceholderRegion: Copy + Debug + Hash + Ord + PlaceholderLike;
 
     // Predicates
-    type Predicate: Clone + Debug + Hash + Eq;
-    type TraitPredicate: Clone + Debug + Hash + Eq;
-    type RegionOutlivesPredicate: Clone + Debug + Hash + Eq;
-    type TypeOutlivesPredicate: Clone + Debug + Hash + Eq;
-    type ProjectionPredicate: Clone + Debug + Hash + Eq;
-    type SubtypePredicate: Clone + Debug + Hash + Eq;
-    type CoercePredicate: Clone + Debug + Hash + Eq;
-    type ClosureKind: Clone + Debug + Hash + Eq;
-
-    fn ty_and_mut_to_parts(ty_and_mut: Self::TypeAndMut) -> (Self::Ty, Mutability);
+    type Predicate: Copy + Debug + Hash + Eq;
+    type TraitPredicate: Copy + Debug + Hash + Eq;
+    type RegionOutlivesPredicate: Copy + Debug + Hash + Eq;
+    type TypeOutlivesPredicate: Copy + Debug + Hash + Eq;
+    type ProjectionPredicate: Copy + Debug + Hash + Eq;
+    type NormalizesTo: Copy + Debug + Hash + Eq;
+    type SubtypePredicate: Copy + Debug + Hash + Eq;
+    type CoercePredicate: Copy + Debug + Hash + Eq;
+    type ClosureKind: Copy + Debug + Hash + Eq;
+
+    fn mk_canonical_var_infos(self, infos: &[CanonicalVarInfo<Self>]) -> Self::CanonicalVars;
+
+    // FIXME: We should not have all these constructors on `Interner`, but as functions on some trait.
+    fn mk_bound_ty(self, debruijn: DebruijnIndex, var: BoundVar) -> Self::Ty;
+    fn mk_bound_region(self, debruijn: DebruijnIndex, var: BoundVar) -> Self::Region;
+    fn mk_bound_const(self, debruijn: DebruijnIndex, var: BoundVar, ty: Self::Ty) -> Self::Const;
+}
+
+/// Common capabilities of placeholder kinds
+pub trait PlaceholderLike {
+    fn universe(self) -> UniverseIndex;
+    fn var(self) -> BoundVar;
+
+    fn with_updated_universe(self, ui: UniverseIndex) -> Self;
+
+    fn new(ui: UniverseIndex, var: BoundVar) -> Self;
+}
+
+pub trait IntoKind {
+    type Kind;
+
+    fn kind(self) -> Self::Kind;
+}
+
+pub trait ConstTy<I: Interner> {
+    fn ty(self) -> I::Ty;
 }
 
 /// Imagine you have a function `F: FnOnce(&[T]) -> R`, plus an iterator `iter`
diff --git a/compiler/rustc_type_ir/src/lib.rs b/compiler/rustc_type_ir/src/lib.rs
index e8785fff2..bff938596 100644
--- a/compiler/rustc_type_ir/src/lib.rs
+++ b/compiler/rustc_type_ir/src/lib.rs
@@ -1,25 +1,29 @@
-#![feature(associated_type_defaults)]
-#![feature(fmt_helpers_for_derive)]
-#![feature(get_mut_unchecked)]
-#![feature(min_specialization)]
-#![feature(never_type)]
-#![feature(new_uninit)]
-#![feature(rustc_attrs)]
-#![feature(unwrap_infallible)]
+#![cfg_attr(
+    feature = "nightly",
+    feature(associated_type_defaults, min_specialization, never_type, rustc_attrs)
+)]
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
-#![allow(internal_features)]
+#![allow(rustc::usage_of_ty_tykind)]
+#![cfg_attr(feature = "nightly", allow(internal_features))]
 
+#[cfg(feature = "nightly")]
 extern crate self as rustc_type_ir;
 
 #[macro_use]
 extern crate bitflags;
+#[cfg(feature = "nightly")]
 #[macro_use]
 extern crate rustc_macros;
 
+#[cfg(feature = "nightly")]
+use rustc_data_structures::sync::Lrc;
 use std::fmt;
 use std::hash::Hash;
+#[cfg(not(feature = "nightly"))]
+use std::sync::Arc as Lrc;
 
+#[cfg(feature = "nightly")]
 pub mod codec;
 pub mod fold;
 pub mod ty_info;
@@ -32,23 +36,28 @@ mod canonical;
 mod const_kind;
 mod debug;
 mod flags;
+mod infcx;
 mod interner;
 mod predicate_kind;
 mod region_kind;
 
 pub use canonical::*;
+#[cfg(feature = "nightly")]
 pub use codec::*;
 pub use const_kind::*;
-pub use debug::{DebugWithInfcx, InferCtxtLike, WithInfcx};
+pub use debug::{DebugWithInfcx, WithInfcx};
 pub use flags::*;
+pub use infcx::InferCtxtLike;
 pub use interner::*;
 pub use predicate_kind::*;
 pub use region_kind::*;
 pub use ty_info::*;
 pub use ty_kind::*;
-
-/// Needed so we can use #[derive(HashStable_Generic)]
-pub trait HashStableContext {}
+pub use AliasKind::*;
+pub use DynKind::*;
+pub use InferTy::*;
+pub use RegionKind::*;
+pub use TyKind::*;
 
 rustc_index::newtype_index! {
     /// A [De Bruijn index][dbi] is a standard means of representing
@@ -90,8 +99,11 @@ rustc_index::newtype_index! {
     /// is the outer fn.
     ///
     /// [dbi]: https://en.wikipedia.org/wiki/De_Bruijn_index
-    #[derive(HashStable_Generic)]
+    #[cfg_attr(feature = "nightly", derive(HashStable_NoContext))]
+    #[encodable]
+    #[orderable]
     #[debug_format = "DebruijnIndex({})"]
+    #[gate_rustc_only]
     pub struct DebruijnIndex {
         const INNERMOST = 0;
     }
@@ -173,8 +185,9 @@ pub fn debug_bound_var<T: std::fmt::Write>(
     }
 }
 
-#[derive(Copy, Clone, PartialEq, Eq, Decodable, Encodable, Hash, HashStable_Generic)]
-#[rustc_pass_by_value]
+#[derive(Copy, Clone, PartialEq, Eq)]
+#[cfg_attr(feature = "nightly", derive(Decodable, Encodable, Hash, HashStable_NoContext))]
+#[cfg_attr(feature = "nightly", rustc_pass_by_value)]
 pub enum Variance {
     Covariant,     // T<A> <: T<B> iff A <: B -- e.g., function return type
     Invariant,     // T<A> <: T<B> iff B == A -- e.g., type of mutable cell
@@ -289,8 +302,11 @@ rustc_index::newtype_index! {
     /// declared, but a type name in a non-zero universe is a placeholder
     /// type -- an idealized representative of "types in general" that we
     /// use for checking generic functions.
-    #[derive(HashStable_Generic)]
+    #[cfg_attr(feature = "nightly", derive(HashStable_NoContext))]
+    #[encodable]
+    #[orderable]
     #[debug_format = "U{}"]
+    #[gate_rustc_only]
     pub struct UniverseIndex {}
 }
 
@@ -328,3 +344,60 @@ impl UniverseIndex {
         self.private < other.private
     }
 }
+
+impl Default for UniverseIndex {
+    fn default() -> Self {
+        Self::ROOT
+    }
+}
+
+rustc_index::newtype_index! {
+    #[cfg_attr(feature = "nightly", derive(HashStable_NoContext))]
+    #[encodable]
+    #[orderable]
+    #[debug_format = "{}"]
+    #[gate_rustc_only]
+    pub struct BoundVar {}
+}
+
+/// Represents the various closure traits in the language. This
+/// will determine the type of the environment (`self`, in the
+/// desugaring) argument that the closure expects.
+///
+/// You can get the environment type of a closure using
+/// `tcx.closure_env_ty()`.
+#[derive(Clone, Copy, PartialOrd, Ord, PartialEq, Eq, Hash, Debug)]
+#[cfg_attr(feature = "nightly", derive(Encodable, Decodable, HashStable_NoContext))]
+pub enum ClosureKind {
+    // Warning: Ordering is significant here! The ordering is chosen
+    // because the trait Fn is a subtrait of FnMut and so in turn, and
+    // hence we order it so that Fn < FnMut < FnOnce.
+    Fn,
+    FnMut,
+    FnOnce,
+}
+
+impl ClosureKind {
+    /// This is the initial value used when doing upvar inference.
+    pub const LATTICE_BOTTOM: ClosureKind = ClosureKind::Fn;
+
+    pub const fn as_str(self) -> &'static str {
+        match self {
+            ClosureKind::Fn => "Fn",
+            ClosureKind::FnMut => "FnMut",
+            ClosureKind::FnOnce => "FnOnce",
+        }
+    }
+
+    /// Returns `true` if a type that impls this closure kind
+    /// must also implement `other`.
+    pub fn extends(self, other: ClosureKind) -> bool {
+        self <= other
+    }
+}
+
+impl fmt::Display for ClosureKind {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        self.as_str().fmt(f)
+    }
+}
diff --git a/compiler/rustc_type_ir/src/macros.rs b/compiler/rustc_type_ir/src/macros.rs
index cfed84a35..82bb1bf29 100644
--- a/compiler/rustc_type_ir/src/macros.rs
+++ b/compiler/rustc_type_ir/src/macros.rs
@@ -51,4 +51,6 @@ TrivialTypeTraversalImpls! {
     crate::DebruijnIndex,
     crate::AliasRelationDirection,
     crate::UniverseIndex,
+    crate::Mutability,
+    crate::Movability,
 }
diff --git a/compiler/rustc_type_ir/src/predicate_kind.rs b/compiler/rustc_type_ir/src/predicate_kind.rs
index 48662d426..8c4d0fda6 100644
--- a/compiler/rustc_type_ir/src/predicate_kind.rs
+++ b/compiler/rustc_type_ir/src/predicate_kind.rs
@@ -1,16 +1,15 @@
-use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use std::fmt;
 use std::ops::ControlFlow;
 
 use crate::fold::{FallibleTypeFolder, TypeFoldable};
 use crate::visit::{TypeVisitable, TypeVisitor};
-use crate::{HashStableContext, Interner};
+use crate::Interner;
 
 /// A clause is something that can appear in where bounds or be inferred
 /// by implied bounds.
 #[derive(derivative::Derivative)]
-#[derivative(Clone(bound = ""), Hash(bound = ""))]
-#[derive(TyEncodable, TyDecodable)]
+#[derivative(Clone(bound = ""), Copy(bound = ""), Hash(bound = ""))]
+#[cfg_attr(feature = "nightly", derive(TyEncodable, TyDecodable, HashStable_NoContext))]
 pub enum ClauseKind<I: Interner> {
     /// Corresponds to `where Foo: Bar<A, B, C>`. `Foo` here would be
     /// the `Self` type of the trait reference and `A`, `B`, and `C`
@@ -38,18 +37,6 @@ pub enum ClauseKind<I: Interner> {
     ConstEvaluatable(I::Const),
 }
 
-impl<I: Interner> Copy for ClauseKind<I>
-where
-    I::Ty: Copy,
-    I::Const: Copy,
-    I::GenericArg: Copy,
-    I::TraitPredicate: Copy,
-    I::ProjectionPredicate: Copy,
-    I::TypeOutlivesPredicate: Copy,
-    I::RegionOutlivesPredicate: Copy,
-{
-}
-
 impl<I: Interner> PartialEq for ClauseKind<I> {
     fn eq(&self, other: &Self) -> bool {
         match (self, other) {
@@ -67,45 +54,6 @@ impl<I: Interner> PartialEq for ClauseKind<I> {
 
 impl<I: Interner> Eq for ClauseKind<I> {}
 
-fn clause_kind_discriminant<I: Interner>(value: &ClauseKind<I>) -> usize {
-    match value {
-        ClauseKind::Trait(_) => 0,
-        ClauseKind::RegionOutlives(_) => 1,
-        ClauseKind::TypeOutlives(_) => 2,
-        ClauseKind::Projection(_) => 3,
-        ClauseKind::ConstArgHasType(_, _) => 4,
-        ClauseKind::WellFormed(_) => 5,
-        ClauseKind::ConstEvaluatable(_) => 6,
-    }
-}
-
-impl<CTX: HashStableContext, I: Interner> HashStable<CTX> for ClauseKind<I>
-where
-    I::Ty: HashStable<CTX>,
-    I::Const: HashStable<CTX>,
-    I::GenericArg: HashStable<CTX>,
-    I::TraitPredicate: HashStable<CTX>,
-    I::ProjectionPredicate: HashStable<CTX>,
-    I::TypeOutlivesPredicate: HashStable<CTX>,
-    I::RegionOutlivesPredicate: HashStable<CTX>,
-{
-    fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) {
-        clause_kind_discriminant(self).hash_stable(hcx, hasher);
-        match self {
-            ClauseKind::Trait(p) => p.hash_stable(hcx, hasher),
-            ClauseKind::RegionOutlives(p) => p.hash_stable(hcx, hasher),
-            ClauseKind::TypeOutlives(p) => p.hash_stable(hcx, hasher),
-            ClauseKind::Projection(p) => p.hash_stable(hcx, hasher),
-            ClauseKind::ConstArgHasType(c, t) => {
-                c.hash_stable(hcx, hasher);
-                t.hash_stable(hcx, hasher);
-            }
-            ClauseKind::WellFormed(t) => t.hash_stable(hcx, hasher),
-            ClauseKind::ConstEvaluatable(c) => c.hash_stable(hcx, hasher),
-        }
-    }
-}
-
 impl<I: Interner> TypeFoldable<I> for ClauseKind<I>
 where
     I::Ty: TypeFoldable<I>,
@@ -160,8 +108,14 @@ where
 }
 
 #[derive(derivative::Derivative)]
-#[derivative(Clone(bound = ""), Hash(bound = ""))]
-#[derive(TyEncodable, TyDecodable)]
+#[derivative(
+    Clone(bound = ""),
+    Copy(bound = ""),
+    Hash(bound = ""),
+    PartialEq(bound = ""),
+    Eq(bound = "")
+)]
+#[cfg_attr(feature = "nightly", derive(TyEncodable, TyDecodable, HashStable_NoContext))]
 pub enum PredicateKind<I: Interner> {
     /// Prove a clause
     Clause(ClauseKind<I>),
@@ -169,11 +123,6 @@ pub enum PredicateKind<I: Interner> {
     /// Trait must be object-safe.
     ObjectSafe(I::DefId),
 
-    /// No direct syntax. May be thought of as `where T: FnFoo<...>`
-    /// for some generic args `...` and `T` being a closure type.
-    /// Satisfied (or refuted) once we know the closure's kind.
-    ClosureKind(I::DefId, I::GenericArgs, I::ClosureKind),
-
     /// `T1 <: T2`
     ///
     /// This obligation is created most often when we have two
@@ -198,6 +147,15 @@ pub enum PredicateKind<I: Interner> {
     /// Used for coherence to mark opaque types as possibly equal to each other but ambiguous.
     Ambiguous,
 
+    /// The alias normalizes to `term`. Unlike `Projection`, this always fails if the alias
+    /// cannot be normalized in the current context.
+    ///
+    /// `Projection(<T as Trait>::Assoc, ?x)` results in `?x == <T as Trait>::Assoc` while
+    /// `NormalizesTo(<T as Trait>::Assoc, ?x)` results in `NoSolution`.
+    ///
+    /// Only used in the new solver.
+    NormalizesTo(I::NormalizesTo),
+
     /// Separate from `ClauseKind::Projection` which is used for normalization in new solver.
     /// This predicate requires two terms to be equal to eachother.
     ///
@@ -205,90 +163,6 @@ pub enum PredicateKind<I: Interner> {
     AliasRelate(I::Term, I::Term, AliasRelationDirection),
 }
 
-impl<I: Interner> Copy for PredicateKind<I>
-where
-    I::DefId: Copy,
-    I::Const: Copy,
-    I::GenericArgs: Copy,
-    I::Term: Copy,
-    I::CoercePredicate: Copy,
-    I::SubtypePredicate: Copy,
-    I::ClosureKind: Copy,
-    ClauseKind<I>: Copy,
-{
-}
-
-impl<I: Interner> PartialEq for PredicateKind<I> {
-    fn eq(&self, other: &Self) -> bool {
-        match (self, other) {
-            (Self::Clause(l0), Self::Clause(r0)) => l0 == r0,
-            (Self::ObjectSafe(l0), Self::ObjectSafe(r0)) => l0 == r0,
-            (Self::ClosureKind(l0, l1, l2), Self::ClosureKind(r0, r1, r2)) => {
-                l0 == r0 && l1 == r1 && l2 == r2
-            }
-            (Self::Subtype(l0), Self::Subtype(r0)) => l0 == r0,
-            (Self::Coerce(l0), Self::Coerce(r0)) => l0 == r0,
-            (Self::ConstEquate(l0, l1), Self::ConstEquate(r0, r1)) => l0 == r0 && l1 == r1,
-            (Self::AliasRelate(l0, l1, l2), Self::AliasRelate(r0, r1, r2)) => {
-                l0 == r0 && l1 == r1 && l2 == r2
-            }
-            _ => core::mem::discriminant(self) == core::mem::discriminant(other),
-        }
-    }
-}
-
-impl<I: Interner> Eq for PredicateKind<I> {}
-
-fn predicate_kind_discriminant<I: Interner>(value: &PredicateKind<I>) -> usize {
-    match value {
-        PredicateKind::Clause(_) => 0,
-        PredicateKind::ObjectSafe(_) => 1,
-        PredicateKind::ClosureKind(_, _, _) => 2,
-        PredicateKind::Subtype(_) => 3,
-        PredicateKind::Coerce(_) => 4,
-        PredicateKind::ConstEquate(_, _) => 5,
-        PredicateKind::Ambiguous => 6,
-        PredicateKind::AliasRelate(_, _, _) => 7,
-    }
-}
-
-impl<CTX: HashStableContext, I: Interner> HashStable<CTX> for PredicateKind<I>
-where
-    I::DefId: HashStable<CTX>,
-    I::Const: HashStable<CTX>,
-    I::GenericArgs: HashStable<CTX>,
-    I::Term: HashStable<CTX>,
-    I::CoercePredicate: HashStable<CTX>,
-    I::SubtypePredicate: HashStable<CTX>,
-    I::ClosureKind: HashStable<CTX>,
-    ClauseKind<I>: HashStable<CTX>,
-{
-    fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) {
-        predicate_kind_discriminant(self).hash_stable(hcx, hasher);
-        match self {
-            PredicateKind::Clause(p) => p.hash_stable(hcx, hasher),
-            PredicateKind::ObjectSafe(d) => d.hash_stable(hcx, hasher),
-            PredicateKind::ClosureKind(d, g, k) => {
-                d.hash_stable(hcx, hasher);
-                g.hash_stable(hcx, hasher);
-                k.hash_stable(hcx, hasher);
-            }
-            PredicateKind::Subtype(p) => p.hash_stable(hcx, hasher),
-            PredicateKind::Coerce(p) => p.hash_stable(hcx, hasher),
-            PredicateKind::ConstEquate(c1, c2) => {
-                c1.hash_stable(hcx, hasher);
-                c2.hash_stable(hcx, hasher);
-            }
-            PredicateKind::Ambiguous => {}
-            PredicateKind::AliasRelate(t1, t2, r) => {
-                t1.hash_stable(hcx, hasher);
-                t2.hash_stable(hcx, hasher);
-                r.hash_stable(hcx, hasher);
-            }
-        }
-    }
-}
-
 impl<I: Interner> TypeFoldable<I> for PredicateKind<I>
 where
     I::DefId: TypeFoldable<I>,
@@ -297,24 +171,20 @@ where
     I::Term: TypeFoldable<I>,
     I::CoercePredicate: TypeFoldable<I>,
     I::SubtypePredicate: TypeFoldable<I>,
-    I::ClosureKind: TypeFoldable<I>,
+    I::NormalizesTo: TypeFoldable<I>,
     ClauseKind<I>: TypeFoldable<I>,
 {
     fn try_fold_with<F: FallibleTypeFolder<I>>(self, folder: &mut F) -> Result<Self, F::Error> {
         Ok(match self {
             PredicateKind::Clause(c) => PredicateKind::Clause(c.try_fold_with(folder)?),
             PredicateKind::ObjectSafe(d) => PredicateKind::ObjectSafe(d.try_fold_with(folder)?),
-            PredicateKind::ClosureKind(d, g, k) => PredicateKind::ClosureKind(
-                d.try_fold_with(folder)?,
-                g.try_fold_with(folder)?,
-                k.try_fold_with(folder)?,
-            ),
             PredicateKind::Subtype(s) => PredicateKind::Subtype(s.try_fold_with(folder)?),
             PredicateKind::Coerce(s) => PredicateKind::Coerce(s.try_fold_with(folder)?),
             PredicateKind::ConstEquate(a, b) => {
                 PredicateKind::ConstEquate(a.try_fold_with(folder)?, b.try_fold_with(folder)?)
             }
             PredicateKind::Ambiguous => PredicateKind::Ambiguous,
+            PredicateKind::NormalizesTo(p) => PredicateKind::NormalizesTo(p.try_fold_with(folder)?),
             PredicateKind::AliasRelate(a, b, d) => PredicateKind::AliasRelate(
                 a.try_fold_with(folder)?,
                 b.try_fold_with(folder)?,
@@ -332,18 +202,13 @@ where
     I::Term: TypeVisitable<I>,
     I::CoercePredicate: TypeVisitable<I>,
     I::SubtypePredicate: TypeVisitable<I>,
-    I::ClosureKind: TypeVisitable<I>,
+    I::NormalizesTo: TypeVisitable<I>,
     ClauseKind<I>: TypeVisitable<I>,
 {
     fn visit_with<V: TypeVisitor<I>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
         match self {
             PredicateKind::Clause(p) => p.visit_with(visitor),
             PredicateKind::ObjectSafe(d) => d.visit_with(visitor),
-            PredicateKind::ClosureKind(d, g, k) => {
-                d.visit_with(visitor)?;
-                g.visit_with(visitor)?;
-                k.visit_with(visitor)
-            }
             PredicateKind::Subtype(s) => s.visit_with(visitor),
             PredicateKind::Coerce(s) => s.visit_with(visitor),
             PredicateKind::ConstEquate(a, b) => {
@@ -351,6 +216,7 @@ where
                 b.visit_with(visitor)
             }
             PredicateKind::Ambiguous => ControlFlow::Continue(()),
+            PredicateKind::NormalizesTo(p) => p.visit_with(visitor),
             PredicateKind::AliasRelate(a, b, d) => {
                 a.visit_with(visitor)?;
                 b.visit_with(visitor)?;
@@ -361,7 +227,7 @@ where
 }
 
 #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Copy)]
-#[derive(HashStable_Generic, Encodable, Decodable)]
+#[cfg_attr(feature = "nightly", derive(HashStable_NoContext, Encodable, Decodable))]
 pub enum AliasRelationDirection {
     Equate,
     Subtype,
@@ -403,11 +269,9 @@ impl<I: Interner> fmt::Debug for PredicateKind<I> {
             PredicateKind::ObjectSafe(trait_def_id) => {
                 write!(f, "ObjectSafe({trait_def_id:?})")
             }
-            PredicateKind::ClosureKind(closure_def_id, closure_args, kind) => {
-                write!(f, "ClosureKind({closure_def_id:?}, {closure_args:?}, {kind:?})")
-            }
             PredicateKind::ConstEquate(c1, c2) => write!(f, "ConstEquate({c1:?}, {c2:?})"),
             PredicateKind::Ambiguous => write!(f, "Ambiguous"),
+            PredicateKind::NormalizesTo(p) => p.fmt(f),
             PredicateKind::AliasRelate(t1, t2, dir) => {
                 write!(f, "AliasRelate({t1:?}, {dir:?}, {t2:?})")
             }
diff --git a/compiler/rustc_type_ir/src/region_kind.rs b/compiler/rustc_type_ir/src/region_kind.rs
index 69ed5bada..3b5e41e8d 100644
--- a/compiler/rustc_type_ir/src/region_kind.rs
+++ b/compiler/rustc_type_ir/src/region_kind.rs
@@ -1,8 +1,8 @@
-use rustc_data_structures::stable_hasher::HashStable;
-use rustc_data_structures::stable_hasher::StableHasher;
+#[cfg(feature = "nightly")]
+use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use std::fmt;
 
-use crate::{DebruijnIndex, DebugWithInfcx, HashStableContext, InferCtxtLike, Interner, WithInfcx};
+use crate::{DebruijnIndex, DebugWithInfcx, InferCtxtLike, Interner, WithInfcx};
 
 use self::RegionKind::*;
 
@@ -22,8 +22,8 @@ use self::RegionKind::*;
 /// ```text
 /// static ----------+-----...------+       (greatest)
 /// |                |              |
-/// early-bound and  |              |
-/// free regions     |              |
+/// param regions    |              |
+/// |                |              |
 /// |                |              |
 /// |                |              |
 /// empty(root)   placeholder(U1)   |
@@ -88,8 +88,8 @@ use self::RegionKind::*;
 /// To do this, we replace the bound regions with placeholder markers,
 /// which don't satisfy any relation not explicitly provided.
 ///
-/// There are two kinds of placeholder regions in rustc: `ReFree` and
-/// `RePlaceholder`. When checking an item's body, `ReFree` is supposed
+/// There are two kinds of placeholder regions in rustc: `ReLateParam` and
+/// `RePlaceholder`. When checking an item's body, `ReLateParam` is supposed
 /// to be used. These also support explicit bounds: both the internally-stored
 /// *scope*, which the region is assumed to outlive, as well as other
 /// relations stored in the `FreeRegionMap`. Note that these relations
@@ -115,27 +115,44 @@ use self::RegionKind::*;
 #[derive(derivative::Derivative)]
 #[derivative(
     Clone(bound = ""),
+    Copy(bound = ""),
     PartialOrd(bound = ""),
     PartialOrd = "feature_allow_slow_enum",
     Ord(bound = ""),
     Ord = "feature_allow_slow_enum",
     Hash(bound = "")
 )]
-#[derive(TyEncodable, TyDecodable)]
+#[cfg_attr(feature = "nightly", derive(TyEncodable, TyDecodable))]
 pub enum RegionKind<I: Interner> {
-    /// Region bound in a type or fn declaration which will be
-    /// substituted 'early' -- that is, at the same time when type
-    /// parameters are substituted.
-    ReEarlyBound(I::EarlyBoundRegion),
+    /// A region parameter; for example `'a` in `impl<'a> Trait for &'a ()`.
+    ///
+    /// There are some important differences between region and type parameters.
+    /// Not all region parameters in the source are represented via `ReEarlyParam`:
+    /// late-bound function parameters are instead lowered to a `ReBound`. Late-bound
+    /// regions get eagerly replaced with `ReLateParam` which behaves in the same way as
+    /// `ReEarlyParam`. Region parameters are also sometimes implicit,
+    /// e.g. in `impl Trait for &()`.
+    ReEarlyParam(I::EarlyParamRegion),
 
-    /// Region bound in a function scope, which will be substituted when the
-    /// function is called.
-    ReLateBound(DebruijnIndex, I::BoundRegion),
+    /// A higher-ranked region. These represent either late-bound function parameters
+    /// or bound variables from a `for<'a>`-binder.
+    ///
+    /// While inside of a function, e.g. during typeck, the late-bound function parameters
+    /// can be converted to `ReLateParam` by calling `tcx.liberate_late_bound_regions`.
+    ///
+    /// Bound regions inside of types **must not** be erased, as they impact trait
+    /// selection and the `TypeId` of that type. `for<'a> fn(&'a ())` and
+    /// `fn(&'static ())` are different types and have to be treated as such.
+    ReBound(DebruijnIndex, I::BoundRegion),
 
-    /// When checking a function body, the types of all arguments and so forth
-    /// that refer to bound region parameters are modified to refer to free
-    /// region parameters.
-    ReFree(I::FreeRegion),
+    /// Late-bound function parameters are represented using a `ReBound`. When
+    /// inside of a function, we convert these bound variables to placeholder
+    /// parameters via `tcx.liberate_late_bound_regions`. They are then treated
+    /// the same way as `ReEarlyParam` while inside of the function.
+    ///
+    /// See <https://rustc-dev-guide.rust-lang.org/early-late-bound-summary.html> for
+    /// more info about early and late bound lifetime parameters.
+    ReLateParam(I::LateParamRegion),
 
     /// Static data that has an "infinite" lifetime. Top in the region lattice.
     ReStatic,
@@ -143,8 +160,11 @@ pub enum RegionKind<I: Interner> {
     /// A region variable. Should not exist outside of type inference.
     ReVar(I::InferRegion),
 
-    /// A placeholder region -- basically, the higher-ranked version of `ReFree`.
+    /// A placeholder region -- the higher-ranked version of `ReLateParam`.
     /// Should not exist outside of type inference.
+    ///
+    /// Used when instantiating a `forall` binder via
+    /// `infcx.instantiate_binder_with_placeholders`.
     RePlaceholder(I::PlaceholderRegion),
 
     /// Erased region, used by trait selection, in MIR and during codegen.
@@ -159,9 +179,9 @@ pub enum RegionKind<I: Interner> {
 #[inline]
 const fn regionkind_discriminant<I: Interner>(value: &RegionKind<I>) -> usize {
     match value {
-        ReEarlyBound(_) => 0,
-        ReLateBound(_, _) => 1,
-        ReFree(_) => 2,
+        ReEarlyParam(_) => 0,
+        ReBound(_, _) => 1,
+        ReLateParam(_) => 2,
         ReStatic => 3,
         ReVar(_) => 4,
         RePlaceholder(_) => 5,
@@ -170,27 +190,15 @@ const fn regionkind_discriminant<I: Interner>(value: &RegionKind<I>) -> usize {
     }
 }
 
-// This is manually implemented because a derive would require `I: Copy`
-impl<I: Interner> Copy for RegionKind<I>
-where
-    I::EarlyBoundRegion: Copy,
-    I::BoundRegion: Copy,
-    I::FreeRegion: Copy,
-    I::InferRegion: Copy,
-    I::PlaceholderRegion: Copy,
-    I::ErrorGuaranteed: Copy,
-{
-}
-
 // This is manually implemented because a derive would require `I: PartialEq`
 impl<I: Interner> PartialEq for RegionKind<I> {
     #[inline]
     fn eq(&self, other: &RegionKind<I>) -> bool {
         regionkind_discriminant(self) == regionkind_discriminant(other)
             && match (self, other) {
-                (ReEarlyBound(a_r), ReEarlyBound(b_r)) => a_r == b_r,
-                (ReLateBound(a_d, a_r), ReLateBound(b_d, b_r)) => a_d == b_d && a_r == b_r,
-                (ReFree(a_r), ReFree(b_r)) => a_r == b_r,
+                (ReEarlyParam(a_r), ReEarlyParam(b_r)) => a_r == b_r,
+                (ReBound(a_d, a_r), ReBound(b_d, b_r)) => a_d == b_d && a_r == b_r,
+                (ReLateParam(a_r), ReLateParam(b_r)) => a_r == b_r,
                 (ReStatic, ReStatic) => true,
                 (ReVar(a_r), ReVar(b_r)) => a_r == b_r,
                 (RePlaceholder(a_r), RePlaceholder(b_r)) => a_r == b_r,
@@ -216,13 +224,13 @@ impl<I: Interner> DebugWithInfcx<I> for RegionKind<I> {
         f: &mut core::fmt::Formatter<'_>,
     ) -> core::fmt::Result {
         match this.data {
-            ReEarlyBound(data) => write!(f, "ReEarlyBound({data:?})"),
+            ReEarlyParam(data) => write!(f, "ReEarlyParam({data:?})"),
 
-            ReLateBound(binder_id, bound_region) => {
-                write!(f, "ReLateBound({binder_id:?}, {bound_region:?})")
+            ReBound(binder_id, bound_region) => {
+                write!(f, "ReBound({binder_id:?}, {bound_region:?})")
             }
 
-            ReFree(fr) => write!(f, "{fr:?}"),
+            ReLateParam(fr) => write!(f, "{fr:?}"),
 
             ReStatic => f.write_str("ReStatic"),
 
@@ -242,12 +250,13 @@ impl<I: Interner> fmt::Debug for RegionKind<I> {
     }
 }
 
+#[cfg(feature = "nightly")]
 // This is not a derived impl because a derive would require `I: HashStable`
-impl<CTX: HashStableContext, I: Interner> HashStable<CTX> for RegionKind<I>
+impl<CTX, I: Interner> HashStable<CTX> for RegionKind<I>
 where
-    I::EarlyBoundRegion: HashStable<CTX>,
+    I::EarlyParamRegion: HashStable<CTX>,
     I::BoundRegion: HashStable<CTX>,
-    I::FreeRegion: HashStable<CTX>,
+    I::LateParamRegion: HashStable<CTX>,
     I::InferRegion: HashStable<CTX>,
     I::PlaceholderRegion: HashStable<CTX>,
 {
@@ -258,14 +267,14 @@ where
             ReErased | ReStatic | ReError(_) => {
                 // No variant fields to hash for these ...
             }
-            ReLateBound(d, r) => {
+            ReBound(d, r) => {
                 d.hash_stable(hcx, hasher);
                 r.hash_stable(hcx, hasher);
             }
-            ReEarlyBound(r) => {
+            ReEarlyParam(r) => {
                 r.hash_stable(hcx, hasher);
             }
-            ReFree(r) => {
+            ReLateParam(r) => {
                 r.hash_stable(hcx, hasher);
             }
             RePlaceholder(r) => {
diff --git a/compiler/rustc_type_ir/src/ty_info.rs b/compiler/rustc_type_ir/src/ty_info.rs
index d986e310c..0e4930552 100644
--- a/compiler/rustc_type_ir/src/ty_info.rs
+++ b/compiler/rustc_type_ir/src/ty_info.rs
@@ -1,4 +1,6 @@
+#[cfg(feature = "nightly")]
 use rustc_data_structures::fingerprint::Fingerprint;
+#[cfg(feature = "nightly")]
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use std::cmp::Ordering;
 use std::hash::{Hash, Hasher};
@@ -16,6 +18,8 @@ use crate::{DebruijnIndex, TypeFlags};
 #[derive(Copy, Clone)]
 pub struct WithCachedTypeInfo<T> {
     pub internee: T,
+
+    #[cfg(feature = "nightly")]
     pub stable_hash: Fingerprint,
 
     /// This field provides fast access to information that is also contained
@@ -81,14 +85,16 @@ impl<T> Deref for WithCachedTypeInfo<T> {
 impl<T: Hash> Hash for WithCachedTypeInfo<T> {
     #[inline]
     fn hash<H: Hasher>(&self, s: &mut H) {
+        #[cfg(feature = "nightly")]
         if self.stable_hash != Fingerprint::ZERO {
-            self.stable_hash.hash(s)
-        } else {
-            self.internee.hash(s)
+            return self.stable_hash.hash(s);
         }
+
+        self.internee.hash(s)
     }
 }
 
+#[cfg(feature = "nightly")]
 impl<T: HashStable<CTX>, CTX> HashStable<CTX> for WithCachedTypeInfo<T> {
     fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) {
         if self.stable_hash == Fingerprint::ZERO || cfg!(debug_assertions) {
diff --git a/compiler/rustc_type_ir/src/ty_kind.rs b/compiler/rustc_type_ir/src/ty_kind.rs
index 09a9a3322..70adfbee2 100644
--- a/compiler/rustc_type_ir/src/ty_kind.rs
+++ b/compiler/rustc_type_ir/src/ty_kind.rs
@@ -1,11 +1,11 @@
-#![allow(rustc::usage_of_ty_tykind)]
-
+#[cfg(feature = "nightly")]
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
+#[cfg(feature = "nightly")]
 use rustc_data_structures::unify::{EqUnifyValue, UnifyKey};
 use std::fmt;
-use std::mem::discriminant;
 
-use crate::HashStableContext;
+use crate::fold::{FallibleTypeFolder, TypeFoldable};
+use crate::visit::{TypeVisitable, TypeVisitor};
 use crate::Interner;
 use crate::{DebruijnIndex, DebugWithInfcx, InferCtxtLike, WithInfcx};
 
@@ -13,8 +13,8 @@ use self::TyKind::*;
 
 /// The movability of a coroutine / closure literal:
 /// whether a coroutine contains self-references, causing it to be `!Unpin`.
-#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Encodable, Decodable, Debug, Copy)]
-#[derive(HashStable_Generic)]
+#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Copy)]
+#[cfg_attr(feature = "nightly", derive(Encodable, Decodable, HashStable_NoContext))]
 pub enum Movability {
     /// May contain self-references, `!Unpin`.
     Static,
@@ -23,7 +23,7 @@ pub enum Movability {
 }
 
 #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Copy)]
-#[derive(HashStable_Generic, Encodable, Decodable)]
+#[cfg_attr(feature = "nightly", derive(Encodable, Decodable, HashStable_NoContext))]
 pub enum Mutability {
     // N.B. Order is deliberate, so that Not < Mut
     Not,
@@ -75,7 +75,7 @@ impl Mutability {
 
 /// Specifies how a trait object is represented.
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
-#[derive(Encodable, Decodable, HashStable_Generic)]
+#[cfg_attr(feature = "nightly", derive(Encodable, Decodable, HashStable_NoContext))]
 pub enum DynKind {
     /// An unsized `dyn Trait` object
     Dyn,
@@ -89,7 +89,7 @@ pub enum DynKind {
 }
 
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
-#[derive(Encodable, Decodable, HashStable_Generic)]
+#[cfg_attr(feature = "nightly", derive(Encodable, Decodable, HashStable_NoContext))]
 pub enum AliasKind {
     /// A projection `<Type as Trait>::AssocType`.
     /// Can get normalized away if monomorphic enough.
@@ -109,17 +109,18 @@ pub enum AliasKind {
 ///
 /// Types written by the user start out as `hir::TyKind` and get
 /// converted to this representation using `AstConv::ast_ty_to_ty`.
-#[rustc_diagnostic_item = "IrTyKind"]
+#[cfg_attr(feature = "nightly", rustc_diagnostic_item = "IrTyKind")]
 #[derive(derivative::Derivative)]
 #[derivative(
     Clone(bound = ""),
+    Copy(bound = ""),
     PartialOrd(bound = ""),
     PartialOrd = "feature_allow_slow_enum",
     Ord(bound = ""),
     Ord = "feature_allow_slow_enum",
     Hash(bound = "")
 )]
-#[derive(TyEncodable, TyDecodable)]
+#[cfg_attr(feature = "nightly", derive(TyEncodable, TyDecodable, HashStable_NoContext))]
 pub enum TyKind<I: Interner> {
     /// The primitive boolean type. Written as `bool`.
     Bool,
@@ -159,7 +160,7 @@ pub enum TyKind<I: Interner> {
     Slice(I::Ty),
 
     /// A raw pointer. Written as `*mut T` or `*const T`
-    RawPtr(I::TypeAndMut),
+    RawPtr(TypeAndMut<I>),
 
     /// A reference; a pointer with an associated lifetime. Written as
     /// `&'a mut T` or `&'a T`.
@@ -281,7 +282,7 @@ pub enum TyKind<I: Interner> {
     /// correctly deal with higher ranked types. Though unlike placeholders,
     /// that universe is stored in the `InferCtxt` instead of directly
     /// inside of the type.
-    Infer(I::InferTy),
+    Infer(InferTy),
 
     /// A placeholder for a type which could not be computed; this is
     /// propagated to avoid useless error messages.
@@ -394,7 +395,7 @@ impl<I: Interner> DebugWithInfcx<I> for TyKind<I> {
             Float(float) => write!(f, "{float:?}"),
             Adt(d, s) => {
                 write!(f, "{d:?}")?;
-                let mut s = s.clone().into_iter();
+                let mut s = s.into_iter();
                 let first = s.next();
                 match first {
                     Some(first) => write!(f, "<{:?}", first)?,
@@ -407,12 +408,11 @@ impl<I: Interner> DebugWithInfcx<I> for TyKind<I> {
 
                 write!(f, ">")
             }
-            Foreign(d) => f.debug_tuple_field1_finish("Foreign", d),
+            Foreign(d) => f.debug_tuple("Foreign").field(d).finish(),
             Str => write!(f, "str"),
             Array(t, c) => write!(f, "[{:?}; {:?}]", &this.wrap(t), &this.wrap(c)),
             Slice(t) => write!(f, "[{:?}]", &this.wrap(t)),
-            RawPtr(p) => {
-                let (ty, mutbl) = I::ty_and_mut_to_parts(p.clone());
+            RawPtr(TypeAndMut { ty, mutbl }) => {
                 match mutbl {
                     Mutability::Mut => write!(f, "*mut "),
                     Mutability::Not => write!(f, "*const "),
@@ -423,7 +423,7 @@ impl<I: Interner> DebugWithInfcx<I> for TyKind<I> {
                 Mutability::Mut => write!(f, "&{:?} mut {:?}", &this.wrap(r), &this.wrap(t)),
                 Mutability::Not => write!(f, "&{:?} {:?}", &this.wrap(r), &this.wrap(t)),
             },
-            FnDef(d, s) => f.debug_tuple_field2_finish("FnDef", d, &this.wrap(s)),
+            FnDef(d, s) => f.debug_tuple("FnDef").field(d).field(&this.wrap(s)).finish(),
             FnPtr(s) => write!(f, "{:?}", &this.wrap(s)),
             Dynamic(p, r, repr) => match repr {
                 DynKind::Dyn => write!(f, "dyn {:?} + {:?}", &this.wrap(p), &this.wrap(r)),
@@ -431,16 +431,18 @@ impl<I: Interner> DebugWithInfcx<I> for TyKind<I> {
                     write!(f, "dyn* {:?} + {:?}", &this.wrap(p), &this.wrap(r))
                 }
             },
-            Closure(d, s) => f.debug_tuple_field2_finish("Closure", d, &this.wrap(s)),
-            Coroutine(d, s, m) => f.debug_tuple_field3_finish("Coroutine", d, &this.wrap(s), m),
+            Closure(d, s) => f.debug_tuple("Closure").field(d).field(&this.wrap(s)).finish(),
+            Coroutine(d, s, m) => {
+                f.debug_tuple("Coroutine").field(d).field(&this.wrap(s)).field(m).finish()
+            }
             CoroutineWitness(d, s) => {
-                f.debug_tuple_field2_finish("CoroutineWitness", d, &this.wrap(s))
+                f.debug_tuple("CoroutineWitness").field(d).field(&this.wrap(s)).finish()
             }
             Never => write!(f, "!"),
             Tuple(t) => {
                 write!(f, "(")?;
                 let mut count = 0;
-                for ty in t.clone() {
+                for ty in *t {
                     if count > 0 {
                         write!(f, ", ")?;
                     }
@@ -453,7 +455,7 @@ impl<I: Interner> DebugWithInfcx<I> for TyKind<I> {
                 }
                 write!(f, ")")
             }
-            Alias(i, a) => f.debug_tuple_field2_finish("Alias", i, &this.wrap(a)),
+            Alias(i, a) => f.debug_tuple("Alias").field(i).field(&this.wrap(a)).finish(),
             Param(p) => write!(f, "{p:?}"),
             Bound(d, b) => crate::debug_bound_var(f, *d, b),
             Placeholder(p) => write!(f, "{p:?}"),
@@ -470,120 +472,8 @@ impl<I: Interner> fmt::Debug for TyKind<I> {
     }
 }
 
-// This is not a derived impl because a derive would require `I: HashStable`
-#[allow(rustc::usage_of_ty_tykind)]
-impl<CTX: HashStableContext, I: Interner> HashStable<CTX> for TyKind<I>
-where
-    I::AdtDef: HashStable<CTX>,
-    I::DefId: HashStable<CTX>,
-    I::GenericArgs: HashStable<CTX>,
-    I::Ty: HashStable<CTX>,
-    I::Const: HashStable<CTX>,
-    I::TypeAndMut: HashStable<CTX>,
-    I::PolyFnSig: HashStable<CTX>,
-    I::BoundExistentialPredicates: HashStable<CTX>,
-    I::Region: HashStable<CTX>,
-    I::Tys: HashStable<CTX>,
-    I::AliasTy: HashStable<CTX>,
-    I::BoundTy: HashStable<CTX>,
-    I::ParamTy: HashStable<CTX>,
-    I::PlaceholderTy: HashStable<CTX>,
-    I::InferTy: HashStable<CTX>,
-    I::ErrorGuaranteed: HashStable<CTX>,
-{
-    #[inline]
-    fn hash_stable(&self, __hcx: &mut CTX, __hasher: &mut StableHasher) {
-        std::mem::discriminant(self).hash_stable(__hcx, __hasher);
-        match self {
-            Bool => {}
-            Char => {}
-            Int(i) => {
-                i.hash_stable(__hcx, __hasher);
-            }
-            Uint(u) => {
-                u.hash_stable(__hcx, __hasher);
-            }
-            Float(f) => {
-                f.hash_stable(__hcx, __hasher);
-            }
-            Adt(adt, args) => {
-                adt.hash_stable(__hcx, __hasher);
-                args.hash_stable(__hcx, __hasher);
-            }
-            Foreign(def_id) => {
-                def_id.hash_stable(__hcx, __hasher);
-            }
-            Str => {}
-            Array(t, c) => {
-                t.hash_stable(__hcx, __hasher);
-                c.hash_stable(__hcx, __hasher);
-            }
-            Slice(t) => {
-                t.hash_stable(__hcx, __hasher);
-            }
-            RawPtr(tam) => {
-                tam.hash_stable(__hcx, __hasher);
-            }
-            Ref(r, t, m) => {
-                r.hash_stable(__hcx, __hasher);
-                t.hash_stable(__hcx, __hasher);
-                m.hash_stable(__hcx, __hasher);
-            }
-            FnDef(def_id, args) => {
-                def_id.hash_stable(__hcx, __hasher);
-                args.hash_stable(__hcx, __hasher);
-            }
-            FnPtr(polyfnsig) => {
-                polyfnsig.hash_stable(__hcx, __hasher);
-            }
-            Dynamic(l, r, repr) => {
-                l.hash_stable(__hcx, __hasher);
-                r.hash_stable(__hcx, __hasher);
-                repr.hash_stable(__hcx, __hasher);
-            }
-            Closure(def_id, args) => {
-                def_id.hash_stable(__hcx, __hasher);
-                args.hash_stable(__hcx, __hasher);
-            }
-            Coroutine(def_id, args, m) => {
-                def_id.hash_stable(__hcx, __hasher);
-                args.hash_stable(__hcx, __hasher);
-                m.hash_stable(__hcx, __hasher);
-            }
-            CoroutineWitness(def_id, args) => {
-                def_id.hash_stable(__hcx, __hasher);
-                args.hash_stable(__hcx, __hasher);
-            }
-            Never => {}
-            Tuple(args) => {
-                args.hash_stable(__hcx, __hasher);
-            }
-            Alias(k, p) => {
-                k.hash_stable(__hcx, __hasher);
-                p.hash_stable(__hcx, __hasher);
-            }
-            Param(p) => {
-                p.hash_stable(__hcx, __hasher);
-            }
-            Bound(d, b) => {
-                d.hash_stable(__hcx, __hasher);
-                b.hash_stable(__hcx, __hasher);
-            }
-            Placeholder(p) => {
-                p.hash_stable(__hcx, __hasher);
-            }
-            Infer(i) => {
-                i.hash_stable(__hcx, __hasher);
-            }
-            Error(d) => {
-                d.hash_stable(__hcx, __hasher);
-            }
-        }
-    }
-}
-
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
-#[derive(Encodable, Decodable, HashStable_Generic)]
+#[cfg_attr(feature = "nightly", derive(Encodable, Decodable, HashStable_NoContext))]
 pub enum IntTy {
     Isize,
     I8,
@@ -641,7 +531,7 @@ impl IntTy {
 }
 
 #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Copy)]
-#[derive(Encodable, Decodable, HashStable_Generic)]
+#[cfg_attr(feature = "nightly", derive(Encodable, Decodable, HashStable_NoContext))]
 pub enum UintTy {
     Usize,
     U8,
@@ -699,7 +589,7 @@ impl UintTy {
 }
 
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
-#[derive(Encodable, Decodable, HashStable_Generic)]
+#[cfg_attr(feature = "nightly", derive(Encodable, Decodable, HashStable_NoContext))]
 pub enum FloatTy {
     F32,
     F64,
@@ -732,19 +622,28 @@ pub struct FloatVarValue(pub FloatTy);
 
 rustc_index::newtype_index! {
     /// A **ty**pe **v**ariable **ID**.
+    #[encodable]
+    #[orderable]
     #[debug_format = "?{}t"]
+    #[gate_rustc_only]
     pub struct TyVid {}
 }
 
 rustc_index::newtype_index! {
     /// An **int**egral (`u32`, `i32`, `usize`, etc.) type **v**ariable **ID**.
+    #[encodable]
+    #[orderable]
     #[debug_format = "?{}i"]
+    #[gate_rustc_only]
     pub struct IntVid {}
 }
 
 rustc_index::newtype_index! {
     /// A **float**ing-point (`f32` or `f64`) type **v**ariable **ID**.
+    #[encodable]
+    #[orderable]
     #[debug_format = "?{}f"]
+    #[gate_rustc_only]
     pub struct FloatVid {}
 }
 
@@ -753,7 +652,8 @@ rustc_index::newtype_index! {
 /// E.g., if we have an empty array (`[]`), then we create a fresh
 /// type variable for the element type since we won't know until it's
 /// used what the element type is supposed to be.
-#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Encodable, Decodable)]
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[cfg_attr(feature = "nightly", derive(Encodable, Decodable))]
 pub enum InferTy {
     /// A type variable.
     TyVar(TyVid),
@@ -786,6 +686,7 @@ pub enum InferTy {
 
 /// Raw `TyVid` are used as the unification key for `sub_relations`;
 /// they carry no values.
+#[cfg(feature = "nightly")]
 impl UnifyKey for TyVid {
     type Value = ();
     #[inline]
@@ -801,8 +702,10 @@ impl UnifyKey for TyVid {
     }
 }
 
+#[cfg(feature = "nightly")]
 impl EqUnifyValue for IntVarValue {}
 
+#[cfg(feature = "nightly")]
 impl UnifyKey for IntVid {
     type Value = Option<IntVarValue>;
     #[inline] // make this function eligible for inlining - it is quite hot.
@@ -818,8 +721,10 @@ impl UnifyKey for IntVid {
     }
 }
 
+#[cfg(feature = "nightly")]
 impl EqUnifyValue for FloatVarValue {}
 
+#[cfg(feature = "nightly")]
 impl UnifyKey for FloatVid {
     type Value = Option<FloatVarValue>;
     #[inline]
@@ -835,10 +740,11 @@ impl UnifyKey for FloatVid {
     }
 }
 
+#[cfg(feature = "nightly")]
 impl<CTX> HashStable<CTX> for InferTy {
     fn hash_stable(&self, ctx: &mut CTX, hasher: &mut StableHasher) {
         use InferTy::*;
-        discriminant(self).hash_stable(ctx, hasher);
+        std::mem::discriminant(self).hash_stable(ctx, hasher);
         match self {
             TyVar(_) | IntVar(_) | FloatVar(_) => {
                 panic!("type variables should not be hashed: {self:?}")
@@ -909,20 +815,59 @@ impl fmt::Debug for InferTy {
     }
 }
 
-impl<I: Interner<InferTy = InferTy>> DebugWithInfcx<I> for InferTy {
+impl<I: Interner> DebugWithInfcx<I> for InferTy {
     fn fmt<Infcx: InferCtxtLike<Interner = I>>(
         this: WithInfcx<'_, Infcx, &Self>,
         f: &mut fmt::Formatter<'_>,
     ) -> fmt::Result {
-        use InferTy::*;
-        match this.infcx.universe_of_ty(*this.data) {
-            None => write!(f, "{:?}", this.data),
-            Some(universe) => match *this.data {
-                TyVar(ty_vid) => write!(f, "?{}_{}t", ty_vid.index(), universe.index()),
-                IntVar(_) | FloatVar(_) | FreshTy(_) | FreshIntTy(_) | FreshFloatTy(_) => {
-                    unreachable!()
+        match this.data {
+            InferTy::TyVar(vid) => {
+                if let Some(universe) = this.infcx.universe_of_ty(*vid) {
+                    write!(f, "?{}_{}t", vid.index(), universe.index())
+                } else {
+                    write!(f, "{:?}", this.data)
                 }
-            },
+            }
+            _ => write!(f, "{:?}", this.data),
         }
     }
 }
+
+#[derive(derivative::Derivative)]
+#[derivative(
+    Clone(bound = ""),
+    Copy(bound = ""),
+    PartialOrd(bound = ""),
+    Ord(bound = ""),
+    PartialEq(bound = ""),
+    Eq(bound = ""),
+    Hash(bound = ""),
+    Debug(bound = "")
+)]
+#[cfg_attr(feature = "nightly", derive(TyEncodable, TyDecodable, HashStable_NoContext))]
+pub struct TypeAndMut<I: Interner> {
+    pub ty: I::Ty,
+    pub mutbl: Mutability,
+}
+
+impl<I: Interner> TypeFoldable<I> for TypeAndMut<I>
+where
+    I::Ty: TypeFoldable<I>,
+{
+    fn try_fold_with<F: FallibleTypeFolder<I>>(self, folder: &mut F) -> Result<Self, F::Error> {
+        Ok(TypeAndMut {
+            ty: self.ty.try_fold_with(folder)?,
+            mutbl: self.mutbl.try_fold_with(folder)?,
+        })
+    }
+}
+
+impl<I: Interner> TypeVisitable<I> for TypeAndMut<I>
+where
+    I::Ty: TypeVisitable<I>,
+{
+    fn visit_with<V: TypeVisitor<I>>(&self, visitor: &mut V) -> std::ops::ControlFlow<V::BreakTy> {
+        self.ty.visit_with(visitor)?;
+        self.mutbl.visit_with(visitor)
+    }
+}
diff --git a/compiler/rustc_type_ir/src/visit.rs b/compiler/rustc_type_ir/src/visit.rs
index 9c7b8156b..7aa990046 100644
--- a/compiler/rustc_type_ir/src/visit.rs
+++ b/compiler/rustc_type_ir/src/visit.rs
@@ -41,12 +41,12 @@
 //! - u.visit_with(visitor)
 //! ```
 
-use rustc_data_structures::sync::Lrc;
 use rustc_index::{Idx, IndexVec};
 use std::fmt;
 use std::ops::ControlFlow;
 
 use crate::Interner;
+use crate::Lrc;
 
 /// This trait is implemented for every type that can be visited,
 /// providing the skeleton of the traversal.
@@ -82,8 +82,12 @@ pub trait TypeSuperVisitable<I: Interner>: TypeVisitable<I> {
 /// method defined for every type of interest. Each such method has a default
 /// that recurses into the type's fields in a non-custom fashion.
 pub trait TypeVisitor<I: Interner>: Sized {
+    #[cfg(feature = "nightly")]
     type BreakTy = !;
 
+    #[cfg(not(feature = "nightly"))]
+    type BreakTy;
+
     fn visit_binder<T: TypeVisitable<I>>(&mut self, t: &I::Binder<T>) -> ControlFlow<Self::BreakTy>
     where
         I::Binder<T>: TypeSuperVisitable<I>,
diff --git a/compiler/stable_mir/src/abi.rs b/compiler/stable_mir/src/abi.rs
new file mode 100644
index 000000000..53dac6abe
--- /dev/null
+++ b/compiler/stable_mir/src/abi.rs
@@ -0,0 +1,283 @@
+use crate::compiler_interface::with;
+use crate::mir::FieldIdx;
+use crate::ty::{Align, IndexedVal, Size, Ty, VariantIdx};
+use crate::Opaque;
+use std::num::NonZeroUsize;
+use std::ops::RangeInclusive;
+
+/// A function ABI definition.
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub struct FnAbi {
+    /// The types of each argument.
+    pub args: Vec<ArgAbi>,
+
+    /// The expected return type.
+    pub ret: ArgAbi,
+
+    /// The count of non-variadic arguments.
+    ///
+    /// Should only be different from `args.len()` when a function is a C variadic function.
+    pub fixed_count: u32,
+
+    /// The ABI convention.
+    pub conv: CallConvention,
+
+    /// Whether this is a variadic C function,
+    pub c_variadic: bool,
+}
+
+/// Information about the ABI of a function's argument, or return value.
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub struct ArgAbi {
+    pub ty: Ty,
+    pub layout: Layout,
+    pub mode: PassMode,
+}
+
+/// How a function argument should be passed in to the target function.
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub enum PassMode {
+    /// Ignore the argument.
+    ///
+    /// The argument is either uninhabited or a ZST.
+    Ignore,
+    /// Pass the argument directly.
+    ///
+    /// The argument has a layout abi of `Scalar` or `Vector`.
+    Direct(Opaque),
+    /// Pass a pair's elements directly in two arguments.
+    ///
+    /// The argument has a layout abi of `ScalarPair`.
+    Pair(Opaque, Opaque),
+    /// Pass the argument after casting it.
+    Cast { pad_i32: bool, cast: Opaque },
+    /// Pass the argument indirectly via a hidden pointer.
+    Indirect { attrs: Opaque, meta_attrs: Opaque, on_stack: bool },
+}
+
+/// The layout of a type, alongside the type itself.
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+pub struct TyAndLayout {
+    pub ty: Ty,
+    pub layout: Layout,
+}
+
+/// The layout of a type in memory.
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub struct LayoutShape {
+    /// The fields location withing the layout
+    pub fields: FieldsShape,
+
+    /// Encodes information about multi-variant layouts.
+    /// Even with `Multiple` variants, a layout still has its own fields! Those are then
+    /// shared between all variants.
+    ///
+    /// To access all fields of this layout, both `fields` and the fields of the active variant
+    /// must be taken into account.
+    pub variants: VariantsShape,
+
+    /// The `abi` defines how this data is passed between functions.
+    pub abi: ValueAbi,
+
+    /// The ABI mandated alignment in bytes.
+    pub abi_align: Align,
+
+    /// The size of this layout in bytes.
+    pub size: Size,
+}
+
+impl LayoutShape {
+    /// Returns `true` if the layout corresponds to an unsized type.
+    #[inline]
+    pub fn is_unsized(&self) -> bool {
+        self.abi.is_unsized()
+    }
+
+    #[inline]
+    pub fn is_sized(&self) -> bool {
+        !self.abi.is_unsized()
+    }
+
+    /// Returns `true` if the type is sized and a 1-ZST (meaning it has size 0 and alignment 1).
+    pub fn is_1zst(&self) -> bool {
+        self.is_sized() && self.size == 0 && self.abi_align == 1
+    }
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+pub struct Layout(usize);
+
+impl Layout {
+    pub fn shape(self) -> LayoutShape {
+        with(|cx| cx.layout_shape(self))
+    }
+}
+
+impl IndexedVal for Layout {
+    fn to_val(index: usize) -> Self {
+        Layout(index)
+    }
+    fn to_index(&self) -> usize {
+        self.0
+    }
+}
+
+/// Describes how the fields of a type are shaped in memory.
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub enum FieldsShape {
+    /// Scalar primitives and `!`, which never have fields.
+    Primitive,
+
+    /// All fields start at no offset. The `usize` is the field count.
+    Union(NonZeroUsize),
+
+    /// Array/vector-like placement, with all fields of identical types.
+    Array { stride: Size, count: u64 },
+
+    /// Struct-like placement, with precomputed offsets.
+    ///
+    /// Fields are guaranteed to not overlap, but note that gaps
+    /// before, between and after all the fields are NOT always
+    /// padding, and as such their contents may not be discarded.
+    /// For example, enum variants leave a gap at the start,
+    /// where the discriminant field in the enum layout goes.
+    Arbitrary {
+        /// Offsets for the first byte of each field,
+        /// ordered to match the source definition order.
+        /// I.e.: It follows the same order as [crate::ty::VariantDef::fields()].
+        /// This vector does not go in increasing order.
+        offsets: Vec<Size>,
+    },
+}
+
+impl FieldsShape {
+    pub fn fields_by_offset_order(&self) -> Vec<FieldIdx> {
+        match self {
+            FieldsShape::Primitive => vec![],
+            FieldsShape::Union(_) | FieldsShape::Array { .. } => (0..self.count()).collect(),
+            FieldsShape::Arbitrary { offsets, .. } => {
+                let mut indices = (0..offsets.len()).collect::<Vec<_>>();
+                indices.sort_by_key(|idx| offsets[*idx]);
+                indices
+            }
+        }
+    }
+
+    pub fn count(&self) -> usize {
+        match self {
+            FieldsShape::Primitive => 0,
+            FieldsShape::Union(count) => count.get(),
+            FieldsShape::Array { count, .. } => *count as usize,
+            FieldsShape::Arbitrary { offsets, .. } => offsets.len(),
+        }
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub enum VariantsShape {
+    /// Single enum variants, structs/tuples, unions, and all non-ADTs.
+    Single { index: VariantIdx },
+
+    /// Enum-likes with more than one inhabited variant: each variant comes with
+    /// a *discriminant* (usually the same as the variant index but the user can
+    /// assign explicit discriminant values). That discriminant is encoded
+    /// as a *tag* on the machine. The layout of each variant is
+    /// a struct, and they all have space reserved for the tag.
+    /// For enums, the tag is the sole field of the layout.
+    Multiple {
+        tag: Scalar,
+        tag_encoding: TagEncoding,
+        tag_field: usize,
+        variants: Vec<LayoutShape>,
+    },
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub enum TagEncoding {
+    /// The tag directly stores the discriminant, but possibly with a smaller layout
+    /// (so converting the tag to the discriminant can require sign extension).
+    Direct,
+
+    /// Niche (values invalid for a type) encoding the discriminant:
+    /// Discriminant and variant index coincide.
+    /// The variant `untagged_variant` contains a niche at an arbitrary
+    /// offset (field `tag_field` of the enum), which for a variant with
+    /// discriminant `d` is set to
+    /// `(d - niche_variants.start).wrapping_add(niche_start)`.
+    ///
+    /// For example, `Option<(usize, &T)>`  is represented such that
+    /// `None` has a null pointer for the second tuple field, and
+    /// `Some` is the identity function (with a non-null reference).
+    Niche {
+        untagged_variant: VariantIdx,
+        niche_variants: RangeInclusive<VariantIdx>,
+        niche_start: u128,
+    },
+}
+
+/// Describes how values of the type are passed by target ABIs,
+/// in terms of categories of C types there are ABI rules for.
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub enum ValueAbi {
+    Uninhabited,
+    Scalar(Scalar),
+    ScalarPair(Scalar, Scalar),
+    Vector {
+        element: Scalar,
+        count: u64,
+    },
+    Aggregate {
+        /// If true, the size is exact, otherwise it's only a lower bound.
+        sized: bool,
+    },
+}
+
+impl ValueAbi {
+    /// Returns `true` if the layout corresponds to an unsized type.
+    pub fn is_unsized(&self) -> bool {
+        match *self {
+            ValueAbi::Uninhabited
+            | ValueAbi::Scalar(_)
+            | ValueAbi::ScalarPair(..)
+            | ValueAbi::Vector { .. } => false,
+            ValueAbi::Aggregate { sized } => !sized,
+        }
+    }
+}
+
+/// We currently do not support `Scalar`, and use opaque instead.
+type Scalar = Opaque;
+
+/// General language calling conventions.
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+pub enum CallConvention {
+    C,
+    Rust,
+
+    Cold,
+    PreserveMost,
+    PreserveAll,
+
+    // Target-specific calling conventions.
+    ArmAapcs,
+    CCmseNonSecureCall,
+
+    Msp430Intr,
+
+    PtxKernel,
+
+    X86Fastcall,
+    X86Intr,
+    X86Stdcall,
+    X86ThisCall,
+    X86VectorCall,
+
+    X86_64SysV,
+    X86_64Win64,
+
+    AmdGpuKernel,
+    AvrInterrupt,
+    AvrNonBlockingInterrupt,
+
+    RiscvInterrupt,
+}
diff --git a/compiler/stable_mir/src/compiler_interface.rs b/compiler/stable_mir/src/compiler_interface.rs
new file mode 100644
index 000000000..f52e50605
--- /dev/null
+++ b/compiler/stable_mir/src/compiler_interface.rs
@@ -0,0 +1,218 @@
+//! Define the interface with the Rust compiler.
+//!
+//! StableMIR users should not use any of the items in this module directly.
+//! These APIs have no stability guarantee.
+
+use std::cell::Cell;
+
+use crate::abi::{FnAbi, Layout, LayoutShape};
+use crate::mir::alloc::{AllocId, GlobalAlloc};
+use crate::mir::mono::{Instance, InstanceDef, StaticDef};
+use crate::mir::Body;
+use crate::target::MachineInfo;
+use crate::ty::{
+    AdtDef, AdtKind, Allocation, ClosureDef, ClosureKind, Const, FieldDef, FnDef, GenericArgs,
+    GenericPredicates, Generics, ImplDef, ImplTrait, LineInfo, PolyFnSig, RigidTy, Span, TraitDecl,
+    TraitDef, Ty, TyKind, VariantDef,
+};
+use crate::{
+    mir, Crate, CrateItem, CrateItems, DefId, Error, Filename, ImplTraitDecls, ItemKind, Symbol,
+    TraitDecls,
+};
+
+/// This trait defines the interface between stable_mir and the Rust compiler.
+/// Do not use this directly.
+pub trait Context {
+    fn entry_fn(&self) -> Option<CrateItem>;
+    /// Retrieve all items of the local crate that have a MIR associated with them.
+    fn all_local_items(&self) -> CrateItems;
+    /// Retrieve the body of a function.
+    /// This function will panic if the body is not available.
+    fn mir_body(&self, item: DefId) -> mir::Body;
+    /// Check whether the body of a function is available.
+    fn has_body(&self, item: DefId) -> bool;
+    fn all_trait_decls(&self) -> TraitDecls;
+    fn trait_decl(&self, trait_def: &TraitDef) -> TraitDecl;
+    fn all_trait_impls(&self) -> ImplTraitDecls;
+    fn trait_impl(&self, trait_impl: &ImplDef) -> ImplTrait;
+    fn generics_of(&self, def_id: DefId) -> Generics;
+    fn predicates_of(&self, def_id: DefId) -> GenericPredicates;
+    fn explicit_predicates_of(&self, def_id: DefId) -> GenericPredicates;
+    /// Get information about the local crate.
+    fn local_crate(&self) -> Crate;
+    /// Retrieve a list of all external crates.
+    fn external_crates(&self) -> Vec<Crate>;
+
+    /// Find a crate with the given name.
+    fn find_crates(&self, name: &str) -> Vec<Crate>;
+
+    /// Returns the name of given `DefId`
+    fn def_name(&self, def_id: DefId, trimmed: bool) -> Symbol;
+
+    /// Returns printable, human readable form of `Span`
+    fn span_to_string(&self, span: Span) -> String;
+
+    /// Return filename from given `Span`, for diagnostic purposes
+    fn get_filename(&self, span: &Span) -> Filename;
+
+    /// Return lines corresponding to this `Span`
+    fn get_lines(&self, span: &Span) -> LineInfo;
+
+    /// Returns the `kind` of given `DefId`
+    fn item_kind(&self, item: CrateItem) -> ItemKind;
+
+    /// Returns whether this is a foreign item.
+    fn is_foreign_item(&self, item: DefId) -> bool;
+
+    /// Returns the kind of a given algebraic data type
+    fn adt_kind(&self, def: AdtDef) -> AdtKind;
+
+    /// Returns if the ADT is a box.
+    fn adt_is_box(&self, def: AdtDef) -> bool;
+
+    /// Returns whether this ADT is simd.
+    fn adt_is_simd(&self, def: AdtDef) -> bool;
+
+    /// Returns whether this definition is a C string.
+    fn adt_is_cstr(&self, def: AdtDef) -> bool;
+
+    /// Retrieve the function signature for the given generic arguments.
+    fn fn_sig(&self, def: FnDef, args: &GenericArgs) -> PolyFnSig;
+
+    /// Retrieve the closure signature for the given generic arguments.
+    fn closure_sig(&self, args: &GenericArgs) -> PolyFnSig;
+
+    /// The number of variants in this ADT.
+    fn adt_variants_len(&self, def: AdtDef) -> usize;
+
+    /// The name of a variant.
+    fn variant_name(&self, def: VariantDef) -> Symbol;
+    fn variant_fields(&self, def: VariantDef) -> Vec<FieldDef>;
+
+    /// Evaluate constant as a target usize.
+    fn eval_target_usize(&self, cnst: &Const) -> Result<u64, Error>;
+
+    /// Create a target usize constant for the given value.
+    fn usize_to_const(&self, val: u64) -> Result<Const, Error>;
+
+    /// Create a new type from the given kind.
+    fn new_rigid_ty(&self, kind: RigidTy) -> Ty;
+
+    /// Create a new box type, `Box<T>`, for the given inner type `T`.
+    fn new_box_ty(&self, ty: Ty) -> Ty;
+
+    /// Returns the type of given crate item.
+    fn def_ty(&self, item: DefId) -> Ty;
+
+    /// Returns the type of given definition instantiated with the given arguments.
+    fn def_ty_with_args(&self, item: DefId, args: &GenericArgs) -> Ty;
+
+    /// Returns literal value of a const as a string.
+    fn const_literal(&self, cnst: &Const) -> String;
+
+    /// `Span` of an item
+    fn span_of_an_item(&self, def_id: DefId) -> Span;
+
+    /// Obtain the representation of a type.
+    fn ty_kind(&self, ty: Ty) -> TyKind;
+
+    // Get the discriminant Ty for this Ty if there's one.
+    fn rigid_ty_discriminant_ty(&self, ty: &RigidTy) -> Ty;
+
+    /// Get the body of an Instance which is already monomorphized.
+    fn instance_body(&self, instance: InstanceDef) -> Option<Body>;
+
+    /// Get the instance type with generic substitutions applied and lifetimes erased.
+    fn instance_ty(&self, instance: InstanceDef) -> Ty;
+
+    /// Get the instantiation types.
+    fn instance_args(&self, def: InstanceDef) -> GenericArgs;
+
+    /// Get the instance.
+    fn instance_def_id(&self, instance: InstanceDef) -> DefId;
+
+    /// Get the instance mangled name.
+    fn instance_mangled_name(&self, instance: InstanceDef) -> Symbol;
+
+    /// Check if this is an empty DropGlue shim.
+    fn is_empty_drop_shim(&self, def: InstanceDef) -> bool;
+
+    /// Convert a non-generic crate item into an instance.
+    /// This function will panic if the item is generic.
+    fn mono_instance(&self, def_id: DefId) -> Instance;
+
+    /// Item requires monomorphization.
+    fn requires_monomorphization(&self, def_id: DefId) -> bool;
+
+    /// Resolve an instance from the given function definition and generic arguments.
+    fn resolve_instance(&self, def: FnDef, args: &GenericArgs) -> Option<Instance>;
+
+    /// Resolve an instance for drop_in_place for the given type.
+    fn resolve_drop_in_place(&self, ty: Ty) -> Instance;
+
+    /// Resolve instance for a function pointer.
+    fn resolve_for_fn_ptr(&self, def: FnDef, args: &GenericArgs) -> Option<Instance>;
+
+    /// Resolve instance for a closure with the requested type.
+    fn resolve_closure(
+        &self,
+        def: ClosureDef,
+        args: &GenericArgs,
+        kind: ClosureKind,
+    ) -> Option<Instance>;
+
+    /// Evaluate a static's initializer.
+    fn eval_static_initializer(&self, def: StaticDef) -> Result<Allocation, Error>;
+
+    /// Try to evaluate an instance into a constant.
+    fn eval_instance(&self, def: InstanceDef, const_ty: Ty) -> Result<Allocation, Error>;
+
+    /// Retrieve global allocation for the given allocation ID.
+    fn global_alloc(&self, id: AllocId) -> GlobalAlloc;
+
+    /// Retrieve the id for the virtual table.
+    fn vtable_allocation(&self, global_alloc: &GlobalAlloc) -> Option<AllocId>;
+    fn krate(&self, def_id: DefId) -> Crate;
+    fn instance_name(&self, def: InstanceDef, trimmed: bool) -> Symbol;
+
+    /// Return information about the target machine.
+    fn target_info(&self) -> MachineInfo;
+
+    /// Get an instance ABI.
+    fn instance_abi(&self, def: InstanceDef) -> Result<FnAbi, Error>;
+
+    /// Get the layout of a type.
+    fn ty_layout(&self, ty: Ty) -> Result<Layout, Error>;
+
+    /// Get the layout shape.
+    fn layout_shape(&self, id: Layout) -> LayoutShape;
+}
+
+// A thread local variable that stores a pointer to the tables mapping between TyCtxt
+// datastructures and stable MIR datastructures
+scoped_thread_local! (static TLV: Cell<*const ()>);
+
+pub fn run<F, T>(context: &dyn Context, f: F) -> Result<T, Error>
+where
+    F: FnOnce() -> T,
+{
+    if TLV.is_set() {
+        Err(Error::from("StableMIR already running"))
+    } else {
+        let ptr: *const () = &context as *const &_ as _;
+        TLV.set(&Cell::new(ptr), || Ok(f()))
+    }
+}
+
+/// Execute the given function with access the compiler [Context].
+///
+/// I.e., This function will load the current context and calls a function with it.
+/// Do not nest these, as that will ICE.
+pub(crate) fn with<R>(f: impl FnOnce(&dyn Context) -> R) -> R {
+    assert!(TLV.is_set());
+    TLV.with(|tlv| {
+        let ptr = tlv.get();
+        assert!(!ptr.is_null());
+        f(unsafe { *(ptr as *const &dyn Context) })
+    })
+}
diff --git a/compiler/stable_mir/src/crate_def.rs b/compiler/stable_mir/src/crate_def.rs
new file mode 100644
index 000000000..70ca9e682
--- /dev/null
+++ b/compiler/stable_mir/src/crate_def.rs
@@ -0,0 +1,69 @@
+//! Module that define a common trait for things that represent a crate definition,
+//! such as, a function, a trait, an enum, and any other definitions.
+
+use crate::ty::Span;
+use crate::{with, Crate, Symbol};
+
+/// A unique identification number for each item accessible for the current compilation unit.
+#[derive(Clone, Copy, PartialEq, Eq, Hash)]
+pub struct DefId(pub(crate) usize);
+
+/// A trait for retrieving information about a particular definition.
+///
+/// Implementors must provide the implementation of `def_id` which will be used to retrieve
+/// information about a crate's definition.
+pub trait CrateDef {
+    /// Retrieve the unique identifier for the current definition.
+    fn def_id(&self) -> DefId;
+
+    /// Return the fully qualified name of the current definition.
+    fn name(&self) -> Symbol {
+        let def_id = self.def_id();
+        with(|cx| cx.def_name(def_id, false))
+    }
+
+    /// Return a trimmed name of this definition.
+    ///
+    /// This can be used to print more user friendly diagnostic messages.
+    ///
+    /// If a symbol name can only be imported from one place for a type, and as
+    /// long as it was not glob-imported anywhere in the current crate, we trim its
+    /// path and print only the name.
+    ///
+    /// For example, this function may shorten `std::vec::Vec` to just `Vec`,
+    /// as long as there is no other `Vec` importable anywhere.
+    fn trimmed_name(&self) -> Symbol {
+        let def_id = self.def_id();
+        with(|cx| cx.def_name(def_id, true))
+    }
+
+    /// Return information about the crate where this definition is declared.
+    ///
+    /// This will return the crate number and its name.
+    fn krate(&self) -> Crate {
+        let def_id = self.def_id();
+        with(|cx| cx.krate(def_id))
+    }
+
+    /// Return the span of this definition.
+    fn span(&self) -> Span {
+        let def_id = self.def_id();
+        with(|cx| cx.span_of_an_item(def_id))
+    }
+}
+
+macro_rules! crate_def {
+    ( $(#[$attr:meta])*
+      $vis:vis $name:ident $(;)?
+    ) => {
+        $(#[$attr])*
+        #[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)]
+        $vis struct $name(pub DefId);
+
+        impl CrateDef for $name {
+            fn def_id(&self) -> DefId {
+                self.0
+            }
+        }
+    };
+}
diff --git a/compiler/stable_mir/src/error.rs b/compiler/stable_mir/src/error.rs
index 199106914..7085fa937 100644
--- a/compiler/stable_mir/src/error.rs
+++ b/compiler/stable_mir/src/error.rs
@@ -4,9 +4,13 @@
 //! - [CompilerError]: This represents errors that can be raised when invoking the compiler.
 //! - [Error]: Generic error that represents the reason why a request that could not be fulfilled.
 
-use std::convert::From;
 use std::fmt::{Debug, Display, Formatter};
-use std::{error, fmt};
+use std::{error, fmt, io};
+
+macro_rules! error {
+     ($fmt: literal $(,)?) => { Error(format!($fmt)) };
+     ($fmt: literal, $($arg:tt)*) => { Error(format!($fmt, $($arg)*)) };
+ }
 
 /// An error type used to represent an error that has already been reported by the compiler.
 #[derive(Clone, Copy, PartialEq, Eq)]
@@ -23,11 +27,11 @@ pub enum CompilerError<T> {
 }
 
 /// A generic error to represent an API request that cannot be fulfilled.
-#[derive(Debug)]
-pub struct Error(String);
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct Error(pub(crate) String);
 
 impl Error {
-    pub(crate) fn new(msg: String) -> Self {
+    pub fn new(msg: String) -> Self {
         Self(msg)
     }
 }
@@ -74,3 +78,9 @@ where
 
 impl error::Error for Error {}
 impl<T> error::Error for CompilerError<T> where T: Display + Debug {}
+
+impl From<io::Error> for Error {
+    fn from(value: io::Error) -> Self {
+        Error(value.to_string())
+    }
+}
diff --git a/compiler/stable_mir/src/lib.rs b/compiler/stable_mir/src/lib.rs
index f316671b2..9194f1e6b 100644
--- a/compiler/stable_mir/src/lib.rs
+++ b/compiler/stable_mir/src/lib.rs
@@ -16,45 +16,45 @@
 //!
 //! The goal is to eventually be published on
 //! [crates.io](https://crates.io).
+#![feature(type_alias_impl_trait)]
+#[macro_use]
+extern crate scoped_tls;
 
-use crate::mir::mono::InstanceDef;
-use crate::mir::Body;
-use std::cell::Cell;
 use std::fmt;
 use std::fmt::Debug;
+use std::io;
 
-use self::ty::{
-    GenericPredicates, Generics, ImplDef, ImplTrait, IndexedVal, LineInfo, Span, TraitDecl,
-    TraitDef, Ty, TyKind,
-};
+use crate::compiler_interface::with;
+pub use crate::crate_def::CrateDef;
+pub use crate::crate_def::DefId;
+pub use crate::error::*;
+use crate::mir::pretty::function_name;
+use crate::mir::Body;
+use crate::mir::Mutability;
+use crate::ty::{ImplDef, ImplTrait, IndexedVal, Span, TraitDecl, TraitDef, Ty};
 
+pub mod abi;
+#[macro_use]
+pub mod crate_def;
+pub mod compiler_interface;
 #[macro_use]
-extern crate scoped_tls;
-
 pub mod error;
 pub mod mir;
+pub mod target;
 pub mod ty;
 pub mod visitor;
 
-pub use error::*;
-use mir::mono::Instance;
-use ty::{FnDef, GenericArgs};
-
 /// Use String for now but we should replace it.
 pub type Symbol = String;
 
 /// The number that identifies a crate.
 pub type CrateNum = usize;
 
-/// A unique identification number for each item accessible for the current compilation unit.
-#[derive(Clone, Copy, PartialEq, Eq)]
-pub struct DefId(usize);
-
 impl Debug for DefId {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         f.debug_struct("DefId")
             .field("id", &self.0)
-            .field("name", &with(|cx| cx.name_of_def_id(*self)))
+            .field("name", &with(|cx| cx.def_name(*self, false)))
             .finish()
     }
 }
@@ -69,19 +69,6 @@ impl IndexedVal for DefId {
     }
 }
 
-/// A unique identification number for each provenance
-#[derive(Clone, Copy, PartialEq, Eq, Debug)]
-pub struct AllocId(usize);
-
-impl IndexedVal for AllocId {
-    fn to_val(index: usize) -> Self {
-        AllocId(index)
-    }
-    fn to_index(&self) -> usize {
-        self.0
-    }
-}
-
 /// A list of crate items.
 pub type CrateItems = Vec<CrateItem>;
 
@@ -99,12 +86,26 @@ pub struct Crate {
     pub is_local: bool,
 }
 
-pub type DefKind = Opaque;
-pub type Filename = Opaque;
+#[derive(Copy, Clone, PartialEq, Eq, Debug, Hash)]
+pub enum ItemKind {
+    Fn,
+    Static,
+    Const,
+    Ctor(CtorKind),
+}
 
-/// Holds information about an item in the crate.
-#[derive(Copy, Clone, PartialEq, Eq, Debug)]
-pub struct CrateItem(pub DefId);
+#[derive(Copy, Clone, PartialEq, Eq, Debug, Hash)]
+pub enum CtorKind {
+    Const,
+    Fn,
+}
+
+pub type Filename = String;
+
+crate_def! {
+    /// Holds information about an item in a crate.
+    pub CrateItem;
+}
 
 impl CrateItem {
     pub fn body(&self) -> mir::Body {
@@ -115,17 +116,26 @@ impl CrateItem {
         with(|cx| cx.span_of_an_item(self.0))
     }
 
-    pub fn name(&self) -> String {
-        with(|cx| cx.name_of_def_id(self.0))
-    }
-
-    pub fn kind(&self) -> DefKind {
-        with(|cx| cx.def_kind(self.0))
+    pub fn kind(&self) -> ItemKind {
+        with(|cx| cx.item_kind(*self))
     }
 
     pub fn requires_monomorphization(&self) -> bool {
         with(|cx| cx.requires_monomorphization(self.0))
     }
+
+    pub fn ty(&self) -> Ty {
+        with(|cx| cx.def_ty(self.0))
+    }
+
+    pub fn is_foreign_item(&self) -> bool {
+        with(|cx| cx.is_foreign_item(self.0))
+    }
+
+    pub fn dump<W: io::Write>(&self, w: &mut W) -> io::Result<()> {
+        writeln!(w, "{}", function_name(*self))?;
+        self.body().dump(w)
+    }
 }
 
 /// Return the function where execution starts if the current
@@ -171,96 +181,8 @@ pub fn trait_impl(trait_impl: &ImplDef) -> ImplTrait {
     with(|cx| cx.trait_impl(trait_impl))
 }
 
-pub trait Context {
-    fn entry_fn(&self) -> Option<CrateItem>;
-    /// Retrieve all items of the local crate that have a MIR associated with them.
-    fn all_local_items(&self) -> CrateItems;
-    fn mir_body(&self, item: DefId) -> mir::Body;
-    fn all_trait_decls(&self) -> TraitDecls;
-    fn trait_decl(&self, trait_def: &TraitDef) -> TraitDecl;
-    fn all_trait_impls(&self) -> ImplTraitDecls;
-    fn trait_impl(&self, trait_impl: &ImplDef) -> ImplTrait;
-    fn generics_of(&self, def_id: DefId) -> Generics;
-    fn predicates_of(&self, def_id: DefId) -> GenericPredicates;
-    fn explicit_predicates_of(&self, def_id: DefId) -> GenericPredicates;
-    /// Get information about the local crate.
-    fn local_crate(&self) -> Crate;
-    /// Retrieve a list of all external crates.
-    fn external_crates(&self) -> Vec<Crate>;
-
-    /// Find a crate with the given name.
-    fn find_crates(&self, name: &str) -> Vec<Crate>;
-
-    /// Returns the name of given `DefId`
-    fn name_of_def_id(&self, def_id: DefId) -> String;
-
-    /// Returns printable, human readable form of `Span`
-    fn span_to_string(&self, span: Span) -> String;
-
-    /// Return filename from given `Span`, for diagnostic purposes
-    fn get_filename(&self, span: &Span) -> Filename;
-
-    /// Return lines corresponding to this `Span`
-    fn get_lines(&self, span: &Span) -> LineInfo;
-
-    /// Returns the `kind` of given `DefId`
-    fn def_kind(&self, def_id: DefId) -> DefKind;
-
-    /// `Span` of an item
-    fn span_of_an_item(&self, def_id: DefId) -> Span;
-
-    /// Obtain the representation of a type.
-    fn ty_kind(&self, ty: Ty) -> TyKind;
-
-    /// Get the body of an Instance.
-    /// FIXME: Monomorphize the body.
-    fn instance_body(&self, instance: InstanceDef) -> Body;
-
-    /// Get the instance type with generic substitutions applied and lifetimes erased.
-    fn instance_ty(&self, instance: InstanceDef) -> Ty;
-
-    /// Get the instance.
-    fn instance_def_id(&self, instance: InstanceDef) -> DefId;
-
-    /// Get the instance mangled name.
-    fn instance_mangled_name(&self, instance: InstanceDef) -> String;
-
-    /// Convert a non-generic crate item into an instance.
-    /// This function will panic if the item is generic.
-    fn mono_instance(&self, item: CrateItem) -> Instance;
-
-    /// Item requires monomorphization.
-    fn requires_monomorphization(&self, def_id: DefId) -> bool;
-
-    /// Resolve an instance from the given function definition and generic arguments.
-    fn resolve_instance(&self, def: FnDef, args: &GenericArgs) -> Option<Instance>;
-}
-
-// A thread local variable that stores a pointer to the tables mapping between TyCtxt
-// datastructures and stable MIR datastructures
-scoped_thread_local! (static TLV: Cell<*const ()>);
-
-pub fn run(context: &dyn Context, f: impl FnOnce()) {
-    assert!(!TLV.is_set());
-    let ptr: *const () = &context as *const &_ as _;
-    TLV.set(&Cell::new(ptr), || {
-        f();
-    });
-}
-
-/// Loads the current context and calls a function with it.
-/// Do not nest these, as that will ICE.
-pub fn with<R>(f: impl FnOnce(&dyn Context) -> R) -> R {
-    assert!(TLV.is_set());
-    TLV.with(|tlv| {
-        let ptr = tlv.get();
-        assert!(!ptr.is_null());
-        f(unsafe { *(ptr as *const &dyn Context) })
-    })
-}
-
 /// A type that provides internal information but that can still be used for debug purpose.
-#[derive(Clone, Eq, PartialEq)]
+#[derive(Clone, PartialEq, Eq, Hash)]
 pub struct Opaque(String);
 
 impl std::fmt::Display for Opaque {
@@ -271,7 +193,7 @@ impl std::fmt::Display for Opaque {
 
 impl std::fmt::Debug for Opaque {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{:?}", self.0)
+        write!(f, "{}", self.0)
     }
 }
 
diff --git a/compiler/stable_mir/src/mir.rs b/compiler/stable_mir/src/mir.rs
index 2e1714b49..82555461d 100644
--- a/compiler/stable_mir/src/mir.rs
+++ b/compiler/stable_mir/src/mir.rs
@@ -1,5 +1,7 @@
+pub mod alloc;
 mod body;
 pub mod mono;
+pub mod pretty;
 pub mod visit;
 
 pub use body::*;
diff --git a/compiler/stable_mir/src/mir/alloc.rs b/compiler/stable_mir/src/mir/alloc.rs
new file mode 100644
index 000000000..c780042ff
--- /dev/null
+++ b/compiler/stable_mir/src/mir/alloc.rs
@@ -0,0 +1,83 @@
+//! This module provides methods to retrieve allocation information, such as static variables.
+use crate::mir::mono::{Instance, StaticDef};
+use crate::target::{Endian, MachineInfo};
+use crate::ty::{Allocation, Binder, ExistentialTraitRef, IndexedVal, Ty};
+use crate::{with, Error};
+use std::io::Read;
+
+/// An allocation in the SMIR global memory can be either a function pointer,
+/// a static, or a "real" allocation with some data in it.
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub enum GlobalAlloc {
+    /// The alloc ID is used as a function pointer.
+    Function(Instance),
+    /// This alloc ID points to a symbolic (not-reified) vtable.
+    /// The `None` trait ref is used to represent auto traits.
+    VTable(Ty, Option<Binder<ExistentialTraitRef>>),
+    /// The alloc ID points to a "lazy" static variable that did not get computed (yet).
+    /// This is also used to break the cycle in recursive statics.
+    Static(StaticDef),
+    /// The alloc ID points to memory.
+    Memory(Allocation),
+}
+
+impl From<AllocId> for GlobalAlloc {
+    fn from(value: AllocId) -> Self {
+        with(|cx| cx.global_alloc(value))
+    }
+}
+
+impl GlobalAlloc {
+    /// Retrieve the allocation id for a global allocation if it exists.
+    ///
+    /// For `[GlobalAlloc::VTable]`, this will return the allocation for the VTable of the given
+    /// type for the optional trait if the type implements the trait.
+    ///
+    /// This method will always return `None` for allocations other than `[GlobalAlloc::VTable]`.
+    pub fn vtable_allocation(&self) -> Option<AllocId> {
+        with(|cx| cx.vtable_allocation(self))
+    }
+}
+
+/// A unique identification number for each provenance
+#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)]
+pub struct AllocId(usize);
+
+impl IndexedVal for AllocId {
+    fn to_val(index: usize) -> Self {
+        AllocId(index)
+    }
+    fn to_index(&self) -> usize {
+        self.0
+    }
+}
+
+/// Utility function used to read an allocation data into a unassigned integer.
+pub(crate) fn read_target_uint(mut bytes: &[u8]) -> Result<u128, Error> {
+    let mut buf = [0u8; std::mem::size_of::<u128>()];
+    match MachineInfo::target_endianess() {
+        Endian::Little => {
+            bytes.read(&mut buf)?;
+            Ok(u128::from_le_bytes(buf))
+        }
+        Endian::Big => {
+            bytes.read(&mut buf[16 - bytes.len()..])?;
+            Ok(u128::from_be_bytes(buf))
+        }
+    }
+}
+
+/// Utility function used to read an allocation data into an assigned integer.
+pub(crate) fn read_target_int(mut bytes: &[u8]) -> Result<i128, Error> {
+    let mut buf = [0u8; std::mem::size_of::<i128>()];
+    match MachineInfo::target_endianess() {
+        Endian::Little => {
+            bytes.read(&mut buf)?;
+            Ok(i128::from_le_bytes(buf))
+        }
+        Endian::Big => {
+            bytes.read(&mut buf[16 - bytes.len()..])?;
+            Ok(i128::from_be_bytes(buf))
+        }
+    }
+}
diff --git a/compiler/stable_mir/src/mir/body.rs b/compiler/stable_mir/src/mir/body.rs
index 069337836..b8fd9370a 100644
--- a/compiler/stable_mir/src/mir/body.rs
+++ b/compiler/stable_mir/src/mir/body.rs
@@ -1,36 +1,59 @@
-use crate::ty::{AdtDef, ClosureDef, Const, CoroutineDef, GenericArgs, Movability, Region, Ty};
-use crate::Opaque;
-use crate::Span;
-
+use crate::mir::pretty::{function_body, pretty_statement, pretty_terminator};
+use crate::ty::{
+    AdtDef, ClosureDef, Const, CoroutineDef, GenericArgs, Movability, Region, RigidTy, Ty, TyKind,
+    VariantIdx,
+};
+use crate::{Error, Opaque, Span, Symbol};
+use std::io;
 /// The SMIR representation of a single function.
 #[derive(Clone, Debug)]
 pub struct Body {
     pub blocks: Vec<BasicBlock>,
 
-    // Declarations of locals within the function.
-    //
-    // The first local is the return value pointer, followed by `arg_count`
-    // locals for the function arguments, followed by any user-declared
-    // variables and temporaries.
+    /// Declarations of locals within the function.
+    ///
+    /// The first local is the return value pointer, followed by `arg_count`
+    /// locals for the function arguments, followed by any user-declared
+    /// variables and temporaries.
     pub(super) locals: LocalDecls,
 
-    // The number of arguments this function takes.
+    /// The number of arguments this function takes.
     pub(super) arg_count: usize,
+
+    /// Debug information pertaining to user variables, including captures.
+    pub var_debug_info: Vec<VarDebugInfo>,
+
+    /// Mark an argument (which must be a tuple) as getting passed as its individual components.
+    ///
+    /// This is used for the "rust-call" ABI such as closures.
+    pub(super) spread_arg: Option<Local>,
+
+    /// The span that covers the entire function body.
+    pub span: Span,
 }
 
+pub type BasicBlockIdx = usize;
+
 impl Body {
     /// Constructs a `Body`.
     ///
     /// A constructor is required to build a `Body` from outside the crate
     /// because the `arg_count` and `locals` fields are private.
-    pub fn new(blocks: Vec<BasicBlock>, locals: LocalDecls, arg_count: usize) -> Self {
+    pub fn new(
+        blocks: Vec<BasicBlock>,
+        locals: LocalDecls,
+        arg_count: usize,
+        var_debug_info: Vec<VarDebugInfo>,
+        spread_arg: Option<Local>,
+        span: Span,
+    ) -> Self {
         // If locals doesn't contain enough entries, it can lead to panics in
         // `ret_local`, `arg_locals`, and `inner_locals`.
         assert!(
             locals.len() > arg_count,
             "A Body must contain at least a local for the return value and each of the function's arguments"
         );
-        Self { blocks, locals, arg_count }
+        Self { blocks, locals, arg_count, var_debug_info, spread_arg, span }
     }
 
     /// Return local that holds this function's return value.
@@ -56,6 +79,44 @@ impl Body {
     pub fn locals(&self) -> &[LocalDecl] {
         &self.locals
     }
+
+    /// Get the local declaration for this local.
+    pub fn local_decl(&self, local: Local) -> Option<&LocalDecl> {
+        self.locals.get(local)
+    }
+
+    /// Get an iterator for all local declarations.
+    pub fn local_decls(&self) -> impl Iterator<Item = (Local, &LocalDecl)> {
+        self.locals.iter().enumerate()
+    }
+
+    pub fn dump<W: io::Write>(&self, w: &mut W) -> io::Result<()> {
+        writeln!(w, "{}", function_body(self))?;
+        self.blocks
+            .iter()
+            .enumerate()
+            .map(|(index, block)| -> io::Result<()> {
+                writeln!(w, "    bb{}: {{", index)?;
+                let _ = block
+                    .statements
+                    .iter()
+                    .map(|statement| -> io::Result<()> {
+                        writeln!(w, "{}", pretty_statement(&statement.kind))?;
+                        Ok(())
+                    })
+                    .collect::<Vec<_>>();
+                pretty_terminator(&block.terminator.kind, w)?;
+                writeln!(w, "").unwrap();
+                writeln!(w, "    }}").unwrap();
+                Ok(())
+            })
+            .collect::<Result<Vec<_>, _>>()?;
+        Ok(())
+    }
+
+    pub fn spread_arg(&self) -> Option<Local> {
+        self.spread_arg
+    }
 }
 
 type LocalDecls = Vec<LocalDecl>;
@@ -64,9 +125,10 @@ type LocalDecls = Vec<LocalDecl>;
 pub struct LocalDecl {
     pub ty: Ty,
     pub span: Span,
+    pub mutability: Mutability,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, PartialEq, Eq, Debug)]
 pub struct BasicBlock {
     pub statements: Vec<Statement>,
     pub terminator: Terminator,
@@ -78,15 +140,22 @@ pub struct Terminator {
     pub span: Span,
 }
 
+impl Terminator {
+    pub fn successors(&self) -> Successors {
+        self.kind.successors()
+    }
+}
+
+pub type Successors = Vec<BasicBlockIdx>;
+
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub enum TerminatorKind {
     Goto {
-        target: usize,
+        target: BasicBlockIdx,
     },
     SwitchInt {
         discr: Operand,
-        targets: Vec<SwitchTarget>,
-        otherwise: usize,
+        targets: SwitchTargets,
     },
     Resume,
     Abort,
@@ -94,34 +163,81 @@ pub enum TerminatorKind {
     Unreachable,
     Drop {
         place: Place,
-        target: usize,
+        target: BasicBlockIdx,
         unwind: UnwindAction,
     },
     Call {
         func: Operand,
         args: Vec<Operand>,
         destination: Place,
-        target: Option<usize>,
+        target: Option<BasicBlockIdx>,
         unwind: UnwindAction,
     },
     Assert {
         cond: Operand,
         expected: bool,
         msg: AssertMessage,
-        target: usize,
+        target: BasicBlockIdx,
         unwind: UnwindAction,
     },
-    CoroutineDrop,
     InlineAsm {
         template: String,
         operands: Vec<InlineAsmOperand>,
         options: String,
         line_spans: String,
-        destination: Option<usize>,
+        destination: Option<BasicBlockIdx>,
         unwind: UnwindAction,
     },
 }
 
+impl TerminatorKind {
+    pub fn successors(&self) -> Successors {
+        use self::TerminatorKind::*;
+        match *self {
+            Call { target: Some(t), unwind: UnwindAction::Cleanup(u), .. }
+            | Drop { target: t, unwind: UnwindAction::Cleanup(u), .. }
+            | Assert { target: t, unwind: UnwindAction::Cleanup(u), .. }
+            | InlineAsm { destination: Some(t), unwind: UnwindAction::Cleanup(u), .. } => {
+                vec![t, u]
+            }
+            Goto { target: t }
+            | Call { target: None, unwind: UnwindAction::Cleanup(t), .. }
+            | Call { target: Some(t), unwind: _, .. }
+            | Drop { target: t, unwind: _, .. }
+            | Assert { target: t, unwind: _, .. }
+            | InlineAsm { destination: None, unwind: UnwindAction::Cleanup(t), .. }
+            | InlineAsm { destination: Some(t), unwind: _, .. } => {
+                vec![t]
+            }
+
+            Return
+            | Resume
+            | Abort
+            | Unreachable
+            | Call { target: None, unwind: _, .. }
+            | InlineAsm { destination: None, unwind: _, .. } => {
+                vec![]
+            }
+            SwitchInt { ref targets, .. } => targets.all_targets(),
+        }
+    }
+
+    pub fn unwind(&self) -> Option<&UnwindAction> {
+        match *self {
+            TerminatorKind::Goto { .. }
+            | TerminatorKind::Return
+            | TerminatorKind::Unreachable
+            | TerminatorKind::Resume
+            | TerminatorKind::Abort
+            | TerminatorKind::SwitchInt { .. } => None,
+            TerminatorKind::Call { ref unwind, .. }
+            | TerminatorKind::Assert { ref unwind, .. }
+            | TerminatorKind::Drop { ref unwind, .. }
+            | TerminatorKind::InlineAsm { ref unwind, .. } => Some(unwind),
+        }
+    }
+}
+
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub struct InlineAsmOperand {
     pub in_value: Option<Operand>,
@@ -131,12 +247,12 @@ pub struct InlineAsmOperand {
     pub raw_rpr: String,
 }
 
-#[derive(Clone, Debug, Eq, PartialEq)]
+#[derive(Copy, Clone, Debug, Eq, PartialEq)]
 pub enum UnwindAction {
     Continue,
     Unreachable,
     Terminate,
-    Cleanup(usize),
+    Cleanup(BasicBlockIdx),
 }
 
 #[derive(Clone, Debug, Eq, PartialEq)]
@@ -151,7 +267,58 @@ pub enum AssertMessage {
     MisalignedPointerDereference { required: Operand, found: Operand },
 }
 
-#[derive(Clone, Debug, Eq, PartialEq)]
+impl AssertMessage {
+    pub fn description(&self) -> Result<&'static str, Error> {
+        match self {
+            AssertMessage::Overflow(BinOp::Add, _, _) => Ok("attempt to add with overflow"),
+            AssertMessage::Overflow(BinOp::Sub, _, _) => Ok("attempt to subtract with overflow"),
+            AssertMessage::Overflow(BinOp::Mul, _, _) => Ok("attempt to multiply with overflow"),
+            AssertMessage::Overflow(BinOp::Div, _, _) => Ok("attempt to divide with overflow"),
+            AssertMessage::Overflow(BinOp::Rem, _, _) => {
+                Ok("attempt to calculate the remainder with overflow")
+            }
+            AssertMessage::OverflowNeg(_) => Ok("attempt to negate with overflow"),
+            AssertMessage::Overflow(BinOp::Shr, _, _) => Ok("attempt to shift right with overflow"),
+            AssertMessage::Overflow(BinOp::Shl, _, _) => Ok("attempt to shift left with overflow"),
+            AssertMessage::Overflow(op, _, _) => Err(error!("`{:?}` cannot overflow", op)),
+            AssertMessage::DivisionByZero(_) => Ok("attempt to divide by zero"),
+            AssertMessage::RemainderByZero(_) => {
+                Ok("attempt to calculate the remainder with a divisor of zero")
+            }
+            AssertMessage::ResumedAfterReturn(CoroutineKind::Coroutine) => {
+                Ok("coroutine resumed after completion")
+            }
+            AssertMessage::ResumedAfterReturn(CoroutineKind::Async(_)) => {
+                Ok("`async fn` resumed after completion")
+            }
+            AssertMessage::ResumedAfterReturn(CoroutineKind::Gen(_)) => {
+                Ok("`async gen fn` resumed after completion")
+            }
+            AssertMessage::ResumedAfterReturn(CoroutineKind::AsyncGen(_)) => {
+                Ok("`gen fn` should just keep returning `AssertMessage::None` after completion")
+            }
+            AssertMessage::ResumedAfterPanic(CoroutineKind::Coroutine) => {
+                Ok("coroutine resumed after panicking")
+            }
+            AssertMessage::ResumedAfterPanic(CoroutineKind::Async(_)) => {
+                Ok("`async fn` resumed after panicking")
+            }
+            AssertMessage::ResumedAfterPanic(CoroutineKind::Gen(_)) => {
+                Ok("`async gen fn` resumed after panicking")
+            }
+            AssertMessage::ResumedAfterPanic(CoroutineKind::AsyncGen(_)) => {
+                Ok("`gen fn` should just keep returning `AssertMessage::None` after panicking")
+            }
+
+            AssertMessage::BoundsCheck { .. } => Ok("index out of bounds"),
+            AssertMessage::MisalignedPointerDereference { .. } => {
+                Ok("misaligned pointer dereference")
+            }
+        }
+    }
+}
+
+#[derive(Copy, Clone, Debug, Eq, PartialEq)]
 pub enum BinOp {
     Add,
     AddUnchecked,
@@ -177,7 +344,47 @@ pub enum BinOp {
     Offset,
 }
 
-#[derive(Clone, Debug, Eq, PartialEq)]
+impl BinOp {
+    /// Return the type of this operation for the given input Ty.
+    /// This function does not perform type checking, and it currently doesn't handle SIMD.
+    pub fn ty(&self, lhs_ty: Ty, rhs_ty: Ty) -> Ty {
+        match self {
+            BinOp::Add
+            | BinOp::AddUnchecked
+            | BinOp::Sub
+            | BinOp::SubUnchecked
+            | BinOp::Mul
+            | BinOp::MulUnchecked
+            | BinOp::Div
+            | BinOp::Rem
+            | BinOp::BitXor
+            | BinOp::BitAnd
+            | BinOp::BitOr => {
+                assert_eq!(lhs_ty, rhs_ty);
+                assert!(lhs_ty.kind().is_primitive());
+                lhs_ty
+            }
+            BinOp::Shl | BinOp::ShlUnchecked | BinOp::Shr | BinOp::ShrUnchecked => {
+                assert!(lhs_ty.kind().is_primitive());
+                assert!(rhs_ty.kind().is_primitive());
+                lhs_ty
+            }
+            BinOp::Offset => {
+                assert!(lhs_ty.kind().is_raw_ptr());
+                assert!(rhs_ty.kind().is_integral());
+                lhs_ty
+            }
+            BinOp::Eq | BinOp::Lt | BinOp::Le | BinOp::Ne | BinOp::Ge | BinOp::Gt => {
+                assert_eq!(lhs_ty, rhs_ty);
+                let lhs_kind = lhs_ty.kind();
+                assert!(lhs_kind.is_primitive() || lhs_kind.is_raw_ptr() || lhs_kind.is_fn_ptr());
+                Ty::bool_ty()
+            }
+        }
+    }
+}
+
+#[derive(Copy, Clone, Debug, Eq, PartialEq)]
 pub enum UnOp {
     Not,
     Neg,
@@ -188,9 +395,10 @@ pub enum CoroutineKind {
     Async(CoroutineSource),
     Coroutine,
     Gen(CoroutineSource),
+    AsyncGen(CoroutineSource),
 }
 
-#[derive(Clone, Debug, Eq, PartialEq)]
+#[derive(Copy, Clone, Debug, Eq, PartialEq)]
 pub enum CoroutineSource {
     Block,
     Closure,
@@ -214,7 +422,7 @@ pub enum FakeReadCause {
 }
 
 /// Describes what kind of retag is to be performed
-#[derive(Clone, Debug, Eq, PartialEq, Hash)]
+#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)]
 pub enum RetagKind {
     FnEntry,
     TwoPhase,
@@ -222,7 +430,7 @@ pub enum RetagKind {
     Default,
 }
 
-#[derive(Clone, Debug, Eq, PartialEq, Hash)]
+#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)]
 pub enum Variance {
     Covariant,
     Invariant,
@@ -378,6 +586,63 @@ pub enum Rvalue {
     Use(Operand),
 }
 
+impl Rvalue {
+    pub fn ty(&self, locals: &[LocalDecl]) -> Result<Ty, Error> {
+        match self {
+            Rvalue::Use(operand) => operand.ty(locals),
+            Rvalue::Repeat(operand, count) => {
+                Ok(Ty::new_array_with_const_len(operand.ty(locals)?, count.clone()))
+            }
+            Rvalue::ThreadLocalRef(did) => Ok(did.ty()),
+            Rvalue::Ref(reg, bk, place) => {
+                let place_ty = place.ty(locals)?;
+                Ok(Ty::new_ref(reg.clone(), place_ty, bk.to_mutable_lossy()))
+            }
+            Rvalue::AddressOf(mutability, place) => {
+                let place_ty = place.ty(locals)?;
+                Ok(Ty::new_ptr(place_ty, *mutability))
+            }
+            Rvalue::Len(..) => Ok(Ty::usize_ty()),
+            Rvalue::Cast(.., ty) => Ok(*ty),
+            Rvalue::BinaryOp(op, lhs, rhs) => {
+                let lhs_ty = lhs.ty(locals)?;
+                let rhs_ty = rhs.ty(locals)?;
+                Ok(op.ty(lhs_ty, rhs_ty))
+            }
+            Rvalue::CheckedBinaryOp(op, lhs, rhs) => {
+                let lhs_ty = lhs.ty(locals)?;
+                let rhs_ty = rhs.ty(locals)?;
+                let ty = op.ty(lhs_ty, rhs_ty);
+                Ok(Ty::new_tuple(&[ty, Ty::bool_ty()]))
+            }
+            Rvalue::UnaryOp(UnOp::Not | UnOp::Neg, operand) => operand.ty(locals),
+            Rvalue::Discriminant(place) => {
+                let place_ty = place.ty(locals)?;
+                place_ty
+                    .kind()
+                    .discriminant_ty()
+                    .ok_or_else(|| error!("Expected a `RigidTy` but found: {place_ty:?}"))
+            }
+            Rvalue::NullaryOp(NullOp::SizeOf | NullOp::AlignOf | NullOp::OffsetOf(..), _) => {
+                Ok(Ty::usize_ty())
+            }
+            Rvalue::Aggregate(ak, ops) => match *ak {
+                AggregateKind::Array(ty) => Ty::try_new_array(ty, ops.len() as u64),
+                AggregateKind::Tuple => Ok(Ty::new_tuple(
+                    &ops.iter().map(|op| op.ty(locals)).collect::<Result<Vec<_>, _>>()?,
+                )),
+                AggregateKind::Adt(def, _, ref args, _, _) => Ok(def.ty_with_args(args)),
+                AggregateKind::Closure(def, ref args) => Ok(Ty::new_closure(def, args.clone())),
+                AggregateKind::Coroutine(def, ref args, mov) => {
+                    Ok(Ty::new_coroutine(def, args.clone(), mov))
+                }
+            },
+            Rvalue::ShallowInitBox(_, ty) => Ok(Ty::new_box(*ty)),
+            Rvalue::CopyForDeref(place) => place.ty(locals),
+        }
+    }
+}
+
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub enum AggregateKind {
     Array(Ty),
@@ -398,23 +663,188 @@ pub enum Operand {
 pub struct Place {
     pub local: Local,
     /// projection out of a place (access a field, deref a pointer, etc)
-    pub projection: String,
+    pub projection: Vec<ProjectionElem>,
+}
+
+impl From<Local> for Place {
+    fn from(local: Local) -> Self {
+        Place { local, projection: vec![] }
+    }
+}
+
+/// Debug information pertaining to a user variable.
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct VarDebugInfo {
+    /// The variable name.
+    pub name: Symbol,
+
+    /// Source info of the user variable, including the scope
+    /// within which the variable is visible (to debuginfo).
+    pub source_info: SourceInfo,
+
+    /// The user variable's data is split across several fragments,
+    /// each described by a `VarDebugInfoFragment`.
+    pub composite: Option<VarDebugInfoFragment>,
+
+    /// Where the data for this user variable is to be found.
+    pub value: VarDebugInfoContents,
+
+    /// When present, indicates what argument number this variable is in the function that it
+    /// originated from (starting from 1). Note, if MIR inlining is enabled, then this is the
+    /// argument number in the original function before it was inlined.
+    pub argument_index: Option<u16>,
+}
+
+impl VarDebugInfo {
+    /// Return a local variable if this info is related to one.
+    pub fn local(&self) -> Option<Local> {
+        match &self.value {
+            VarDebugInfoContents::Place(place) if place.projection.is_empty() => Some(place.local),
+            VarDebugInfoContents::Place(_) | VarDebugInfoContents::Const(_) => None,
+        }
+    }
+
+    /// Return a constant if this info is related to one.
+    pub fn constant(&self) -> Option<&ConstOperand> {
+        match &self.value {
+            VarDebugInfoContents::Place(_) => None,
+            VarDebugInfoContents::Const(const_op) => Some(const_op),
+        }
+    }
+}
+
+pub type SourceScope = u32;
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct SourceInfo {
+    pub span: Span,
+    pub scope: SourceScope,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct VarDebugInfoFragment {
+    pub ty: Ty,
+    pub projection: Vec<ProjectionElem>,
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub enum VarDebugInfoContents {
+    Place(Place),
+    Const(ConstOperand),
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct ConstOperand {
+    pub span: Span,
+    pub user_ty: Option<UserTypeAnnotationIndex>,
+    pub const_: Const,
+}
+
+// In MIR ProjectionElem is parameterized on the second Field argument and the Index argument. This
+// is so it can be used for both Places (for which the projection elements are of type
+// ProjectionElem<Local, Ty>) and user-provided type annotations (for which the projection elements
+// are of type ProjectionElem<(), ()>). In SMIR we don't need this generality, so we just use
+// ProjectionElem for Places.
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub enum ProjectionElem {
+    /// Dereference projections (e.g. `*_1`) project to the address referenced by the base place.
+    Deref,
+
+    /// A field projection (e.g., `f` in `_1.f`) project to a field in the base place. The field is
+    /// referenced by source-order index rather than the name of the field. The fields type is also
+    /// given.
+    Field(FieldIdx, Ty),
+
+    /// Index into a slice/array. The value of the index is computed at runtime using the `V`
+    /// argument.
+    ///
+    /// Note that this does not also dereference, and so it does not exactly correspond to slice
+    /// indexing in Rust. In other words, in the below Rust code:
+    ///
+    /// ```rust
+    /// let x = &[1, 2, 3, 4];
+    /// let i = 2;
+    /// x[i];
+    /// ```
+    ///
+    /// The `x[i]` is turned into a `Deref` followed by an `Index`, not just an `Index`. The same
+    /// thing is true of the `ConstantIndex` and `Subslice` projections below.
+    Index(Local),
+
+    /// Index into a slice/array given by offsets.
+    ///
+    /// These indices are generated by slice patterns. Easiest to explain by example:
+    ///
+    /// ```ignore (illustrative)
+    /// [X, _, .._, _, _] => { offset: 0, min_length: 4, from_end: false },
+    /// [_, X, .._, _, _] => { offset: 1, min_length: 4, from_end: false },
+    /// [_, _, .._, X, _] => { offset: 2, min_length: 4, from_end: true },
+    /// [_, _, .._, _, X] => { offset: 1, min_length: 4, from_end: true },
+    /// ```
+    ConstantIndex {
+        /// index or -index (in Python terms), depending on from_end
+        offset: u64,
+        /// The thing being indexed must be at least this long. For arrays this
+        /// is always the exact length.
+        min_length: u64,
+        /// Counting backwards from end? This is always false when indexing an
+        /// array.
+        from_end: bool,
+    },
+
+    /// Projects a slice from the base place.
+    ///
+    /// These indices are generated by slice patterns. If `from_end` is true, this represents
+    /// `slice[from..slice.len() - to]`. Otherwise it represents `array[from..to]`.
+    Subslice {
+        from: u64,
+        to: u64,
+        /// Whether `to` counts from the start or end of the array/slice.
+        from_end: bool,
+    },
+
+    /// "Downcast" to a variant of an enum or a coroutine.
+    Downcast(VariantIdx),
+
+    /// Like an explicit cast from an opaque type to a concrete type, but without
+    /// requiring an intermediate variable.
+    OpaqueCast(Ty),
+
+    /// A `Subtype(T)` projection is applied to any `StatementKind::Assign` where
+    /// type of lvalue doesn't match the type of rvalue, the primary goal is making subtyping
+    /// explicit during optimizations and codegen.
+    ///
+    /// This projection doesn't impact the runtime behavior of the program except for potentially changing
+    /// some type metadata of the interpreter or codegen backend.
+    Subtype(Ty),
 }
 
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub struct UserTypeProjection {
     pub base: UserTypeAnnotationIndex,
-    pub projection: String,
+
+    pub projection: Opaque,
 }
 
 pub type Local = usize;
 
 pub const RETURN_LOCAL: Local = 0;
 
-type FieldIdx = usize;
-
-/// The source-order index of a variant in a type.
-pub type VariantIdx = usize;
+/// The source-order index of a field in a variant.
+///
+/// For example, in the following types,
+/// ```ignore(illustrative)
+/// enum Demo1 {
+///    Variant0 { a: bool, b: i32 },
+///    Variant1 { c: u8, d: u64 },
+/// }
+/// struct Demo2 { e: u8, f: u16, g: u8 }
+/// ```
+/// `a`'s `FieldIdx` is `0`,
+/// `b`'s `FieldIdx` is `1`,
+/// `c`'s `FieldIdx` is `0`, and
+/// `g`'s `FieldIdx` is `2`.
+pub type FieldIdx = usize;
 
 type UserTypeAnnotationIndex = usize;
 
@@ -425,13 +855,45 @@ pub struct Constant {
     pub literal: Const,
 }
 
+/// The possible branch sites of a [TerminatorKind::SwitchInt].
 #[derive(Clone, Debug, Eq, PartialEq)]
-pub struct SwitchTarget {
-    pub value: u128,
-    pub target: usize,
+pub struct SwitchTargets {
+    /// The conditional branches where the first element represents the value that guards this
+    /// branch, and the second element is the branch target.
+    branches: Vec<(u128, BasicBlockIdx)>,
+    /// The `otherwise` branch which will be taken in case none of the conditional branches are
+    /// satisfied.
+    otherwise: BasicBlockIdx,
 }
 
-#[derive(Clone, Debug, Eq, PartialEq)]
+impl SwitchTargets {
+    /// All possible targets including the `otherwise` target.
+    pub fn all_targets(&self) -> Successors {
+        self.branches.iter().map(|(_, target)| *target).chain(Some(self.otherwise)).collect()
+    }
+
+    /// The `otherwise` branch target.
+    pub fn otherwise(&self) -> BasicBlockIdx {
+        self.otherwise
+    }
+
+    /// The conditional targets which are only taken if the pattern matches the given value.
+    pub fn branches(&self) -> impl Iterator<Item = (u128, BasicBlockIdx)> + '_ {
+        self.branches.iter().copied()
+    }
+
+    /// The number of targets including `otherwise`.
+    pub fn len(&self) -> usize {
+        self.branches.len() + 1
+    }
+
+    /// Create a new SwitchTargets from the given branches and `otherwise` target.
+    pub fn new(branches: Vec<(u128, BasicBlockIdx)>, otherwise: BasicBlockIdx) -> SwitchTargets {
+        SwitchTargets { branches, otherwise }
+    }
+}
+
+#[derive(Copy, Clone, Debug, Eq, PartialEq)]
 pub enum BorrowKind {
     /// Data must be immutable and is aliasable.
     Shared,
@@ -449,14 +911,25 @@ pub enum BorrowKind {
     },
 }
 
-#[derive(Clone, Debug, Eq, PartialEq)]
+impl BorrowKind {
+    pub fn to_mutable_lossy(self) -> Mutability {
+        match self {
+            BorrowKind::Mut { .. } => Mutability::Mut,
+            BorrowKind::Shared => Mutability::Not,
+            // FIXME: There's no type corresponding to a shallow borrow, so use `&` as an approximation.
+            BorrowKind::Fake => Mutability::Not,
+        }
+    }
+}
+
+#[derive(Copy, Clone, Debug, Eq, PartialEq)]
 pub enum MutBorrowKind {
     Default,
     TwoPhaseBorrow,
     ClosureCapture,
 }
 
-#[derive(Clone, Debug, PartialEq, Eq)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
 pub enum Mutability {
     Not,
     Mut,
@@ -468,7 +941,7 @@ pub enum Safety {
     Normal,
 }
 
-#[derive(Clone, Debug, Eq, PartialEq)]
+#[derive(Copy, Clone, Debug, Eq, PartialEq)]
 pub enum PointerCoercion {
     /// Go from a fn-item type to a fn-pointer type.
     ReifyFnPointer,
@@ -495,7 +968,7 @@ pub enum PointerCoercion {
     Unsize,
 }
 
-#[derive(Clone, Debug, Eq, PartialEq)]
+#[derive(Copy, Clone, Debug, Eq, PartialEq)]
 pub enum CastKind {
     PointerExposeAddress,
     PointerFromExposedAddress,
@@ -521,10 +994,16 @@ pub enum NullOp {
 }
 
 impl Operand {
-    pub fn ty(&self, locals: &[LocalDecl]) -> Ty {
+    /// Get the type of an operand relative to the local declaration.
+    ///
+    /// In order to retrieve the correct type, the `locals` argument must match the list of all
+    /// locals from the function body where this operand originates from.
+    ///
+    /// Errors indicate a malformed operand or incompatible locals list.
+    pub fn ty(&self, locals: &[LocalDecl]) -> Result<Ty, Error> {
         match self {
             Operand::Copy(place) | Operand::Move(place) => place.ty(locals),
-            Operand::Constant(c) => c.ty(),
+            Operand::Constant(c) => Ok(c.ty()),
         }
     }
 }
@@ -536,8 +1015,57 @@ impl Constant {
 }
 
 impl Place {
-    pub fn ty(&self, locals: &[LocalDecl]) -> Ty {
-        let _start_ty = locals[self.local].ty;
-        todo!("Implement projection")
+    /// Resolve down the chain of projections to get the type referenced at the end of it.
+    /// E.g.:
+    /// Calling `ty()` on `var.field` should return the type of `field`.
+    ///
+    /// In order to retrieve the correct type, the `locals` argument must match the list of all
+    /// locals from the function body where this place originates from.
+    pub fn ty(&self, locals: &[LocalDecl]) -> Result<Ty, Error> {
+        let start_ty = locals[self.local].ty;
+        self.projection.iter().fold(Ok(start_ty), |place_ty, elem| {
+            let ty = place_ty?;
+            match elem {
+                ProjectionElem::Deref => Self::deref_ty(ty),
+                ProjectionElem::Field(_idx, fty) => Ok(*fty),
+                ProjectionElem::Index(_) | ProjectionElem::ConstantIndex { .. } => {
+                    Self::index_ty(ty)
+                }
+                ProjectionElem::Subslice { from, to, from_end } => {
+                    Self::subslice_ty(ty, from, to, from_end)
+                }
+                ProjectionElem::Downcast(_) => Ok(ty),
+                ProjectionElem::OpaqueCast(ty) | ProjectionElem::Subtype(ty) => Ok(*ty),
+            }
+        })
+    }
+
+    fn index_ty(ty: Ty) -> Result<Ty, Error> {
+        ty.kind().builtin_index().ok_or_else(|| error!("Cannot index non-array type: {ty:?}"))
+    }
+
+    fn subslice_ty(ty: Ty, from: &u64, to: &u64, from_end: &bool) -> Result<Ty, Error> {
+        let ty_kind = ty.kind();
+        match ty_kind {
+            TyKind::RigidTy(RigidTy::Slice(..)) => Ok(ty),
+            TyKind::RigidTy(RigidTy::Array(inner, _)) if !from_end => Ty::try_new_array(
+                inner,
+                to.checked_sub(*from).ok_or_else(|| error!("Subslice overflow: {from}..{to}"))?,
+            ),
+            TyKind::RigidTy(RigidTy::Array(inner, size)) => {
+                let size = size.eval_target_usize()?;
+                let len = size - from - to;
+                Ty::try_new_array(inner, len)
+            }
+            _ => Err(Error(format!("Cannot subslice non-array type: `{ty_kind:?}`"))),
+        }
+    }
+
+    fn deref_ty(ty: Ty) -> Result<Ty, Error> {
+        let deref_ty = ty
+            .kind()
+            .builtin_deref(true)
+            .ok_or_else(|| error!("Cannot dereference type: {ty:?}"))?;
+        Ok(deref_ty.ty)
     }
 }
diff --git a/compiler/stable_mir/src/mir/mono.rs b/compiler/stable_mir/src/mir/mono.rs
index 8f5333498..6c791ae85 100644
--- a/compiler/stable_mir/src/mir/mono.rs
+++ b/compiler/stable_mir/src/mir/mono.rs
@@ -1,16 +1,18 @@
+use crate::abi::FnAbi;
+use crate::crate_def::CrateDef;
 use crate::mir::Body;
-use crate::ty::{FnDef, GenericArgs, IndexedVal, Ty};
-use crate::{with, CrateItem, DefId, Error, Opaque};
-use std::fmt::Debug;
+use crate::ty::{Allocation, ClosureDef, ClosureKind, FnDef, GenericArgs, IndexedVal, Ty};
+use crate::{with, CrateItem, DefId, Error, ItemKind, Opaque, Symbol};
+use std::fmt::{Debug, Formatter};
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
 pub enum MonoItem {
     Fn(Instance),
     Static(StaticDef),
     GlobalAsm(Opaque),
 }
 
-#[derive(Copy, Clone, Debug)]
+#[derive(Copy, Clone, PartialEq, Eq, Hash)]
 pub struct Instance {
     /// The type of instance.
     pub kind: InstanceKind,
@@ -19,33 +21,75 @@ pub struct Instance {
     pub def: InstanceDef,
 }
 
-#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
 pub enum InstanceKind {
     /// A user defined item.
     Item,
     /// A compiler intrinsic function.
     Intrinsic,
     /// A virtual function definition stored in a VTable.
-    Virtual,
+    /// The `idx` field indicates the position in the VTable for this instance.
+    Virtual { idx: usize },
     /// A compiler generated shim.
     Shim,
 }
 
 impl Instance {
+    /// Get the arguments this instance was instantiated with.
+    pub fn args(&self) -> GenericArgs {
+        with(|cx| cx.instance_args(self.def))
+    }
+
     /// Get the body of an Instance. The body will be eagerly monomorphized.
-    pub fn body(&self) -> Body {
+    pub fn body(&self) -> Option<Body> {
         with(|context| context.instance_body(self.def))
     }
 
+    /// Check whether this instance has a body available.
+    ///
+    /// This call is much cheaper than `instance.body().is_some()`, since it doesn't try to build
+    /// the StableMIR body.
+    pub fn has_body(&self) -> bool {
+        with(|cx| cx.has_body(self.def.def_id()))
+    }
+
+    pub fn is_foreign_item(&self) -> bool {
+        with(|cx| cx.is_foreign_item(self.def.def_id()))
+    }
+
     /// Get the instance type with generic substitutions applied and lifetimes erased.
     pub fn ty(&self) -> Ty {
         with(|context| context.instance_ty(self.def))
     }
 
-    pub fn mangled_name(&self) -> String {
+    /// Retrieve information about this instance binary interface.
+    pub fn fn_abi(&self) -> Result<FnAbi, Error> {
+        with(|cx| cx.instance_abi(self.def))
+    }
+
+    /// Retrieve the instance's mangled name used for calling the given instance.
+    ///
+    /// This will also look up the correct name of instances from upstream crates.
+    pub fn mangled_name(&self) -> Symbol {
         with(|context| context.instance_mangled_name(self.def))
     }
 
+    /// Retrieve the instance name for diagnostic messages.
+    ///
+    /// This will return the specialized name, e.g., `std::vec::Vec<u8>::new`.
+    pub fn name(&self) -> Symbol {
+        with(|context| context.instance_name(self.def, false))
+    }
+
+    /// Return a trimmed name of the given instance including its args.
+    ///
+    /// If a symbol name can only be imported from one place for a type, and as
+    /// long as it was not glob-imported anywhere in the current crate, we trim its
+    /// path and print only the name.
+    pub fn trimmed_name(&self) -> Symbol {
+        with(|context| context.instance_name(self.def, true))
+    }
+
     /// Resolve an instance starting from a function definition and generic arguments.
     pub fn resolve(def: FnDef, args: &GenericArgs) -> Result<Instance, crate::Error> {
         with(|context| {
@@ -54,6 +98,64 @@ impl Instance {
             })
         })
     }
+
+    /// Resolve the drop in place for a given type.
+    pub fn resolve_drop_in_place(ty: Ty) -> Instance {
+        with(|cx| cx.resolve_drop_in_place(ty))
+    }
+
+    /// Resolve an instance for a given function pointer.
+    pub fn resolve_for_fn_ptr(def: FnDef, args: &GenericArgs) -> Result<Instance, crate::Error> {
+        with(|context| {
+            context.resolve_for_fn_ptr(def, args).ok_or_else(|| {
+                crate::Error::new(format!("Failed to resolve `{def:?}` with `{args:?}`"))
+            })
+        })
+    }
+
+    /// Resolve a closure with the expected kind.
+    pub fn resolve_closure(
+        def: ClosureDef,
+        args: &GenericArgs,
+        kind: ClosureKind,
+    ) -> Result<Instance, crate::Error> {
+        with(|context| {
+            context.resolve_closure(def, args, kind).ok_or_else(|| {
+                crate::Error::new(format!("Failed to resolve `{def:?}` with `{args:?}`"))
+            })
+        })
+    }
+
+    /// Check whether this instance is an empty shim.
+    ///
+    /// Allow users to check if this shim can be ignored when called directly.
+    ///
+    /// We have decided not to export different types of Shims to StableMIR users, however, this
+    /// is a query that can be very helpful for users when processing DropGlue.
+    ///
+    /// When generating code for a Drop terminator, users can ignore an empty drop glue.
+    /// These shims are only needed to generate a valid Drop call done via VTable.
+    pub fn is_empty_shim(&self) -> bool {
+        self.kind == InstanceKind::Shim && with(|cx| cx.is_empty_drop_shim(self.def))
+    }
+
+    /// Try to constant evaluate the instance into a constant with the given type.
+    ///
+    /// This can be used to retrieve a constant that represents an intrinsic return such as
+    /// `type_id`.
+    pub fn try_const_eval(&self, const_ty: Ty) -> Result<Allocation, Error> {
+        with(|cx| cx.eval_instance(self.def, const_ty))
+    }
+}
+
+impl Debug for Instance {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("Instance")
+            .field("kind", &self.kind)
+            .field("def", &self.mangled_name())
+            .field("args", &self.args())
+            .finish()
+    }
 }
 
 /// Try to convert a crate item into an instance.
@@ -63,8 +165,9 @@ impl TryFrom<CrateItem> for Instance {
 
     fn try_from(item: CrateItem) -> Result<Self, Self::Error> {
         with(|context| {
-            if !context.requires_monomorphization(item.0) {
-                Ok(context.mono_instance(item))
+            let def_id = item.def_id();
+            if !context.requires_monomorphization(def_id) {
+                Ok(context.mono_instance(def_id))
             } else {
                 Err(Error::new("Item requires monomorphization".to_string()))
             }
@@ -78,19 +181,86 @@ impl TryFrom<Instance> for CrateItem {
     type Error = crate::Error;
 
     fn try_from(value: Instance) -> Result<Self, Self::Error> {
-        if value.kind == InstanceKind::Item {
-            Ok(CrateItem(with(|context| context.instance_def_id(value.def))))
+        with(|context| {
+            if value.kind == InstanceKind::Item && context.has_body(value.def.def_id()) {
+                Ok(CrateItem(context.instance_def_id(value.def)))
+            } else {
+                Err(Error::new(format!("Item kind `{:?}` cannot be converted", value.kind)))
+            }
+        })
+    }
+}
+
+impl From<Instance> for MonoItem {
+    fn from(value: Instance) -> Self {
+        MonoItem::Fn(value)
+    }
+}
+
+impl From<StaticDef> for MonoItem {
+    fn from(value: StaticDef) -> Self {
+        MonoItem::Static(value)
+    }
+}
+
+impl From<StaticDef> for CrateItem {
+    fn from(value: StaticDef) -> Self {
+        CrateItem(value.0)
+    }
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub struct InstanceDef(usize);
+
+impl CrateDef for InstanceDef {
+    fn def_id(&self) -> DefId {
+        with(|context| context.instance_def_id(*self))
+    }
+}
+
+crate_def! {
+    /// Holds information about a static variable definition.
+    pub StaticDef;
+}
+
+impl TryFrom<CrateItem> for StaticDef {
+    type Error = crate::Error;
+
+    fn try_from(value: CrateItem) -> Result<Self, Self::Error> {
+        if matches!(value.kind(), ItemKind::Static) {
+            Ok(StaticDef(value.0))
         } else {
-            Err(Error::new(format!("Item kind `{:?}` cannot be converted", value.kind)))
+            Err(Error::new(format!("Expected a static item, but found: {value:?}")))
         }
     }
 }
 
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub struct InstanceDef(usize);
+impl TryFrom<Instance> for StaticDef {
+    type Error = crate::Error;
+
+    fn try_from(value: Instance) -> Result<Self, Self::Error> {
+        StaticDef::try_from(CrateItem::try_from(value)?)
+    }
+}
 
-#[derive(Clone, Copy, PartialEq, Eq, Debug)]
-pub struct StaticDef(pub DefId);
+impl From<StaticDef> for Instance {
+    fn from(value: StaticDef) -> Self {
+        // A static definition should always be convertible to an instance.
+        with(|cx| cx.mono_instance(value.def_id()))
+    }
+}
+
+impl StaticDef {
+    /// Return the type of this static definition.
+    pub fn ty(&self) -> Ty {
+        with(|cx| cx.def_ty(self.0))
+    }
+
+    /// Evaluate a static's initializer, returning the allocation of the initializer's memory.
+    pub fn eval_initializer(&self) -> Result<Allocation, Error> {
+        with(|cx| cx.eval_static_initializer(*self))
+    }
+}
 
 impl IndexedVal for InstanceDef {
     fn to_val(index: usize) -> Self {
diff --git a/compiler/stable_mir/src/mir/pretty.rs b/compiler/stable_mir/src/mir/pretty.rs
new file mode 100644
index 000000000..8b7b488d3
--- /dev/null
+++ b/compiler/stable_mir/src/mir/pretty.rs
@@ -0,0 +1,483 @@
+use crate::crate_def::CrateDef;
+use crate::mir::{Operand, Rvalue, StatementKind, UnwindAction};
+use crate::ty::{DynKind, FloatTy, IntTy, RigidTy, TyKind, UintTy};
+use crate::{with, Body, CrateItem, Mutability};
+use std::io::Write;
+use std::{io, iter};
+
+use super::{AssertMessage, BinOp, TerminatorKind};
+
+pub fn function_name(item: CrateItem) -> String {
+    let mut pretty_name = String::new();
+    let body = item.body();
+    pretty_name.push_str("fn ");
+    pretty_name.push_str(item.name().as_str());
+    if body.arg_locals().is_empty() {
+        pretty_name.push_str("()");
+    } else {
+        pretty_name.push_str("(");
+    }
+    body.arg_locals().iter().enumerate().for_each(|(index, local)| {
+        pretty_name.push_str(format!("_{}: ", index).as_str());
+        pretty_name.push_str(&pretty_ty(local.ty.kind()));
+    });
+    if !body.arg_locals().is_empty() {
+        pretty_name.push_str(")");
+    }
+    let return_local = body.ret_local();
+    pretty_name.push_str(" -> ");
+    pretty_name.push_str(&pretty_ty(return_local.ty.kind()));
+    pretty_name.push_str(" {");
+    pretty_name
+}
+
+pub fn function_body(body: &Body) -> String {
+    let mut pretty_body = String::new();
+    body.inner_locals().iter().enumerate().for_each(|(index, local)| {
+        pretty_body.push_str("    ");
+        pretty_body.push_str(format!("let {}", ret_mutability(&local.mutability)).as_str());
+        pretty_body.push_str(format!("_{}: ", index).as_str());
+        pretty_body.push_str(format!("{}", pretty_ty(local.ty.kind())).as_str());
+        pretty_body.push_str(";\n");
+    });
+    pretty_body.push_str("}");
+    pretty_body
+}
+
+pub fn ret_mutability(mutability: &Mutability) -> String {
+    match mutability {
+        Mutability::Not => "".to_string(),
+        Mutability::Mut => "mut ".to_string(),
+    }
+}
+
+pub fn pretty_statement(statement: &StatementKind) -> String {
+    let mut pretty = String::new();
+    match statement {
+        StatementKind::Assign(place, rval) => {
+            pretty.push_str(format!("        _{} = ", place.local).as_str());
+            pretty.push_str(format!("{}", &pretty_rvalue(rval)).as_str());
+        }
+        // FIXME: Add rest of the statements
+        StatementKind::FakeRead(_, _) => {
+            return String::from("StatementKind::FakeRead:Unimplemented");
+        }
+        StatementKind::SetDiscriminant { .. } => {
+            return String::from("StatementKind::SetDiscriminant:Unimplemented");
+        }
+        StatementKind::Deinit(_) => return String::from("StatementKind::Deinit:Unimplemented"),
+        StatementKind::StorageLive(_) => {
+            return String::from("StatementKind::StorageLive:Unimplemented");
+        }
+        StatementKind::StorageDead(_) => {
+            return String::from("StatementKind::StorageDead:Unimplemented");
+        }
+        StatementKind::Retag(_, _) => return String::from("StatementKind::Retag:Unimplemented"),
+        StatementKind::PlaceMention(_) => {
+            return String::from("StatementKind::PlaceMention:Unimplemented");
+        }
+        StatementKind::AscribeUserType { .. } => {
+            return String::from("StatementKind::AscribeUserType:Unimplemented");
+        }
+        StatementKind::Coverage(_) => return String::from("StatementKind::Coverage:Unimplemented"),
+        StatementKind::Intrinsic(_) => {
+            return String::from("StatementKind::Intrinsic:Unimplemented");
+        }
+        StatementKind::ConstEvalCounter => {
+            return String::from("StatementKind::ConstEvalCounter:Unimplemented");
+        }
+        StatementKind::Nop => return String::from("StatementKind::Nop:Unimplemented"),
+    }
+    pretty
+}
+
+pub fn pretty_terminator<W: io::Write>(terminator: &TerminatorKind, w: &mut W) -> io::Result<()> {
+    write!(w, "{}", pretty_terminator_head(terminator))?;
+    let successors = terminator.successors();
+    let successor_count = successors.len();
+    let labels = pretty_successor_labels(terminator);
+
+    let show_unwind = !matches!(terminator.unwind(), None | Some(UnwindAction::Cleanup(_)));
+    let fmt_unwind = |fmt: &mut dyn Write| -> io::Result<()> {
+        write!(fmt, "unwind ")?;
+        match terminator.unwind() {
+            None | Some(UnwindAction::Cleanup(_)) => unreachable!(),
+            Some(UnwindAction::Continue) => write!(fmt, "continue"),
+            Some(UnwindAction::Unreachable) => write!(fmt, "unreachable"),
+            Some(UnwindAction::Terminate) => write!(fmt, "terminate"),
+        }
+    };
+
+    match (successor_count, show_unwind) {
+        (0, false) => Ok(()),
+        (0, true) => {
+            write!(w, " -> ")?;
+            fmt_unwind(w)?;
+            Ok(())
+        }
+        (1, false) => {
+            write!(w, " -> {:?}", successors[0])?;
+            Ok(())
+        }
+        _ => {
+            write!(w, " -> [")?;
+            for (i, target) in successors.iter().enumerate() {
+                if i > 0 {
+                    write!(w, ", ")?;
+                }
+                write!(w, "{}: bb{:?}", labels[i], target)?;
+            }
+            if show_unwind {
+                write!(w, ", ")?;
+                fmt_unwind(w)?;
+            }
+            write!(w, "]")
+        }
+    }?;
+
+    Ok(())
+}
+
+pub fn pretty_terminator_head(terminator: &TerminatorKind) -> String {
+    use self::TerminatorKind::*;
+    let mut pretty = String::new();
+    match terminator {
+        Goto { .. } => format!("        goto"),
+        SwitchInt { discr, .. } => {
+            format!("        switchInt(_{})", pretty_operand(discr))
+        }
+        Resume => format!("        resume"),
+        Abort => format!("        abort"),
+        Return => format!("        return"),
+        Unreachable => format!("        unreachable"),
+        Drop { place, .. } => format!("        drop(_{:?})", place.local),
+        Call { func, args, destination, .. } => {
+            pretty.push_str("        ");
+            pretty.push_str(format!("_{} = ", destination.local).as_str());
+            pretty.push_str(&pretty_operand(func));
+            pretty.push_str("(");
+            args.iter().enumerate().for_each(|(i, arg)| {
+                if i > 0 {
+                    pretty.push_str(", ");
+                }
+                pretty.push_str(&pretty_operand(arg));
+            });
+            pretty.push_str(")");
+            pretty
+        }
+        Assert { cond, expected, msg, target: _, unwind: _ } => {
+            pretty.push_str("        assert(");
+            if !expected {
+                pretty.push_str("!");
+            }
+            pretty.push_str(format!("{} bool),", &pretty_operand(cond)).as_str());
+            pretty.push_str(&pretty_assert_message(msg));
+            pretty.push_str(")");
+            pretty
+        }
+        InlineAsm { .. } => todo!(),
+    }
+}
+
+pub fn pretty_successor_labels(terminator: &TerminatorKind) -> Vec<String> {
+    use self::TerminatorKind::*;
+    match terminator {
+        Resume | Abort | Return | Unreachable => vec![],
+        Goto { .. } => vec!["".to_string()],
+        SwitchInt { targets, .. } => targets
+            .branches()
+            .map(|(val, _target)| format!("{val}"))
+            .chain(iter::once("otherwise".into()))
+            .collect(),
+        Drop { unwind: UnwindAction::Cleanup(_), .. } => vec!["return".into(), "unwind".into()],
+        Drop { unwind: _, .. } => vec!["return".into()],
+        Call { target: Some(_), unwind: UnwindAction::Cleanup(_), .. } => {
+            vec!["return".into(), "unwind".into()]
+        }
+        Call { target: Some(_), unwind: _, .. } => vec!["return".into()],
+        Call { target: None, unwind: UnwindAction::Cleanup(_), .. } => vec!["unwind".into()],
+        Call { target: None, unwind: _, .. } => vec![],
+        Assert { unwind: UnwindAction::Cleanup(_), .. } => {
+            vec!["success".into(), "unwind".into()]
+        }
+        Assert { unwind: _, .. } => vec!["success".into()],
+        InlineAsm { .. } => todo!(),
+    }
+}
+
+pub fn pretty_assert_message(msg: &AssertMessage) -> String {
+    let mut pretty = String::new();
+    match msg {
+        AssertMessage::BoundsCheck { len, index } => {
+            let pretty_len = pretty_operand(len);
+            let pretty_index = pretty_operand(index);
+            pretty.push_str(format!("\"index out of bounds: the length is {{}} but the index is {{}}\", {pretty_len}, {pretty_index}").as_str());
+            pretty
+        }
+        AssertMessage::Overflow(BinOp::Add, l, r) => {
+            let pretty_l = pretty_operand(l);
+            let pretty_r = pretty_operand(r);
+            pretty.push_str(format!("\"attempt to compute `{{}} + {{}}`, which would overflow\", {pretty_l}, {pretty_r}").as_str());
+            pretty
+        }
+        AssertMessage::Overflow(BinOp::Sub, l, r) => {
+            let pretty_l = pretty_operand(l);
+            let pretty_r = pretty_operand(r);
+            pretty.push_str(format!("\"attempt to compute `{{}} - {{}}`, which would overflow\", {pretty_l}, {pretty_r}").as_str());
+            pretty
+        }
+        AssertMessage::Overflow(BinOp::Mul, l, r) => {
+            let pretty_l = pretty_operand(l);
+            let pretty_r = pretty_operand(r);
+            pretty.push_str(format!("\"attempt to compute `{{}} * {{}}`, which would overflow\", {pretty_l}, {pretty_r}").as_str());
+            pretty
+        }
+        AssertMessage::Overflow(BinOp::Div, l, r) => {
+            let pretty_l = pretty_operand(l);
+            let pretty_r = pretty_operand(r);
+            pretty.push_str(format!("\"attempt to compute `{{}} / {{}}`, which would overflow\", {pretty_l}, {pretty_r}").as_str());
+            pretty
+        }
+        AssertMessage::Overflow(BinOp::Rem, l, r) => {
+            let pretty_l = pretty_operand(l);
+            let pretty_r = pretty_operand(r);
+            pretty.push_str(format!("\"attempt to compute `{{}} % {{}}`, which would overflow\", {pretty_l}, {pretty_r}").as_str());
+            pretty
+        }
+        AssertMessage::Overflow(BinOp::Shr, _, r) => {
+            let pretty_r = pretty_operand(r);
+            pretty.push_str(
+                format!("\"attempt to shift right by `{{}}`, which would overflow\", {pretty_r}")
+                    .as_str(),
+            );
+            pretty
+        }
+        AssertMessage::Overflow(BinOp::Shl, _, r) => {
+            let pretty_r = pretty_operand(r);
+            pretty.push_str(
+                format!("\"attempt to shift left by `{{}}`, which would overflow\", {pretty_r}")
+                    .as_str(),
+            );
+            pretty
+        }
+        AssertMessage::Overflow(op, _, _) => unreachable!("`{:?}` cannot overflow", op),
+        AssertMessage::OverflowNeg(op) => {
+            let pretty_op = pretty_operand(op);
+            pretty.push_str(
+                format!("\"attempt to negate `{{}}`, which would overflow\", {pretty_op}").as_str(),
+            );
+            pretty
+        }
+        AssertMessage::DivisionByZero(op) => {
+            let pretty_op = pretty_operand(op);
+            pretty.push_str(format!("\"attempt to divide `{{}}` by zero\", {pretty_op}").as_str());
+            pretty
+        }
+        AssertMessage::RemainderByZero(op) => {
+            let pretty_op = pretty_operand(op);
+            pretty.push_str(
+                format!("\"attempt to calculate the remainder of `{{}}` with a divisor of zero\", {pretty_op}").as_str(),
+            );
+            pretty
+        }
+        AssertMessage::MisalignedPointerDereference { required, found } => {
+            let pretty_required = pretty_operand(required);
+            let pretty_found = pretty_operand(found);
+            pretty.push_str(format!("\"misaligned pointer dereference: address must be a multiple of {{}} but is {{}}\",{pretty_required}, {pretty_found}").as_str());
+            pretty
+        }
+        AssertMessage::ResumedAfterReturn(_) | AssertMessage::ResumedAfterPanic(_) => {
+            msg.description().unwrap().to_string()
+        }
+    }
+}
+
+pub fn pretty_operand(operand: &Operand) -> String {
+    let mut pretty = String::new();
+    match operand {
+        Operand::Copy(copy) => {
+            pretty.push_str("");
+            pretty.push_str(format!("{}", copy.local).as_str());
+        }
+        Operand::Move(mv) => {
+            pretty.push_str("move ");
+            pretty.push_str(format!("_{}", mv.local).as_str());
+        }
+        Operand::Constant(cnst) => {
+            pretty.push_str("const ");
+            pretty.push_str(with(|cx| cx.const_literal(&cnst.literal)).as_str());
+        }
+    }
+    pretty
+}
+
+pub fn pretty_rvalue(rval: &Rvalue) -> String {
+    let mut pretty = String::new();
+    match rval {
+        Rvalue::AddressOf(muta, addr) => {
+            pretty.push_str("&raw ");
+            pretty.push_str(&ret_mutability(muta));
+            pretty.push_str(format!("(*_{})", addr.local).as_str());
+        }
+        Rvalue::Aggregate(aggregatekind, operands) => {
+            pretty.push_str(format!("{:#?}", aggregatekind).as_str());
+            pretty.push_str("(");
+            operands.iter().enumerate().for_each(|(i, op)| {
+                pretty.push_str(&pretty_operand(op));
+                if i != operands.len() - 1 {
+                    pretty.push_str(", ");
+                }
+            });
+            pretty.push_str(")");
+        }
+        Rvalue::BinaryOp(bin, op, op2) => {
+            pretty.push_str(&pretty_operand(op));
+            pretty.push_str(" ");
+            pretty.push_str(format!("{:#?}", bin).as_str());
+            pretty.push_str(" ");
+            pretty.push_str(&pretty_operand(op2));
+        }
+        Rvalue::Cast(_, op, ty) => {
+            pretty.push_str(&pretty_operand(op));
+            pretty.push_str(" as ");
+            pretty.push_str(&pretty_ty(ty.kind()));
+        }
+        Rvalue::CheckedBinaryOp(bin, op1, op2) => {
+            pretty.push_str(&pretty_operand(op1));
+            pretty.push_str(" ");
+            pretty.push_str(format!("{:#?}", bin).as_str());
+            pretty.push_str(" ");
+            pretty.push_str(&pretty_operand(op2));
+        }
+        Rvalue::CopyForDeref(deref) => {
+            pretty.push_str("CopyForDeref");
+            pretty.push_str(format!("{}", deref.local).as_str());
+        }
+        Rvalue::Discriminant(place) => {
+            pretty.push_str("discriminant");
+            pretty.push_str(format!("{}", place.local).as_str());
+        }
+        Rvalue::Len(len) => {
+            pretty.push_str("len");
+            pretty.push_str(format!("{}", len.local).as_str());
+        }
+        Rvalue::Ref(_, borrowkind, place) => {
+            pretty.push_str("ref");
+            pretty.push_str(format!("{:#?}", borrowkind).as_str());
+            pretty.push_str(format!("{}", place.local).as_str());
+        }
+        Rvalue::Repeat(op, cnst) => {
+            pretty.push_str(&pretty_operand(op));
+            pretty.push_str(" ");
+            pretty.push_str(&pretty_ty(cnst.ty().kind()));
+        }
+        Rvalue::ShallowInitBox(_, _) => (),
+        Rvalue::ThreadLocalRef(item) => {
+            pretty.push_str("thread_local_ref");
+            pretty.push_str(format!("{:#?}", item).as_str());
+        }
+        Rvalue::NullaryOp(nul, ty) => {
+            pretty.push_str(format!("{:#?}", nul).as_str());
+            pretty.push_str(&pretty_ty(ty.kind()));
+            pretty.push_str(" ");
+        }
+        Rvalue::UnaryOp(un, op) => {
+            pretty.push_str(&pretty_operand(op));
+            pretty.push_str(" ");
+            pretty.push_str(format!("{:#?}", un).as_str());
+        }
+        Rvalue::Use(op) => pretty.push_str(&pretty_operand(op)),
+    }
+    pretty
+}
+
+pub fn pretty_ty(ty: TyKind) -> String {
+    let mut pretty = String::new();
+    match ty {
+        TyKind::RigidTy(rigid_ty) => match rigid_ty {
+            RigidTy::Bool => "bool".to_string(),
+            RigidTy::Char => "char".to_string(),
+            RigidTy::Int(i) => match i {
+                IntTy::Isize => "isize".to_string(),
+                IntTy::I8 => "i8".to_string(),
+                IntTy::I16 => "i16".to_string(),
+                IntTy::I32 => "i32".to_string(),
+                IntTy::I64 => "i64".to_string(),
+                IntTy::I128 => "i128".to_string(),
+            },
+            RigidTy::Uint(u) => match u {
+                UintTy::Usize => "usize".to_string(),
+                UintTy::U8 => "u8".to_string(),
+                UintTy::U16 => "u16".to_string(),
+                UintTy::U32 => "u32".to_string(),
+                UintTy::U64 => "u64".to_string(),
+                UintTy::U128 => "u128".to_string(),
+            },
+            RigidTy::Float(f) => match f {
+                FloatTy::F32 => "f32".to_string(),
+                FloatTy::F64 => "f64".to_string(),
+            },
+            RigidTy::Adt(def, _) => {
+                format!("{:#?}", with(|cx| cx.def_ty(def.0)))
+            }
+            RigidTy::Str => "str".to_string(),
+            RigidTy::Array(ty, len) => {
+                format!("[{}; {}]", pretty_ty(ty.kind()), with(|cx| cx.const_literal(&len)))
+            }
+            RigidTy::Slice(ty) => {
+                format!("[{}]", pretty_ty(ty.kind()))
+            }
+            RigidTy::RawPtr(ty, mutability) => {
+                pretty.push_str("*");
+                match mutability {
+                    Mutability::Not => pretty.push_str("const "),
+                    Mutability::Mut => pretty.push_str("mut "),
+                }
+                pretty.push_str(&pretty_ty(ty.kind()));
+                pretty
+            }
+            RigidTy::Ref(_, ty, mutability) => match mutability {
+                Mutability::Not => format!("&{}", pretty_ty(ty.kind())),
+                Mutability::Mut => format!("&mut {}", pretty_ty(ty.kind())),
+            },
+            RigidTy::FnDef(_, _) => format!("{:#?}", rigid_ty),
+            RigidTy::FnPtr(_) => format!("{:#?}", rigid_ty),
+            RigidTy::Closure(_, _) => format!("{:#?}", rigid_ty),
+            RigidTy::Coroutine(_, _, _) => format!("{:#?}", rigid_ty),
+            RigidTy::Dynamic(data, region, repr) => {
+                // FIXME: Fix binder printing, it looks ugly now
+                pretty.push_str("(");
+                match repr {
+                    DynKind::Dyn => pretty.push_str("dyn "),
+                    DynKind::DynStar => pretty.push_str("dyn* "),
+                }
+                pretty.push_str(format!("{:#?}", data).as_str());
+                pretty.push_str(format!(" +  {:#?} )", region).as_str());
+                pretty
+            }
+            RigidTy::Never => "!".to_string(),
+            RigidTy::Tuple(tuple) => {
+                if tuple.is_empty() {
+                    "()".to_string()
+                } else {
+                    let mut tuple_str = String::new();
+                    tuple_str.push_str("(");
+                    tuple.iter().enumerate().for_each(|(i, ty)| {
+                        tuple_str.push_str(&pretty_ty(ty.kind()));
+                        if i != tuple.len() - 1 {
+                            tuple_str.push_str(", ");
+                        }
+                    });
+                    tuple_str.push_str(")");
+                    tuple_str
+                }
+            }
+            _ => format!("{:#?}", rigid_ty),
+        },
+        TyKind::Alias(_, _) => format!("{:#?}", ty),
+        TyKind::Param(param_ty) => {
+            format!("{:#?}", param_ty.name)
+        }
+        TyKind::Bound(_, _) => format!("{:#?}", ty),
+    }
+}
diff --git a/compiler/stable_mir/src/mir/visit.rs b/compiler/stable_mir/src/mir/visit.rs
index 806dced71..ab57ff0f8 100644
--- a/compiler/stable_mir/src/mir/visit.rs
+++ b/compiler/stable_mir/src/mir/visit.rs
@@ -76,6 +76,15 @@ pub trait MirVisitor {
         self.super_place(place, ptx, location)
     }
 
+    fn visit_projection_elem(
+        &mut self,
+        elem: &ProjectionElem,
+        ptx: PlaceContext,
+        location: Location,
+    ) {
+        self.super_projection_elem(elem, ptx, location);
+    }
+
     fn visit_local(&mut self, local: &Local, ptx: PlaceContext, location: Location) {
         let _ = (local, ptx, location);
     }
@@ -119,8 +128,12 @@ pub trait MirVisitor {
         self.super_assert_msg(msg, location)
     }
 
+    fn visit_var_debug_info(&mut self, var_debug_info: &VarDebugInfo) {
+        self.super_var_debug_info(var_debug_info);
+    }
+
     fn super_body(&mut self, body: &Body) {
-        let Body { blocks, locals: _, arg_count } = body;
+        let Body { blocks, locals: _, arg_count, var_debug_info, spread_arg: _, span } = body;
 
         for bb in blocks {
             self.visit_basic_block(bb);
@@ -133,9 +146,15 @@ pub trait MirVisitor {
         }
 
         let local_start = arg_count + 1;
-        for (idx, arg) in body.arg_locals().iter().enumerate() {
+        for (idx, arg) in body.inner_locals().iter().enumerate() {
             self.visit_local_decl(idx + local_start, arg)
         }
+
+        for info in var_debug_info.iter() {
+            self.visit_var_debug_info(info);
+        }
+
+        self.visit_span(span)
     }
 
     fn super_basic_block(&mut self, bb: &BasicBlock) {
@@ -148,7 +167,7 @@ pub trait MirVisitor {
 
     fn super_local_decl(&mut self, local: Local, decl: &LocalDecl) {
         let _ = local;
-        let LocalDecl { ty, span } = decl;
+        let LocalDecl { ty, span, .. } = decl;
         self.visit_ty(ty, Location(*span));
     }
 
@@ -215,13 +234,12 @@ pub trait MirVisitor {
 
     fn super_terminator(&mut self, term: &Terminator, location: Location) {
         let Terminator { kind, span } = term;
-        self.visit_span(&span);
+        self.visit_span(span);
         match kind {
             TerminatorKind::Goto { .. }
             | TerminatorKind::Resume
             | TerminatorKind::Abort
-            | TerminatorKind::Unreachable
-            | TerminatorKind::CoroutineDrop => {}
+            | TerminatorKind::Unreachable => {}
             TerminatorKind::Assert { cond, expected: _, msg, target: _, unwind: _ } => {
                 self.visit_operand(cond, location);
                 self.visit_assert_msg(msg, location);
@@ -251,7 +269,7 @@ pub trait MirVisitor {
                 let local = RETURN_LOCAL;
                 self.visit_local(&local, PlaceContext::NON_MUTATING, location);
             }
-            TerminatorKind::SwitchInt { discr, targets: _, otherwise: _ } => {
+            TerminatorKind::SwitchInt { discr, targets: _ } => {
                 self.visit_operand(discr, location);
             }
         }
@@ -264,7 +282,29 @@ pub trait MirVisitor {
     fn super_place(&mut self, place: &Place, ptx: PlaceContext, location: Location) {
         let _ = location;
         let _ = ptx;
-        visit_opaque(&Opaque(place.projection.clone()));
+        self.visit_local(&place.local, ptx, location);
+
+        for elem in &place.projection {
+            self.visit_projection_elem(elem, ptx, location);
+        }
+    }
+
+    fn super_projection_elem(
+        &mut self,
+        elem: &ProjectionElem,
+        ptx: PlaceContext,
+        location: Location,
+    ) {
+        match elem {
+            ProjectionElem::Deref => {}
+            ProjectionElem::Field(_idx, ty) => self.visit_ty(ty, location),
+            ProjectionElem::Index(local) => self.visit_local(local, ptx, location),
+            ProjectionElem::ConstantIndex { offset: _, min_length: _, from_end: _ } => {}
+            ProjectionElem::Subslice { from: _, to: _, from_end: _ } => {}
+            ProjectionElem::Downcast(_idx) => {}
+            ProjectionElem::OpaqueCast(ty) => self.visit_ty(ty, location),
+            ProjectionElem::Subtype(ty) => self.visit_ty(ty, location),
+        }
     }
 
     fn super_rvalue(&mut self, rvalue: &Rvalue, location: Location) {
@@ -351,6 +391,24 @@ pub trait MirVisitor {
         let _ = args;
     }
 
+    fn super_var_debug_info(&mut self, var_debug_info: &VarDebugInfo) {
+        let VarDebugInfo { source_info, composite, value, name: _, argument_index: _ } =
+            var_debug_info;
+        self.visit_span(&source_info.span);
+        let location = Location(source_info.span);
+        if let Some(composite) = composite {
+            self.visit_ty(&composite.ty, location);
+        }
+        match value {
+            VarDebugInfoContents::Place(place) => {
+                self.visit_place(place, PlaceContext::NON_USE, location);
+            }
+            VarDebugInfoContents::Const(constant) => {
+                self.visit_const(&constant.const_, location);
+            }
+        }
+    }
+
     fn super_assert_msg(&mut self, msg: &AssertMessage, location: Location) {
         match msg {
             AssertMessage::BoundsCheck { len, index } => {
@@ -386,7 +444,7 @@ pub trait MirVisitor {
 fn visit_opaque(_: &Opaque) {}
 
 /// The location of a statement / terminator in the code and the CFG.
-#[derive(Clone, Copy, PartialEq, Eq)]
+#[derive(Clone, Copy, PartialEq, Eq, Debug)]
 pub struct Location(Span);
 
 impl Location {
@@ -396,7 +454,7 @@ impl Location {
 }
 
 /// Information about a place's usage.
-#[derive(Copy, Clone, PartialEq, Eq, Hash)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
 pub struct PlaceContext {
     /// Whether the access is mutable or not. Keep this private so we can increment the type in a
     /// backward compatible manner.
diff --git a/compiler/stable_mir/src/target.rs b/compiler/stable_mir/src/target.rs
new file mode 100644
index 000000000..41ec205cf
--- /dev/null
+++ b/compiler/stable_mir/src/target.rs
@@ -0,0 +1,50 @@
+//! Provide information about the machine that this is being compiled into.
+
+use crate::compiler_interface::with;
+
+/// The properties of the target machine being compiled into.
+#[derive(Clone, PartialEq, Eq)]
+pub struct MachineInfo {
+    pub endian: Endian,
+    pub pointer_width: MachineSize,
+}
+
+impl MachineInfo {
+    pub fn target() -> MachineInfo {
+        with(|cx| cx.target_info())
+    }
+
+    pub fn target_endianess() -> Endian {
+        with(|cx| cx.target_info().endian)
+    }
+
+    pub fn target_pointer_width() -> MachineSize {
+        with(|cx| cx.target_info().pointer_width)
+    }
+}
+
+#[derive(Copy, Clone, PartialEq, Eq)]
+pub enum Endian {
+    Little,
+    Big,
+}
+
+/// Represent the size of a component.
+#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord)]
+pub struct MachineSize {
+    num_bits: usize,
+}
+
+impl MachineSize {
+    pub fn bytes(self) -> usize {
+        self.num_bits / 8
+    }
+
+    pub fn bits(self) -> usize {
+        self.num_bits
+    }
+
+    pub fn from_bits(num_bits: usize) -> MachineSize {
+        MachineSize { num_bits }
+    }
+}
diff --git a/compiler/stable_mir/src/ty.rs b/compiler/stable_mir/src/ty.rs
index 5dfaa0fd8..1d4d7b6d3 100644
--- a/compiler/stable_mir/src/ty.rs
+++ b/compiler/stable_mir/src/ty.rs
@@ -1,10 +1,15 @@
 use super::{
     mir::Safety,
     mir::{Body, Mutability},
-    with, AllocId, DefId, Symbol,
+    with, DefId, Error, Symbol,
 };
+use crate::abi::Layout;
+use crate::crate_def::CrateDef;
+use crate::mir::alloc::{read_target_int, read_target_uint, AllocId};
+use crate::target::MachineInfo;
 use crate::{Filename, Opaque};
-use std::fmt::{self, Debug, Formatter};
+use std::fmt::{self, Debug, Display, Formatter};
+use std::ops::Range;
 
 #[derive(Copy, Clone, Eq, PartialEq, Hash)]
 pub struct Ty(pub usize);
@@ -15,6 +20,79 @@ impl Debug for Ty {
     }
 }
 
+/// Constructors for `Ty`.
+impl Ty {
+    /// Create a new type from a given kind.
+    pub fn from_rigid_kind(kind: RigidTy) -> Ty {
+        with(|cx| cx.new_rigid_ty(kind))
+    }
+
+    /// Create a new array type.
+    pub fn try_new_array(elem_ty: Ty, size: u64) -> Result<Ty, Error> {
+        Ok(Ty::from_rigid_kind(RigidTy::Array(elem_ty, Const::try_from_target_usize(size)?)))
+    }
+
+    /// Create a new array type from Const length.
+    pub fn new_array_with_const_len(elem_ty: Ty, len: Const) -> Ty {
+        Ty::from_rigid_kind(RigidTy::Array(elem_ty, len))
+    }
+
+    /// Create a new pointer type.
+    pub fn new_ptr(pointee_ty: Ty, mutability: Mutability) -> Ty {
+        Ty::from_rigid_kind(RigidTy::RawPtr(pointee_ty, mutability))
+    }
+
+    /// Create a new reference type.
+    pub fn new_ref(reg: Region, pointee_ty: Ty, mutability: Mutability) -> Ty {
+        Ty::from_rigid_kind(RigidTy::Ref(reg, pointee_ty, mutability))
+    }
+
+    /// Create a new pointer type.
+    pub fn new_tuple(tys: &[Ty]) -> Ty {
+        Ty::from_rigid_kind(RigidTy::Tuple(Vec::from(tys)))
+    }
+
+    /// Create a new closure type.
+    pub fn new_closure(def: ClosureDef, args: GenericArgs) -> Ty {
+        Ty::from_rigid_kind(RigidTy::Closure(def, args))
+    }
+
+    /// Create a new coroutine type.
+    pub fn new_coroutine(def: CoroutineDef, args: GenericArgs, mov: Movability) -> Ty {
+        Ty::from_rigid_kind(RigidTy::Coroutine(def, args, mov))
+    }
+
+    /// Create a new box type that represents `Box<T>`, for the given inner type `T`.
+    pub fn new_box(inner_ty: Ty) -> Ty {
+        with(|cx| cx.new_box_ty(inner_ty))
+    }
+
+    /// Create a type representing `usize`.
+    pub fn usize_ty() -> Ty {
+        Ty::from_rigid_kind(RigidTy::Uint(UintTy::Usize))
+    }
+
+    /// Create a type representing `bool`.
+    pub fn bool_ty() -> Ty {
+        Ty::from_rigid_kind(RigidTy::Bool)
+    }
+
+    /// Create a type representing a signed integer.
+    pub fn signed_ty(inner: IntTy) -> Ty {
+        Ty::from_rigid_kind(RigidTy::Int(inner))
+    }
+
+    /// Create a type representing an unsigned integer.
+    pub fn unsigned_ty(inner: UintTy) -> Ty {
+        Ty::from_rigid_kind(RigidTy::Uint(inner))
+    }
+
+    /// Get a type layout.
+    pub fn layout(self) -> Result<Layout, Error> {
+        with(|cx| cx.ty_layout(self))
+    }
+}
+
 impl Ty {
     pub fn kind(&self) -> TyKind {
         with(|context| context.ty_kind(*self))
@@ -47,6 +125,16 @@ impl Const {
     pub fn ty(&self) -> Ty {
         self.ty
     }
+
+    /// Creates an interned usize constant.
+    fn try_from_target_usize(val: u64) -> Result<Self, Error> {
+        with(|cx| cx.usize_to_const(val))
+    }
+
+    /// Try to evaluate to a target `usize`.
+    pub fn eval_target_usize(&self) -> Result<u64, Error> {
+        with(|cx| cx.eval_target_usize(self))
+    }
 }
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -61,8 +149,8 @@ pub struct Region {
 
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub enum RegionKind {
-    ReEarlyBound(EarlyBoundRegion),
-    ReLateBound(DebruijnIndex, BoundRegion),
+    ReEarlyParam(EarlyParamRegion),
+    ReBound(DebruijnIndex, BoundRegion),
     ReStatic,
     RePlaceholder(Placeholder<BoundRegion>),
     ReErased,
@@ -71,7 +159,7 @@ pub enum RegionKind {
 pub(crate) type DebruijnIndex = u32;
 
 #[derive(Clone, Debug, Eq, PartialEq)]
-pub struct EarlyBoundRegion {
+pub struct EarlyParamRegion {
     pub def_id: RegionDef,
     pub index: u32,
     pub name: Symbol,
@@ -113,7 +201,7 @@ impl Span {
 
     /// Return lines that corespond to this `Span`
     pub fn get_lines(&self) -> LineInfo {
-        with(|c| c.get_lines(&self))
+        with(|c| c.get_lines(self))
     }
 }
 
@@ -135,6 +223,226 @@ pub enum TyKind {
     Bound(usize, BoundTy),
 }
 
+impl TyKind {
+    pub fn rigid(&self) -> Option<&RigidTy> {
+        if let TyKind::RigidTy(inner) = self { Some(inner) } else { None }
+    }
+
+    #[inline]
+    pub fn is_unit(&self) -> bool {
+        matches!(self, TyKind::RigidTy(RigidTy::Tuple(data)) if data.is_empty())
+    }
+
+    #[inline]
+    pub fn is_bool(&self) -> bool {
+        matches!(self, TyKind::RigidTy(RigidTy::Bool))
+    }
+
+    #[inline]
+    pub fn is_char(&self) -> bool {
+        matches!(self, TyKind::RigidTy(RigidTy::Char))
+    }
+
+    #[inline]
+    pub fn is_trait(&self) -> bool {
+        matches!(self, TyKind::RigidTy(RigidTy::Dynamic(_, _, DynKind::Dyn)))
+    }
+
+    #[inline]
+    pub fn is_enum(&self) -> bool {
+        matches!(self, TyKind::RigidTy(RigidTy::Adt(def, _)) if def.kind() == AdtKind::Enum)
+    }
+
+    #[inline]
+    pub fn is_struct(&self) -> bool {
+        matches!(self, TyKind::RigidTy(RigidTy::Adt(def, _)) if def.kind() == AdtKind::Struct)
+    }
+
+    #[inline]
+    pub fn is_union(&self) -> bool {
+        matches!(self, TyKind::RigidTy(RigidTy::Adt(def, _)) if def.kind() == AdtKind::Union)
+    }
+
+    #[inline]
+    pub fn is_adt(&self) -> bool {
+        matches!(self, TyKind::RigidTy(RigidTy::Adt(..)))
+    }
+
+    #[inline]
+    pub fn is_ref(&self) -> bool {
+        matches!(self, TyKind::RigidTy(RigidTy::Ref(..)))
+    }
+
+    #[inline]
+    pub fn is_fn(&self) -> bool {
+        matches!(self, TyKind::RigidTy(RigidTy::FnDef(..)))
+    }
+
+    #[inline]
+    pub fn is_fn_ptr(&self) -> bool {
+        matches!(self, TyKind::RigidTy(RigidTy::FnPtr(..)))
+    }
+
+    #[inline]
+    pub fn is_primitive(&self) -> bool {
+        matches!(
+            self,
+            TyKind::RigidTy(
+                RigidTy::Bool
+                    | RigidTy::Char
+                    | RigidTy::Int(_)
+                    | RigidTy::Uint(_)
+                    | RigidTy::Float(_)
+            )
+        )
+    }
+
+    #[inline]
+    pub fn is_float(&self) -> bool {
+        matches!(self, TyKind::RigidTy(RigidTy::Float(_)))
+    }
+
+    #[inline]
+    pub fn is_integral(&self) -> bool {
+        matches!(self, TyKind::RigidTy(RigidTy::Int(_) | RigidTy::Uint(_)))
+    }
+
+    #[inline]
+    pub fn is_numeric(&self) -> bool {
+        self.is_integral() || self.is_float()
+    }
+
+    #[inline]
+    pub fn is_signed(&self) -> bool {
+        matches!(self, TyKind::RigidTy(RigidTy::Int(_)))
+    }
+
+    #[inline]
+    pub fn is_str(&self) -> bool {
+        *self == TyKind::RigidTy(RigidTy::Str)
+    }
+
+    #[inline]
+    pub fn is_cstr(&self) -> bool {
+        let TyKind::RigidTy(RigidTy::Adt(def, _)) = self else { return false };
+        with(|cx| cx.adt_is_cstr(*def))
+    }
+
+    #[inline]
+    pub fn is_slice(&self) -> bool {
+        matches!(self, TyKind::RigidTy(RigidTy::Slice(_)))
+    }
+
+    #[inline]
+    pub fn is_array(&self) -> bool {
+        matches!(self, TyKind::RigidTy(RigidTy::Array(..)))
+    }
+
+    #[inline]
+    pub fn is_mutable_ptr(&self) -> bool {
+        matches!(
+            self,
+            TyKind::RigidTy(RigidTy::RawPtr(_, Mutability::Mut))
+                | TyKind::RigidTy(RigidTy::Ref(_, _, Mutability::Mut))
+        )
+    }
+
+    #[inline]
+    pub fn is_raw_ptr(&self) -> bool {
+        matches!(self, TyKind::RigidTy(RigidTy::RawPtr(..)))
+    }
+
+    /// Tests if this is any kind of primitive pointer type (reference, raw pointer, fn pointer).
+    #[inline]
+    pub fn is_any_ptr(&self) -> bool {
+        self.is_ref() || self.is_raw_ptr() || self.is_fn_ptr()
+    }
+
+    #[inline]
+    pub fn is_coroutine(&self) -> bool {
+        matches!(self, TyKind::RigidTy(RigidTy::Coroutine(..)))
+    }
+
+    #[inline]
+    pub fn is_closure(&self) -> bool {
+        matches!(self, TyKind::RigidTy(RigidTy::Closure(..)))
+    }
+
+    #[inline]
+    pub fn is_box(&self) -> bool {
+        match self {
+            TyKind::RigidTy(RigidTy::Adt(def, _)) => def.is_box(),
+            _ => false,
+        }
+    }
+
+    #[inline]
+    pub fn is_simd(&self) -> bool {
+        matches!(self, TyKind::RigidTy(RigidTy::Adt(def, _)) if def.is_simd())
+    }
+
+    pub fn trait_principal(&self) -> Option<Binder<ExistentialTraitRef>> {
+        if let TyKind::RigidTy(RigidTy::Dynamic(predicates, _, _)) = self {
+            if let Some(Binder { value: ExistentialPredicate::Trait(trait_ref), bound_vars }) =
+                predicates.first()
+            {
+                Some(Binder { value: trait_ref.clone(), bound_vars: bound_vars.clone() })
+            } else {
+                None
+            }
+        } else {
+            None
+        }
+    }
+
+    /// Returns the type of `ty[i]` for builtin types.
+    pub fn builtin_index(&self) -> Option<Ty> {
+        match self.rigid()? {
+            RigidTy::Array(ty, _) | RigidTy::Slice(ty) => Some(*ty),
+            _ => None,
+        }
+    }
+
+    /// Returns the type and mutability of `*ty` for builtin types.
+    ///
+    /// The parameter `explicit` indicates if this is an *explicit* dereference.
+    /// Some types -- notably unsafe ptrs -- can only be dereferenced explicitly.
+    pub fn builtin_deref(&self, explicit: bool) -> Option<TypeAndMut> {
+        match self.rigid()? {
+            RigidTy::Adt(def, args) if def.is_box() => {
+                Some(TypeAndMut { ty: *args.0.first()?.ty()?, mutability: Mutability::Not })
+            }
+            RigidTy::Ref(_, ty, mutability) => {
+                Some(TypeAndMut { ty: *ty, mutability: *mutability })
+            }
+            RigidTy::RawPtr(ty, mutability) if explicit => {
+                Some(TypeAndMut { ty: *ty, mutability: *mutability })
+            }
+            _ => None,
+        }
+    }
+
+    /// Get the function signature for function like types (Fn, FnPtr, and Closure)
+    pub fn fn_sig(&self) -> Option<PolyFnSig> {
+        match self {
+            TyKind::RigidTy(RigidTy::FnDef(def, args)) => Some(with(|cx| cx.fn_sig(*def, args))),
+            TyKind::RigidTy(RigidTy::FnPtr(sig)) => Some(sig.clone()),
+            TyKind::RigidTy(RigidTy::Closure(_def, args)) => Some(with(|cx| cx.closure_sig(args))),
+            _ => None,
+        }
+    }
+
+    /// Get the discriminant type for this type.
+    pub fn discriminant_ty(&self) -> Option<Ty> {
+        self.rigid().map(|ty| with(|cx| cx.rigid_ty_discriminant_ty(ty)))
+    }
+}
+
+pub struct TypeAndMut {
+    pub ty: Ty,
+    pub mutability: Mutability,
+}
+
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub enum RigidTy {
     Bool,
@@ -156,6 +464,20 @@ pub enum RigidTy {
     Dynamic(Vec<Binder<ExistentialPredicate>>, Region, DynKind),
     Never,
     Tuple(Vec<Ty>),
+    CoroutineWitness(CoroutineWitnessDef, GenericArgs),
+}
+
+impl RigidTy {
+    /// Get the discriminant type for this type.
+    pub fn discriminant_ty(&self) -> Ty {
+        with(|cx| cx.rigid_ty_discriminant_ty(self))
+    }
+}
+
+impl From<RigidTy> for TyKind {
+    fn from(value: RigidTy) -> Self {
+        TyKind::RigidTy(value)
+    }
 }
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -168,6 +490,19 @@ pub enum IntTy {
     I128,
 }
 
+impl IntTy {
+    pub fn num_bytes(self) -> usize {
+        match self {
+            IntTy::Isize => crate::target::MachineInfo::target_pointer_width().bytes().into(),
+            IntTy::I8 => 1,
+            IntTy::I16 => 2,
+            IntTy::I32 => 4,
+            IntTy::I64 => 8,
+            IntTy::I128 => 16,
+        }
+    }
+}
+
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub enum UintTy {
     Usize,
@@ -178,6 +513,19 @@ pub enum UintTy {
     U128,
 }
 
+impl UintTy {
+    pub fn num_bytes(self) -> usize {
+        match self {
+            UintTy::Usize => crate::target::MachineInfo::target_pointer_width().bytes().into(),
+            UintTy::U8 => 1,
+            UintTy::U16 => 2,
+            UintTy::U32 => 4,
+            UintTy::U64 => 8,
+            UintTy::U128 => 16,
+        }
+    }
+}
+
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub enum FloatTy {
     F32,
@@ -190,50 +538,203 @@ pub enum Movability {
     Movable,
 }
 
-#[derive(Clone, Copy, PartialEq, Eq, Debug)]
-pub struct ForeignDef(pub DefId);
+crate_def! {
+    /// Hold information about a ForeignItem in a crate.
+    pub ForeignDef;
+}
 
-#[derive(Clone, Copy, PartialEq, Eq, Debug)]
-pub struct FnDef(pub DefId);
+crate_def! {
+    /// Hold information about a function definition in a crate.
+    pub FnDef;
+}
 
 impl FnDef {
-    pub fn body(&self) -> Body {
-        with(|ctx| ctx.mir_body(self.0))
+    // Get the function body if available.
+    pub fn body(&self) -> Option<Body> {
+        with(|ctx| ctx.has_body(self.0).then(|| ctx.mir_body(self.0)))
     }
 }
 
-#[derive(Clone, Copy, PartialEq, Eq, Debug)]
-pub struct ClosureDef(pub DefId);
+crate_def! {
+    pub ClosureDef;
+}
 
-#[derive(Clone, Copy, PartialEq, Eq, Debug)]
-pub struct CoroutineDef(pub DefId);
+crate_def! {
+    pub CoroutineDef;
+}
 
-#[derive(Clone, Copy, PartialEq, Eq, Debug)]
-pub struct ParamDef(pub DefId);
+crate_def! {
+    pub ParamDef;
+}
 
-#[derive(Clone, Copy, PartialEq, Eq, Debug)]
-pub struct BrNamedDef(pub DefId);
+crate_def! {
+    pub BrNamedDef;
+}
 
-#[derive(Clone, Copy, PartialEq, Eq, Debug)]
-pub struct AdtDef(pub DefId);
+crate_def! {
+    pub AdtDef;
+}
 
-#[derive(Clone, Copy, PartialEq, Eq, Debug)]
-pub struct AliasDef(pub DefId);
+#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)]
+pub enum AdtKind {
+    Enum,
+    Union,
+    Struct,
+}
 
-#[derive(Clone, Copy, PartialEq, Eq, Debug)]
-pub struct TraitDef(pub DefId);
+impl AdtDef {
+    pub fn kind(&self) -> AdtKind {
+        with(|cx| cx.adt_kind(*self))
+    }
 
-#[derive(Clone, Copy, PartialEq, Eq, Debug)]
-pub struct GenericDef(pub DefId);
+    /// Retrieve the type of this Adt.
+    pub fn ty(&self) -> Ty {
+        with(|cx| cx.def_ty(self.0))
+    }
 
-#[derive(Clone, Copy, PartialEq, Eq, Debug)]
-pub struct ConstDef(pub DefId);
+    /// Retrieve the type of this Adt instantiating the type with the given arguments.
+    ///
+    /// This will assume the type can be instantiated with these arguments.
+    pub fn ty_with_args(&self, args: &GenericArgs) -> Ty {
+        with(|cx| cx.def_ty_with_args(self.0, args))
+    }
 
-#[derive(Clone, PartialEq, Eq, Debug)]
-pub struct ImplDef(pub DefId);
+    pub fn is_box(&self) -> bool {
+        with(|cx| cx.adt_is_box(*self))
+    }
 
-#[derive(Clone, PartialEq, Eq, Debug)]
-pub struct RegionDef(pub DefId);
+    pub fn is_simd(&self) -> bool {
+        with(|cx| cx.adt_is_simd(*self))
+    }
+
+    /// The number of variants in this ADT.
+    pub fn num_variants(&self) -> usize {
+        with(|cx| cx.adt_variants_len(*self))
+    }
+
+    /// Retrieve the variants in this ADT.
+    pub fn variants(&self) -> Vec<VariantDef> {
+        self.variants_iter().collect()
+    }
+
+    /// Iterate over the variants in this ADT.
+    pub fn variants_iter(&self) -> impl Iterator<Item = VariantDef> + '_ {
+        (0..self.num_variants())
+            .map(|idx| VariantDef { idx: VariantIdx::to_val(idx), adt_def: *self })
+    }
+
+    pub fn variant(&self, idx: VariantIdx) -> Option<VariantDef> {
+        (idx.to_index() < self.num_variants()).then_some(VariantDef { idx, adt_def: *self })
+    }
+}
+
+/// Definition of a variant, which can be either a struct / union field or an enum variant.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub struct VariantDef {
+    /// The variant index.
+    ///
+    /// ## Warning
+    /// Do not access this field directly!
+    pub idx: VariantIdx,
+    /// The data type where this variant comes from.
+    /// For now, we use this to retrieve information about the variant itself so we don't need to
+    /// cache more information.
+    ///
+    /// ## Warning
+    /// Do not access this field directly!
+    pub adt_def: AdtDef,
+}
+
+impl VariantDef {
+    pub fn name(&self) -> Symbol {
+        with(|cx| cx.variant_name(*self))
+    }
+
+    /// Retrieve all the fields in this variant.
+    // We expect user to cache this and use it directly since today it is expensive to generate all
+    // fields name.
+    pub fn fields(&self) -> Vec<FieldDef> {
+        with(|cx| cx.variant_fields(*self))
+    }
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct FieldDef {
+    /// The field definition.
+    ///
+    /// ## Warning
+    /// Do not access this field directly! This is public for the compiler to have access to it.
+    pub def: DefId,
+
+    /// The field name.
+    pub name: Symbol,
+}
+
+impl FieldDef {
+    /// Retrieve the type of this field instantiating the type with the given arguments.
+    ///
+    /// This will assume the type can be instantiated with these arguments.
+    pub fn ty_with_args(&self, args: &GenericArgs) -> Ty {
+        with(|cx| cx.def_ty_with_args(self.def, args))
+    }
+
+    /// Retrieve the type of this field.
+    pub fn ty(&self) -> Ty {
+        with(|cx| cx.def_ty(self.def))
+    }
+}
+
+impl Display for AdtKind {
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+        f.write_str(match self {
+            AdtKind::Enum => "enum",
+            AdtKind::Union => "union",
+            AdtKind::Struct => "struct",
+        })
+    }
+}
+
+impl AdtKind {
+    pub fn is_enum(&self) -> bool {
+        matches!(self, AdtKind::Enum)
+    }
+
+    pub fn is_struct(&self) -> bool {
+        matches!(self, AdtKind::Struct)
+    }
+
+    pub fn is_union(&self) -> bool {
+        matches!(self, AdtKind::Union)
+    }
+}
+
+crate_def! {
+    pub AliasDef;
+}
+
+crate_def! {
+    pub TraitDef;
+}
+
+crate_def! {
+    pub GenericDef;
+}
+
+crate_def! {
+    pub ConstDef;
+}
+
+crate_def! {
+    pub ImplDef;
+}
+
+crate_def! {
+    pub RegionDef;
+}
+
+crate_def! {
+    pub CoroutineWitnessDef;
+}
 
 /// A list of generic arguments.
 #[derive(Clone, Debug, Eq, PartialEq)]
@@ -282,6 +783,14 @@ impl GenericArgKind {
             _ => panic!("{self:?}"),
         }
     }
+
+    /// Return the generic argument type if applicable, otherwise return `None`.
+    pub fn ty(&self) -> Option<&Ty> {
+        match self {
+            GenericArgKind::Type(ty) => Some(ty),
+            _ => None,
+        }
+    }
 }
 
 #[derive(Clone, Debug, Eq, PartialEq)]
@@ -314,6 +823,16 @@ pub struct FnSig {
     pub abi: Abi,
 }
 
+impl FnSig {
+    pub fn output(&self) -> Ty {
+        self.inputs_and_output[self.inputs_and_output.len() - 1]
+    }
+
+    pub fn inputs(&self) -> &[Ty] {
+        &self.inputs_and_output[..self.inputs_and_output.len() - 1]
+    }
+}
+
 #[derive(Clone, PartialEq, Eq, Debug)]
 pub enum Abi {
     Rust,
@@ -345,12 +864,47 @@ pub enum Abi {
     RiscvInterruptS,
 }
 
+/// A binder represents a possibly generic type and its bound vars.
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub struct Binder<T> {
     pub value: T,
     pub bound_vars: Vec<BoundVariableKind>,
 }
 
+impl<T> Binder<T> {
+    /// Create a new binder with the given bound vars.
+    pub fn bind_with_vars(value: T, bound_vars: Vec<BoundVariableKind>) -> Self {
+        Binder { value, bound_vars }
+    }
+
+    /// Create a new binder with no bounded variable.
+    pub fn dummy(value: T) -> Self {
+        Binder { value, bound_vars: vec![] }
+    }
+
+    pub fn skip_binder(self) -> T {
+        self.value
+    }
+
+    pub fn map_bound_ref<F, U>(&self, f: F) -> Binder<U>
+    where
+        F: FnOnce(&T) -> U,
+    {
+        let Binder { value, bound_vars } = self;
+        let new_value = f(value);
+        Binder { value: new_value, bound_vars: bound_vars.clone() }
+    }
+
+    pub fn map_bound<F, U>(self, f: F) -> Binder<U>
+    where
+        F: FnOnce(T) -> U,
+    {
+        let Binder { value, bound_vars } = self;
+        let new_value = f(value);
+        Binder { value: new_value, bound_vars }
+    }
+}
+
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub struct EarlyBinder<T> {
     pub value: T,
@@ -389,12 +943,27 @@ pub enum ExistentialPredicate {
     AutoTrait(TraitDef),
 }
 
+/// An existential reference to a trait where `Self` is not included.
+///
+/// The `generic_args` will include any other known argument.
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub struct ExistentialTraitRef {
     pub def_id: TraitDef,
     pub generic_args: GenericArgs,
 }
 
+impl Binder<ExistentialTraitRef> {
+    pub fn with_self_ty(&self, self_ty: Ty) -> Binder<TraitRef> {
+        self.map_bound_ref(|trait_ref| trait_ref.with_self_ty(self_ty))
+    }
+}
+
+impl ExistentialTraitRef {
+    pub fn with_self_ty(&self, self_ty: Ty) -> TraitRef {
+        TraitRef::new(self.def_id, self_ty, &self.generic_args)
+    }
+}
+
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub struct ExistentialProjection {
     pub def_id: TraitDef,
@@ -417,21 +986,21 @@ pub struct BoundTy {
 pub type Bytes = Vec<Option<u8>>;
 pub type Size = usize;
 
-#[derive(Clone, Copy, PartialEq, Eq, Debug)]
+#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)]
 pub struct Prov(pub AllocId);
 pub type Align = u64;
 pub type Promoted = u32;
 pub type InitMaskMaterialized = Vec<u64>;
 
 /// Stores the provenance information of pointers stored in memory.
-#[derive(Clone, Debug, Eq, PartialEq)]
+#[derive(Clone, Debug, Eq, PartialEq, Hash)]
 pub struct ProvenanceMap {
     /// Provenance in this map applies from the given offset for an entire pointer-size worth of
     /// bytes. Two entries in this map are always at least a pointer size apart.
     pub ptrs: Vec<(Size, Prov)>,
 }
 
-#[derive(Clone, Debug, Eq, PartialEq)]
+#[derive(Clone, Debug, Eq, PartialEq, Hash)]
 pub struct Allocation {
     pub bytes: Bytes,
     pub provenance: ProvenanceMap,
@@ -439,6 +1008,74 @@ pub struct Allocation {
     pub mutability: Mutability,
 }
 
+impl Allocation {
+    /// Get a vector of bytes for an Allocation that has been fully initialized
+    pub fn raw_bytes(&self) -> Result<Vec<u8>, Error> {
+        self.bytes
+            .iter()
+            .copied()
+            .collect::<Option<Vec<_>>>()
+            .ok_or_else(|| error!("Found uninitialized bytes: `{:?}`", self.bytes))
+    }
+
+    /// Read a uint value from the specified range.
+    pub fn read_partial_uint(&self, range: Range<usize>) -> Result<u128, Error> {
+        if range.end - range.start > 16 {
+            return Err(error!("Allocation is bigger than largest integer"));
+        }
+        if range.end > self.bytes.len() {
+            return Err(error!(
+                "Range is out of bounds. Allocation length is `{}`, but requested range `{:?}`",
+                self.bytes.len(),
+                range
+            ));
+        }
+        let raw = self.bytes[range]
+            .iter()
+            .copied()
+            .collect::<Option<Vec<_>>>()
+            .ok_or_else(|| error!("Found uninitialized bytes: `{:?}`", self.bytes))?;
+        read_target_uint(&raw)
+    }
+
+    /// Read this allocation and try to convert it to an unassigned integer.
+    pub fn read_uint(&self) -> Result<u128, Error> {
+        if self.bytes.len() > 16 {
+            return Err(error!("Allocation is bigger than largest integer"));
+        }
+        let raw = self.raw_bytes()?;
+        read_target_uint(&raw)
+    }
+
+    /// Read this allocation and try to convert it to a signed integer.
+    pub fn read_int(&self) -> Result<i128, Error> {
+        if self.bytes.len() > 16 {
+            return Err(error!("Allocation is bigger than largest integer"));
+        }
+        let raw = self.raw_bytes()?;
+        read_target_int(&raw)
+    }
+
+    /// Read this allocation and try to convert it to a boolean.
+    pub fn read_bool(&self) -> Result<bool, Error> {
+        match self.read_int()? {
+            0 => Ok(false),
+            1 => Ok(true),
+            val @ _ => Err(error!("Unexpected value for bool: `{val}`")),
+        }
+    }
+
+    /// Read this allocation as a pointer and return whether it represents a `null` pointer.
+    pub fn is_null(&self) -> Result<bool, Error> {
+        let len = self.bytes.len();
+        let ptr_len = MachineInfo::target_pointer_width().bytes();
+        if len != ptr_len {
+            return Err(error!("Expected width of pointer (`{ptr_len}`), but found: `{len}`"));
+        }
+        Ok(self.read_uint()? == 0 && self.provenance.ptrs.is_empty())
+    }
+}
+
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub enum ConstantKind {
     Allocated(Allocation),
@@ -500,10 +1137,39 @@ impl TraitDecl {
 
 pub type ImplTrait = EarlyBinder<TraitRef>;
 
+/// A complete reference to a trait, i.e., one where `Self` is known.
 #[derive(Clone, Debug, Eq, PartialEq)]
 pub struct TraitRef {
     pub def_id: TraitDef,
-    pub args: GenericArgs,
+    /// The generic arguments for this definition.
+    /// The first element must always be type, and it represents `Self`.
+    args: GenericArgs,
+}
+
+impl TraitRef {
+    pub fn new(def_id: TraitDef, self_ty: Ty, gen_args: &GenericArgs) -> TraitRef {
+        let mut args = vec![GenericArgKind::Type(self_ty)];
+        args.extend_from_slice(&gen_args.0);
+        TraitRef { def_id, args: GenericArgs(args) }
+    }
+
+    pub fn try_new(def_id: TraitDef, args: GenericArgs) -> Result<TraitRef, ()> {
+        match &args.0[..] {
+            [GenericArgKind::Type(_), ..] => Ok(TraitRef { def_id, args }),
+            _ => Err(()),
+        }
+    }
+
+    pub fn args(&self) -> &GenericArgs {
+        &self.args
+    }
+
+    pub fn self_ty(&self) -> Ty {
+        let GenericArgKind::Type(self_ty) = self.args.0[0] else {
+            panic!("Self must be a type, but found: {:?}", self.args.0[0])
+        };
+        self_ty
+    }
 }
 
 #[derive(Clone, Debug, Eq, PartialEq)]
@@ -542,7 +1208,6 @@ pub struct GenericPredicates {
 pub enum PredicateKind {
     Clause(ClauseKind),
     ObjectSafe(TraitDef),
-    ClosureKind(ClosureDef, GenericArgs, ClosureKind),
     SubType(SubtypePredicate),
     Coerce(CoercePredicate),
     ConstEquate(Const, Const),
@@ -633,3 +1298,21 @@ macro_rules! index_impl {
 index_impl!(ConstId);
 index_impl!(Ty);
 index_impl!(Span);
+
+/// The source-order index of a variant in a type.
+///
+/// For example, in the following types,
+/// ```ignore(illustrative)
+/// enum Demo1 {
+///    Variant0 { a: bool, b: i32 },
+///    Variant1 { c: u8, d: u64 },
+/// }
+/// struct Demo2 { e: u8, f: u16, g: u8 }
+/// ```
+/// `a` is in the variant with the `VariantIdx` of `0`,
+/// `c` is in the variant with the `VariantIdx` of `1`, and
+/// `g` is in the variant with the `VariantIdx` of `0`.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub struct VariantIdx(usize);
+
+index_impl!(VariantIdx);
diff --git a/compiler/stable_mir/src/visitor.rs b/compiler/stable_mir/src/visitor.rs
index 05e0b9b4d..65e42879d 100644
--- a/compiler/stable_mir/src/visitor.rs
+++ b/compiler/stable_mir/src/visitor.rs
@@ -149,6 +149,7 @@ impl Visitable for RigidTy {
             RigidTy::FnPtr(sig) => sig.visit(visitor),
             RigidTy::Closure(_, args) => args.visit(visitor),
             RigidTy::Coroutine(_, args, _) => args.visit(visitor),
+            RigidTy::CoroutineWitness(_, args) => args.visit(visitor),
             RigidTy::Dynamic(pred, r, _) => {
                 pred.visit(visitor)?;
                 r.visit(visitor)