16 files changed, 597 insertions, 613 deletions

diff --git a/compiler/rustc_middle/src/mir/basic_blocks.rs b/compiler/rustc_middle/src/mir/basic_blocks.rs
index 9d70dbfa0..7722e7b47 100644
--- a/compiler/rustc_middle/src/mir/basic_blocks.rs
+++ b/compiler/rustc_middle/src/mir/basic_blocks.rs
@@ -26,7 +26,7 @@ struct Cache {
     predecessors: OnceCell<Predecessors>,
     switch_sources: OnceCell<SwitchSources>,
     is_cyclic: OnceCell<bool>,
-    postorder: OnceCell<Vec<BasicBlock>>,
+    reverse_postorder: OnceCell<Vec<BasicBlock>>,
     dominators: OnceCell<Dominators<BasicBlock>>,
 }
 
@@ -62,11 +62,14 @@ impl<'tcx> BasicBlocks<'tcx> {
         })
     }
 
-    /// Returns basic blocks in a postorder.
+    /// Returns basic blocks in a reverse postorder.
     #[inline]
-    pub fn postorder(&self) -> &[BasicBlock] {
-        self.cache.postorder.get_or_init(|| {
-            Postorder::new(&self.basic_blocks, START_BLOCK).map(|(bb, _)| bb).collect()
+    pub fn reverse_postorder(&self) -> &[BasicBlock] {
+        self.cache.reverse_postorder.get_or_init(|| {
+            let mut rpo: Vec<_> =
+                Postorder::new(&self.basic_blocks, START_BLOCK).map(|(bb, _)| bb).collect();
+            rpo.reverse();
+            rpo
         })
     }
 
diff --git a/compiler/rustc_middle/src/mir/interpret/allocation.rs b/compiler/rustc_middle/src/mir/interpret/allocation.rs
index 1a8e48264..b8030d9db 100644
--- a/compiler/rustc_middle/src/mir/interpret/allocation.rs
+++ b/compiler/rustc_middle/src/mir/interpret/allocation.rs
@@ -296,25 +296,13 @@ impl<Prov: Provenance, Bytes: AllocBytes> Allocation<Prov, (), Bytes> {
         Allocation::from_bytes(slice, Align::ONE, Mutability::Not)
     }
 
-    /// Try to create an Allocation of `size` bytes, failing if there is not enough memory
-    /// available to the compiler to do so.
-    ///
-    /// If `panic_on_fail` is true, this will never return `Err`.
-    pub fn uninit<'tcx>(size: Size, align: Align, panic_on_fail: bool) -> InterpResult<'tcx, Self> {
-        let bytes = Bytes::zeroed(size, align).ok_or_else(|| {
-            // This results in an error that can happen non-deterministically, since the memory
-            // available to the compiler can change between runs. Normally queries are always
-            // deterministic. However, we can be non-deterministic here because all uses of const
-            // evaluation (including ConstProp!) will make compilation fail (via hard error
-            // or ICE) upon encountering a `MemoryExhausted` error.
-            if panic_on_fail {
-                panic!("Allocation::uninit called with panic_on_fail had allocation failure")
-            }
-            ty::tls::with(|tcx| {
-                tcx.sess.delay_span_bug(DUMMY_SP, "exhausted memory during interpretation")
-            });
-            InterpError::ResourceExhaustion(ResourceExhaustionInfo::MemoryExhausted)
-        })?;
+    fn uninit_inner<R>(size: Size, align: Align, fail: impl FnOnce() -> R) -> Result<Self, R> {
+        // This results in an error that can happen non-deterministically, since the memory
+        // available to the compiler can change between runs. Normally queries are always
+        // deterministic. However, we can be non-deterministic here because all uses of const
+        // evaluation (including ConstProp!) will make compilation fail (via hard error
+        // or ICE) upon encountering a `MemoryExhausted` error.
+        let bytes = Bytes::zeroed(size, align).ok_or_else(fail)?;
 
         Ok(Allocation {
             bytes,
@@ -325,6 +313,28 @@ impl<Prov: Provenance, Bytes: AllocBytes> Allocation<Prov, (), Bytes> {
             extra: (),
         })
     }
+
+    /// Try to create an Allocation of `size` bytes, failing if there is not enough memory
+    /// available to the compiler to do so.
+    pub fn try_uninit<'tcx>(size: Size, align: Align) -> InterpResult<'tcx, Self> {
+        Self::uninit_inner(size, align, || {
+            ty::tls::with(|tcx| {
+                tcx.sess.delay_span_bug(DUMMY_SP, "exhausted memory during interpretation")
+            });
+            InterpError::ResourceExhaustion(ResourceExhaustionInfo::MemoryExhausted).into()
+        })
+    }
+
+    /// Try to create an Allocation of `size` bytes, panics if there is not enough memory
+    /// available to the compiler to do so.
+    pub fn uninit(size: Size, align: Align) -> Self {
+        match Self::uninit_inner(size, align, || {
+            panic!("Allocation::uninit called with panic_on_fail had allocation failure");
+        }) {
+            Ok(x) => x,
+            Err(x) => x,
+        }
+    }
 }
 
 impl<Bytes: AllocBytes> Allocation<AllocId, (), Bytes> {
diff --git a/compiler/rustc_middle/src/mir/interpret/error.rs b/compiler/rustc_middle/src/mir/interpret/error.rs
index 055d8e9a3..2435bc59e 100644
--- a/compiler/rustc_middle/src/mir/interpret/error.rs
+++ b/compiler/rustc_middle/src/mir/interpret/error.rs
@@ -5,11 +5,15 @@ use crate::query::TyCtxtAt;
 use crate::ty::{layout, tls, Ty, ValTree};
 
 use rustc_data_structures::sync::Lock;
-use rustc_errors::{pluralize, struct_span_err, DiagnosticBuilder, ErrorGuaranteed};
+use rustc_errors::{
+    struct_span_err, DiagnosticArgValue, DiagnosticBuilder, DiagnosticMessage, ErrorGuaranteed,
+    IntoDiagnosticArg,
+};
 use rustc_macros::HashStable;
 use rustc_session::CtfeBacktrace;
 use rustc_span::def_id::DefId;
-use rustc_target::abi::{call, Align, Size};
+use rustc_target::abi::{call, Align, Size, WrappingRange};
+use std::borrow::Cow;
 use std::{any::Any, backtrace::Backtrace, fmt};
 
 #[derive(Debug, Copy, Clone, PartialEq, Eq, HashStable, TyEncodable, TyDecodable)]
@@ -91,21 +95,54 @@ pub struct InterpErrorInfo<'tcx>(Box<InterpErrorInfoInner<'tcx>>);
 #[derive(Debug)]
 struct InterpErrorInfoInner<'tcx> {
     kind: InterpError<'tcx>,
+    backtrace: InterpErrorBacktrace,
+}
+
+#[derive(Debug)]
+pub struct InterpErrorBacktrace {
     backtrace: Option<Box<Backtrace>>,
 }
 
-impl fmt::Display for InterpErrorInfo<'_> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{}", self.0.kind)
+impl InterpErrorBacktrace {
+    pub fn new() -> InterpErrorBacktrace {
+        let capture_backtrace = tls::with_opt(|tcx| {
+            if let Some(tcx) = tcx {
+                *Lock::borrow(&tcx.sess.ctfe_backtrace)
+            } else {
+                CtfeBacktrace::Disabled
+            }
+        });
+
+        let backtrace = match capture_backtrace {
+            CtfeBacktrace::Disabled => None,
+            CtfeBacktrace::Capture => Some(Box::new(Backtrace::force_capture())),
+            CtfeBacktrace::Immediate => {
+                // Print it now.
+                let backtrace = Backtrace::force_capture();
+                print_backtrace(&backtrace);
+                None
+            }
+        };
+
+        InterpErrorBacktrace { backtrace }
     }
-}
 
-impl<'tcx> InterpErrorInfo<'tcx> {
     pub fn print_backtrace(&self) {
-        if let Some(backtrace) = self.0.backtrace.as_ref() {
+        if let Some(backtrace) = self.backtrace.as_ref() {
             print_backtrace(backtrace);
         }
     }
+}
+
+impl<'tcx> InterpErrorInfo<'tcx> {
+    pub fn from_parts(kind: InterpError<'tcx>, backtrace: InterpErrorBacktrace) -> Self {
+        Self(Box::new(InterpErrorInfoInner { kind, backtrace }))
+    }
+
+    pub fn into_parts(self) -> (InterpError<'tcx>, InterpErrorBacktrace) {
+        let InterpErrorInfo(box InterpErrorInfoInner { kind, backtrace }) = self;
+        (kind, backtrace)
+    }
 
     pub fn into_kind(self) -> InterpError<'tcx> {
         let InterpErrorInfo(box InterpErrorInfoInner { kind, .. }) = self;
@@ -130,32 +167,17 @@ impl From<ErrorGuaranteed> for InterpErrorInfo<'_> {
 
 impl<'tcx> From<InterpError<'tcx>> for InterpErrorInfo<'tcx> {
     fn from(kind: InterpError<'tcx>) -> Self {
-        let capture_backtrace = tls::with_opt(|tcx| {
-            if let Some(tcx) = tcx {
-                *Lock::borrow(&tcx.sess.ctfe_backtrace)
-            } else {
-                CtfeBacktrace::Disabled
-            }
-        });
-
-        let backtrace = match capture_backtrace {
-            CtfeBacktrace::Disabled => None,
-            CtfeBacktrace::Capture => Some(Box::new(Backtrace::force_capture())),
-            CtfeBacktrace::Immediate => {
-                // Print it now.
-                let backtrace = Backtrace::force_capture();
-                print_backtrace(&backtrace);
-                None
-            }
-        };
-
-        InterpErrorInfo(Box::new(InterpErrorInfoInner { kind, backtrace }))
+        InterpErrorInfo(Box::new(InterpErrorInfoInner {
+            kind,
+            backtrace: InterpErrorBacktrace::new(),
+        }))
     }
 }
 
 /// Error information for when the program we executed turned out not to actually be a valid
 /// program. This cannot happen in stand-alone Miri, but it can happen during CTFE/ConstProp
 /// where we work on generic code or execution does not have all information available.
+#[derive(Debug)]
 pub enum InvalidProgramInfo<'tcx> {
     /// Resolution can fail if we are in a too generic context.
     TooGeneric,
@@ -174,25 +196,6 @@ pub enum InvalidProgramInfo<'tcx> {
     UninitUnsizedLocal,
 }
 
-impl fmt::Display for InvalidProgramInfo<'_> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        use InvalidProgramInfo::*;
-        match self {
-            TooGeneric => write!(f, "encountered overly generic constant"),
-            AlreadyReported(_) => {
-                write!(
-                    f,
-                    "an error has already been reported elsewhere (this should not usually be printed)"
-                )
-            }
-            Layout(ref err) => write!(f, "{err}"),
-            FnAbiAdjustForForeignAbi(ref err) => write!(f, "{err}"),
-            SizeOfUnsizedType(ty) => write!(f, "size_of called on unsized type `{ty}`"),
-            UninitUnsizedLocal => write!(f, "unsized local is used while uninitialized"),
-        }
-    }
-}
-
 /// Details of why a pointer had to be in-bounds.
 #[derive(Debug, Copy, Clone, TyEncodable, TyDecodable, HashStable)]
 pub enum CheckInAllocMsg {
@@ -208,26 +211,25 @@ pub enum CheckInAllocMsg {
     InboundsTest,
 }
 
-impl fmt::Display for CheckInAllocMsg {
-    /// When this is printed as an error the context looks like this:
-    /// "{msg}{pointer} is a dangling pointer".
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(
-            f,
-            "{}",
-            match *self {
-                CheckInAllocMsg::DerefTest => "dereferencing pointer failed: ",
-                CheckInAllocMsg::MemoryAccessTest => "memory access failed: ",
-                CheckInAllocMsg::PointerArithmeticTest => "out-of-bounds pointer arithmetic: ",
-                CheckInAllocMsg::OffsetFromTest => "out-of-bounds offset_from: ",
-                CheckInAllocMsg::InboundsTest => "out-of-bounds pointer use: ",
-            }
-        )
+#[derive(Debug, Copy, Clone, TyEncodable, TyDecodable, HashStable)]
+pub enum InvalidMetaKind {
+    /// Size of a `[T]` is too big
+    SliceTooBig,
+    /// Size of a DST is too big
+    TooBig,
+}
+
+impl IntoDiagnosticArg for InvalidMetaKind {
+    fn into_diagnostic_arg(self) -> DiagnosticArgValue<'static> {
+        DiagnosticArgValue::Str(Cow::Borrowed(match self {
+            InvalidMetaKind::SliceTooBig => "slice_too_big",
+            InvalidMetaKind::TooBig => "too_big",
+        }))
     }
 }
 
 /// Details of an access to uninitialized bytes where it is not allowed.
-#[derive(Debug)]
+#[derive(Debug, Clone, Copy)]
 pub struct UninitBytesAccess {
     /// Range of the original memory access.
     pub access: AllocRange,
@@ -242,17 +244,32 @@ pub struct ScalarSizeMismatch {
     pub data_size: u64,
 }
 
+macro_rules! impl_into_diagnostic_arg_through_debug {
+    ($($ty:ty),*$(,)?) => {$(
+        impl IntoDiagnosticArg for $ty {
+            fn into_diagnostic_arg(self) -> DiagnosticArgValue<'static> {
+                DiagnosticArgValue::Str(Cow::Owned(format!("{self:?}")))
+            }
+        }
+    )*}
+}
+
+// These types have nice `Debug` output so we can just use them in diagnostics.
+impl_into_diagnostic_arg_through_debug! {
+    AllocId,
+    Pointer,
+    AllocRange,
+}
+
 /// Error information for when the program caused Undefined Behavior.
-pub enum UndefinedBehaviorInfo {
-    /// Free-form case. Only for errors that are never caught!
+#[derive(Debug)]
+pub enum UndefinedBehaviorInfo<'a> {
+    /// Free-form case. Only for errors that are never caught! Used by miri
     Ub(String),
     /// Unreachable code was executed.
     Unreachable,
     /// A slice/array index projection went out-of-bounds.
-    BoundsCheckFailed {
-        len: u64,
-        index: u64,
-    },
+    BoundsCheckFailed { len: u64, index: u64 },
     /// Something was divided by 0 (x / 0).
     DivisionByZero,
     /// Something was "remainded" by 0 (x % 0).
@@ -263,8 +280,8 @@ pub enum UndefinedBehaviorInfo {
     RemainderOverflow,
     /// Overflowing inbounds pointer arithmetic.
     PointerArithOverflow,
-    /// Invalid metadata in a wide pointer (using `str` to avoid allocations).
-    InvalidMeta(&'static str),
+    /// Invalid metadata in a wide pointer
+    InvalidMeta(InvalidMetaKind),
     /// Reading a C string that does not end within its allocation.
     UnterminatedCString(Pointer),
     /// Dereferencing a dangling pointer after it got freed.
@@ -281,25 +298,13 @@ pub enum UndefinedBehaviorInfo {
     /// Using an integer as a pointer in the wrong way.
     DanglingIntPointer(u64, CheckInAllocMsg),
     /// Used a pointer with bad alignment.
-    AlignmentCheckFailed {
-        required: Align,
-        has: Align,
-    },
+    AlignmentCheckFailed { required: Align, has: Align },
     /// Writing to read-only memory.
     WriteToReadOnly(AllocId),
-    // Trying to access the data behind a function pointer.
+    /// Trying to access the data behind a function pointer.
     DerefFunctionPointer(AllocId),
-    // Trying to access the data behind a vtable pointer.
+    /// Trying to access the data behind a vtable pointer.
     DerefVTablePointer(AllocId),
-    /// The value validity check found a problem.
-    /// Should only be thrown by `validity.rs` and always point out which part of the value
-    /// is the problem.
-    ValidationFailure {
-        /// The "path" to the value in question, e.g. `.0[5].field` for a struct
-        /// field in the 6th element of an array that is the first element of a tuple.
-        path: Option<String>,
-        msg: String,
-    },
     /// Using a non-boolean `u8` as bool.
     InvalidBool(u8),
     /// Using a non-character `u32` as character.
@@ -320,110 +325,100 @@ pub enum UndefinedBehaviorInfo {
     ScalarSizeMismatch(ScalarSizeMismatch),
     /// A discriminant of an uninhabited enum variant is written.
     UninhabitedEnumVariantWritten,
+    /// Validation error.
+    Validation(ValidationErrorInfo<'a>),
+    // FIXME(fee1-dead) these should all be actual variants of the enum instead of dynamically
+    // dispatched
+    /// A custom (free-form) error, created by `err_ub_custom!`.
+    Custom(crate::error::CustomSubdiagnostic<'a>),
+}
+
+#[derive(Debug, Clone, Copy)]
+pub enum PointerKind {
+    Ref,
+    Box,
+}
+
+impl IntoDiagnosticArg for PointerKind {
+    fn into_diagnostic_arg(self) -> DiagnosticArgValue<'static> {
+        DiagnosticArgValue::Str(
+            match self {
+                Self::Ref => "ref",
+                Self::Box => "box",
+            }
+            .into(),
+        )
+    }
 }
 
-impl fmt::Display for UndefinedBehaviorInfo {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        use UndefinedBehaviorInfo::*;
-        match self {
-            Ub(msg) => write!(f, "{msg}"),
-            Unreachable => write!(f, "entering unreachable code"),
-            BoundsCheckFailed { ref len, ref index } => {
-                write!(f, "indexing out of bounds: the len is {len} but the index is {index}")
-            }
-            DivisionByZero => write!(f, "dividing by zero"),
-            RemainderByZero => write!(f, "calculating the remainder with a divisor of zero"),
-            DivisionOverflow => write!(f, "overflow in signed division (dividing MIN by -1)"),
-            RemainderOverflow => write!(f, "overflow in signed remainder (dividing MIN by -1)"),
-            PointerArithOverflow => write!(f, "overflowing in-bounds pointer arithmetic"),
-            InvalidMeta(msg) => write!(f, "invalid metadata in wide pointer: {msg}"),
-            UnterminatedCString(p) => write!(
-                f,
-                "reading a null-terminated string starting at {p:?} with no null found before end of allocation",
-            ),
-            PointerUseAfterFree(a) => {
-                write!(f, "pointer to {a:?} was dereferenced after this allocation got freed")
-            }
-            PointerOutOfBounds { alloc_id, alloc_size, ptr_offset, ptr_size: Size::ZERO, msg } => {
-                write!(
-                    f,
-                    "{msg}{alloc_id:?} has size {alloc_size}, so pointer at offset {ptr_offset} is out-of-bounds",
-                    alloc_size = alloc_size.bytes(),
-                )
-            }
-            PointerOutOfBounds { alloc_id, alloc_size, ptr_offset, ptr_size, msg } => write!(
-                f,
-                "{msg}{alloc_id:?} has size {alloc_size}, so pointer to {ptr_size} byte{ptr_size_p} starting at offset {ptr_offset} is out-of-bounds",
-                alloc_size = alloc_size.bytes(),
-                ptr_size = ptr_size.bytes(),
-                ptr_size_p = pluralize!(ptr_size.bytes()),
-            ),
-            DanglingIntPointer(i, msg) => {
-                write!(
-                    f,
-                    "{msg}{pointer} is a dangling pointer (it has no provenance)",
-                    pointer = Pointer::<Option<AllocId>>::from_addr_invalid(*i),
-                )
-            }
-            AlignmentCheckFailed { required, has } => write!(
-                f,
-                "accessing memory with alignment {has}, but alignment {required} is required",
-                has = has.bytes(),
-                required = required.bytes()
-            ),
-            WriteToReadOnly(a) => write!(f, "writing to {a:?} which is read-only"),
-            DerefFunctionPointer(a) => write!(f, "accessing {a:?} which contains a function"),
-            DerefVTablePointer(a) => write!(f, "accessing {a:?} which contains a vtable"),
-            ValidationFailure { path: None, msg } => {
-                write!(f, "constructing invalid value: {msg}")
-            }
-            ValidationFailure { path: Some(path), msg } => {
-                write!(f, "constructing invalid value at {path}: {msg}")
-            }
-            InvalidBool(b) => {
-                write!(f, "interpreting an invalid 8-bit value as a bool: 0x{b:02x}")
-            }
-            InvalidChar(c) => {
-                write!(f, "interpreting an invalid 32-bit value as a char: 0x{c:08x}")
-            }
-            InvalidTag(val) => write!(f, "enum value has invalid tag: {val:x}"),
-            InvalidFunctionPointer(p) => {
-                write!(f, "using {p:?} as function pointer but it does not point to a function")
-            }
-            InvalidVTablePointer(p) => {
-                write!(f, "using {p:?} as vtable pointer but it does not point to a vtable")
-            }
-            InvalidStr(err) => write!(f, "this string is not valid UTF-8: {err}"),
-            InvalidUninitBytes(Some((alloc, info))) => write!(
-                f,
-                "reading memory at {alloc:?}{access:?}, \
-                 but memory is uninitialized at {uninit:?}, \
-                 and this operation requires initialized memory",
-                access = info.access,
-                uninit = info.uninit,
-            ),
-            InvalidUninitBytes(None) => write!(
-                f,
-                "using uninitialized data, but this operation requires initialized memory"
-            ),
-            DeadLocal => write!(f, "accessing a dead local variable"),
-            ScalarSizeMismatch(self::ScalarSizeMismatch { target_size, data_size }) => write!(
-                f,
-                "scalar size mismatch: expected {target_size} bytes but got {data_size} bytes instead",
-            ),
-            UninhabitedEnumVariantWritten => {
-                write!(f, "writing discriminant of an uninhabited enum")
-            }
+#[derive(Debug)]
+pub struct ValidationErrorInfo<'tcx> {
+    pub path: Option<String>,
+    pub kind: ValidationErrorKind<'tcx>,
+}
+
+#[derive(Debug)]
+pub enum ExpectedKind {
+    Reference,
+    Box,
+    RawPtr,
+    InitScalar,
+    Bool,
+    Char,
+    Float,
+    Int,
+    FnPtr,
+}
+
+impl From<PointerKind> for ExpectedKind {
+    fn from(x: PointerKind) -> ExpectedKind {
+        match x {
+            PointerKind::Box => ExpectedKind::Box,
+            PointerKind::Ref => ExpectedKind::Reference,
         }
     }
 }
 
+#[derive(Debug)]
+pub enum ValidationErrorKind<'tcx> {
+    PtrToUninhabited { ptr_kind: PointerKind, ty: Ty<'tcx> },
+    PtrToStatic { ptr_kind: PointerKind },
+    PtrToMut { ptr_kind: PointerKind },
+    ExpectedNonPtr { value: String },
+    MutableRefInConst,
+    NullFnPtr,
+    NeverVal,
+    NullablePtrOutOfRange { range: WrappingRange, max_value: u128 },
+    PtrOutOfRange { range: WrappingRange, max_value: u128 },
+    OutOfRange { value: String, range: WrappingRange, max_value: u128 },
+    UnsafeCell,
+    UninhabitedVal { ty: Ty<'tcx> },
+    InvalidEnumTag { value: String },
+    UninitEnumTag,
+    UninitStr,
+    Uninit { expected: ExpectedKind },
+    UninitVal,
+    InvalidVTablePtr { value: String },
+    InvalidMetaSliceTooLarge { ptr_kind: PointerKind },
+    InvalidMetaTooLarge { ptr_kind: PointerKind },
+    UnalignedPtr { ptr_kind: PointerKind, required_bytes: u64, found_bytes: u64 },
+    NullPtr { ptr_kind: PointerKind },
+    DanglingPtrNoProvenance { ptr_kind: PointerKind, pointer: String },
+    DanglingPtrOutOfBounds { ptr_kind: PointerKind },
+    DanglingPtrUseAfterFree { ptr_kind: PointerKind },
+    InvalidBool { value: String },
+    InvalidChar { value: String },
+    InvalidFnPtr { value: String },
+}
+
 /// Error information for when the program did something that might (or might not) be correct
 /// to do according to the Rust spec, but due to limitations in the interpreter, the
 /// operation could not be carried out. These limitations can differ between CTFE and the
 /// Miri engine, e.g., CTFE does not support dereferencing pointers at integral addresses.
+#[derive(Debug)]
 pub enum UnsupportedOpInfo {
     /// Free-form case. Only for errors that are never caught!
+    // FIXME still use translatable diagnostics
     Unsupported(String),
     //
     // The variants below are only reachable from CTFE/const prop, miri will never emit them.
@@ -442,83 +437,42 @@ pub enum UnsupportedOpInfo {
     ReadExternStatic(DefId),
 }
 
-impl fmt::Display for UnsupportedOpInfo {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        use UnsupportedOpInfo::*;
-        match self {
-            Unsupported(ref msg) => write!(f, "{msg}"),
-            PartialPointerOverwrite(ptr) => {
-                write!(f, "unable to overwrite parts of a pointer in memory at {ptr:?}")
-            }
-            PartialPointerCopy(ptr) => {
-                write!(f, "unable to copy parts of a pointer from memory at {ptr:?}")
-            }
-            ReadPointerAsBytes => write!(f, "unable to turn pointer into raw bytes"),
-            ThreadLocalStatic(did) => write!(f, "cannot access thread local static ({did:?})"),
-            ReadExternStatic(did) => write!(f, "cannot read from extern static ({did:?})"),
-        }
-    }
-}
-
 /// Error information for when the program exhausted the resources granted to it
 /// by the interpreter.
+#[derive(Debug)]
 pub enum ResourceExhaustionInfo {
     /// The stack grew too big.
     StackFrameLimitReached,
-    /// The program ran for too long.
-    ///
-    /// The exact limit is set by the `const_eval_limit` attribute.
-    StepLimitReached,
     /// There is not enough memory (on the host) to perform an allocation.
     MemoryExhausted,
     /// The address space (of the target) is full.
     AddressSpaceFull,
 }
 
-impl fmt::Display for ResourceExhaustionInfo {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        use ResourceExhaustionInfo::*;
-        match self {
-            StackFrameLimitReached => {
-                write!(f, "reached the configured maximum number of stack frames")
-            }
-            StepLimitReached => {
-                write!(f, "exceeded interpreter step limit (see `#[const_eval_limit]`)")
-            }
-            MemoryExhausted => {
-                write!(f, "tried to allocate more memory than available to compiler")
-            }
-            AddressSpaceFull => {
-                write!(f, "there are no more free addresses in the address space")
-            }
-        }
-    }
-}
-
-/// A trait to work around not having trait object upcasting.
-pub trait AsAny: Any {
-    fn as_any(&self) -> &dyn Any;
-}
-impl<T: Any> AsAny for T {
-    #[inline(always)]
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-}
-
 /// A trait for machine-specific errors (or other "machine stop" conditions).
-pub trait MachineStopType: AsAny + fmt::Display + Send {}
+pub trait MachineStopType: Any + fmt::Debug + Send {
+    /// The diagnostic message for this error
+    fn diagnostic_message(&self) -> DiagnosticMessage;
+    /// Add diagnostic arguments by passing name and value pairs to `adder`, which are passed to
+    /// fluent for formatting the translated diagnostic message.
+    fn add_args(
+        self: Box<Self>,
+        adder: &mut dyn FnMut(Cow<'static, str>, DiagnosticArgValue<'static>),
+    );
+}
 
 impl dyn MachineStopType {
     #[inline(always)]
     pub fn downcast_ref<T: Any>(&self) -> Option<&T> {
-        self.as_any().downcast_ref()
+        let x: &dyn Any = self;
+        x.downcast_ref()
     }
 }
 
+#[derive(Debug)]
 pub enum InterpError<'tcx> {
     /// The program caused undefined behavior.
-    UndefinedBehavior(UndefinedBehaviorInfo),
+    UndefinedBehavior(UndefinedBehaviorInfo<'tcx>),
     /// The program did something the interpreter does not support (some of these *might* be UB
     /// but the interpreter is not sure).
     Unsupported(UnsupportedOpInfo),
@@ -534,26 +488,6 @@ pub enum InterpError<'tcx> {
 
 pub type InterpResult<'tcx, T = ()> = Result<T, InterpErrorInfo<'tcx>>;
 
-impl fmt::Display for InterpError<'_> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        use InterpError::*;
-        match *self {
-            Unsupported(ref msg) => write!(f, "{msg}"),
-            InvalidProgram(ref msg) => write!(f, "{msg}"),
-            UndefinedBehavior(ref msg) => write!(f, "{msg}"),
-            ResourceExhaustion(ref msg) => write!(f, "{msg}"),
-            MachineStop(ref msg) => write!(f, "{msg}"),
-        }
-    }
-}
-
-// Forward `Debug` to `Display`, so it does not look awful.
-impl fmt::Debug for InterpError<'_> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        fmt::Display::fmt(self, f)
-    }
-}
-
 impl InterpError<'_> {
     /// Some errors do string formatting even if the error is never printed.
     /// To avoid performance issues, there are places where we want to be sure to never raise these formatting errors,
@@ -562,7 +496,7 @@ impl InterpError<'_> {
         matches!(
             self,
             InterpError::Unsupported(UnsupportedOpInfo::Unsupported(_))
-                | InterpError::UndefinedBehavior(UndefinedBehaviorInfo::ValidationFailure { .. })
+                | InterpError::UndefinedBehavior(UndefinedBehaviorInfo::Validation { .. })
                 | InterpError::UndefinedBehavior(UndefinedBehaviorInfo::Ub(_))
         )
     }
diff --git a/compiler/rustc_middle/src/mir/interpret/mod.rs b/compiler/rustc_middle/src/mir/interpret/mod.rs
index 3620385fa..2d2cfee1b 100644
--- a/compiler/rustc_middle/src/mir/interpret/mod.rs
+++ b/compiler/rustc_middle/src/mir/interpret/mod.rs
@@ -89,6 +89,30 @@ macro_rules! throw_machine_stop {
     ($($tt:tt)*) => { do yeet err_machine_stop!($($tt)*) };
 }
 
+#[macro_export]
+macro_rules! err_ub_custom {
+    ($msg:expr $(, $($name:ident = $value:expr),* $(,)?)?) => {{
+        $(
+            let ($($name,)*) = ($($value,)*);
+        )?
+        err_ub!(Custom(
+            rustc_middle::error::CustomSubdiagnostic {
+                msg: || $msg,
+                add_args: Box::new(move |mut set_arg| {
+                    $($(
+                        set_arg(stringify!($name).into(), rustc_errors::IntoDiagnosticArg::into_diagnostic_arg($name));
+                    )*)?
+                })
+            }
+        ))
+    }};
+}
+
+#[macro_export]
+macro_rules! throw_ub_custom {
+    ($($tt:tt)*) => { do yeet err_ub_custom!($($tt)*) };
+}
+
 mod allocation;
 mod error;
 mod pointer;
@@ -119,9 +143,10 @@ use crate::ty::{self, Instance, Ty, TyCtxt};
 
 pub use self::error::{
     struct_error, CheckInAllocMsg, ErrorHandled, EvalToAllocationRawResult, EvalToConstValueResult,
-    EvalToValTreeResult, InterpError, InterpErrorInfo, InterpResult, InvalidProgramInfo,
-    MachineStopType, ReportedErrorInfo, ResourceExhaustionInfo, ScalarSizeMismatch,
-    UndefinedBehaviorInfo, UninitBytesAccess, UnsupportedOpInfo,
+    EvalToValTreeResult, ExpectedKind, InterpError, InterpErrorInfo, InterpResult, InvalidMetaKind,
+    InvalidProgramInfo, MachineStopType, PointerKind, ReportedErrorInfo, ResourceExhaustionInfo,
+    ScalarSizeMismatch, UndefinedBehaviorInfo, UninitBytesAccess, UnsupportedOpInfo,
+    ValidationErrorInfo, ValidationErrorKind,
 };
 
 pub use self::value::{get_slice_bytes, ConstAlloc, ConstValue, Scalar};
diff --git a/compiler/rustc_middle/src/mir/interpret/queries.rs b/compiler/rustc_middle/src/mir/interpret/queries.rs
index f53dc8cb0..9c97431f3 100644
--- a/compiler/rustc_middle/src/mir/interpret/queries.rs
+++ b/compiler/rustc_middle/src/mir/interpret/queries.rs
@@ -95,8 +95,15 @@ impl<'tcx> TyCtxt<'tcx> {
                     // used generic parameters is a bug of evaluation, so checking for it
                     // here does feel somewhat sensible.
                     if !self.features().generic_const_exprs && ct.substs.has_non_region_param() {
-                        assert!(matches!(self.def_kind(ct.def), DefKind::AnonConst));
-                        let mir_body = self.mir_for_ctfe(ct.def);
+                        let def_kind = self.def_kind(instance.def_id());
+                        assert!(
+                            matches!(
+                                def_kind,
+                                DefKind::InlineConst | DefKind::AnonConst | DefKind::AssocConst
+                            ),
+                            "{cid:?} is {def_kind:?}",
+                        );
+                        let mir_body = self.mir_for_ctfe(instance.def_id());
                         if mir_body.is_polymorphic {
                             let Some(local_def_id) = ct.def.as_local() else { return };
                             self.struct_span_lint_hir(
@@ -236,15 +243,3 @@ impl<'tcx> TyCtxtEnsure<'tcx> {
         self.eval_to_allocation_raw(param_env.and(gid))
     }
 }
-
-impl<'tcx> TyCtxt<'tcx> {
-    /// Destructure a mir constant ADT or array into its variant index and its field values.
-    /// Panics if the destructuring fails, use `try_destructure_mir_constant` for fallible version.
-    pub fn destructure_mir_constant(
-        self,
-        param_env: ty::ParamEnv<'tcx>,
-        constant: mir::ConstantKind<'tcx>,
-    ) -> mir::DestructuredConstant<'tcx> {
-        self.try_destructure_mir_constant(param_env.and(constant)).unwrap()
-    }
-}
diff --git a/compiler/rustc_middle/src/mir/interpret/value.rs b/compiler/rustc_middle/src/mir/interpret/value.rs
index 36dbbe4bf..0416411df 100644
--- a/compiler/rustc_middle/src/mir/interpret/value.rs
+++ b/compiler/rustc_middle/src/mir/interpret/value.rs
@@ -97,6 +97,10 @@ impl<'tcx> ConstValue<'tcx> {
         ConstValue::Scalar(Scalar::from_u64(i))
     }
 
+    pub fn from_u128(i: u128) -> Self {
+        ConstValue::Scalar(Scalar::from_u128(i))
+    }
+
     pub fn from_target_usize(i: u64, cx: &impl HasDataLayout) -> Self {
         ConstValue::Scalar(Scalar::from_target_usize(i, cx))
     }
@@ -241,6 +245,11 @@ impl<Prov> Scalar<Prov> {
     }
 
     #[inline]
+    pub fn from_u128(i: u128) -> Self {
+        Scalar::Int(i.into())
+    }
+
+    #[inline]
     pub fn from_target_usize(i: u64, cx: &impl HasDataLayout) -> Self {
         Self::from_uint(i, cx.data_layout().pointer_size)
     }
@@ -375,7 +384,8 @@ impl<'tcx, Prov: Provenance> Scalar<Prov> {
     #[inline(always)]
     #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
     pub fn assert_bits(self, target_size: Size) -> u128 {
-        self.to_bits(target_size).unwrap()
+        self.to_bits(target_size)
+            .unwrap_or_else(|_| panic!("assertion failed: {self:?} fits {target_size:?}"))
     }
 
     pub fn to_bool(self) -> InterpResult<'tcx, bool> {
diff --git a/compiler/rustc_middle/src/mir/mod.rs b/compiler/rustc_middle/src/mir/mod.rs
index 5c71910a9..28c505878 100644
--- a/compiler/rustc_middle/src/mir/mod.rs
+++ b/compiler/rustc_middle/src/mir/mod.rs
@@ -3,7 +3,7 @@
 //! [rustc dev guide]: https://rustc-dev-guide.rust-lang.org/mir/index.html
 
 use crate::mir::interpret::{
-    AllocRange, ConstAllocation, ConstValue, ErrorHandled, GlobalAlloc, LitToConstInput, Scalar,
+    AllocRange, ConstAllocation, ConstValue, ErrorHandled, GlobalAlloc, Scalar,
 };
 use crate::mir::visit::MirVisitable;
 use crate::ty::codec::{TyDecoder, TyEncoder};
@@ -15,7 +15,7 @@ use crate::ty::{AdtDef, InstanceDef, ScalarInt, UserTypeAnnotationIndex};
 use crate::ty::{GenericArg, InternalSubsts, SubstsRef};
 
 use rustc_data_structures::captures::Captures;
-use rustc_errors::ErrorGuaranteed;
+use rustc_errors::{DiagnosticArgValue, DiagnosticMessage, ErrorGuaranteed, IntoDiagnosticArg};
 use rustc_hir::def::{CtorKind, Namespace};
 use rustc_hir::def_id::{DefId, LocalDefId, CRATE_DEF_ID};
 use rustc_hir::{self, GeneratorKind, ImplicitSelfKind};
@@ -145,7 +145,7 @@ impl MirPhase {
             }
             "analysis" => Self::Analysis(AnalysisPhase::parse(phase)),
             "runtime" => Self::Runtime(RuntimePhase::parse(phase)),
-            _ => panic!("Unknown MIR dialect {}", dialect),
+            _ => bug!("Unknown MIR dialect: '{}'", dialect),
         }
     }
 }
@@ -159,7 +159,7 @@ impl AnalysisPhase {
         match &*phase.to_ascii_lowercase() {
             "initial" => Self::Initial,
             "post_cleanup" | "post-cleanup" | "postcleanup" => Self::PostCleanup,
-            _ => panic!("Unknown analysis phase {}", phase),
+            _ => bug!("Unknown analysis phase: '{}'", phase),
         }
     }
 }
@@ -174,7 +174,7 @@ impl RuntimePhase {
             "initial" => Self::Initial,
             "post_cleanup" | "post-cleanup" | "postcleanup" => Self::PostCleanup,
             "optimized" => Self::Optimized,
-            _ => panic!("Unknown runtime phase {}", phase),
+            _ => bug!("Unknown runtime phase: '{}'", phase),
         }
     }
 }
@@ -476,7 +476,7 @@ impl<'tcx> Body<'tcx> {
     /// Returns the return type; it always return first element from `local_decls` array.
     #[inline]
     pub fn bound_return_ty(&self) -> ty::EarlyBinder<Ty<'tcx>> {
-        ty::EarlyBinder(self.local_decls[RETURN_PLACE].ty)
+        ty::EarlyBinder::bind(self.local_decls[RETURN_PLACE].ty)
     }
 
     /// Gets the location of the terminator for the given block.
@@ -1371,55 +1371,61 @@ impl<O> AssertKind<O> {
             _ => write!(f, "\"{}\"", self.description()),
         }
     }
-}
 
-impl<O: fmt::Debug> fmt::Debug for AssertKind<O> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+    pub fn diagnostic_message(&self) -> DiagnosticMessage {
+        use crate::fluent_generated::*;
         use AssertKind::*;
+
         match self {
-            BoundsCheck { ref len, ref index } => write!(
-                f,
-                "index out of bounds: the length is {:?} but the index is {:?}",
-                len, index
-            ),
-            OverflowNeg(op) => write!(f, "attempt to negate `{:#?}`, which would overflow", op),
-            DivisionByZero(op) => write!(f, "attempt to divide `{:#?}` by zero", op),
-            RemainderByZero(op) => write!(
-                f,
-                "attempt to calculate the remainder of `{:#?}` with a divisor of zero",
-                op
-            ),
-            Overflow(BinOp::Add, l, r) => {
-                write!(f, "attempt to compute `{:#?} + {:#?}`, which would overflow", l, r)
-            }
-            Overflow(BinOp::Sub, l, r) => {
-                write!(f, "attempt to compute `{:#?} - {:#?}`, which would overflow", l, r)
-            }
-            Overflow(BinOp::Mul, l, r) => {
-                write!(f, "attempt to compute `{:#?} * {:#?}`, which would overflow", l, r)
-            }
-            Overflow(BinOp::Div, l, r) => {
-                write!(f, "attempt to compute `{:#?} / {:#?}`, which would overflow", l, r)
+            BoundsCheck { .. } => middle_bounds_check,
+            Overflow(BinOp::Shl, _, _) => middle_assert_shl_overflow,
+            Overflow(BinOp::Shr, _, _) => middle_assert_shr_overflow,
+            Overflow(_, _, _) => middle_assert_op_overflow,
+            OverflowNeg(_) => middle_assert_overflow_neg,
+            DivisionByZero(_) => middle_assert_divide_by_zero,
+            RemainderByZero(_) => middle_assert_remainder_by_zero,
+            ResumedAfterReturn(GeneratorKind::Async(_)) => middle_assert_async_resume_after_return,
+            ResumedAfterReturn(GeneratorKind::Gen) => middle_assert_generator_resume_after_return,
+            ResumedAfterPanic(GeneratorKind::Async(_)) => middle_assert_async_resume_after_panic,
+            ResumedAfterPanic(GeneratorKind::Gen) => middle_assert_generator_resume_after_panic,
+
+            MisalignedPointerDereference { .. } => middle_assert_misaligned_ptr_deref,
+        }
+    }
+
+    pub fn add_args(self, adder: &mut dyn FnMut(Cow<'static, str>, DiagnosticArgValue<'static>))
+    where
+        O: fmt::Debug,
+    {
+        use AssertKind::*;
+
+        macro_rules! add {
+            ($name: expr, $value: expr) => {
+                adder($name.into(), $value.into_diagnostic_arg());
+            };
+        }
+
+        match self {
+            BoundsCheck { len, index } => {
+                add!("len", format!("{len:?}"));
+                add!("index", format!("{index:?}"));
             }
-            Overflow(BinOp::Rem, l, r) => write!(
-                f,
-                "attempt to compute the remainder of `{:#?} % {:#?}`, which would overflow",
-                l, r
-            ),
-            Overflow(BinOp::Shr, _, r) => {
-                write!(f, "attempt to shift right by `{:#?}`, which would overflow", r)
+            Overflow(BinOp::Shl | BinOp::Shr, _, val)
+            | DivisionByZero(val)
+            | RemainderByZero(val)
+            | OverflowNeg(val) => {
+                add!("val", format!("{val:#?}"));
             }
-            Overflow(BinOp::Shl, _, r) => {
-                write!(f, "attempt to shift left by `{:#?}`, which would overflow", r)
+            Overflow(binop, left, right) => {
+                add!("op", binop.to_hir_binop().as_str());
+                add!("left", format!("{left:#?}"));
+                add!("right", format!("{right:#?}"));
             }
+            ResumedAfterReturn(_) | ResumedAfterPanic(_) => {}
             MisalignedPointerDereference { required, found } => {
-                write!(
-                    f,
-                    "misaligned pointer dereference: address must be a multiple of {:?} but is {:?}",
-                    required, found
-                )
+                add!("required", format!("{required:#?}"));
+                add!("found", format!("{found:#?}"));
             }
-            _ => write!(f, "{}", self.description()),
         }
     }
 }
@@ -1907,7 +1913,7 @@ impl<'tcx> Operand<'tcx> {
         substs: impl IntoIterator<Item = GenericArg<'tcx>>,
         span: Span,
     ) -> Self {
-        let ty = tcx.mk_fn_def(def_id, substs);
+        let ty = Ty::new_fn_def(tcx, def_id, substs);
         Operand::Constant(Box::new(Constant {
             span,
             user_ty: None,
@@ -2008,7 +2014,7 @@ impl<'tcx> Rvalue<'tcx> {
                 | CastKind::IntToFloat
                 | CastKind::FnPtrToPtr
                 | CastKind::PtrToPtr
-                | CastKind::Pointer(_)
+                | CastKind::PointerCoercion(_)
                 | CastKind::PointerFromExposedAddress
                 | CastKind::DynStar
                 | CastKind::Transmute,
@@ -2029,23 +2035,19 @@ impl<'tcx> Rvalue<'tcx> {
 impl BorrowKind {
     pub fn mutability(&self) -> Mutability {
         match *self {
-            BorrowKind::Shared | BorrowKind::Shallow | BorrowKind::Unique => Mutability::Not,
+            BorrowKind::Shared | BorrowKind::Shallow => Mutability::Not,
             BorrowKind::Mut { .. } => Mutability::Mut,
         }
     }
 
     pub fn allows_two_phase_borrow(&self) -> bool {
         match *self {
-            BorrowKind::Shared | BorrowKind::Shallow | BorrowKind::Unique => false,
-            BorrowKind::Mut { allow_two_phase_borrow } => allow_two_phase_borrow,
-        }
-    }
-
-    // FIXME: won't be used after diagnostic migration
-    pub fn describe_mutability(&self) -> &str {
-        match *self {
-            BorrowKind::Shared | BorrowKind::Shallow | BorrowKind::Unique => "immutable",
-            BorrowKind::Mut { .. } => "mutable",
+            BorrowKind::Shared
+            | BorrowKind::Shallow
+            | BorrowKind::Mut { kind: MutBorrowKind::Default | MutBorrowKind::ClosureCapture } => {
+                false
+            }
+            BorrowKind::Mut { kind: MutBorrowKind::TwoPhaseBorrow } => true,
         }
     }
 }
@@ -2084,7 +2086,7 @@ impl<'tcx> Debug for Rvalue<'tcx> {
                 let kind_str = match borrow_kind {
                     BorrowKind::Shared => "",
                     BorrowKind::Shallow => "shallow ",
-                    BorrowKind::Mut { .. } | BorrowKind::Unique => "mut ",
+                    BorrowKind::Mut { .. } => "mut ",
                 };
 
                 // When printing regions, add trailing space if necessary.
@@ -2327,10 +2329,10 @@ impl<'tcx> ConstantKind<'tcx> {
     pub fn eval(self, tcx: TyCtxt<'tcx>, param_env: ty::ParamEnv<'tcx>) -> Self {
         match self {
             Self::Ty(c) => {
-                if let Some(val) = c.kind().try_eval_for_mir(tcx, param_env) {
+                if let Some(val) = c.try_eval_for_mir(tcx, param_env) {
                     match val {
                         Ok(val) => Self::Val(val, c.ty()),
-                        Err(guar) => Self::Ty(tcx.const_error(self.ty(), guar)),
+                        Err(guar) => Self::Ty(ty::Const::new_error(tcx, guar, self.ty())),
                     }
                 } else {
                     self
@@ -2342,7 +2344,9 @@ impl<'tcx> ConstantKind<'tcx> {
                 match tcx.const_eval_resolve(param_env, uneval, None) {
                     Ok(val) => Self::Val(val, ty),
                     Err(ErrorHandled::TooGeneric) => self,
-                    Err(ErrorHandled::Reported(guar)) => Self::Ty(tcx.const_error(ty, guar.into())),
+                    Err(ErrorHandled::Reported(guar)) => {
+                        Self::Ty(ty::Const::new_error(tcx, guar.into(), ty))
+                    }
                 }
             }
         }
@@ -2461,51 +2465,6 @@ impl<'tcx> ConstantKind<'tcx> {
         Self::Val(val, ty)
     }
 
-    #[instrument(skip(tcx), level = "debug", ret)]
-    pub fn from_inline_const(tcx: TyCtxt<'tcx>, def_id: LocalDefId) -> Self {
-        let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
-        let body_id = match tcx.hir().get(hir_id) {
-            hir::Node::AnonConst(ac) => ac.body,
-            _ => span_bug!(
-                tcx.def_span(def_id.to_def_id()),
-                "from_inline_const can only process anonymous constants"
-            ),
-        };
-        let expr = &tcx.hir().body(body_id).value;
-        let ty = tcx.typeck(def_id).node_type(hir_id);
-
-        let lit_input = match expr.kind {
-            hir::ExprKind::Lit(ref lit) => Some(LitToConstInput { lit: &lit.node, ty, neg: false }),
-            hir::ExprKind::Unary(hir::UnOp::Neg, ref expr) => match expr.kind {
-                hir::ExprKind::Lit(ref lit) => {
-                    Some(LitToConstInput { lit: &lit.node, ty, neg: true })
-                }
-                _ => None,
-            },
-            _ => None,
-        };
-        if let Some(lit_input) = lit_input {
-            // If an error occurred, ignore that it's a literal and leave reporting the error up to
-            // mir.
-            match tcx.at(expr.span).lit_to_mir_constant(lit_input) {
-                Ok(c) => return c,
-                Err(_) => {}
-            }
-        }
-
-        let typeck_root_def_id = tcx.typeck_root_def_id(def_id.to_def_id());
-        let parent_substs =
-            tcx.erase_regions(InternalSubsts::identity_for_item(tcx, typeck_root_def_id));
-        let substs =
-            ty::InlineConstSubsts::new(tcx, ty::InlineConstSubstsParts { parent_substs, ty })
-                .substs;
-
-        let uneval = UnevaluatedConst { def: def_id.to_def_id(), substs, promoted: None };
-        debug_assert!(!uneval.has_free_regions());
-
-        Self::Unevaluated(uneval, ty)
-    }
-
     /// Literals are converted to `ConstantKindVal`, const generic parameters are eagerly
     /// converted to a constant, everything else becomes `Unevaluated`.
     #[instrument(skip(tcx), level = "debug", ret)]
@@ -2553,7 +2512,7 @@ impl<'tcx> ConstantKind<'tcx> {
                 let generics = tcx.generics_of(item_def_id);
                 let index = generics.param_def_id_to_index[&def_id];
                 let name = tcx.item_name(def_id);
-                let ty_const = tcx.mk_const(ty::ParamConst::new(index, name), ty);
+                let ty_const = ty::Const::new_param(tcx, ty::ParamConst::new(index, name), ty);
                 debug!(?ty_const);
 
                 return Self::Ty(ty_const);
@@ -2624,10 +2583,9 @@ pub struct UnevaluatedConst<'tcx> {
 }
 
 impl<'tcx> UnevaluatedConst<'tcx> {
-    // FIXME: probably should get rid of this method. It's also wrong to
-    // shrink and then later expand a promoted.
     #[inline]
     pub fn shrink(self) -> ty::UnevaluatedConst<'tcx> {
+        assert_eq!(self.promoted, None);
         ty::UnevaluatedConst { def: self.def, substs: self.substs }
     }
 }
@@ -2818,7 +2776,7 @@ impl<'tcx> Display for ConstantKind<'tcx> {
     fn fmt(&self, fmt: &mut Formatter<'_>) -> fmt::Result {
         match *self {
             ConstantKind::Ty(c) => pretty_print_const(c, fmt, true),
-            ConstantKind::Val(val, ty) => pretty_print_const_value(val, ty, fmt, true),
+            ConstantKind::Val(val, ty) => pretty_print_const_value(val, ty, fmt),
             // FIXME(valtrees): Correctly print mir constants.
             ConstantKind::Unevaluated(..) => {
                 fmt.write_str("_")?;
@@ -2848,13 +2806,16 @@ fn pretty_print_byte_str(fmt: &mut Formatter<'_>, byte_str: &[u8]) -> fmt::Resul
     write!(fmt, "b\"{}\"", byte_str.escape_ascii())
 }
 
-fn comma_sep<'tcx>(fmt: &mut Formatter<'_>, elems: Vec<ConstantKind<'tcx>>) -> fmt::Result {
+fn comma_sep<'tcx>(
+    fmt: &mut Formatter<'_>,
+    elems: Vec<(ConstValue<'tcx>, Ty<'tcx>)>,
+) -> fmt::Result {
     let mut first = true;
-    for elem in elems {
+    for (ct, ty) in elems {
         if !first {
             fmt.write_str(", ")?;
         }
-        fmt.write_str(&format!("{}", elem))?;
+        pretty_print_const_value(ct, ty, fmt)?;
         first = false;
     }
     Ok(())
@@ -2865,7 +2826,6 @@ fn pretty_print_const_value<'tcx>(
     ct: ConstValue<'tcx>,
     ty: Ty<'tcx>,
     fmt: &mut Formatter<'_>,
-    print_ty: bool,
 ) -> fmt::Result {
     use crate::ty::print::PrettyPrinter;
 
@@ -2909,7 +2869,7 @@ fn pretty_print_const_value<'tcx>(
                 }
             }
             (ConstValue::ByRef { alloc, offset }, ty::Array(t, n)) if *t == u8_type => {
-                let n = n.kind().try_to_bits(tcx.data_layout.pointer_size).unwrap();
+                let n = n.try_to_bits(tcx.data_layout.pointer_size).unwrap();
                 // cast is ok because we already checked for pointer size (32 or 64 bit) above
                 let range = AllocRange { start: offset, size: Size::from_bytes(n) };
                 let byte_str = alloc.inner().get_bytes_strip_provenance(&tcx, range).unwrap();
@@ -2924,16 +2884,11 @@ fn pretty_print_const_value<'tcx>(
             // introducing ICEs (e.g. via `layout_of`) from missing bounds.
             // E.g. `transmute([0usize; 2]): (u8, *mut T)` needs to know `T: Sized`
             // to be able to destructure the tuple into `(0u8, *mut T)`
-            //
-            // FIXME(eddyb) for `--emit=mir`/`-Z dump-mir`, we should provide the
-            // correct `ty::ParamEnv` to allow printing *all* constant values.
             (_, ty::Array(..) | ty::Tuple(..) | ty::Adt(..)) if !ty.has_non_region_param() => {
                 let ct = tcx.lift(ct).unwrap();
                 let ty = tcx.lift(ty).unwrap();
-                if let Some(contents) = tcx.try_destructure_mir_constant(
-                    ty::ParamEnv::reveal_all().and(ConstantKind::Val(ct, ty)),
-                ) {
-                    let fields = contents.fields.to_vec();
+                if let Some(contents) = tcx.try_destructure_mir_constant_for_diagnostics((ct, ty)) {
+                    let fields: Vec<(ConstValue<'_>, Ty<'_>)> = contents.fields.to_vec();
                     match *ty.kind() {
                         ty::Array(..) => {
                             fmt.write_str("[")?;
@@ -2972,12 +2927,14 @@ fn pretty_print_const_value<'tcx>(
                                 None => {
                                     fmt.write_str(" {{ ")?;
                                     let mut first = true;
-                                    for (field_def, field) in iter::zip(&variant_def.fields, fields)
+                                    for (field_def, (ct, ty)) in
+                                        iter::zip(&variant_def.fields, fields)
                                     {
                                         if !first {
                                             fmt.write_str(", ")?;
                                         }
-                                        fmt.write_str(&format!("{}: {}", field_def.name, field))?;
+                                        write!(fmt, "{}: ", field_def.name)?;
+                                        pretty_print_const_value(ct, ty, fmt)?;
                                         first = false;
                                     }
                                     fmt.write_str(" }}")?;
@@ -2987,20 +2944,13 @@ fn pretty_print_const_value<'tcx>(
                         _ => unreachable!(),
                     }
                     return Ok(());
-                } else {
-                    // Fall back to debug pretty printing for invalid constants.
-                    fmt.write_str(&format!("{:?}", ct))?;
-                    if print_ty {
-                        fmt.write_str(&format!(": {}", ty))?;
-                    }
-                    return Ok(());
-                };
+                }
             }
             (ConstValue::Scalar(scalar), _) => {
                 let mut cx = FmtPrinter::new(tcx, Namespace::ValueNS);
                 cx.print_alloc_ids = true;
                 let ty = tcx.lift(ty).unwrap();
-                cx = cx.pretty_print_const_scalar(scalar, ty, print_ty)?;
+                cx = cx.pretty_print_const_scalar(scalar, ty)?;
                 fmt.write_str(&cx.into_buffer())?;
                 return Ok(());
             }
@@ -3015,12 +2965,8 @@ fn pretty_print_const_value<'tcx>(
             // their fields instead of just dumping the memory.
             _ => {}
         }
-        // fallback
-        fmt.write_str(&format!("{:?}", ct))?;
-        if print_ty {
-            fmt.write_str(&format!(": {}", ty))?;
-        }
-        Ok(())
+        // Fall back to debug pretty printing for invalid constants.
+        write!(fmt, "{ct:?}: {ty}")
     })
 }
 
diff --git a/compiler/rustc_middle/src/mir/mono.rs b/compiler/rustc_middle/src/mir/mono.rs
index f31b343c9..ca735d523 100644
--- a/compiler/rustc_middle/src/mir/mono.rs
+++ b/compiler/rustc_middle/src/mir/mono.rs
@@ -231,7 +231,7 @@ pub struct CodegenUnit<'tcx> {
     /// as well as the crate name and disambiguator.
     name: Symbol,
     items: FxHashMap<MonoItem<'tcx>, (Linkage, Visibility)>,
-    size_estimate: Option<usize>,
+    size_estimate: usize,
     primary: bool,
     /// True if this is CGU is used to hold code coverage information for dead code,
     /// false otherwise.
@@ -269,7 +269,7 @@ impl<'tcx> CodegenUnit<'tcx> {
         CodegenUnit {
             name,
             items: Default::default(),
-            size_estimate: None,
+            size_estimate: 0,
             primary: false,
             is_code_coverage_dead_code_cgu: false,
         }
@@ -291,10 +291,12 @@ impl<'tcx> CodegenUnit<'tcx> {
         self.primary = true;
     }
 
+    /// The order of these items is non-determinstic.
     pub fn items(&self) -> &FxHashMap<MonoItem<'tcx>, (Linkage, Visibility)> {
         &self.items
     }
 
+    /// The order of these items is non-determinstic.
     pub fn items_mut(&mut self) -> &mut FxHashMap<MonoItem<'tcx>, (Linkage, Visibility)> {
         &mut self.items
     }
@@ -318,23 +320,21 @@ impl<'tcx> CodegenUnit<'tcx> {
         base_n::encode(hash, base_n::CASE_INSENSITIVE)
     }
 
-    pub fn create_size_estimate(&mut self, tcx: TyCtxt<'tcx>) {
+    pub fn compute_size_estimate(&mut self, tcx: TyCtxt<'tcx>) {
         // Estimate the size of a codegen unit as (approximately) the number of MIR
         // statements it corresponds to.
-        self.size_estimate = Some(self.items.keys().map(|mi| mi.size_estimate(tcx)).sum());
+        self.size_estimate = self.items.keys().map(|mi| mi.size_estimate(tcx)).sum();
     }
 
     #[inline]
-    /// Should only be called if [`create_size_estimate`] has previously been called.
+    /// Should only be called if [`compute_size_estimate`] has previously been called.
     ///
-    /// [`create_size_estimate`]: Self::create_size_estimate
+    /// [`compute_size_estimate`]: Self::compute_size_estimate
     pub fn size_estimate(&self) -> usize {
+        // Items are never zero-sized, so if we have items the estimate must be
+        // non-zero, unless we forgot to call `compute_size_estimate` first.
+        assert!(self.items.is_empty() || self.size_estimate != 0);
         self.size_estimate
-            .expect("create_size_estimate must be called before getting a size_estimate")
-    }
-
-    pub fn modify_size_estimate(&mut self, delta: usize) {
-        *self.size_estimate.as_mut().unwrap() += delta;
     }
 
     pub fn contains_item(&self, item: &MonoItem<'tcx>) -> bool {
diff --git a/compiler/rustc_middle/src/mir/pretty.rs b/compiler/rustc_middle/src/mir/pretty.rs
index 62c3d8cf2..ffa7a5400 100644
--- a/compiler/rustc_middle/src/mir/pretty.rs
+++ b/compiler/rustc_middle/src/mir/pretty.rs
@@ -353,14 +353,22 @@ where
     for statement in &data.statements {
         extra_data(PassWhere::BeforeLocation(current_location), w)?;
         let indented_body = format!("{0}{0}{1:?};", INDENT, statement);
-        writeln!(
-            w,
-            "{:A$} // {}{}",
-            indented_body,
-            if tcx.sess.verbose() { format!("{:?}: ", current_location) } else { String::new() },
-            comment(tcx, statement.source_info, body.span),
-            A = ALIGN,
-        )?;
+        if tcx.sess.opts.unstable_opts.mir_include_spans {
+            writeln!(
+                w,
+                "{:A$} // {}{}",
+                indented_body,
+                if tcx.sess.verbose() {
+                    format!("{:?}: ", current_location)
+                } else {
+                    String::new()
+                },
+                comment(tcx, statement.source_info),
+                A = ALIGN,
+            )?;
+        } else {
+            writeln!(w, "{}", indented_body)?;
+        }
 
         write_extra(tcx, w, |visitor| {
             visitor.visit_statement(statement, current_location);
@@ -374,14 +382,18 @@ where
     // Terminator at the bottom.
     extra_data(PassWhere::BeforeLocation(current_location), w)?;
     let indented_terminator = format!("{0}{0}{1:?};", INDENT, data.terminator().kind);
-    writeln!(
-        w,
-        "{:A$} // {}{}",
-        indented_terminator,
-        if tcx.sess.verbose() { format!("{:?}: ", current_location) } else { String::new() },
-        comment(tcx, data.terminator().source_info, body.span),
-        A = ALIGN,
-    )?;
+    if tcx.sess.opts.unstable_opts.mir_include_spans {
+        writeln!(
+            w,
+            "{:A$} // {}{}",
+            indented_terminator,
+            if tcx.sess.verbose() { format!("{:?}: ", current_location) } else { String::new() },
+            comment(tcx, data.terminator().source_info),
+            A = ALIGN,
+        )?;
+    } else {
+        writeln!(w, "{}", indented_terminator)?;
+    }
 
     write_extra(tcx, w, |visitor| {
         visitor.visit_terminator(data.terminator(), current_location);
@@ -400,10 +412,12 @@ fn write_extra<'tcx, F>(tcx: TyCtxt<'tcx>, write: &mut dyn Write, mut visit_op:
 where
     F: FnMut(&mut ExtraComments<'tcx>),
 {
-    let mut extra_comments = ExtraComments { tcx, comments: vec![] };
-    visit_op(&mut extra_comments);
-    for comment in extra_comments.comments {
-        writeln!(write, "{:A$} // {}", "", comment, A = ALIGN)?;
+    if tcx.sess.opts.unstable_opts.mir_include_spans {
+        let mut extra_comments = ExtraComments { tcx, comments: vec![] };
+        visit_op(&mut extra_comments);
+        for comment in extra_comments.comments {
+            writeln!(write, "{:A$} // {}", "", comment, A = ALIGN)?;
+        }
     }
     Ok(())
 }
@@ -522,13 +536,8 @@ impl<'tcx> Visitor<'tcx> for ExtraComments<'tcx> {
     }
 }
 
-fn comment(tcx: TyCtxt<'_>, SourceInfo { span, scope }: SourceInfo, function_span: Span) -> String {
-    let location = if tcx.sess.opts.unstable_opts.mir_pretty_relative_line_numbers {
-        tcx.sess.source_map().span_to_relative_line_string(span, function_span)
-    } else {
-        tcx.sess.source_map().span_to_embeddable_string(span)
-    };
-
+fn comment(tcx: TyCtxt<'_>, SourceInfo { span, scope }: SourceInfo) -> String {
+    let location = tcx.sess.source_map().span_to_embeddable_string(span);
     format!("scope {} at {}", scope.index(), location,)
 }
 
@@ -560,13 +569,17 @@ fn write_scope_tree(
             var_debug_info.value,
         );
 
-        writeln!(
-            w,
-            "{0:1$} // in {2}",
-            indented_debug_info,
-            ALIGN,
-            comment(tcx, var_debug_info.source_info, body.span),
-        )?;
+        if tcx.sess.opts.unstable_opts.mir_include_spans {
+            writeln!(
+                w,
+                "{0:1$} // in {2}",
+                indented_debug_info,
+                ALIGN,
+                comment(tcx, var_debug_info.source_info),
+            )?;
+        } else {
+            writeln!(w, "{}", indented_debug_info)?;
+        }
     }
 
     // Local variable types.
@@ -594,14 +607,18 @@ fn write_scope_tree(
 
         let local_name = if local == RETURN_PLACE { " return place" } else { "" };
 
-        writeln!(
-            w,
-            "{0:1$} //{2} in {3}",
-            indented_decl,
-            ALIGN,
-            local_name,
-            comment(tcx, local_decl.source_info, body.span),
-        )?;
+        if tcx.sess.opts.unstable_opts.mir_include_spans {
+            writeln!(
+                w,
+                "{0:1$} //{2} in {3}",
+                indented_decl,
+                ALIGN,
+                local_name,
+                comment(tcx, local_decl.source_info),
+            )?;
+        } else {
+            writeln!(w, "{}", indented_decl,)?;
+        }
     }
 
     let Some(children) = scope_tree.get(&parent) else {
@@ -627,14 +644,18 @@ fn write_scope_tree(
 
         let indented_header = format!("{0:1$}scope {2}{3} {{", "", indent, child.index(), special);
 
-        if let Some(span) = span {
-            writeln!(
-                w,
-                "{0:1$} // at {2}",
-                indented_header,
-                ALIGN,
-                tcx.sess.source_map().span_to_embeddable_string(span),
-            )?;
+        if tcx.sess.opts.unstable_opts.mir_include_spans {
+            if let Some(span) = span {
+                writeln!(
+                    w,
+                    "{0:1$} // at {2}",
+                    indented_header,
+                    ALIGN,
+                    tcx.sess.source_map().span_to_embeddable_string(span),
+                )?;
+            } else {
+                writeln!(w, "{}", indented_header)?;
+            }
         } else {
             writeln!(w, "{}", indented_header)?;
         }
@@ -846,7 +867,7 @@ fn write_allocation_newline(
 /// The `prefix` argument allows callers to add an arbitrary prefix before each line (even if there
 /// is only one line). Note that your prefix should contain a trailing space as the lines are
 /// printed directly after it.
-fn write_allocation_bytes<'tcx, Prov: Provenance, Extra, Bytes: AllocBytes>(
+pub fn write_allocation_bytes<'tcx, Prov: Provenance, Extra, Bytes: AllocBytes>(
     tcx: TyCtxt<'tcx>,
     alloc: &Allocation<Prov, Extra, Bytes>,
     w: &mut dyn std::fmt::Write,
diff --git a/compiler/rustc_middle/src/mir/query.rs b/compiler/rustc_middle/src/mir/query.rs
index 53fd2dd23..613b132ff 100644
--- a/compiler/rustc_middle/src/mir/query.rs
+++ b/compiler/rustc_middle/src/mir/query.rs
@@ -1,6 +1,6 @@
 //! Values computed by queries that use MIR.
 
-use crate::mir::ConstantKind;
+use crate::mir::interpret::ConstValue;
 use crate::ty::{self, OpaqueHiddenType, Ty, TyCtxt};
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_data_structures::unord::UnordSet;
@@ -9,6 +9,7 @@ use rustc_hir as hir;
 use rustc_hir::def_id::LocalDefId;
 use rustc_index::bit_set::BitMatrix;
 use rustc_index::{Idx, IndexVec};
+use rustc_span::symbol::Symbol;
 use rustc_span::Span;
 use rustc_target::abi::{FieldIdx, VariantIdx};
 use smallvec::SmallVec;
@@ -150,6 +151,9 @@ pub struct GeneratorLayout<'tcx> {
     /// The type of every local stored inside the generator.
     pub field_tys: IndexVec<GeneratorSavedLocal, GeneratorSavedTy<'tcx>>,
 
+    /// The name for debuginfo.
+    pub field_names: IndexVec<GeneratorSavedLocal, Option<Symbol>>,
+
     /// Which of the above fields are in each variant. Note that one field may
     /// be stored in multiple variants.
     pub variant_fields: IndexVec<VariantIdx, IndexVec<FieldIdx, GeneratorSavedLocal>>,
@@ -413,7 +417,7 @@ impl<'tcx> ClosureOutlivesSubjectTy<'tcx> {
             ty::ReVar(vid) => {
                 let br =
                     ty::BoundRegion { var: ty::BoundVar::new(vid.index()), kind: ty::BrAnon(None) };
-                tcx.mk_re_late_bound(depth, br)
+                ty::Region::new_late_bound(tcx, depth, br)
             }
             _ => bug!("unexpected region in ClosureOutlivesSubjectTy: {r:?}"),
         });
@@ -440,7 +444,7 @@ impl<'tcx> ClosureOutlivesSubjectTy<'tcx> {
 #[derive(Copy, Clone, Debug, HashStable)]
 pub struct DestructuredConstant<'tcx> {
     pub variant: Option<VariantIdx>,
-    pub fields: &'tcx [ConstantKind<'tcx>],
+    pub fields: &'tcx [(ConstValue<'tcx>, Ty<'tcx>)],
 }
 
 /// Coverage information summarized from a MIR if instrumented for source code coverage (see
diff --git a/compiler/rustc_middle/src/mir/spanview.rs b/compiler/rustc_middle/src/mir/spanview.rs
index 2165403da..730c55157 100644
--- a/compiler/rustc_middle/src/mir/spanview.rs
+++ b/compiler/rustc_middle/src/mir/spanview.rs
@@ -3,7 +3,7 @@ use rustc_middle::hir;
 use rustc_middle::mir::*;
 use rustc_middle::ty::TyCtxt;
 use rustc_session::config::MirSpanview;
-use rustc_span::{BytePos, Pos, Span, SyntaxContext};
+use rustc_span::{BytePos, Pos, Span};
 
 use std::cmp;
 use std::io::{self, Write};
@@ -15,8 +15,9 @@ const ANNOTATION_LEFT_BRACKET: char = '\u{298a}'; // Unicode `Z NOTATION RIGHT B
 const ANNOTATION_RIGHT_BRACKET: char = '\u{2989}'; // Unicode `Z NOTATION LEFT BINDING BRACKET`
 const NEW_LINE_SPAN: &str = "</span>\n<span class=\"line\">";
 const HEADER: &str = r#"<!DOCTYPE html>
-<html>
-<head>"#;
+<html lang="en">
+<head>
+<meta charset="utf-8">"#;
 const START_BODY: &str = r#"</head>
 <body>"#;
 const FOOTER: &str = r#"</body>
@@ -327,7 +328,7 @@ fn compute_block_span(data: &BasicBlockData<'_>, body_span: Span) -> Span {
     let mut span = data.terminator().source_info.span;
     for statement_span in data.statements.iter().map(|statement| statement.source_info.span) {
         // Only combine Spans from the root context, and within the function's body_span.
-        if statement_span.ctxt() == SyntaxContext::root() && body_span.contains(statement_span) {
+        if statement_span.ctxt().is_root() && body_span.contains(statement_span) {
             span = span.to(statement_span);
         }
     }
diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs
index 3e474c1d3..7f1d38203 100644
--- a/compiler/rustc_middle/src/mir/syntax.rs
+++ b/compiler/rustc_middle/src/mir/syntax.rs
@@ -7,7 +7,7 @@ use super::{BasicBlock, Constant, Local, SwitchTargets, UserTypeProjection};
 
 use crate::mir::coverage::{CodeRegion, CoverageKind};
 use crate::traits::Reveal;
-use crate::ty::adjustment::PointerCast;
+use crate::ty::adjustment::PointerCoercion;
 use crate::ty::subst::SubstsRef;
 use crate::ty::{self, List, Ty};
 use crate::ty::{Region, UserTypeAnnotationIndex};
@@ -182,6 +182,16 @@ pub enum BorrowKind {
     /// We can also report errors with this kind of borrow differently.
     Shallow,
 
+    /// Data is mutable and not aliasable.
+    Mut { kind: MutBorrowKind },
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, TyEncodable, TyDecodable)]
+#[derive(Hash, HashStable)]
+pub enum MutBorrowKind {
+    Default,
+    /// This borrow arose from method-call auto-ref. (i.e., `adjustment::Adjust::Borrow`)
+    TwoPhaseBorrow,
     /// Data must be immutable but not aliasable. This kind of borrow
     /// cannot currently be expressed by the user and is used only in
     /// implicit closure bindings. It is needed when the closure is
@@ -216,23 +226,14 @@ pub enum BorrowKind {
     /// user code, if awkward, but extra weird for closures, since the
     /// borrow is hidden.
     ///
-    /// So we introduce a "unique imm" borrow -- the referent is
-    /// immutable, but not aliasable. This solves the problem. For
-    /// simplicity, we don't give users the way to express this
-    /// borrow, it's just used when translating closures.
+    /// So we introduce a `ClosureCapture` borrow -- user will not have to mark the variable
+    /// containing the mutable reference as `mut`, as they didn't ever
+    /// intend to mutate the mutable reference itself. We still mutable capture it in order to
+    /// mutate the pointed value through it (but not mutating the reference itself).
     ///
-    // FIXME(#112072): This is wrong. Unique borrows are mutable borrows except
-    // that they do not require their pointee to be marked as a mutable.
-    // They should still be treated as mutable borrows in every other way,
-    // e.g. for variance or overlap checking.
-    Unique,
-
-    /// Data is mutable and not aliasable.
-    Mut {
-        /// `true` if this borrow arose from method-call auto-ref
-        /// (i.e., `adjustment::Adjust::Borrow`).
-        allow_two_phase_borrow: bool,
-    },
+    /// This solves the problem. For simplicity, we don't give users the way to express this
+    /// borrow, it's just used when translating closures.
+    ClosureCapture,
 }
 
 ///////////////////////////////////////////////////////////////////////////
@@ -512,6 +513,31 @@ pub struct CopyNonOverlapping<'tcx> {
     pub count: Operand<'tcx>,
 }
 
+/// Represents how a `TerminatorKind::Call` was constructed, used for diagnostics
+#[derive(Clone, Copy, TyEncodable, TyDecodable, Debug, PartialEq, Hash, HashStable)]
+#[derive(TypeFoldable, TypeVisitable)]
+pub enum CallSource {
+    /// This came from something such as `a > b` or `a + b`. In THIR, if `from_hir_call`
+    /// is false then this is the desugaring.
+    OverloadedOperator,
+    /// This was from comparison generated by a match, used by const-eval for better errors
+    /// when the comparison cannot be done in compile time.
+    ///
+    /// (see <https://github.com/rust-lang/rust/issues/90237>)
+    MatchCmp,
+    /// Other types of desugaring that did not come from the HIR, but we don't care about
+    /// for diagnostics (yet).
+    Misc,
+    /// Normal function call, no special source
+    Normal,
+}
+
+impl CallSource {
+    pub fn from_hir_call(self) -> bool {
+        matches!(self, CallSource::Normal)
+    }
+}
+
 ///////////////////////////////////////////////////////////////////////////
 // Terminators
 
@@ -638,11 +664,10 @@ pub enum TerminatorKind<'tcx> {
         target: Option<BasicBlock>,
         /// Action to be taken if the call unwinds.
         unwind: UnwindAction,
-        /// `true` if this is from a call in HIR rather than from an overloaded
-        /// operator. True for overloaded function call.
-        from_hir_call: bool,
+        /// Where this call came from in HIR/THIR.
+        call_source: CallSource,
         /// This `Span` is the span of the function, without the dot and receiver
-        /// (e.g. `foo(a, b)` in `x.foo(a, b)`
+        /// e.g. `foo(a, b)` in `x.foo(a, b)`
         fn_span: Span,
     },
 
@@ -801,7 +826,8 @@ pub enum UnwindAction {
 }
 
 /// Information about an assertion failure.
-#[derive(Clone, TyEncodable, TyDecodable, Hash, HashStable, PartialEq, TypeFoldable, TypeVisitable)]
+#[derive(Clone, Hash, HashStable, PartialEq, Debug)]
+#[derive(TyEncodable, TyDecodable, TypeFoldable, TypeVisitable)]
 pub enum AssertKind<O> {
     BoundsCheck { len: O, index: O },
     Overflow(BinOp, O, O),
@@ -1204,9 +1230,9 @@ pub enum CastKind {
     /// An address-to-pointer cast that picks up an exposed provenance.
     /// See the docs on `from_exposed_addr` for more details.
     PointerFromExposedAddress,
-    /// All sorts of pointer-to-pointer casts. Note that reference-to-raw-ptr casts are
+    /// Pointer related casts that are done by coercions. Note that reference-to-raw-ptr casts are
     /// translated into `&raw mut/const *r`, i.e., they are not actually casts.
-    Pointer(PointerCast),
+    PointerCoercion(PointerCoercion),
     /// Cast into a dyn* object.
     DynStar,
     IntToInt,
@@ -1266,19 +1292,30 @@ pub enum UnOp {
 pub enum BinOp {
     /// The `+` operator (addition)
     Add,
+    /// Like `Add`, but with UB on overflow.  (Integers only.)
+    AddUnchecked,
     /// The `-` operator (subtraction)
     Sub,
+    /// Like `Sub`, but with UB on overflow.  (Integers only.)
+    SubUnchecked,
     /// The `*` operator (multiplication)
     Mul,
+    /// Like `Mul`, but with UB on overflow.  (Integers only.)
+    MulUnchecked,
     /// The `/` operator (division)
     ///
-    /// Division by zero is UB, because the compiler should have inserted checks
-    /// prior to this.
+    /// For integer types, division by zero is UB, as is `MIN / -1` for signed.
+    /// The compiler should have inserted checks prior to this.
+    ///
+    /// Floating-point division by zero is safe, and does not need guards.
     Div,
     /// The `%` operator (modulus)
     ///
-    /// Using zero as the modulus (second operand) is UB, because the compiler
-    /// should have inserted checks prior to this.
+    /// For integer types, using zero as the modulus (second operand) is UB,
+    /// as is `MIN % -1` for signed.
+    /// The compiler should have inserted checks prior to this.
+    ///
+    /// Floating-point remainder by zero is safe, and does not need guards.
     Rem,
     /// The `^` operator (bitwise xor)
     BitXor,
@@ -1290,10 +1327,17 @@ pub enum BinOp {
     ///
     /// The offset is truncated to the size of the first operand before shifting.
     Shl,
+    /// Like `Shl`, but is UB if the RHS >= LHS::BITS
+    ShlUnchecked,
     /// The `>>` operator (shift right)
     ///
     /// The offset is truncated to the size of the first operand before shifting.
+    ///
+    /// This is an arithmetic shift if the LHS is signed
+    /// and a logical shift if the LHS is unsigned.
     Shr,
+    /// Like `Shl`, but is UB if the RHS >= LHS::BITS
+    ShrUnchecked,
     /// The `==` operator (equality)
     Eq,
     /// The `<` operator (less than)
diff --git a/compiler/rustc_middle/src/mir/tcx.rs b/compiler/rustc_middle/src/mir/tcx.rs
index 5ca824134..8618a5315 100644
--- a/compiler/rustc_middle/src/mir/tcx.rs
+++ b/compiler/rustc_middle/src/mir/tcx.rs
@@ -95,11 +95,13 @@ impl<'tcx> PlaceTy<'tcx> {
             ProjectionElem::Subslice { from, to, from_end } => {
                 PlaceTy::from_ty(match self.ty.kind() {
                     ty::Slice(..) => self.ty,
-                    ty::Array(inner, _) if !from_end => tcx.mk_array(*inner, (to - from) as u64),
+                    ty::Array(inner, _) if !from_end => {
+                        Ty::new_array(tcx, *inner, (to - from) as u64)
+                    }
                     ty::Array(inner, size) if from_end => {
                         let size = size.eval_target_usize(tcx, param_env);
                         let len = size - from - to;
-                        tcx.mk_array(*inner, len)
+                        Ty::new_array(tcx, *inner, len)
                     }
                     _ => bug!("cannot subslice non-array type: `{:?}`", self),
                 })
@@ -162,16 +164,16 @@ impl<'tcx> Rvalue<'tcx> {
         match *self {
             Rvalue::Use(ref operand) => operand.ty(local_decls, tcx),
             Rvalue::Repeat(ref operand, count) => {
-                tcx.mk_array_with_const_len(operand.ty(local_decls, tcx), count)
+                Ty::new_array_with_const_len(tcx, operand.ty(local_decls, tcx), count)
             }
             Rvalue::ThreadLocalRef(did) => tcx.thread_local_ptr_ty(did),
             Rvalue::Ref(reg, bk, ref place) => {
                 let place_ty = place.ty(local_decls, tcx).ty;
-                tcx.mk_ref(reg, ty::TypeAndMut { ty: place_ty, mutbl: bk.to_mutbl_lossy() })
+                Ty::new_ref(tcx, reg, ty::TypeAndMut { ty: place_ty, mutbl: bk.to_mutbl_lossy() })
             }
             Rvalue::AddressOf(mutability, ref place) => {
                 let place_ty = place.ty(local_decls, tcx).ty;
-                tcx.mk_ptr(ty::TypeAndMut { ty: place_ty, mutbl: mutability })
+                Ty::new_ptr(tcx, ty::TypeAndMut { ty: place_ty, mutbl: mutability })
             }
             Rvalue::Len(..) => tcx.types.usize,
             Rvalue::Cast(.., ty) => ty,
@@ -184,7 +186,7 @@ impl<'tcx> Rvalue<'tcx> {
                 let lhs_ty = lhs.ty(local_decls, tcx);
                 let rhs_ty = rhs.ty(local_decls, tcx);
                 let ty = op.ty(tcx, lhs_ty, rhs_ty);
-                tcx.mk_tup(&[ty, tcx.types.bool])
+                Ty::new_tup(tcx, &[ty, tcx.types.bool])
             }
             Rvalue::UnaryOp(UnOp::Not | UnOp::Neg, ref operand) => operand.ty(local_decls, tcx),
             Rvalue::Discriminant(ref place) => place.ty(local_decls, tcx).ty.discriminant_ty(tcx),
@@ -192,17 +194,17 @@ impl<'tcx> Rvalue<'tcx> {
                 tcx.types.usize
             }
             Rvalue::Aggregate(ref ak, ref ops) => match **ak {
-                AggregateKind::Array(ty) => tcx.mk_array(ty, ops.len() as u64),
+                AggregateKind::Array(ty) => Ty::new_array(tcx, ty, ops.len() as u64),
                 AggregateKind::Tuple => {
-                    tcx.mk_tup_from_iter(ops.iter().map(|op| op.ty(local_decls, tcx)))
+                    Ty::new_tup_from_iter(tcx, ops.iter().map(|op| op.ty(local_decls, tcx)))
                 }
                 AggregateKind::Adt(did, _, substs, _, _) => tcx.type_of(did).subst(tcx, substs),
-                AggregateKind::Closure(did, substs) => tcx.mk_closure(did, substs),
+                AggregateKind::Closure(did, substs) => Ty::new_closure(tcx, did, substs),
                 AggregateKind::Generator(did, substs, movability) => {
-                    tcx.mk_generator(did, substs, movability)
+                    Ty::new_generator(tcx, did, substs, movability)
                 }
             },
-            Rvalue::ShallowInitBox(_, ty) => tcx.mk_box(ty),
+            Rvalue::ShallowInitBox(_, ty) => Ty::new_box(tcx, ty),
             Rvalue::CopyForDeref(ref place) => place.ty(local_decls, tcx).ty,
         }
     }
@@ -235,8 +237,11 @@ impl<'tcx> BinOp {
         // FIXME: handle SIMD correctly
         match self {
             &BinOp::Add
+            | &BinOp::AddUnchecked
             | &BinOp::Sub
+            | &BinOp::SubUnchecked
             | &BinOp::Mul
+            | &BinOp::MulUnchecked
             | &BinOp::Div
             | &BinOp::Rem
             | &BinOp::BitXor
@@ -246,7 +251,11 @@ impl<'tcx> BinOp {
                 assert_eq!(lhs_ty, rhs_ty);
                 lhs_ty
             }
-            &BinOp::Shl | &BinOp::Shr | &BinOp::Offset => {
+            &BinOp::Shl
+            | &BinOp::ShlUnchecked
+            | &BinOp::Shr
+            | &BinOp::ShrUnchecked
+            | &BinOp::Offset => {
                 lhs_ty // lhs_ty can be != rhs_ty
             }
             &BinOp::Eq | &BinOp::Lt | &BinOp::Le | &BinOp::Ne | &BinOp::Ge | &BinOp::Gt => {
@@ -262,11 +271,6 @@ impl BorrowKind {
             BorrowKind::Mut { .. } => hir::Mutability::Mut,
             BorrowKind::Shared => hir::Mutability::Not,
 
-            // We have no type corresponding to a unique imm borrow, so
-            // use `&mut`. It gives all the capabilities of a `&uniq`
-            // and hence is a safe "over approximation".
-            BorrowKind::Unique => hir::Mutability::Mut,
-
             // We have no type corresponding to a shallow borrow, so use
             // `&` as an approximation.
             BorrowKind::Shallow => hir::Mutability::Not,
@@ -293,7 +297,14 @@ impl BinOp {
             BinOp::Gt => hir::BinOpKind::Gt,
             BinOp::Le => hir::BinOpKind::Le,
             BinOp::Ge => hir::BinOpKind::Ge,
-            BinOp::Offset => unreachable!(),
+            BinOp::AddUnchecked
+            | BinOp::SubUnchecked
+            | BinOp::MulUnchecked
+            | BinOp::ShlUnchecked
+            | BinOp::ShrUnchecked
+            | BinOp::Offset => {
+                unreachable!()
+            }
         }
     }
 }
diff --git a/compiler/rustc_middle/src/mir/terminator.rs b/compiler/rustc_middle/src/mir/terminator.rs
index 2c6126cdd..1b9c1438f 100644
--- a/compiler/rustc_middle/src/mir/terminator.rs
+++ b/compiler/rustc_middle/src/mir/terminator.rs
@@ -105,7 +105,7 @@ pub struct Terminator<'tcx> {
     pub kind: TerminatorKind<'tcx>,
 }
 
-pub type Successors<'a> = impl Iterator<Item = BasicBlock> + 'a;
+pub type Successors<'a> = impl DoubleEndedIterator<Item = BasicBlock> + 'a;
 pub type SuccessorsMut<'a> =
     iter::Chain<std::option::IntoIter<&'a mut BasicBlock>, slice::IterMut<'a, BasicBlock>>;
 
@@ -272,7 +272,8 @@ impl<'tcx> Debug for TerminatorKind<'tcx> {
 
         let unwind = match self.unwind() {
             // Not needed or included in successors
-            None | Some(UnwindAction::Continue) | Some(UnwindAction::Cleanup(_)) => None,
+            None | Some(UnwindAction::Cleanup(_)) => None,
+            Some(UnwindAction::Continue) => Some("unwind continue"),
             Some(UnwindAction::Unreachable) => Some("unwind unreachable"),
             Some(UnwindAction::Terminate) => Some("unwind terminate"),
         };
diff --git a/compiler/rustc_middle/src/mir/traversal.rs b/compiler/rustc_middle/src/mir/traversal.rs
index 7d247eeb6..ec16a8470 100644
--- a/compiler/rustc_middle/src/mir/traversal.rs
+++ b/compiler/rustc_middle/src/mir/traversal.rs
@@ -149,7 +149,7 @@ impl<'a, 'tcx> Postorder<'a, 'tcx> {
         //      B     C
         //      |     |
         //      |     |
-        //      D     |
+        //      |     D
         //       \   /
         //        \ /
         //         E
@@ -159,26 +159,26 @@ impl<'a, 'tcx> Postorder<'a, 'tcx> {
         //
         // When the first call to `traverse_successor` happens, the following happens:
         //
-        //     [(B, [D]),  // `B` taken from the successors of `A`, pushed to the
-        //                 // top of the stack along with the successors of `B`
-        //      (A, [C])]
+        //     [(C, [D]),  // `C` taken from the successors of `A`, pushed to the
+        //                 // top of the stack along with the successors of `C`
+        //      (A, [B])]
         //
-        //     [(D, [E]),  // `D` taken from successors of `B`, pushed to stack
-        //      (B, []),
-        //      (A, [C])]
+        //     [(D, [E]),  // `D` taken from successors of `C`, pushed to stack
+        //      (C, []),
+        //      (A, [B])]
         //
         //     [(E, []),   // `E` taken from successors of `D`, pushed to stack
         //      (D, []),
-        //      (B, []),
-        //      (A, [C])]
+        //      (C, []),
+        //      (A, [B])]
         //
         // Now that the top of the stack has no successors we can traverse, each item will
-        // be popped off during iteration until we get back to `A`. This yields [E, D, B].
+        // be popped off during iteration until we get back to `A`. This yields [E, D, C].
         //
-        // When we yield `B` and call `traverse_successor`, we push `C` to the stack, but
+        // When we yield `C` and call `traverse_successor`, we push `B` to the stack, but
         // since we've already visited `E`, that child isn't added to the stack. The last
-        // two iterations yield `C` and finally `A` for a final traversal of [E, D, B, C, A]
-        while let Some(&mut (_, ref mut iter)) = self.visit_stack.last_mut() && let Some(bb) = iter.next() {
+        // two iterations yield `B` and finally `A` for a final traversal of [E, D, C, B, A]
+        while let Some(&mut (_, ref mut iter)) = self.visit_stack.last_mut() && let Some(bb) = iter.next_back() {
             if self.visited.insert(bb) {
                 if let Some(term) = &self.basic_blocks[bb].terminator {
                     self.visit_stack.push((bb, term.successors()));
@@ -188,10 +188,6 @@ impl<'a, 'tcx> Postorder<'a, 'tcx> {
     }
 }
 
-pub fn postorder<'a, 'tcx>(body: &'a Body<'tcx>) -> Postorder<'a, 'tcx> {
-    Postorder::new(&body.basic_blocks, START_BLOCK)
-}
-
 impl<'a, 'tcx> Iterator for Postorder<'a, 'tcx> {
     type Item = (BasicBlock, &'a BasicBlockData<'tcx>);
 
@@ -219,6 +215,17 @@ impl<'a, 'tcx> Iterator for Postorder<'a, 'tcx> {
     }
 }
 
+/// Creates an iterator over the `Body`'s basic blocks, that:
+/// - returns basic blocks in a postorder,
+/// - traverses the `BasicBlocks` CFG cache's reverse postorder backwards, and does not cache the
+///   postorder itself.
+pub fn postorder<'a, 'tcx>(
+    body: &'a Body<'tcx>,
+) -> impl Iterator<Item = (BasicBlock, &'a BasicBlockData<'tcx>)> + ExactSizeIterator + DoubleEndedIterator
+{
+    reverse_postorder(body).rev()
+}
+
 /// Reverse postorder traversal of a graph
 ///
 /// Reverse postorder is the reverse order of a postorder traversal.
@@ -295,34 +302,12 @@ pub fn reachable_as_bitset(body: &Body<'_>) -> BitSet<BasicBlock> {
     iter.visited
 }
 
-#[derive(Clone)]
-pub struct ReversePostorderIter<'a, 'tcx> {
+/// Creates an iterator over the `Body`'s basic blocks, that:
+/// - returns basic blocks in a reverse postorder,
+/// - makes use of the `BasicBlocks` CFG cache's reverse postorder.
+pub fn reverse_postorder<'a, 'tcx>(
     body: &'a Body<'tcx>,
-    blocks: &'a [BasicBlock],
-    idx: usize,
-}
-
-impl<'a, 'tcx> Iterator for ReversePostorderIter<'a, 'tcx> {
-    type Item = (BasicBlock, &'a BasicBlockData<'tcx>);
-
-    fn next(&mut self) -> Option<(BasicBlock, &'a BasicBlockData<'tcx>)> {
-        if self.idx == 0 {
-            return None;
-        }
-        self.idx -= 1;
-
-        self.blocks.get(self.idx).map(|&bb| (bb, &self.body[bb]))
-    }
-
-    fn size_hint(&self) -> (usize, Option<usize>) {
-        (self.idx, Some(self.idx))
-    }
-}
-
-impl<'a, 'tcx> ExactSizeIterator for ReversePostorderIter<'a, 'tcx> {}
-
-pub fn reverse_postorder<'a, 'tcx>(body: &'a Body<'tcx>) -> ReversePostorderIter<'a, 'tcx> {
-    let blocks = body.basic_blocks.postorder();
-    let len = blocks.len();
-    ReversePostorderIter { body, blocks, idx: len }
+) -> impl Iterator<Item = (BasicBlock, &'a BasicBlockData<'tcx>)> + ExactSizeIterator + DoubleEndedIterator
+{
+    body.basic_blocks.reverse_postorder().iter().map(|&bb| (bb, &body.basic_blocks[bb]))
 }
diff --git a/compiler/rustc_middle/src/mir/visit.rs b/compiler/rustc_middle/src/mir/visit.rs
index 8d44e929a..205dc9ec7 100644
--- a/compiler/rustc_middle/src/mir/visit.rs
+++ b/compiler/rustc_middle/src/mir/visit.rs
@@ -519,7 +519,7 @@ macro_rules! make_mir_visitor {
                         destination,
                         target: _,
                         unwind: _,
-                        from_hir_call: _,
+                        call_source: _,
                         fn_span: _
                     } => {
                         self.visit_operand(func, location);
@@ -650,9 +650,6 @@ macro_rules! make_mir_visitor {
                             BorrowKind::Shallow => PlaceContext::NonMutatingUse(
                                 NonMutatingUseContext::ShallowBorrow
                             ),
-                            BorrowKind::Unique => PlaceContext::MutatingUse(
-                                MutatingUseContext::Borrow
-                            ),
                             BorrowKind::Mut { .. } =>
                                 PlaceContext::MutatingUse(MutatingUseContext::Borrow),
                         };
@@ -1136,13 +1133,12 @@ macro_rules! visit_place_fns {
 
         fn visit_projection_elem(
             &mut self,
-            local: Local,
-            proj_base: &[PlaceElem<'tcx>],
+            place_ref: PlaceRef<'tcx>,
             elem: PlaceElem<'tcx>,
             context: PlaceContext,
             location: Location,
         ) {
-            self.super_projection_elem(local, proj_base, elem, context, location);
+            self.super_projection_elem(place_ref, elem, context, location);
         }
 
         fn super_place(&mut self, place: &Place<'tcx>, context: PlaceContext, location: Location) {
@@ -1171,15 +1167,13 @@ macro_rules! visit_place_fns {
             location: Location,
         ) {
             for (base, elem) in place_ref.iter_projections().rev() {
-                let base_proj = base.projection;
-                self.visit_projection_elem(place_ref.local, base_proj, elem, context, location);
+                self.visit_projection_elem(base, elem, context, location);
             }
         }
 
         fn super_projection_elem(
             &mut self,
-            _local: Local,
-            _proj_base: &[PlaceElem<'tcx>],
+            _place_ref: PlaceRef<'tcx>,
             elem: PlaceElem<'tcx>,
             _context: PlaceContext,
             location: Location,