Adding debian version 1.65.0+dfsg1-2.debian/1.65.0+dfsg1-2

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

74 files changed, 2530 insertions, 2473 deletions

diff --git a/compiler/rustc_middle/Cargo.toml b/compiler/rustc_middle/Cargo.toml
index 008d2c709..cca17a4ec 100644
--- a/compiler/rustc_middle/Cargo.toml
+++ b/compiler/rustc_middle/Cargo.toml
@@ -7,34 +7,35 @@ edition = "2021"
 doctest = false
 
 [dependencies]
-rustc_arena = { path = "../rustc_arena" }
 bitflags = "1.2.1"
+chalk-ir = "0.80.0"
 either = "1.5.0"
 gsgdt = "0.1.2"
-tracing = "0.1"
-rustc-rayon = { version = "0.4.0", optional = true }
-rustc-rayon-core = { version = "0.4.0", optional = true }
 polonius-engine = "0.13.0"
+rand = "0.8.4"
+rand_xoshiro = "0.6.0"
 rustc_apfloat = { path = "../rustc_apfloat" }
+rustc_arena = { path = "../rustc_arena" }
+rustc_ast = { path = "../rustc_ast" }
 rustc_attr = { path = "../rustc_attr" }
-rustc_feature = { path = "../rustc_feature" }
-rustc_hir = { path = "../rustc_hir" }
-rustc_target = { path = "../rustc_target" }
-rustc_macros = { path = "../rustc_macros" }
 rustc_data_structures = { path = "../rustc_data_structures" }
-rustc_query_system = { path = "../rustc_query_system" }
 rustc_errors = { path = "../rustc_errors" }
+rustc_feature = { path = "../rustc_feature" }
 rustc_graphviz = { path = "../rustc_graphviz" }
+rustc_hir = { path = "../rustc_hir" }
 rustc_index = { path = "../rustc_index" }
+rustc_macros = { path = "../rustc_macros" }
+rustc_query_system = { path = "../rustc_query_system" }
+rustc-rayon-core = { version = "0.4.0", optional = true }
+rustc-rayon = { version = "0.4.0", optional = true }
 rustc_serialize = { path = "../rustc_serialize" }
-rustc_ast = { path = "../rustc_ast" }
-rustc_span = { path = "../rustc_span" }
-chalk-ir = "0.80.0"
-smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
 rustc_session = { path = "../rustc_session" }
+rustc_span = { path = "../rustc_span" }
+rustc_target = { path = "../rustc_target" }
 rustc_type_ir = { path = "../rustc_type_ir" }
-rand = "0.8.4"
-rand_xoshiro = "0.6.0"
+smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
+thin-vec = "0.2.8"
+tracing = "0.1"
 
 [features]
 rustc_use_parallel_compiler = ["rustc-rayon", "rustc-rayon-core"]
diff --git a/compiler/rustc_middle/src/arena.rs b/compiler/rustc_middle/src/arena.rs
index b94de537d..6bdf5a023 100644
--- a/compiler/rustc_middle/src/arena.rs
+++ b/compiler/rustc_middle/src/arena.rs
@@ -100,7 +100,9 @@ macro_rules! arena_types {
             [decode] is_late_bound_map: rustc_data_structures::fx::FxIndexSet<rustc_hir::def_id::LocalDefId>,
             [decode] impl_source: rustc_middle::traits::ImplSource<'tcx, ()>,
 
-            [] dep_kind: rustc_middle::dep_graph::DepKindStruct,
+            [] dep_kind: rustc_middle::dep_graph::DepKindStruct<'tcx>,
+
+            [decode] trait_impl_trait_tys: rustc_data_structures::fx::FxHashMap<rustc_hir::def_id::DefId, rustc_middle::ty::Ty<'tcx>>,
         ]);
     )
 }
diff --git a/compiler/rustc_middle/src/dep_graph/dep_node.rs b/compiler/rustc_middle/src/dep_graph/dep_node.rs
index 2d095438f..1fa0c6bab 100644
--- a/compiler/rustc_middle/src/dep_graph/dep_node.rs
+++ b/compiler/rustc_middle/src/dep_graph/dep_node.rs
@@ -74,7 +74,7 @@ pub use rustc_query_system::dep_graph::{DepContext, DepNodeParams};
 /// Information is retrieved by indexing the `DEP_KINDS` array using the integer value
 /// of the `DepKind`. Overall, this allows to implement `DepContext` using this manual
 /// jump table instead of large matches.
-pub struct DepKindStruct {
+pub struct DepKindStruct<'tcx> {
     /// Anonymous queries cannot be replayed from one compiler invocation to the next.
     /// When their result is needed, it is recomputed. They are useful for fine-grained
     /// dependency tracking, and caching within one compiler invocation.
@@ -124,10 +124,10 @@ pub struct DepKindStruct {
     /// with kind `MirValidated`, we know that the GUID/fingerprint of the `DepNode`
     /// is actually a `DefPathHash`, and can therefore just look up the corresponding
     /// `DefId` in `tcx.def_path_hash_to_def_id`.
-    pub force_from_dep_node: Option<fn(tcx: TyCtxt<'_>, dep_node: DepNode) -> bool>,
+    pub force_from_dep_node: Option<fn(tcx: TyCtxt<'tcx>, dep_node: DepNode) -> bool>,
 
     /// Invoke a query to put the on-disk cached value in memory.
-    pub try_load_from_on_disk_cache: Option<fn(TyCtxt<'_>, DepNode)>,
+    pub try_load_from_on_disk_cache: Option<fn(TyCtxt<'tcx>, DepNode)>,
 }
 
 impl DepKind {
@@ -143,12 +143,10 @@ impl DepKind {
 }
 
 macro_rules! define_dep_nodes {
-    (<$tcx:tt>
-    $(
-        [$($attrs:tt)*]
-        $variant:ident $(( $tuple_arg_ty:ty $(,)? ))*
-      ,)*
-    ) => (
+    (
+     $($(#[$attr:meta])*
+        [$($modifiers:tt)*] fn $variant:ident($($K:tt)*) -> $V:ty,)*) => {
+
         #[macro_export]
         macro_rules! make_dep_kind_array {
             ($mod:ident) => {[ $($mod::$variant()),* ]};
@@ -158,7 +156,7 @@ macro_rules! define_dep_nodes {
         #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Encodable, Decodable)]
         #[allow(non_camel_case_types)]
         pub enum DepKind {
-            $($variant),*
+            $( $( #[$attr] )* $variant),*
         }
 
         fn dep_kind_from_label_string(label: &str) -> Result<DepKind, ()> {
@@ -176,24 +174,17 @@ macro_rules! define_dep_nodes {
                 pub const $variant: &str = stringify!($variant);
             )*
         }
-    );
+    };
 }
 
-rustc_dep_node_append!([define_dep_nodes!][ <'tcx>
-    // We use this for most things when incr. comp. is turned off.
-    [] Null,
-
-    // We use this to create a forever-red node.
-    [] Red,
-
-    [anon] TraitSelect,
-
-    // WARNING: if `Symbol` is changed, make sure you update `make_compile_codegen_unit` below.
-    [] CompileCodegenUnit(Symbol),
-
-    // WARNING: if `MonoItem` is changed, make sure you update `make_compile_mono_item` below.
-    // Only used by rustc_codegen_cranelift
-    [] CompileMonoItem(MonoItem),
+rustc_query_append!(define_dep_nodes![
+    /// We use this for most things when incr. comp. is turned off.
+    [] fn Null() -> (),
+    /// We use this to create a forever-red node.
+    [] fn Red() -> (),
+    [] fn TraitSelect() -> (),
+    [] fn CompileCodegenUnit() -> (),
+    [] fn CompileMonoItem() -> (),
 ]);
 
 // WARNING: `construct` is generic and does not know that `CompileCodegenUnit` takes `Symbol`s as keys.
diff --git a/compiler/rustc_middle/src/error.rs b/compiler/rustc_middle/src/error.rs
new file mode 100644
index 000000000..18b31a75b
--- /dev/null
+++ b/compiler/rustc_middle/src/error.rs
@@ -0,0 +1,50 @@
+use rustc_macros::SessionDiagnostic;
+use rustc_span::Span;
+
+use crate::ty::Ty;
+
+#[derive(SessionDiagnostic)]
+#[diag(middle::drop_check_overflow, code = "E0320")]
+#[note]
+pub struct DropCheckOverflow<'tcx> {
+    #[primary_span]
+    pub span: Span,
+    pub ty: Ty<'tcx>,
+    pub overflow_ty: Ty<'tcx>,
+}
+
+#[derive(SessionDiagnostic)]
+#[diag(middle::opaque_hidden_type_mismatch)]
+pub struct OpaqueHiddenTypeMismatch<'tcx> {
+    pub self_ty: Ty<'tcx>,
+    pub other_ty: Ty<'tcx>,
+    #[primary_span]
+    #[label]
+    pub other_span: Span,
+    #[subdiagnostic]
+    pub sub: TypeMismatchReason,
+}
+
+#[derive(SessionSubdiagnostic)]
+pub enum TypeMismatchReason {
+    #[label(middle::conflict_types)]
+    ConflictType {
+        #[primary_span]
+        span: Span,
+    },
+    #[note(middle::previous_use_here)]
+    PreviousUse {
+        #[primary_span]
+        span: Span,
+    },
+}
+
+#[derive(SessionDiagnostic)]
+#[diag(middle::limit_invalid)]
+pub struct LimitInvalid<'a> {
+    #[primary_span]
+    pub span: Span,
+    #[label]
+    pub value_span: Span,
+    pub error_str: &'a str,
+}
diff --git a/compiler/rustc_middle/src/hir/map/mod.rs b/compiler/rustc_middle/src/hir/map/mod.rs
index 47b04c33e..30a23c342 100644
--- a/compiler/rustc_middle/src/hir/map/mod.rs
+++ b/compiler/rustc_middle/src/hir/map/mod.rs
@@ -17,28 +17,6 @@ use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::Span;
 use rustc_target::spec::abi::Abi;
 
-fn fn_decl<'hir>(node: Node<'hir>) -> Option<&'hir FnDecl<'hir>> {
-    match node {
-        Node::Item(Item { kind: ItemKind::Fn(sig, _, _), .. })
-        | Node::TraitItem(TraitItem { kind: TraitItemKind::Fn(sig, _), .. })
-        | Node::ImplItem(ImplItem { kind: ImplItemKind::Fn(sig, _), .. }) => Some(&sig.decl),
-        Node::Expr(Expr { kind: ExprKind::Closure(Closure { fn_decl, .. }), .. })
-        | Node::ForeignItem(ForeignItem { kind: ForeignItemKind::Fn(fn_decl, ..), .. }) => {
-            Some(fn_decl)
-        }
-        _ => None,
-    }
-}
-
-pub fn fn_sig<'hir>(node: Node<'hir>) -> Option<&'hir FnSig<'hir>> {
-    match &node {
-        Node::Item(Item { kind: ItemKind::Fn(sig, _, _), .. })
-        | Node::TraitItem(TraitItem { kind: TraitItemKind::Fn(sig, _), .. })
-        | Node::ImplItem(ImplItem { kind: ImplItemKind::Fn(sig, _), .. }) => Some(sig),
-        _ => None,
-    }
-}
-
 #[inline]
 pub fn associated_body<'hir>(node: Node<'hir>) -> Option<BodyId> {
     match node {
@@ -146,10 +124,12 @@ impl<'hir> Iterator for ParentOwnerIterator<'hir> {
 }
 
 impl<'hir> Map<'hir> {
+    #[inline]
     pub fn krate(self) -> &'hir Crate<'hir> {
         self.tcx.hir_crate(())
     }
 
+    #[inline]
     pub fn root_module(self) -> &'hir Mod<'hir> {
         match self.tcx.hir_owner(CRATE_DEF_ID).map(|o| o.node) {
             Some(OwnerNode::Crate(item)) => item,
@@ -157,14 +137,17 @@ impl<'hir> Map<'hir> {
         }
     }
 
+    #[inline]
     pub fn items(self) -> impl Iterator<Item = ItemId> + 'hir {
         self.tcx.hir_crate_items(()).items.iter().copied()
     }
 
+    #[inline]
     pub fn module_items(self, module: LocalDefId) -> impl Iterator<Item = ItemId> + 'hir {
         self.tcx.hir_module_items(module).items()
     }
 
+    #[inline]
     pub fn par_for_each_item(self, f: impl Fn(ItemId) + Sync + Send) {
         par_for_each_in(&self.tcx.hir_crate_items(()).items[..], |id| f(*id));
     }
@@ -229,7 +212,13 @@ impl<'hir> Map<'hir> {
                 ItemKind::Fn(..) => DefKind::Fn,
                 ItemKind::Macro(_, macro_kind) => DefKind::Macro(macro_kind),
                 ItemKind::Mod(..) => DefKind::Mod,
-                ItemKind::OpaqueTy(..) => DefKind::OpaqueTy,
+                ItemKind::OpaqueTy(ref opaque) => {
+                    if opaque.in_trait {
+                        DefKind::ImplTraitPlaceholder
+                    } else {
+                        DefKind::OpaqueTy
+                    }
+                }
                 ItemKind::TyAlias(..) => DefKind::TyAlias,
                 ItemKind::Enum(..) => DefKind::Enum,
                 ItemKind::Struct(..) => DefKind::Struct,
@@ -297,6 +286,8 @@ impl<'hir> Map<'hir> {
             | Node::Infer(_)
             | Node::TraitRef(_)
             | Node::Pat(_)
+            | Node::PatField(_)
+            | Node::ExprField(_)
             | Node::Local(_)
             | Node::Param(_)
             | Node::Arm(_)
@@ -306,6 +297,9 @@ impl<'hir> Map<'hir> {
         Some(def_kind)
     }
 
+    /// Finds the id of the parent node to this one.
+    ///
+    /// If calling repeatedly and iterating over parents, prefer [`Map::parent_iter`].
     pub fn find_parent_node(self, id: HirId) -> Option<HirId> {
         if id.local_id == ItemLocalId::from_u32(0) {
             Some(self.tcx.hir_owner_parent(id.owner))
@@ -313,6 +307,8 @@ impl<'hir> Map<'hir> {
             let owner = self.tcx.hir_owner_nodes(id.owner).as_owner()?;
             let node = owner.nodes[id.local_id].as_ref()?;
             let hir_id = HirId { owner: id.owner, local_id: node.parent };
+            // HIR indexing should have checked that.
+            debug_assert_ne!(id.local_id, node.parent);
             Some(hir_id)
         }
     }
@@ -382,7 +378,7 @@ impl<'hir> Map<'hir> {
 
     pub fn fn_decl_by_hir_id(self, hir_id: HirId) -> Option<&'hir FnDecl<'hir>> {
         if let Some(node) = self.find(hir_id) {
-            fn_decl(node)
+            node.fn_decl()
         } else {
             bug!("no node for hir_id `{}`", hir_id)
         }
@@ -390,7 +386,7 @@ impl<'hir> Map<'hir> {
 
     pub fn fn_sig_by_hir_id(self, hir_id: HirId) -> Option<&'hir FnSig<'hir>> {
         if let Some(node) = self.find(hir_id) {
-            fn_sig(node)
+            node.fn_sig()
         } else {
             bug!("no node for hir_id `{}`", hir_id)
         }
@@ -487,11 +483,13 @@ impl<'hir> Map<'hir> {
     /// Returns an iterator of the `DefId`s for all body-owners in this
     /// crate. If you would prefer to iterate over the bodies
     /// themselves, you can do `self.hir().krate().body_ids.iter()`.
+    #[inline]
     pub fn body_owners(self) -> impl Iterator<Item = LocalDefId> + 'hir {
         self.tcx.hir_crate_items(()).body_owners.iter().copied()
     }
 
-    pub fn par_body_owners<F: Fn(LocalDefId) + Sync + Send>(self, f: F) {
+    #[inline]
+    pub fn par_body_owners(self, f: impl Fn(LocalDefId) + Sync + Send) {
         par_for_each_in(&self.tcx.hir_crate_items(()).body_owners[..], |&def_id| f(def_id));
     }
 
@@ -499,7 +497,9 @@ impl<'hir> Map<'hir> {
         let def_kind = self.tcx.def_kind(def_id);
         match def_kind {
             DefKind::Trait | DefKind::TraitAlias => def_id,
-            DefKind::TyParam | DefKind::ConstParam => self.tcx.local_parent(def_id),
+            DefKind::LifetimeParam | DefKind::TyParam | DefKind::ConstParam => {
+                self.tcx.local_parent(def_id)
+            }
             _ => bug!("ty_param_owner: {:?} is a {:?} not a type parameter", def_id, def_kind),
         }
     }
@@ -508,7 +508,9 @@ impl<'hir> Map<'hir> {
         let def_kind = self.tcx.def_kind(def_id);
         match def_kind {
             DefKind::Trait | DefKind::TraitAlias => kw::SelfUpper,
-            DefKind::TyParam | DefKind::ConstParam => self.tcx.item_name(def_id.to_def_id()),
+            DefKind::LifetimeParam | DefKind::TyParam | DefKind::ConstParam => {
+                self.tcx.item_name(def_id.to_def_id())
+            }
             _ => bug!("ty_param_name: {:?} is a {:?} not a type parameter", def_id, def_kind),
         }
     }
@@ -624,35 +626,22 @@ impl<'hir> Map<'hir> {
         }
     }
 
-    #[cfg(not(parallel_compiler))]
     #[inline]
-    pub fn par_for_each_module(self, f: impl Fn(LocalDefId)) {
-        self.for_each_module(f)
-    }
-
-    #[cfg(parallel_compiler)]
-    pub fn par_for_each_module(self, f: impl Fn(LocalDefId) + Sync) {
-        use rustc_data_structures::sync::{par_iter, ParallelIterator};
-        par_iter_submodules(self.tcx, CRATE_DEF_ID, &f);
-
-        fn par_iter_submodules<F>(tcx: TyCtxt<'_>, module: LocalDefId, f: &F)
-        where
-            F: Fn(LocalDefId) + Sync,
-        {
-            (*f)(module);
-            let items = tcx.hir_module_items(module);
-            par_iter(&items.submodules[..]).for_each(|&sm| par_iter_submodules(tcx, sm, f));
-        }
+    pub fn par_for_each_module(self, f: impl Fn(LocalDefId) + Sync + Send) {
+        let crate_items = self.tcx.hir_crate_items(());
+        par_for_each_in(&crate_items.submodules[..], |module| f(*module))
     }
 
     /// Returns an iterator for the nodes in the ancestor tree of the `current_id`
     /// until the crate root is reached. Prefer this over your own loop using `get_parent_node`.
+    #[inline]
     pub fn parent_iter(self, current_id: HirId) -> ParentHirIterator<'hir> {
         ParentHirIterator { current_id, map: self }
     }
 
     /// Returns an iterator for the nodes in the ancestor tree of the `current_id`
     /// until the crate root is reached. Prefer this over your own loop using `get_parent_node`.
+    #[inline]
     pub fn parent_owner_iter(self, current_id: HirId) -> ParentOwnerIterator<'hir> {
         ParentOwnerIterator { current_id, map: self }
     }
@@ -1020,6 +1009,7 @@ impl<'hir> Map<'hir> {
             Node::Field(field) => field.span,
             Node::AnonConst(constant) => self.body(constant.body).value.span,
             Node::Expr(expr) => expr.span,
+            Node::ExprField(field) => field.span,
             Node::Stmt(stmt) => stmt.span,
             Node::PathSegment(seg) => {
                 let ident_span = seg.ident.span;
@@ -1030,6 +1020,7 @@ impl<'hir> Map<'hir> {
             Node::TypeBinding(tb) => tb.span,
             Node::TraitRef(tr) => tr.path.span,
             Node::Pat(pat) => pat.span,
+            Node::PatField(field) => field.span,
             Node::Arm(arm) => arm.span,
             Node::Block(block) => block.span,
             Node::Ctor(..) => self.span_with_body(self.get_parent_node(hir_id)),
@@ -1204,7 +1195,13 @@ fn hir_id_to_string(map: Map<'_>, id: HirId) -> String {
                 ItemKind::ForeignMod { .. } => "foreign mod",
                 ItemKind::GlobalAsm(..) => "global asm",
                 ItemKind::TyAlias(..) => "ty",
-                ItemKind::OpaqueTy(..) => "opaque type",
+                ItemKind::OpaqueTy(ref opaque) => {
+                    if opaque.in_trait {
+                        "opaque type in trait"
+                    } else {
+                        "opaque type"
+                    }
+                }
                 ItemKind::Enum(..) => "enum",
                 ItemKind::Struct(..) => "struct",
                 ItemKind::Union(..) => "union",
@@ -1241,12 +1238,14 @@ fn hir_id_to_string(map: Map<'_>, id: HirId) -> String {
         }
         Some(Node::AnonConst(_)) => node_str("const"),
         Some(Node::Expr(_)) => node_str("expr"),
+        Some(Node::ExprField(_)) => node_str("expr field"),
         Some(Node::Stmt(_)) => node_str("stmt"),
         Some(Node::PathSegment(_)) => node_str("path segment"),
         Some(Node::Ty(_)) => node_str("type"),
         Some(Node::TypeBinding(_)) => node_str("type binding"),
         Some(Node::TraitRef(_)) => node_str("trait ref"),
         Some(Node::Pat(_)) => node_str("pat"),
+        Some(Node::PatField(_)) => node_str("pattern field"),
         Some(Node::Param(_)) => node_str("param"),
         Some(Node::Arm(_)) => node_str("arm"),
         Some(Node::Block(_)) => node_str("block"),
diff --git a/compiler/rustc_middle/src/infer/canonical.rs b/compiler/rustc_middle/src/infer/canonical.rs
index 200de9079..d3cf519b6 100644
--- a/compiler/rustc_middle/src/infer/canonical.rs
+++ b/compiler/rustc_middle/src/infer/canonical.rs
@@ -22,6 +22,7 @@
 //! [c]: https://rust-lang.github.io/chalk/book/canonical_queries/canonicalization.html
 
 use crate::infer::MemberConstraint;
+use crate::mir::ConstraintCategory;
 use crate::ty::subst::GenericArg;
 use crate::ty::{self, BoundVar, List, Region, Ty, TyCtxt};
 use rustc_index::vec::IndexVec;
@@ -43,6 +44,15 @@ pub struct Canonical<'tcx, V> {
 
 pub type CanonicalVarInfos<'tcx> = &'tcx List<CanonicalVarInfo<'tcx>>;
 
+impl<'tcx> ty::TypeFoldable<'tcx> for CanonicalVarInfos<'tcx> {
+    fn try_fold_with<F: ty::FallibleTypeFolder<'tcx>>(
+        self,
+        folder: &mut F,
+    ) -> Result<Self, F::Error> {
+        ty::util::fold_list(self, folder, |tcx, v| tcx.intern_canonical_var_infos(v))
+    }
+}
+
 /// A set of values corresponding to the canonical variables from some
 /// `Canonical`. You can give these values to
 /// `canonical_value.substitute` to substitute them into the canonical
@@ -89,6 +99,7 @@ impl<'tcx> Default for OriginalQueryValues<'tcx> {
 /// a copy of the canonical value in some other inference context,
 /// with fresh inference variables replacing the canonical values.
 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, TyDecodable, TyEncodable, HashStable)]
+#[derive(TypeFoldable, TypeVisitable)]
 pub struct CanonicalVarInfo<'tcx> {
     pub kind: CanonicalVarKind<'tcx>,
 }
@@ -114,6 +125,7 @@ impl<'tcx> CanonicalVarInfo<'tcx> {
 /// in the type-theory sense of the term -- i.e., a "meta" type system
 /// that analyzes type-like values.
 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, TyDecodable, TyEncodable, HashStable)]
+#[derive(TypeFoldable, TypeVisitable)]
 pub enum CanonicalVarKind<'tcx> {
     /// Some kind of type inference variable.
     Ty(CanonicalTyVarKind),
@@ -290,20 +302,15 @@ impl<'tcx, V> Canonical<'tcx, V> {
     }
 }
 
-pub type QueryOutlivesConstraint<'tcx> =
-    ty::Binder<'tcx, ty::OutlivesPredicate<GenericArg<'tcx>, Region<'tcx>>>;
+pub type QueryOutlivesConstraint<'tcx> = (
+    ty::Binder<'tcx, ty::OutlivesPredicate<GenericArg<'tcx>, Region<'tcx>>>,
+    ConstraintCategory<'tcx>,
+);
 
 TrivialTypeTraversalAndLiftImpls! {
     for <'tcx> {
         crate::infer::canonical::Certainty,
-        crate::infer::canonical::CanonicalVarInfo<'tcx>,
-        crate::infer::canonical::CanonicalVarKind<'tcx>,
-    }
-}
-
-TrivialTypeTraversalImpls! {
-    for <'tcx> {
-        crate::infer::canonical::CanonicalVarInfos<'tcx>,
+        crate::infer::canonical::CanonicalTyVarKind,
     }
 }
 
diff --git a/compiler/rustc_middle/src/lib.rs b/compiler/rustc_middle/src/lib.rs
index ef06c457b..01b9277b9 100644
--- a/compiler/rustc_middle/src/lib.rs
+++ b/compiler/rustc_middle/src/lib.rs
@@ -26,13 +26,12 @@
 #![feature(allocator_api)]
 #![feature(array_windows)]
 #![feature(assert_matches)]
-#![feature(backtrace)]
 #![feature(box_patterns)]
 #![feature(core_intrinsics)]
 #![feature(discriminant_kind)]
 #![feature(exhaustive_patterns)]
 #![feature(get_mut_unchecked)]
-#![feature(generic_associated_types)]
+#![cfg_attr(bootstrap, feature(generic_associated_types))]
 #![feature(if_let_guard)]
 #![feature(map_first_last)]
 #![feature(negative_impls)]
@@ -41,7 +40,7 @@
 #![feature(new_uninit)]
 #![feature(once_cell)]
 #![feature(let_chains)]
-#![feature(let_else)]
+#![cfg_attr(bootstrap, feature(let_else))]
 #![feature(min_specialization)]
 #![feature(trusted_len)]
 #![feature(type_alias_impl_trait)]
@@ -87,6 +86,7 @@ pub mod query;
 pub mod arena;
 #[macro_use]
 pub mod dep_graph;
+pub(crate) mod error;
 pub mod hir;
 pub mod infer;
 pub mod lint;
@@ -96,6 +96,7 @@ pub mod mir;
 pub mod thir;
 pub mod traits;
 pub mod ty;
+mod values;
 
 pub mod util {
     pub mod bug;
diff --git a/compiler/rustc_middle/src/metadata.rs b/compiler/rustc_middle/src/metadata.rs
index c8e78747d..5ff014c78 100644
--- a/compiler/rustc_middle/src/metadata.rs
+++ b/compiler/rustc_middle/src/metadata.rs
@@ -2,6 +2,7 @@ use crate::ty;
 
 use rustc_hir::def::Res;
 use rustc_macros::HashStable;
+use rustc_span::def_id::DefId;
 use rustc_span::symbol::Ident;
 use rustc_span::Span;
 
@@ -18,7 +19,7 @@ pub struct ModChild {
     /// Local variables cannot be exported, so this `Res` doesn't need the ID parameter.
     pub res: Res<!>,
     /// Visibility of the item.
-    pub vis: ty::Visibility,
+    pub vis: ty::Visibility<DefId>,
     /// Span of the item.
     pub span: Span,
     /// A proper `macro_rules` item (not a reexport).
diff --git a/compiler/rustc_middle/src/middle/lang_items.rs b/compiler/rustc_middle/src/middle/lang_items.rs
index cc9706f2d..31c20fa14 100644
--- a/compiler/rustc_middle/src/middle/lang_items.rs
+++ b/compiler/rustc_middle/src/middle/lang_items.rs
@@ -18,11 +18,11 @@ impl<'tcx> TyCtxt<'tcx> {
     /// Returns the `DefId` for a given `LangItem`.
     /// If not found, fatally aborts compilation.
     pub fn require_lang_item(self, lang_item: LangItem, span: Option<Span>) -> DefId {
-        self.lang_items().require(lang_item).unwrap_or_else(|msg| {
+        self.lang_items().require(lang_item).unwrap_or_else(|err| {
             if let Some(span) = span {
-                self.sess.span_fatal(span, &msg)
+                self.sess.span_fatal(span, err.to_string())
             } else {
-                self.sess.fatal(&msg)
+                self.sess.fatal(err.to_string())
             }
         })
     }
diff --git a/compiler/rustc_middle/src/middle/limits.rs b/compiler/rustc_middle/src/middle/limits.rs
index acced0492..53c4d9267 100644
--- a/compiler/rustc_middle/src/middle/limits.rs
+++ b/compiler/rustc_middle/src/middle/limits.rs
@@ -10,6 +10,7 @@
 //! just peeks and looks for that attribute.
 
 use crate::bug;
+use crate::error::LimitInvalid;
 use crate::ty;
 use rustc_ast::Attribute;
 use rustc_session::Session;
@@ -56,9 +57,6 @@ fn get_limit(krate_attrs: &[Attribute], sess: &Session, name: Symbol, default: u
             match s.as_str().parse() {
                 Ok(n) => return Limit::new(n),
                 Err(e) => {
-                    let mut err =
-                        sess.struct_span_err(attr.span, "`limit` must be a non-negative integer");
-
                     let value_span = attr
                         .meta()
                         .and_then(|meta| meta.name_value_literal_span())
@@ -74,9 +72,7 @@ fn get_limit(krate_attrs: &[Attribute], sess: &Session, name: Symbol, default: u
                         IntErrorKind::Zero => bug!("zero is a valid `limit`"),
                         kind => bug!("unimplemented IntErrorKind variant: {:?}", kind),
                     };
-
-                    err.span_label(value_span, error_str);
-                    err.emit();
+                    sess.emit_err(LimitInvalid { span: attr.span, value_span, error_str });
                 }
             }
         }
diff --git a/compiler/rustc_middle/src/middle/resolve_lifetime.rs b/compiler/rustc_middle/src/middle/resolve_lifetime.rs
index 9b2f44567..a171f5711 100644
--- a/compiler/rustc_middle/src/middle/resolve_lifetime.rs
+++ b/compiler/rustc_middle/src/middle/resolve_lifetime.rs
@@ -10,7 +10,7 @@ use rustc_macros::HashStable;
 #[derive(Clone, Copy, PartialEq, Eq, Hash, TyEncodable, TyDecodable, Debug, HashStable)]
 pub enum Region {
     Static,
-    EarlyBound(/* index */ u32, /* lifetime decl */ DefId),
+    EarlyBound(/* lifetime decl */ DefId),
     LateBound(ty::DebruijnIndex, /* late-bound index */ u32, /* lifetime decl */ DefId),
     Free(DefId, /* lifetime decl */ DefId),
 }
@@ -35,7 +35,13 @@ impl<T: PartialEq> Set1<T> {
     }
 }
 
-pub type ObjectLifetimeDefault = Set1<Region>;
+#[derive(Copy, Clone, Debug, HashStable, Encodable, Decodable)]
+pub enum ObjectLifetimeDefault {
+    Empty,
+    Static,
+    Ambiguous,
+    Param(DefId),
+}
 
 /// Maps the id of each lifetime reference to the lifetime decl
 /// that it corresponds to.
diff --git a/compiler/rustc_middle/src/middle/stability.rs b/compiler/rustc_middle/src/middle/stability.rs
index 414912dd0..d182929c4 100644
--- a/compiler/rustc_middle/src/middle/stability.rs
+++ b/compiler/rustc_middle/src/middle/stability.rs
@@ -5,7 +5,7 @@ pub use self::StabilityLevel::*;
 
 use crate::ty::{self, DefIdTree, TyCtxt};
 use rustc_ast::NodeId;
-use rustc_attr::{self as attr, ConstStability, Deprecation, Stability};
+use rustc_attr::{self as attr, ConstStability, DefaultBodyStability, Deprecation, Stability};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_errors::{Applicability, Diagnostic};
 use rustc_feature::GateIssue;
@@ -61,6 +61,7 @@ pub struct Index {
     /// are filled by the annotator.
     pub stab_map: FxHashMap<LocalDefId, Stability>,
     pub const_stab_map: FxHashMap<LocalDefId, ConstStability>,
+    pub default_body_stab_map: FxHashMap<LocalDefId, DefaultBodyStability>,
     pub depr_map: FxHashMap<LocalDefId, DeprecationEntry>,
     /// Mapping from feature name to feature name based on the `implied_by` field of `#[unstable]`
     /// attributes. If a `#[unstable(feature = "implier", implied_by = "impliee")]` attribute
@@ -86,6 +87,10 @@ impl Index {
         self.const_stab_map.get(&def_id).copied()
     }
 
+    pub fn local_default_body_stability(&self, def_id: LocalDefId) -> Option<DefaultBodyStability> {
+        self.default_body_stab_map.get(&def_id).copied()
+    }
+
     pub fn local_deprecation_entry(&self, def_id: LocalDefId) -> Option<DeprecationEntry> {
         self.depr_map.get(&def_id).cloned()
     }
@@ -288,7 +293,7 @@ fn skip_stability_check_due_to_privacy(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
 
         // These are not visible outside crate; therefore
         // stability markers are irrelevant, if even present.
-        ty::Visibility::Restricted(..) | ty::Visibility::Invisible => true,
+        ty::Visibility::Restricted(..) => true,
     }
 }
 
@@ -416,6 +421,12 @@ impl<'tcx> TyCtxt<'tcx> {
             return EvalResult::Allow;
         }
 
+        // Only the cross-crate scenario matters when checking unstable APIs
+        let cross_crate = !def_id.is_local();
+        if !cross_crate {
+            return EvalResult::Allow;
+        }
+
         let stability = self.lookup_stability(def_id);
         debug!(
             "stability: \
@@ -423,12 +434,6 @@ impl<'tcx> TyCtxt<'tcx> {
             def_id, span, stability
         );
 
-        // Only the cross-crate scenario matters when checking unstable APIs
-        let cross_crate = !def_id.is_local();
-        if !cross_crate {
-            return EvalResult::Allow;
-        }
-
         // Issue #38412: private items lack stability markers.
         if skip_stability_check_due_to_privacy(self, def_id) {
             return EvalResult::Allow;
@@ -492,6 +497,62 @@ impl<'tcx> TyCtxt<'tcx> {
         }
     }
 
+    /// Evaluates the default-impl stability of an item.
+    ///
+    /// Returns `EvalResult::Allow` if the item's default implementation is stable, or unstable but the corresponding
+    /// `#![feature]` has been provided. Returns `EvalResult::Deny` which describes the offending
+    /// unstable feature otherwise.
+    pub fn eval_default_body_stability(self, def_id: DefId, span: Span) -> EvalResult {
+        let is_staged_api = self.lookup_stability(def_id.krate.as_def_id()).is_some();
+        if !is_staged_api {
+            return EvalResult::Allow;
+        }
+
+        // Only the cross-crate scenario matters when checking unstable APIs
+        let cross_crate = !def_id.is_local();
+        if !cross_crate {
+            return EvalResult::Allow;
+        }
+
+        let stability = self.lookup_default_body_stability(def_id);
+        debug!(
+            "body stability: inspecting def_id={def_id:?} span={span:?} of stability={stability:?}"
+        );
+
+        // Issue #38412: private items lack stability markers.
+        if skip_stability_check_due_to_privacy(self, def_id) {
+            return EvalResult::Allow;
+        }
+
+        match stability {
+            Some(DefaultBodyStability {
+                level: attr::Unstable { reason, issue, is_soft, .. },
+                feature,
+            }) => {
+                if span.allows_unstable(feature) {
+                    debug!("body stability: skipping span={:?} since it is internal", span);
+                    return EvalResult::Allow;
+                }
+                if self.features().active(feature) {
+                    return EvalResult::Allow;
+                }
+
+                EvalResult::Deny {
+                    feature,
+                    reason: reason.to_opt_reason(),
+                    issue,
+                    suggestion: None,
+                    is_soft,
+                }
+            }
+            Some(_) => {
+                // Stable APIs are always ok to call
+                EvalResult::Allow
+            }
+            None => EvalResult::Unmarked,
+        }
+    }
+
     /// Checks if an item is stable or error out.
     ///
     /// If the item defined by `def_id` is unstable and the corresponding `#![feature]` does not
diff --git a/compiler/rustc_middle/src/mir/basic_blocks.rs b/compiler/rustc_middle/src/mir/basic_blocks.rs
index 78080fcd5..752cbdeae 100644
--- a/compiler/rustc_middle/src/mir/basic_blocks.rs
+++ b/compiler/rustc_middle/src/mir/basic_blocks.rs
@@ -86,7 +86,7 @@ impl<'tcx> BasicBlocks<'tcx> {
     ///
     /// You will only ever need this if you have also called [`BasicBlocks::as_mut_preserves_cfg`].
     /// All other methods that allow you to mutate the basic blocks also call this method
-    /// themselves, thereby avoiding any risk of accidentaly cache invalidation.
+    /// themselves, thereby avoiding any risk of accidentally cache invalidation.
     pub fn invalidate_cfg_cache(&mut self) {
         self.predecessor_cache.invalidate();
         self.switch_source_cache.invalidate();
diff --git a/compiler/rustc_middle/src/mir/generic_graph.rs b/compiler/rustc_middle/src/mir/generic_graph.rs
index f3621cd99..d1f3561c0 100644
--- a/compiler/rustc_middle/src/mir/generic_graph.rs
+++ b/compiler/rustc_middle/src/mir/generic_graph.rs
@@ -12,14 +12,14 @@ pub fn mir_fn_to_generic_graph<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'_>) -> Grap
 
     // Nodes
     let nodes: Vec<Node> = body
-        .basic_blocks()
+        .basic_blocks
         .iter_enumerated()
         .map(|(block, _)| bb_to_graph_node(block, body, dark_mode))
         .collect();
 
     // Edges
     let mut edges = Vec::new();
-    for (source, _) in body.basic_blocks().iter_enumerated() {
+    for (source, _) in body.basic_blocks.iter_enumerated() {
         let def_id = body.source.def_id();
         let terminator = body[source].terminator();
         let labels = terminator.kind.fmt_successor_labels();
diff --git a/compiler/rustc_middle/src/mir/interpret/allocation.rs b/compiler/rustc_middle/src/mir/interpret/allocation.rs
index db7e0fb8a..37ec04b07 100644
--- a/compiler/rustc_middle/src/mir/interpret/allocation.rs
+++ b/compiler/rustc_middle/src/mir/interpret/allocation.rs
@@ -16,8 +16,8 @@ use rustc_target::abi::{Align, HasDataLayout, Size};
 
 use super::{
     read_target_uint, write_target_uint, AllocId, InterpError, InterpResult, Pointer, Provenance,
-    ResourceExhaustionInfo, Scalar, ScalarMaybeUninit, ScalarSizeMismatch, UndefinedBehaviorInfo,
-    UninitBytesAccess, UnsupportedOpInfo,
+    ResourceExhaustionInfo, Scalar, ScalarSizeMismatch, UndefinedBehaviorInfo, UninitBytesAccess,
+    UnsupportedOpInfo,
 };
 use crate::ty;
 
@@ -34,11 +34,11 @@ pub struct Allocation<Prov = AllocId, Extra = ()> {
     /// The actual bytes of the allocation.
     /// Note that the bytes of a pointer represent the offset of the pointer.
     bytes: Box<[u8]>,
-    /// Maps from byte addresses to extra data for each pointer.
+    /// Maps from byte addresses to extra provenance data for each pointer.
     /// Only the first byte of a pointer is inserted into the map; i.e.,
     /// every entry in this map applies to `pointer_size` consecutive bytes starting
     /// at the given offset.
-    relocations: Relocations<Prov>,
+    provenance: ProvenanceMap<Prov>,
     /// Denotes which part of this allocation is initialized.
     init_mask: InitMask,
     /// The alignment of the allocation to detect unaligned reads.
@@ -84,7 +84,7 @@ impl hash::Hash for Allocation {
         }
 
         // Hash the other fields as usual.
-        self.relocations.hash(state);
+        self.provenance.hash(state);
         self.init_mask.hash(state);
         self.align.hash(state);
         self.mutability.hash(state);
@@ -130,6 +130,8 @@ pub enum AllocError {
     ReadPointerAsBytes,
     /// Partially overwriting a pointer.
     PartialPointerOverwrite(Size),
+    /// Partially copying a pointer.
+    PartialPointerCopy(Size),
     /// Using uninitialized data where it is not allowed.
     InvalidUninitBytes(Option<UninitBytesAccess>),
 }
@@ -152,6 +154,9 @@ impl AllocError {
             PartialPointerOverwrite(offset) => InterpError::Unsupported(
                 UnsupportedOpInfo::PartialPointerOverwrite(Pointer::new(alloc_id, offset)),
             ),
+            PartialPointerCopy(offset) => InterpError::Unsupported(
+                UnsupportedOpInfo::PartialPointerCopy(Pointer::new(alloc_id, offset)),
+            ),
             InvalidUninitBytes(info) => InterpError::UndefinedBehavior(
                 UndefinedBehaviorInfo::InvalidUninitBytes(info.map(|b| (alloc_id, b))),
             ),
@@ -211,7 +216,7 @@ impl<Prov> Allocation<Prov> {
         let size = Size::from_bytes(bytes.len());
         Self {
             bytes,
-            relocations: Relocations::new(),
+            provenance: ProvenanceMap::new(),
             init_mask: InitMask::new(size, true),
             align,
             mutability,
@@ -246,7 +251,7 @@ impl<Prov> Allocation<Prov> {
         let bytes = unsafe { bytes.assume_init() };
         Ok(Allocation {
             bytes,
-            relocations: Relocations::new(),
+            provenance: ProvenanceMap::new(),
             init_mask: InitMask::new(size, false),
             align,
             mutability: Mutability::Mut,
@@ -266,22 +271,22 @@ impl Allocation {
     ) -> Result<Allocation<Prov, Extra>, Err> {
         // Compute new pointer provenance, which also adjusts the bytes.
         let mut bytes = self.bytes;
-        let mut new_relocations = Vec::with_capacity(self.relocations.0.len());
+        let mut new_provenance = Vec::with_capacity(self.provenance.0.len());
         let ptr_size = cx.data_layout().pointer_size.bytes_usize();
         let endian = cx.data_layout().endian;
-        for &(offset, alloc_id) in self.relocations.iter() {
+        for &(offset, alloc_id) in self.provenance.iter() {
             let idx = offset.bytes_usize();
             let ptr_bytes = &mut bytes[idx..idx + ptr_size];
             let bits = read_target_uint(endian, ptr_bytes).unwrap();
             let (ptr_prov, ptr_offset) =
                 adjust_ptr(Pointer::new(alloc_id, Size::from_bytes(bits)))?.into_parts();
             write_target_uint(endian, ptr_bytes, ptr_offset.bytes().into()).unwrap();
-            new_relocations.push((offset, ptr_prov));
+            new_provenance.push((offset, ptr_prov));
         }
         // Create allocation.
         Ok(Allocation {
             bytes,
-            relocations: Relocations::from_presorted(new_relocations),
+            provenance: ProvenanceMap::from_presorted(new_provenance),
             init_mask: self.init_mask,
             align: self.align,
             mutability: self.mutability,
@@ -300,8 +305,8 @@ impl<Prov, Extra> Allocation<Prov, Extra> {
         Size::from_bytes(self.len())
     }
 
-    /// Looks at a slice which may describe uninitialized bytes or describe a relocation. This differs
-    /// from `get_bytes_with_uninit_and_ptr` in that it does no relocation checks (even on the
+    /// Looks at a slice which may contain uninitialized bytes or provenance. This differs
+    /// from `get_bytes_with_uninit_and_ptr` in that it does no provenance checks (even on the
     /// edges) at all.
     /// This must not be used for reads affecting the interpreter execution.
     pub fn inspect_with_uninit_and_ptr_outside_interpreter(&self, range: Range<usize>) -> &[u8] {
@@ -313,74 +318,47 @@ impl<Prov, Extra> Allocation<Prov, Extra> {
         &self.init_mask
     }
 
-    /// Returns the relocation list.
-    pub fn relocations(&self) -> &Relocations<Prov> {
-        &self.relocations
+    /// Returns the provenance map.
+    pub fn provenance(&self) -> &ProvenanceMap<Prov> {
+        &self.provenance
     }
 }
 
 /// Byte accessors.
 impl<Prov: Provenance, Extra> Allocation<Prov, Extra> {
     /// This is the entirely abstraction-violating way to just grab the raw bytes without
-    /// caring about relocations. It just deduplicates some code between `read_scalar`
-    /// and `get_bytes_internal`.
-    fn get_bytes_even_more_internal(&self, range: AllocRange) -> &[u8] {
-        &self.bytes[range.start.bytes_usize()..range.end().bytes_usize()]
-    }
-
-    /// The last argument controls whether we error out when there are uninitialized or pointer
-    /// bytes. However, we *always* error when there are relocations overlapping the edges of the
-    /// range.
-    ///
-    /// You should never call this, call `get_bytes` or `get_bytes_with_uninit_and_ptr` instead,
+    /// caring about provenance or initialization.
     ///
     /// This function also guarantees that the resulting pointer will remain stable
     /// even when new allocations are pushed to the `HashMap`. `mem_copy_repeatedly` relies
     /// on that.
-    ///
-    /// It is the caller's responsibility to check bounds and alignment beforehand.
-    fn get_bytes_internal(
-        &self,
-        cx: &impl HasDataLayout,
-        range: AllocRange,
-        check_init_and_ptr: bool,
-    ) -> AllocResult<&[u8]> {
-        if check_init_and_ptr {
-            self.check_init(range)?;
-            self.check_relocations(cx, range)?;
-        } else {
-            // We still don't want relocations on the *edges*.
-            self.check_relocation_edges(cx, range)?;
-        }
-
-        Ok(self.get_bytes_even_more_internal(range))
+    #[inline]
+    pub fn get_bytes_unchecked(&self, range: AllocRange) -> &[u8] {
+        &self.bytes[range.start.bytes_usize()..range.end().bytes_usize()]
     }
 
-    /// Checks that these bytes are initialized and not pointer bytes, and then return them
-    /// as a slice.
+    /// Checks that these bytes are initialized, and then strip provenance (if possible) and return
+    /// them.
     ///
     /// It is the caller's responsibility to check bounds and alignment beforehand.
     /// Most likely, you want to use the `PlaceTy` and `OperandTy`-based methods
     /// on `InterpCx` instead.
     #[inline]
-    pub fn get_bytes(&self, cx: &impl HasDataLayout, range: AllocRange) -> AllocResult<&[u8]> {
-        self.get_bytes_internal(cx, range, true)
-    }
-
-    /// It is the caller's responsibility to handle uninitialized and pointer bytes.
-    /// However, this still checks that there are no relocations on the *edges*.
-    ///
-    /// It is the caller's responsibility to check bounds and alignment beforehand.
-    #[inline]
-    pub fn get_bytes_with_uninit_and_ptr(
+    pub fn get_bytes_strip_provenance(
         &self,
         cx: &impl HasDataLayout,
         range: AllocRange,
     ) -> AllocResult<&[u8]> {
-        self.get_bytes_internal(cx, range, false)
+        self.check_init(range)?;
+        if !Prov::OFFSET_IS_ADDR {
+            if self.range_has_provenance(cx, range) {
+                return Err(AllocError::ReadPointerAsBytes);
+            }
+        }
+        Ok(self.get_bytes_unchecked(range))
     }
 
-    /// Just calling this already marks everything as defined and removes relocations,
+    /// Just calling this already marks everything as defined and removes provenance,
     /// so be sure to actually put data there!
     ///
     /// It is the caller's responsibility to check bounds and alignment beforehand.
@@ -392,7 +370,7 @@ impl<Prov: Provenance, Extra> Allocation<Prov, Extra> {
         range: AllocRange,
     ) -> AllocResult<&mut [u8]> {
         self.mark_init(range, true);
-        self.clear_relocations(cx, range)?;
+        self.clear_provenance(cx, range)?;
 
         Ok(&mut self.bytes[range.start.bytes_usize()..range.end().bytes_usize()])
     }
@@ -404,7 +382,7 @@ impl<Prov: Provenance, Extra> Allocation<Prov, Extra> {
         range: AllocRange,
     ) -> AllocResult<*mut [u8]> {
         self.mark_init(range, true);
-        self.clear_relocations(cx, range)?;
+        self.clear_provenance(cx, range)?;
 
         assert!(range.end().bytes_usize() <= self.bytes.len()); // need to do our own bounds-check
         let begin_ptr = self.bytes.as_mut_ptr().wrapping_add(range.start.bytes_usize());
@@ -415,28 +393,6 @@ impl<Prov: Provenance, Extra> Allocation<Prov, Extra> {
 
 /// Reading and writing.
 impl<Prov: Provenance, Extra> Allocation<Prov, Extra> {
-    /// Validates that `ptr.offset` and `ptr.offset + size` do not point to the middle of a
-    /// relocation. If `allow_uninit`/`allow_ptr` is `false`, also enforces that the memory in the
-    /// given range contains no uninitialized bytes/relocations.
-    pub fn check_bytes(
-        &self,
-        cx: &impl HasDataLayout,
-        range: AllocRange,
-        allow_uninit: bool,
-        allow_ptr: bool,
-    ) -> AllocResult {
-        // Check bounds and relocations on the edges.
-        self.get_bytes_with_uninit_and_ptr(cx, range)?;
-        // Check uninit and ptr.
-        if !allow_uninit {
-            self.check_init(range)?;
-        }
-        if !allow_ptr {
-            self.check_relocations(cx, range)?;
-        }
-        Ok(())
-    }
-
     /// Reads a *non-ZST* scalar.
     ///
     /// If `read_provenance` is `true`, this will also read provenance; otherwise (if the machine
@@ -452,47 +408,55 @@ impl<Prov: Provenance, Extra> Allocation<Prov, Extra> {
         cx: &impl HasDataLayout,
         range: AllocRange,
         read_provenance: bool,
-    ) -> AllocResult<ScalarMaybeUninit<Prov>> {
-        if read_provenance {
-            assert_eq!(range.size, cx.data_layout().pointer_size);
-        }
-
+    ) -> AllocResult<Scalar<Prov>> {
         // First and foremost, if anything is uninit, bail.
         if self.is_init(range).is_err() {
-            // This inflates uninitialized bytes to the entire scalar, even if only a few
-            // bytes are uninitialized.
-            return Ok(ScalarMaybeUninit::Uninit);
+            return Err(AllocError::InvalidUninitBytes(None));
         }
 
-        // If we are doing a pointer read, and there is a relocation exactly where we
-        // are reading, then we can put data and relocation back together and return that.
-        if read_provenance && let Some(&prov) = self.relocations.get(&range.start) {
-            // We already checked init and relocations, so we can use this function.
-            let bytes = self.get_bytes_even_more_internal(range);
-            let bits = read_target_uint(cx.data_layout().endian, bytes).unwrap();
-            let ptr = Pointer::new(prov, Size::from_bytes(bits));
-            return Ok(ScalarMaybeUninit::from_pointer(ptr, cx));
-        }
+        // Get the integer part of the result. We HAVE TO check provenance before returning this!
+        let bytes = self.get_bytes_unchecked(range);
+        let bits = read_target_uint(cx.data_layout().endian, bytes).unwrap();
 
-        // If we are *not* reading a pointer, and we can just ignore relocations,
-        // then do exactly that.
-        if !read_provenance && Prov::OFFSET_IS_ADDR {
-            // We just strip provenance.
-            let bytes = self.get_bytes_even_more_internal(range);
-            let bits = read_target_uint(cx.data_layout().endian, bytes).unwrap();
-            return Ok(ScalarMaybeUninit::Scalar(Scalar::from_uint(bits, range.size)));
+        if read_provenance {
+            assert_eq!(range.size, cx.data_layout().pointer_size);
+
+            // When reading data with provenance, the easy case is finding provenance exactly where we
+            // are reading, then we can put data and provenance back together and return that.
+            if let Some(&prov) = self.provenance.get(&range.start) {
+                // Now we can return the bits, with their appropriate provenance.
+                let ptr = Pointer::new(prov, Size::from_bytes(bits));
+                return Ok(Scalar::from_pointer(ptr, cx));
+            }
+
+            // If we can work on pointers byte-wise, join the byte-wise provenances.
+            if Prov::OFFSET_IS_ADDR {
+                let mut prov = self.offset_get_provenance(cx, range.start);
+                for offset in 1..range.size.bytes() {
+                    let this_prov =
+                        self.offset_get_provenance(cx, range.start + Size::from_bytes(offset));
+                    prov = Prov::join(prov, this_prov);
+                }
+                // Now use this provenance.
+                let ptr = Pointer::new(prov, Size::from_bytes(bits));
+                return Ok(Scalar::from_maybe_pointer(ptr, cx));
+            }
+        } else {
+            // We are *not* reading a pointer.
+            // If we can just ignore provenance, do exactly that.
+            if Prov::OFFSET_IS_ADDR {
+                // We just strip provenance.
+                return Ok(Scalar::from_uint(bits, range.size));
+            }
         }
 
-        // It's complicated. Better make sure there is no provenance anywhere.
-        // FIXME: If !OFFSET_IS_ADDR, this is the best we can do. But if OFFSET_IS_ADDR, then
-        // `read_pointer` is true and we ideally would distinguish the following two cases:
-        // - The entire `range` is covered by 2 relocations for the same provenance.
-        //   Then we should return a pointer with that provenance.
-        // - The range has inhomogeneous provenance. Then we should return just the
-        //   underlying bits.
-        let bytes = self.get_bytes(cx, range)?;
-        let bits = read_target_uint(cx.data_layout().endian, bytes).unwrap();
-        Ok(ScalarMaybeUninit::Scalar(Scalar::from_uint(bits, range.size)))
+        // Fallback path for when we cannot treat provenance bytewise or ignore it.
+        assert!(!Prov::OFFSET_IS_ADDR);
+        if self.range_has_provenance(cx, range) {
+            return Err(AllocError::ReadPointerAsBytes);
+        }
+        // There is no provenance, we can just return the bits.
+        Ok(Scalar::from_uint(bits, range.size))
     }
 
     /// Writes a *non-ZST* scalar.
@@ -507,17 +471,10 @@ impl<Prov: Provenance, Extra> Allocation<Prov, Extra> {
         &mut self,
         cx: &impl HasDataLayout,
         range: AllocRange,
-        val: ScalarMaybeUninit<Prov>,
+        val: Scalar<Prov>,
     ) -> AllocResult {
         assert!(self.mutability == Mutability::Mut);
 
-        let val = match val {
-            ScalarMaybeUninit::Scalar(scalar) => scalar,
-            ScalarMaybeUninit::Uninit => {
-                return self.write_uninit(cx, range);
-            }
-        };
-
         // `to_bits_or_ptr_internal` is the right method because we just want to store this data
         // as-is into memory.
         let (bytes, provenance) = match val.to_bits_or_ptr_internal(range.size)? {
@@ -532,9 +489,9 @@ impl<Prov: Provenance, Extra> Allocation<Prov, Extra> {
         let dst = self.get_bytes_mut(cx, range)?;
         write_target_uint(endian, dst, bytes).unwrap();
 
-        // See if we have to also write a relocation.
+        // See if we have to also store some provenance.
         if let Some(provenance) = provenance {
-            self.relocations.0.insert(range.start, provenance);
+            self.provenance.0.insert(range.start, provenance);
         }
 
         Ok(())
@@ -543,64 +500,65 @@ impl<Prov: Provenance, Extra> Allocation<Prov, Extra> {
     /// Write "uninit" to the given memory range.
     pub fn write_uninit(&mut self, cx: &impl HasDataLayout, range: AllocRange) -> AllocResult {
         self.mark_init(range, false);
-        self.clear_relocations(cx, range)?;
+        self.clear_provenance(cx, range)?;
         return Ok(());
     }
 }
 
-/// Relocations.
+/// Provenance.
 impl<Prov: Copy, Extra> Allocation<Prov, Extra> {
-    /// Returns all relocations overlapping with the given pointer-offset pair.
-    fn get_relocations(&self, cx: &impl HasDataLayout, range: AllocRange) -> &[(Size, Prov)] {
+    /// Returns all provenance overlapping with the given pointer-offset pair.
+    fn range_get_provenance(&self, cx: &impl HasDataLayout, range: AllocRange) -> &[(Size, Prov)] {
         // We have to go back `pointer_size - 1` bytes, as that one would still overlap with
         // the beginning of this range.
         let start = range.start.bytes().saturating_sub(cx.data_layout().pointer_size.bytes() - 1);
-        self.relocations.range(Size::from_bytes(start)..range.end())
+        self.provenance.range(Size::from_bytes(start)..range.end())
     }
 
-    /// Returns whether this allocation has relocations overlapping with the given range.
-    ///
-    /// Note: this function exists to allow `get_relocations` to be private, in order to somewhat
-    /// limit access to relocations outside of the `Allocation` abstraction.
-    ///
-    pub fn has_relocations(&self, cx: &impl HasDataLayout, range: AllocRange) -> bool {
-        !self.get_relocations(cx, range).is_empty()
+    /// Get the provenance of a single byte.
+    fn offset_get_provenance(&self, cx: &impl HasDataLayout, offset: Size) -> Option<Prov> {
+        let prov = self.range_get_provenance(cx, alloc_range(offset, Size::from_bytes(1)));
+        assert!(prov.len() <= 1);
+        prov.first().map(|(_offset, prov)| *prov)
     }
 
-    /// Checks that there are no relocations overlapping with the given range.
-    #[inline(always)]
-    fn check_relocations(&self, cx: &impl HasDataLayout, range: AllocRange) -> AllocResult {
-        if self.has_relocations(cx, range) { Err(AllocError::ReadPointerAsBytes) } else { Ok(()) }
+    /// Returns whether this allocation has progrnance overlapping with the given range.
+    ///
+    /// Note: this function exists to allow `range_get_provenance` to be private, in order to somewhat
+    /// limit access to provenance outside of the `Allocation` abstraction.
+    ///
+    pub fn range_has_provenance(&self, cx: &impl HasDataLayout, range: AllocRange) -> bool {
+        !self.range_get_provenance(cx, range).is_empty()
     }
 
-    /// Removes all relocations inside the given range.
-    /// If there are relocations overlapping with the edges, they
+    /// Removes all provenance inside the given range.
+    /// If there is provenance overlapping with the edges, it
     /// are removed as well *and* the bytes they cover are marked as
     /// uninitialized. This is a somewhat odd "spooky action at a distance",
     /// but it allows strictly more code to run than if we would just error
     /// immediately in that case.
-    fn clear_relocations(&mut self, cx: &impl HasDataLayout, range: AllocRange) -> AllocResult
+    fn clear_provenance(&mut self, cx: &impl HasDataLayout, range: AllocRange) -> AllocResult
     where
         Prov: Provenance,
     {
-        // Find the start and end of the given range and its outermost relocations.
+        // Find the start and end of the given range and its outermost provenance.
         let (first, last) = {
-            // Find all relocations overlapping the given range.
-            let relocations = self.get_relocations(cx, range);
-            if relocations.is_empty() {
+            // Find all provenance overlapping the given range.
+            let provenance = self.range_get_provenance(cx, range);
+            if provenance.is_empty() {
                 return Ok(());
             }
 
             (
-                relocations.first().unwrap().0,
-                relocations.last().unwrap().0 + cx.data_layout().pointer_size,
+                provenance.first().unwrap().0,
+                provenance.last().unwrap().0 + cx.data_layout().pointer_size,
             )
         };
         let start = range.start;
         let end = range.end();
 
-        // We need to handle clearing the relocations from parts of a pointer.
-        // FIXME: Miri should preserve partial relocations; see
+        // We need to handle clearing the provenance from parts of a pointer.
+        // FIXME: Miri should preserve partial provenance; see
         // https://github.com/rust-lang/miri/issues/2181.
         if first < start {
             if Prov::ERR_ON_PARTIAL_PTR_OVERWRITE {
@@ -623,41 +581,32 @@ impl<Prov: Copy, Extra> Allocation<Prov, Extra> {
             self.init_mask.set_range(end, last, false);
         }
 
-        // Forget all the relocations.
-        // Since relocations do not overlap, we know that removing until `last` (exclusive) is fine,
-        // i.e., this will not remove any other relocations just after the ones we care about.
-        self.relocations.0.remove_range(first..last);
+        // Forget all the provenance.
+        // Since provenance do not overlap, we know that removing until `last` (exclusive) is fine,
+        // i.e., this will not remove any other provenance just after the ones we care about.
+        self.provenance.0.remove_range(first..last);
 
         Ok(())
     }
-
-    /// Errors if there are relocations overlapping with the edges of the
-    /// given memory range.
-    #[inline]
-    fn check_relocation_edges(&self, cx: &impl HasDataLayout, range: AllocRange) -> AllocResult {
-        self.check_relocations(cx, alloc_range(range.start, Size::ZERO))?;
-        self.check_relocations(cx, alloc_range(range.end(), Size::ZERO))?;
-        Ok(())
-    }
 }
 
-/// "Relocations" stores the provenance information of pointers stored in memory.
+/// Stores the provenance information of pointers stored in memory.
 #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, TyEncodable, TyDecodable)]
-pub struct Relocations<Prov = AllocId>(SortedMap<Size, Prov>);
+pub struct ProvenanceMap<Prov = AllocId>(SortedMap<Size, Prov>);
 
-impl<Prov> Relocations<Prov> {
+impl<Prov> ProvenanceMap<Prov> {
     pub fn new() -> Self {
-        Relocations(SortedMap::new())
+        ProvenanceMap(SortedMap::new())
     }
 
-    // The caller must guarantee that the given relocations are already sorted
+    // The caller must guarantee that the given provenance list is already sorted
     // by address and contain no duplicates.
     pub fn from_presorted(r: Vec<(Size, Prov)>) -> Self {
-        Relocations(SortedMap::from_presorted_elements(r))
+        ProvenanceMap(SortedMap::from_presorted_elements(r))
     }
 }
 
-impl<Prov> Deref for Relocations<Prov> {
+impl<Prov> Deref for ProvenanceMap<Prov> {
     type Target = SortedMap<Size, Prov>;
 
     fn deref(&self) -> &Self::Target {
@@ -665,36 +614,36 @@ impl<Prov> Deref for Relocations<Prov> {
     }
 }
 
-/// A partial, owned list of relocations to transfer into another allocation.
+/// A partial, owned list of provenance to transfer into another allocation.
 ///
 /// Offsets are already adjusted to the destination allocation.
-pub struct AllocationRelocations<Prov> {
-    dest_relocations: Vec<(Size, Prov)>,
+pub struct AllocationProvenance<Prov> {
+    dest_provenance: Vec<(Size, Prov)>,
 }
 
 impl<Prov: Copy, Extra> Allocation<Prov, Extra> {
-    pub fn prepare_relocation_copy(
+    pub fn prepare_provenance_copy(
         &self,
         cx: &impl HasDataLayout,
         src: AllocRange,
         dest: Size,
         count: u64,
-    ) -> AllocationRelocations<Prov> {
-        let relocations = self.get_relocations(cx, src);
-        if relocations.is_empty() {
-            return AllocationRelocations { dest_relocations: Vec::new() };
+    ) -> AllocationProvenance<Prov> {
+        let provenance = self.range_get_provenance(cx, src);
+        if provenance.is_empty() {
+            return AllocationProvenance { dest_provenance: Vec::new() };
         }
 
         let size = src.size;
-        let mut new_relocations = Vec::with_capacity(relocations.len() * (count as usize));
+        let mut new_provenance = Vec::with_capacity(provenance.len() * (count as usize));
 
         // If `count` is large, this is rather wasteful -- we are allocating a big array here, which
         // is mostly filled with redundant information since it's just N copies of the same `Prov`s
-        // at slightly adjusted offsets. The reason we do this is so that in `mark_relocation_range`
+        // at slightly adjusted offsets. The reason we do this is so that in `mark_provenance_range`
         // we can use `insert_presorted`. That wouldn't work with an `Iterator` that just produces
-        // the right sequence of relocations for all N copies.
+        // the right sequence of provenance for all N copies.
         for i in 0..count {
-            new_relocations.extend(relocations.iter().map(|&(offset, reloc)| {
+            new_provenance.extend(provenance.iter().map(|&(offset, reloc)| {
                 // compute offset for current repetition
                 let dest_offset = dest + size * i; // `Size` operations
                 (
@@ -705,17 +654,17 @@ impl<Prov: Copy, Extra> Allocation<Prov, Extra> {
             }));
         }
 
-        AllocationRelocations { dest_relocations: new_relocations }
+        AllocationProvenance { dest_provenance: new_provenance }
     }
 
-    /// Applies a relocation copy.
-    /// The affected range, as defined in the parameters to `prepare_relocation_copy` is expected
-    /// to be clear of relocations.
+    /// Applies a provenance copy.
+    /// The affected range, as defined in the parameters to `prepare_provenance_copy` is expected
+    /// to be clear of provenance.
     ///
     /// This is dangerous to use as it can violate internal `Allocation` invariants!
     /// It only exists to support an efficient implementation of `mem_copy_repeatedly`.
-    pub fn mark_relocation_range(&mut self, relocations: AllocationRelocations<Prov>) {
-        self.relocations.0.insert_presorted(relocations.dest_relocations);
+    pub fn mark_provenance_range(&mut self, provenance: AllocationProvenance<Prov>) {
+        self.provenance.0.insert_presorted(provenance.dest_provenance);
     }
 }
 
diff --git a/compiler/rustc_middle/src/mir/interpret/error.rs b/compiler/rustc_middle/src/mir/interpret/error.rs
index cecb55578..e4039cc7c 100644
--- a/compiler/rustc_middle/src/mir/interpret/error.rs
+++ b/compiler/rustc_middle/src/mir/interpret/error.rs
@@ -401,14 +401,18 @@ impl fmt::Display for UndefinedBehaviorInfo {
 pub enum UnsupportedOpInfo {
     /// Free-form case. Only for errors that are never caught!
     Unsupported(String),
-    /// Encountered a pointer where we needed raw bytes.
-    ReadPointerAsBytes,
     /// Overwriting parts of a pointer; the resulting state cannot be represented in our
     /// `Allocation` data structure. See <https://github.com/rust-lang/miri/issues/2181>.
     PartialPointerOverwrite(Pointer<AllocId>),
+    /// Attempting to `copy` parts of a pointer to somewhere else; the resulting state cannot be
+    /// represented in our `Allocation` data structure. See
+    /// <https://github.com/rust-lang/miri/issues/2181>.
+    PartialPointerCopy(Pointer<AllocId>),
     //
     // The variants below are only reachable from CTFE/const prop, miri will never emit them.
     //
+    /// Encountered a pointer where we needed raw bytes.
+    ReadPointerAsBytes,
     /// Accessing thread local statics
     ThreadLocalStatic(DefId),
     /// Accessing an unsupported extern static.
@@ -420,10 +424,13 @@ impl fmt::Display for UnsupportedOpInfo {
         use UnsupportedOpInfo::*;
         match self {
             Unsupported(ref msg) => write!(f, "{msg}"),
-            ReadPointerAsBytes => write!(f, "unable to turn pointer into raw bytes"),
             PartialPointerOverwrite(ptr) => {
                 write!(f, "unable to overwrite parts of a pointer in memory at {ptr:?}")
             }
+            PartialPointerCopy(ptr) => {
+                write!(f, "unable to copy parts of a pointer from memory at {ptr:?}")
+            }
+            ReadPointerAsBytes => write!(f, "unable to turn pointer into raw bytes"),
             ThreadLocalStatic(did) => write!(f, "cannot access thread local static ({did:?})"),
             ReadExternStatic(did) => write!(f, "cannot read from extern static ({did:?})"),
         }
diff --git a/compiler/rustc_middle/src/mir/interpret/mod.rs b/compiler/rustc_middle/src/mir/interpret/mod.rs
index 967f8ece1..5e3dfcbcc 100644
--- a/compiler/rustc_middle/src/mir/interpret/mod.rs
+++ b/compiler/rustc_middle/src/mir/interpret/mod.rs
@@ -124,11 +124,11 @@ pub use self::error::{
     UninitBytesAccess, UnsupportedOpInfo,
 };
 
-pub use self::value::{get_slice_bytes, ConstAlloc, ConstValue, Scalar, ScalarMaybeUninit};
+pub use self::value::{get_slice_bytes, ConstAlloc, ConstValue, Scalar};
 
 pub use self::allocation::{
     alloc_range, AllocRange, Allocation, ConstAllocation, InitChunk, InitChunkIter, InitMask,
-    Relocations,
+    ProvenanceMap,
 };
 
 pub use self::pointer::{Pointer, PointerArithmetic, Provenance};
@@ -137,7 +137,7 @@ pub use self::pointer::{Pointer, PointerArithmetic, Provenance};
 /// - A constant
 /// - A static
 #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, TyEncodable, TyDecodable)]
-#[derive(HashStable, Lift)]
+#[derive(HashStable, Lift, TypeFoldable, TypeVisitable)]
 pub struct GlobalId<'tcx> {
     /// For a constant or static, the `Instance` of the item itself.
     /// For a promoted global, the `Instance` of the function they belong to.
diff --git a/compiler/rustc_middle/src/mir/interpret/pointer.rs b/compiler/rustc_middle/src/mir/interpret/pointer.rs
index 384954cbb..95e52e391 100644
--- a/compiler/rustc_middle/src/mir/interpret/pointer.rs
+++ b/compiler/rustc_middle/src/mir/interpret/pointer.rs
@@ -107,8 +107,12 @@ impl<T: HasDataLayout> PointerArithmetic for T {}
 /// pointer), but `derive` adds some unnecessary bounds.
 pub trait Provenance: Copy + fmt::Debug {
     /// Says whether the `offset` field of `Pointer`s with this provenance is the actual physical address.
-    /// If `true, ptr-to-int casts work by simply discarding the provenance.
-    /// If `false`, ptr-to-int casts are not supported. The offset *must* be relative in that case.
+    /// - If `false`, the offset *must* be relative. This means the bytes representing a pointer are
+    ///   different from what the Abstract Machine prescribes, so the interpreter must prevent any
+    ///   operation that would inspect the underlying bytes of a pointer, such as ptr-to-int
+    ///   transmutation. A `ReadPointerAsBytes` error will be raised in such situations.
+    /// - If `true`, the interpreter will permit operations to inspect the underlying bytes of a
+    ///   pointer, and implement ptr-to-int transmutation by stripping provenance.
     const OFFSET_IS_ADDR: bool;
 
     /// We also use this trait to control whether to abort execution when a pointer is being partially overwritten
@@ -125,6 +129,9 @@ pub trait Provenance: Copy + fmt::Debug {
     /// Otherwise this function is best-effort (but must agree with `Machine::ptr_get_alloc`).
     /// (Identifying the offset in that allocation, however, is harder -- use `Memory::ptr_get_alloc` for that.)
     fn get_alloc_id(self) -> Option<AllocId>;
+
+    /// Defines the 'join' of provenance: what happens when doing a pointer load and different bytes have different provenance.
+    fn join(left: Option<Self>, right: Option<Self>) -> Option<Self>;
 }
 
 impl Provenance for AllocId {
@@ -152,6 +159,10 @@ impl Provenance for AllocId {
     fn get_alloc_id(self) -> Option<AllocId> {
         Some(self)
     }
+
+    fn join(_left: Option<Self>, _right: Option<Self>) -> Option<Self> {
+        panic!("merging provenance is not supported when `OFFSET_IS_ADDR` is false")
+    }
 }
 
 /// Represents a pointer in the Miri engine.
diff --git a/compiler/rustc_middle/src/mir/interpret/queries.rs b/compiler/rustc_middle/src/mir/interpret/queries.rs
index 786927e2d..4207988d7 100644
--- a/compiler/rustc_middle/src/mir/interpret/queries.rs
+++ b/compiler/rustc_middle/src/mir/interpret/queries.rs
@@ -63,7 +63,7 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn const_eval_resolve_for_typeck(
         self,
         param_env: ty::ParamEnv<'tcx>,
-        ct: ty::Unevaluated<'tcx>,
+        ct: ty::Unevaluated<'tcx, ()>,
         span: Option<Span>,
     ) -> EvalToValTreeResult<'tcx> {
         // Cannot resolve `Unevaluated` constants that contain inference
@@ -78,7 +78,7 @@ impl<'tcx> TyCtxt<'tcx> {
 
         match ty::Instance::resolve_opt_const_arg(self, param_env, ct.def, ct.substs) {
             Ok(Some(instance)) => {
-                let cid = GlobalId { instance, promoted: ct.promoted };
+                let cid = GlobalId { instance, promoted: None };
                 self.const_eval_global_id_for_typeck(param_env, cid, span)
             }
             Ok(None) => Err(ErrorHandled::TooGeneric),
diff --git a/compiler/rustc_middle/src/mir/interpret/value.rs b/compiler/rustc_middle/src/mir/interpret/value.rs
index 834c114ee..ac5fddb7a 100644
--- a/compiler/rustc_middle/src/mir/interpret/value.rs
+++ b/compiler/rustc_middle/src/mir/interpret/value.rs
@@ -8,7 +8,7 @@ use rustc_apfloat::{
 use rustc_macros::HashStable;
 use rustc_target::abi::{HasDataLayout, Size};
 
-use crate::ty::{Lift, ParamEnv, ScalarInt, Ty, TyCtxt};
+use crate::ty::{ParamEnv, ScalarInt, Ty, TyCtxt};
 
 use super::{
     AllocId, AllocRange, ConstAllocation, InterpResult, Pointer, PointerArithmetic, Provenance,
@@ -27,7 +27,7 @@ pub struct ConstAlloc<'tcx> {
 /// Represents a constant value in Rust. `Scalar` and `Slice` are optimizations for
 /// array length computations, enum discriminants and the pattern matching logic.
 #[derive(Copy, Clone, Debug, Eq, PartialEq, PartialOrd, Ord, TyEncodable, TyDecodable, Hash)]
-#[derive(HashStable)]
+#[derive(HashStable, Lift)]
 pub enum ConstValue<'tcx> {
     /// Used only for types with `layout::abi::Scalar` ABI.
     ///
@@ -53,22 +53,6 @@ pub enum ConstValue<'tcx> {
 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 static_assert_size!(ConstValue<'_>, 32);
 
-impl<'a, 'tcx> Lift<'tcx> for ConstValue<'a> {
-    type Lifted = ConstValue<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<ConstValue<'tcx>> {
-        Some(match self {
-            ConstValue::Scalar(s) => ConstValue::Scalar(s),
-            ConstValue::ZeroSized => ConstValue::ZeroSized,
-            ConstValue::Slice { data, start, end } => {
-                ConstValue::Slice { data: tcx.lift(data)?, start, end }
-            }
-            ConstValue::ByRef { alloc, offset } => {
-                ConstValue::ByRef { alloc: tcx.lift(alloc)?, offset }
-            }
-        })
-    }
-}
-
 impl<'tcx> ConstValue<'tcx> {
     #[inline]
     pub fn try_to_scalar(&self) -> Option<Scalar<AllocId>> {
@@ -79,7 +63,7 @@ impl<'tcx> ConstValue<'tcx> {
     }
 
     pub fn try_to_scalar_int(&self) -> Option<ScalarInt> {
-        Some(self.try_to_scalar()?.assert_int())
+        self.try_to_scalar()?.try_to_int().ok()
     }
 
     pub fn try_to_bits(&self, size: Size) -> Option<u128> {
@@ -130,9 +114,7 @@ pub enum Scalar<Prov = AllocId> {
     /// The raw bytes of a simple value.
     Int(ScalarInt),
 
-    /// A pointer into an `Allocation`. An `Allocation` in the `memory` module has a list of
-    /// relocations, but a `Scalar` is only large enough to contain one, so we just represent the
-    /// relocation and its associated offset together as a `Pointer` here.
+    /// A pointer.
     ///
     /// We also store the size of the pointer, such that a `Scalar` always knows how big it is.
     /// The size is always the pointer size of the current target, but this is not information
@@ -368,6 +350,7 @@ impl<'tcx, Prov: Provenance> Scalar<Prov> {
     }
 
     #[inline(always)]
+    #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
     pub fn assert_int(self) -> ScalarInt {
         self.try_to_int().unwrap()
     }
@@ -389,6 +372,7 @@ impl<'tcx, Prov: Provenance> Scalar<Prov> {
     }
 
     #[inline(always)]
+    #[cfg_attr(debug_assertions, track_caller)] // only in debug builds due to perf (see #98980)
     pub fn assert_bits(self, target_size: Size) -> u128 {
         self.to_bits(target_size).unwrap()
     }
@@ -502,145 +486,12 @@ impl<'tcx, Prov: Provenance> Scalar<Prov> {
     }
 }
 
-#[derive(Clone, Copy, Eq, PartialEq, TyEncodable, TyDecodable, HashStable, Hash)]
-pub enum ScalarMaybeUninit<Prov = AllocId> {
-    Scalar(Scalar<Prov>),
-    Uninit,
-}
-
-#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
-static_assert_size!(ScalarMaybeUninit, 24);
-
-impl<Prov> From<Scalar<Prov>> for ScalarMaybeUninit<Prov> {
-    #[inline(always)]
-    fn from(s: Scalar<Prov>) -> Self {
-        ScalarMaybeUninit::Scalar(s)
-    }
-}
-
-// We want the `Debug` output to be readable as it is used by `derive(Debug)` for
-// all the Miri types.
-impl<Prov: Provenance> fmt::Debug for ScalarMaybeUninit<Prov> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            ScalarMaybeUninit::Uninit => write!(f, "<uninitialized>"),
-            ScalarMaybeUninit::Scalar(s) => write!(f, "{:?}", s),
-        }
-    }
-}
-
-impl<Prov: Provenance> fmt::LowerHex for ScalarMaybeUninit<Prov> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            ScalarMaybeUninit::Uninit => write!(f, "uninitialized bytes"),
-            ScalarMaybeUninit::Scalar(s) => write!(f, "{:x}", s),
-        }
-    }
-}
-
-impl<Prov> ScalarMaybeUninit<Prov> {
-    #[inline]
-    pub fn from_pointer(ptr: Pointer<Prov>, cx: &impl HasDataLayout) -> Self {
-        ScalarMaybeUninit::Scalar(Scalar::from_pointer(ptr, cx))
-    }
-
-    #[inline]
-    pub fn from_maybe_pointer(ptr: Pointer<Option<Prov>>, cx: &impl HasDataLayout) -> Self {
-        ScalarMaybeUninit::Scalar(Scalar::from_maybe_pointer(ptr, cx))
-    }
-
-    #[inline]
-    pub fn check_init<'tcx>(self) -> InterpResult<'tcx, Scalar<Prov>> {
-        match self {
-            ScalarMaybeUninit::Scalar(scalar) => Ok(scalar),
-            ScalarMaybeUninit::Uninit => throw_ub!(InvalidUninitBytes(None)),
-        }
-    }
-}
-
-impl<'tcx, Prov: Provenance> ScalarMaybeUninit<Prov> {
-    #[inline(always)]
-    pub fn to_pointer(self, cx: &impl HasDataLayout) -> InterpResult<'tcx, Pointer<Option<Prov>>> {
-        self.check_init()?.to_pointer(cx)
-    }
-
-    #[inline(always)]
-    pub fn to_bool(self) -> InterpResult<'tcx, bool> {
-        self.check_init()?.to_bool()
-    }
-
-    #[inline(always)]
-    pub fn to_char(self) -> InterpResult<'tcx, char> {
-        self.check_init()?.to_char()
-    }
-
-    #[inline(always)]
-    pub fn to_f32(self) -> InterpResult<'tcx, Single> {
-        self.check_init()?.to_f32()
-    }
-
-    #[inline(always)]
-    pub fn to_f64(self) -> InterpResult<'tcx, Double> {
-        self.check_init()?.to_f64()
-    }
-
-    #[inline(always)]
-    pub fn to_u8(self) -> InterpResult<'tcx, u8> {
-        self.check_init()?.to_u8()
-    }
-
-    #[inline(always)]
-    pub fn to_u16(self) -> InterpResult<'tcx, u16> {
-        self.check_init()?.to_u16()
-    }
-
-    #[inline(always)]
-    pub fn to_u32(self) -> InterpResult<'tcx, u32> {
-        self.check_init()?.to_u32()
-    }
-
-    #[inline(always)]
-    pub fn to_u64(self) -> InterpResult<'tcx, u64> {
-        self.check_init()?.to_u64()
-    }
-
-    #[inline(always)]
-    pub fn to_machine_usize(self, cx: &impl HasDataLayout) -> InterpResult<'tcx, u64> {
-        self.check_init()?.to_machine_usize(cx)
-    }
-
-    #[inline(always)]
-    pub fn to_i8(self) -> InterpResult<'tcx, i8> {
-        self.check_init()?.to_i8()
-    }
-
-    #[inline(always)]
-    pub fn to_i16(self) -> InterpResult<'tcx, i16> {
-        self.check_init()?.to_i16()
-    }
-
-    #[inline(always)]
-    pub fn to_i32(self) -> InterpResult<'tcx, i32> {
-        self.check_init()?.to_i32()
-    }
-
-    #[inline(always)]
-    pub fn to_i64(self) -> InterpResult<'tcx, i64> {
-        self.check_init()?.to_i64()
-    }
-
-    #[inline(always)]
-    pub fn to_machine_isize(self, cx: &impl HasDataLayout) -> InterpResult<'tcx, i64> {
-        self.check_init()?.to_machine_isize(cx)
-    }
-}
-
 /// Gets the bytes of a constant slice value.
 pub fn get_slice_bytes<'tcx>(cx: &impl HasDataLayout, val: ConstValue<'tcx>) -> &'tcx [u8] {
     if let ConstValue::Slice { data, start, end } = val {
         let len = end - start;
         data.inner()
-            .get_bytes(
+            .get_bytes_strip_provenance(
                 cx,
                 AllocRange { start: Size::from_bytes(start), size: Size::from_bytes(len) },
             )
diff --git a/compiler/rustc_middle/src/mir/mod.rs b/compiler/rustc_middle/src/mir/mod.rs
index 7ab71f900..3d7a6230e 100644
--- a/compiler/rustc_middle/src/mir/mod.rs
+++ b/compiler/rustc_middle/src/mir/mod.rs
@@ -3,7 +3,7 @@
 //! [rustc dev guide]: https://rustc-dev-guide.rust-lang.org/mir/index.html
 
 use crate::mir::interpret::{
-    AllocRange, ConstAllocation, ConstValue, GlobalAlloc, LitToConstInput, Scalar,
+    AllocRange, ConstAllocation, ConstValue, ErrorHandled, GlobalAlloc, LitToConstInput, Scalar,
 };
 use crate::mir::visit::MirVisitable;
 use crate::ty::codec::{TyDecoder, TyEncoder};
@@ -18,7 +18,7 @@ use rustc_data_structures::captures::Captures;
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir::def::{CtorKind, Namespace};
 use rustc_hir::def_id::{DefId, LocalDefId, CRATE_DEF_ID};
-use rustc_hir::{self, GeneratorKind};
+use rustc_hir::{self, GeneratorKind, ImplicitSelfKind};
 use rustc_hir::{self as hir, HirId};
 use rustc_session::Session;
 use rustc_target::abi::{Size, VariantIdx};
@@ -128,8 +128,20 @@ pub trait MirPass<'tcx> {
 
 impl MirPhase {
     /// Gets the index of the current MirPhase within the set of all `MirPhase`s.
+    ///
+    /// FIXME(JakobDegen): Return a `(usize, usize)` instead.
     pub fn phase_index(&self) -> usize {
-        *self as usize
+        const BUILT_PHASE_COUNT: usize = 1;
+        const ANALYSIS_PHASE_COUNT: usize = 2;
+        match self {
+            MirPhase::Built => 1,
+            MirPhase::Analysis(analysis_phase) => {
+                1 + BUILT_PHASE_COUNT + (*analysis_phase as usize)
+            }
+            MirPhase::Runtime(runtime_phase) => {
+                1 + BUILT_PHASE_COUNT + ANALYSIS_PHASE_COUNT + (*runtime_phase as usize)
+            }
+        }
     }
 }
 
@@ -332,11 +344,6 @@ impl<'tcx> Body<'tcx> {
     }
 
     #[inline]
-    pub fn basic_blocks(&self) -> &IndexVec<BasicBlock, BasicBlockData<'tcx>> {
-        &self.basic_blocks
-    }
-
-    #[inline]
     pub fn basic_blocks_mut(&mut self) -> &mut IndexVec<BasicBlock, BasicBlockData<'tcx>> {
         self.basic_blocks.as_mut()
     }
@@ -490,7 +497,7 @@ impl<'tcx> Index<BasicBlock> for Body<'tcx> {
 
     #[inline]
     fn index(&self, index: BasicBlock) -> &BasicBlockData<'tcx> {
-        &self.basic_blocks()[index]
+        &self.basic_blocks[index]
     }
 }
 
@@ -646,22 +653,6 @@ pub enum BindingForm<'tcx> {
     RefForGuard,
 }
 
-/// Represents what type of implicit self a function has, if any.
-#[derive(Clone, Copy, PartialEq, Debug, TyEncodable, TyDecodable, HashStable)]
-pub enum ImplicitSelfKind {
-    /// Represents a `fn x(self);`.
-    Imm,
-    /// Represents a `fn x(mut self);`.
-    Mut,
-    /// Represents a `fn x(&self);`.
-    ImmRef,
-    /// Represents a `fn x(&mut self);`.
-    MutRef,
-    /// Represents when a function does not have a self argument or
-    /// when a function has a `self: X` argument.
-    None,
-}
-
 TrivialTypeTraversalAndLiftImpls! { BindingForm<'tcx>, }
 
 mod binding_form_impl {
@@ -832,10 +823,6 @@ pub struct LocalDecl<'tcx> {
     pub source_info: SourceInfo,
 }
 
-// `LocalDecl` is used a lot. Make sure it doesn't unintentionally get bigger.
-#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
-static_assert_size!(LocalDecl<'_>, 56);
-
 /// Extra information about a some locals that's used for diagnostics and for
 /// classifying variables into local variables, statics, etc, which is needed e.g.
 /// for unsafety checking.
@@ -1310,10 +1297,6 @@ pub struct Statement<'tcx> {
     pub kind: StatementKind<'tcx>,
 }
 
-// `Statement` is used a lot. Make sure it doesn't unintentionally get bigger.
-#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
-static_assert_size!(Statement<'_>, 32);
-
 impl Statement<'_> {
     /// Changes a statement to a nop. This is both faster than deleting instructions and avoids
     /// invalidating statement indices in `Location`s.
@@ -1363,13 +1346,7 @@ impl Debug for Statement<'_> {
                 write!(fmt, "Coverage::{:?} for {:?}", kind, rgn)
             }
             Coverage(box ref coverage) => write!(fmt, "Coverage::{:?}", coverage.kind),
-            CopyNonOverlapping(box crate::mir::CopyNonOverlapping {
-                ref src,
-                ref dst,
-                ref count,
-            }) => {
-                write!(fmt, "copy_nonoverlapping(src={:?}, dst={:?}, count={:?})", src, dst, count)
-            }
+            Intrinsic(box ref intrinsic) => write!(fmt, "{intrinsic}"),
             Nop => write!(fmt, "nop"),
         }
     }
@@ -1450,7 +1427,7 @@ pub struct PlaceRef<'tcx> {
 // Once we stop implementing `Ord` for `DefId`,
 // this impl will be unnecessary. Until then, we'll
 // leave this impl in place to prevent re-adding a
-// dependnecy on the `Ord` impl for `DefId`
+// dependency on the `Ord` impl for `DefId`
 impl<'tcx> !PartialOrd for PlaceRef<'tcx> {}
 
 impl<'tcx> Place<'tcx> {
@@ -1471,7 +1448,9 @@ impl<'tcx> Place<'tcx> {
     /// It's guaranteed to be in the first place
     pub fn has_deref(&self) -> bool {
         // To make sure this is not accidently used in wrong mir phase
-        debug_assert!(!self.projection[1..].contains(&PlaceElem::Deref));
+        debug_assert!(
+            self.projection.is_empty() || !self.projection[1..].contains(&PlaceElem::Deref)
+        );
         self.projection.first() == Some(&PlaceElem::Deref)
     }
 
@@ -1531,6 +1510,7 @@ impl<'tcx> Place<'tcx> {
 }
 
 impl From<Local> for Place<'_> {
+    #[inline]
     fn from(local: Local) -> Self {
         Place { local, projection: List::empty() }
     }
@@ -1838,6 +1818,7 @@ impl<'tcx> Rvalue<'tcx> {
             // While the model is undecided, we should be conservative. See
             // <https://www.ralfj.de/blog/2022/04/11/provenance-exposed.html>
             Rvalue::Cast(CastKind::PointerExposeAddress, _, _) => false,
+            Rvalue::Cast(CastKind::DynStar, _, _) => false,
 
             Rvalue::Use(_)
             | Rvalue::CopyForDeref(_)
@@ -2047,6 +2028,7 @@ impl<'tcx> Debug for Rvalue<'tcx> {
 /// particular, one must be wary of `NaN`!
 
 #[derive(Clone, Copy, PartialEq, TyEncodable, TyDecodable, Hash, HashStable)]
+#[derive(TypeFoldable, TypeVisitable)]
 pub struct Constant<'tcx> {
     pub span: Span,
 
@@ -2065,6 +2047,10 @@ pub struct Constant<'tcx> {
 pub enum ConstantKind<'tcx> {
     /// This constant came from the type system
     Ty(ty::Const<'tcx>),
+
+    /// An unevaluated mir constant which is not part of the type system.
+    Unevaluated(ty::Unevaluated<'tcx, Option<Promoted>>, Ty<'tcx>),
+
     /// This constant cannot go back into the type system, as it represents
     /// something the type system cannot handle (e.g. pointers).
     Val(interpret::ConstValue<'tcx>, Ty<'tcx>),
@@ -2090,20 +2076,11 @@ impl<'tcx> Constant<'tcx> {
 }
 
 impl<'tcx> ConstantKind<'tcx> {
-    /// Returns `None` if the constant is not trivially safe for use in the type system.
-    #[inline]
-    pub fn const_for_ty(&self) -> Option<ty::Const<'tcx>> {
-        match self {
-            ConstantKind::Ty(c) => Some(*c),
-            ConstantKind::Val(..) => None,
-        }
-    }
-
     #[inline(always)]
     pub fn ty(&self) -> Ty<'tcx> {
         match self {
             ConstantKind::Ty(c) => c.ty(),
-            ConstantKind::Val(_, ty) => *ty,
+            ConstantKind::Val(_, ty) | ConstantKind::Unevaluated(_, ty) => *ty,
         }
     }
 
@@ -2115,6 +2092,7 @@ impl<'tcx> ConstantKind<'tcx> {
                 _ => None,
             },
             ConstantKind::Val(val, _) => Some(val),
+            ConstantKind::Unevaluated(..) => None,
         }
     }
 
@@ -2129,6 +2107,7 @@ impl<'tcx> ConstantKind<'tcx> {
                 _ => None,
             },
             ConstantKind::Val(val, _) => val.try_to_scalar(),
+            ConstantKind::Unevaluated(..) => None,
         }
     }
 
@@ -2161,6 +2140,14 @@ impl<'tcx> ConstantKind<'tcx> {
                 }
             }
             Self::Val(_, _) => self,
+            Self::Unevaluated(uneval, ty) => {
+                // FIXME: We might want to have a `try_eval`-like function on `Unevaluated`
+                match tcx.const_eval_resolve(param_env, uneval, None) {
+                    Ok(val) => Self::Val(val, ty),
+                    Err(ErrorHandled::TooGeneric | ErrorHandled::Linted) => self,
+                    Err(_) => Self::Ty(tcx.const_error(ty)),
+                }
+            }
         }
     }
 
@@ -2186,6 +2173,18 @@ impl<'tcx> ConstantKind<'tcx> {
                     tcx.layout_of(param_env.with_reveal_all_normalized(tcx).and(ty)).ok()?.size;
                 val.try_to_bits(size)
             }
+            Self::Unevaluated(uneval, ty) => {
+                match tcx.const_eval_resolve(param_env, *uneval, None) {
+                    Ok(val) => {
+                        let size = tcx
+                            .layout_of(param_env.with_reveal_all_normalized(tcx).and(*ty))
+                            .ok()?
+                            .size;
+                        val.try_to_bits(size)
+                    }
+                    Err(_) => None,
+                }
+            }
         }
     }
 
@@ -2194,6 +2193,12 @@ impl<'tcx> ConstantKind<'tcx> {
         match self {
             Self::Ty(ct) => ct.try_eval_bool(tcx, param_env),
             Self::Val(val, _) => val.try_to_bool(),
+            Self::Unevaluated(uneval, _) => {
+                match tcx.const_eval_resolve(param_env, *uneval, None) {
+                    Ok(val) => val.try_to_bool(),
+                    Err(_) => None,
+                }
+            }
         }
     }
 
@@ -2202,6 +2207,12 @@ impl<'tcx> ConstantKind<'tcx> {
         match self {
             Self::Ty(ct) => ct.try_eval_usize(tcx, param_env),
             Self::Val(val, _) => val.try_to_machine_usize(tcx),
+            Self::Unevaluated(uneval, _) => {
+                match tcx.const_eval_resolve(param_env, *uneval, None) {
+                    Ok(val) => val.try_to_machine_usize(tcx),
+                    Err(_) => None,
+                }
+            }
         }
     }
 
@@ -2259,7 +2270,7 @@ impl<'tcx> ConstantKind<'tcx> {
         Self::from_opt_const_arg_anon_const(tcx, ty::WithOptConstParam::unknown(def_id), param_env)
     }
 
-    #[instrument(skip(tcx), level = "debug")]
+    #[instrument(skip(tcx), level = "debug", ret)]
     pub fn from_inline_const(tcx: TyCtxt<'tcx>, def_id: LocalDefId) -> Self {
         let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
         let body_id = match tcx.hir().get(hir_id) {
@@ -2297,21 +2308,19 @@ impl<'tcx> ConstantKind<'tcx> {
         let substs =
             ty::InlineConstSubsts::new(tcx, ty::InlineConstSubstsParts { parent_substs, ty })
                 .substs;
-        let uneval_const = tcx.mk_const(ty::ConstS {
-            kind: ty::ConstKind::Unevaluated(ty::Unevaluated {
-                def: ty::WithOptConstParam::unknown(def_id).to_global(),
-                substs,
-                promoted: None,
-            }),
-            ty,
-        });
-        debug!(?uneval_const);
-        debug_assert!(!uneval_const.has_free_regions());
 
-        Self::Ty(uneval_const)
+        let uneval = ty::Unevaluated {
+            def: ty::WithOptConstParam::unknown(def_id).to_global(),
+            substs,
+            promoted: None,
+        };
+
+        debug_assert!(!uneval.has_free_regions());
+
+        Self::Unevaluated(uneval, ty)
     }
 
-    #[instrument(skip(tcx), level = "debug")]
+    #[instrument(skip(tcx), level = "debug", ret)]
     fn from_opt_const_arg_anon_const(
         tcx: TyCtxt<'tcx>,
         def: ty::WithOptConstParam<LocalDefId>,
@@ -2394,24 +2403,21 @@ impl<'tcx> ConstantKind<'tcx> {
 
         match tcx.const_eval_resolve(param_env, uneval, Some(span)) {
             Ok(val) => {
-                debug!("evaluated const value: {:?}", val);
+                debug!("evaluated const value");
                 Self::Val(val, ty)
             }
             Err(_) => {
                 debug!("error encountered during evaluation");
                 // Error was handled in `const_eval_resolve`. Here we just create a
                 // new unevaluated const and error hard later in codegen
-                let ty_const = tcx.mk_const(ty::ConstS {
-                    kind: ty::ConstKind::Unevaluated(ty::Unevaluated {
+                Self::Unevaluated(
+                    ty::Unevaluated {
                         def: def.to_global(),
                         substs: InternalSubsts::identity_for_item(tcx, def.did.to_def_id()),
                         promoted: None,
-                    }),
+                    },
                     ty,
-                });
-                debug!(?ty_const);
-
-                Self::Ty(ty_const)
+                )
             }
         }
     }
@@ -2422,6 +2428,7 @@ impl<'tcx> ConstantKind<'tcx> {
                 let const_val = tcx.valtree_to_const_val((c.ty(), valtree));
                 Self::Val(const_val, c.ty())
             }
+            ty::ConstKind::Unevaluated(uv) => Self::Unevaluated(uv.expand(), c.ty()),
             _ => Self::Ty(c),
         }
     }
@@ -2576,8 +2583,6 @@ impl UserTypeProjection {
     }
 }
 
-TrivialTypeTraversalAndLiftImpls! { ProjectionKind, }
-
 impl<'tcx> TypeFoldable<'tcx> for UserTypeProjection {
     fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
         Ok(UserTypeProjection {
@@ -2622,6 +2627,11 @@ impl<'tcx> Display for ConstantKind<'tcx> {
         match *self {
             ConstantKind::Ty(c) => pretty_print_const(c, fmt, true),
             ConstantKind::Val(val, ty) => pretty_print_const_value(val, ty, fmt, true),
+            // FIXME(valtrees): Correctly print mir constants.
+            ConstantKind::Unevaluated(..) => {
+                fmt.write_str("_")?;
+                Ok(())
+            }
         }
     }
 }
@@ -2643,15 +2653,7 @@ fn pretty_print_const<'tcx>(
 }
 
 fn pretty_print_byte_str(fmt: &mut Formatter<'_>, byte_str: &[u8]) -> fmt::Result {
-    fmt.write_str("b\"")?;
-    for &c in byte_str {
-        for e in std::ascii::escape_default(c) {
-            fmt.write_char(e as char)?;
-        }
-    }
-    fmt.write_str("\"")?;
-
-    Ok(())
+    write!(fmt, "b\"{}\"", byte_str.escape_ascii())
 }
 
 fn comma_sep<'tcx>(fmt: &mut Formatter<'_>, elems: Vec<ConstantKind<'tcx>>) -> fmt::Result {
@@ -2691,8 +2693,8 @@ fn pretty_print_const_value<'tcx>(
                 match inner.kind() {
                     ty::Slice(t) => {
                         if *t == u8_type {
-                            // The `inspect` here is okay since we checked the bounds, and there are
-                            // no relocations (we have an active slice reference here). We don't use
+                            // The `inspect` here is okay since we checked the bounds, and `u8` carries
+                            // no provenance (we have an active slice reference here). We don't use
                             // this result to affect interpreter execution.
                             let byte_str = data
                                 .inner()
@@ -2702,8 +2704,8 @@ fn pretty_print_const_value<'tcx>(
                         }
                     }
                     ty::Str => {
-                        // The `inspect` here is okay since we checked the bounds, and there are no
-                        // relocations (we have an active `str` reference here). We don't use this
+                        // The `inspect` here is okay since we checked the bounds, and `str` carries
+                        // no provenance (we have an active `str` reference here). We don't use this
                         // result to affect interpreter execution.
                         let slice = data
                             .inner()
@@ -2718,7 +2720,7 @@ fn pretty_print_const_value<'tcx>(
                 let n = n.kind().try_to_bits(tcx.data_layout.pointer_size).unwrap();
                 // cast is ok because we already checked for pointer size (32 or 64 bit) above
                 let range = AllocRange { start: offset, size: Size::from_bytes(n) };
-                let byte_str = alloc.inner().get_bytes(&tcx, range).unwrap();
+                let byte_str = alloc.inner().get_bytes_strip_provenance(&tcx, range).unwrap();
                 fmt.write_str("*")?;
                 pretty_print_byte_str(fmt, byte_str)?;
                 return Ok(());
@@ -2898,3 +2900,17 @@ impl Location {
         }
     }
 }
+
+// Some nodes are used a lot. Make sure they don't unintentionally get bigger.
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
+mod size_asserts {
+    use super::*;
+    use rustc_data_structures::static_assert_size;
+    // These are in alphabetical order, which is easy to maintain.
+    static_assert_size!(BasicBlockData<'_>, 144);
+    static_assert_size!(LocalDecl<'_>, 56);
+    static_assert_size!(Statement<'_>, 32);
+    static_assert_size!(StatementKind<'_>, 16);
+    static_assert_size!(Terminator<'_>, 112);
+    static_assert_size!(TerminatorKind<'_>, 96);
+}
diff --git a/compiler/rustc_middle/src/mir/patch.rs b/compiler/rustc_middle/src/mir/patch.rs
index 15496842d..24fe3b472 100644
--- a/compiler/rustc_middle/src/mir/patch.rs
+++ b/compiler/rustc_middle/src/mir/patch.rs
@@ -19,7 +19,7 @@ pub struct MirPatch<'tcx> {
 impl<'tcx> MirPatch<'tcx> {
     pub fn new(body: &Body<'tcx>) -> Self {
         let mut result = MirPatch {
-            patch_map: IndexVec::from_elem(None, body.basic_blocks()),
+            patch_map: IndexVec::from_elem(None, &body.basic_blocks),
             new_blocks: vec![],
             new_statements: vec![],
             new_locals: vec![],
@@ -29,7 +29,7 @@ impl<'tcx> MirPatch<'tcx> {
         };
 
         // Check if we already have a resume block
-        for (bb, block) in body.basic_blocks().iter_enumerated() {
+        for (bb, block) in body.basic_blocks.iter_enumerated() {
             if let TerminatorKind::Resume = block.terminator().kind && block.statements.is_empty() {
                 result.resume_block = Some(bb);
                 break;
@@ -61,14 +61,14 @@ impl<'tcx> MirPatch<'tcx> {
     }
 
     pub fn terminator_loc(&self, body: &Body<'tcx>, bb: BasicBlock) -> Location {
-        let offset = match bb.index().checked_sub(body.basic_blocks().len()) {
+        let offset = match bb.index().checked_sub(body.basic_blocks.len()) {
             Some(index) => self.new_blocks[index].statements.len(),
             None => body[bb].statements.len(),
         };
         Location { block: bb, statement_index: offset }
     }
 
-    pub fn new_local_with_info(
+    pub fn new_internal_with_info(
         &mut self,
         ty: Ty<'tcx>,
         span: Span,
@@ -76,14 +76,17 @@ impl<'tcx> MirPatch<'tcx> {
     ) -> Local {
         let index = self.next_local;
         self.next_local += 1;
-        let mut new_decl = LocalDecl::new(ty, span);
+        let mut new_decl = LocalDecl::new(ty, span).internal();
         new_decl.local_info = local_info;
         self.new_locals.push(new_decl);
         Local::new(index as usize)
     }
 
     pub fn new_temp(&mut self, ty: Ty<'tcx>, span: Span) -> Local {
-        self.new_local_with_info(ty, span, None)
+        let index = self.next_local;
+        self.next_local += 1;
+        self.new_locals.push(LocalDecl::new(ty, span));
+        Local::new(index as usize)
     }
 
     pub fn new_internal(&mut self, ty: Ty<'tcx>, span: Span) -> Local {
@@ -126,7 +129,7 @@ impl<'tcx> MirPatch<'tcx> {
         debug!(
             "MirPatch: {} new blocks, starting from index {}",
             self.new_blocks.len(),
-            body.basic_blocks().len()
+            body.basic_blocks.len()
         );
         let bbs = if self.patch_map.is_empty() && self.new_blocks.is_empty() {
             body.basic_blocks.as_mut_preserves_cfg()
@@ -147,7 +150,6 @@ impl<'tcx> MirPatch<'tcx> {
 
         let mut delta = 0;
         let mut last_bb = START_BLOCK;
-        let mut stmts_and_targets: Vec<(Statement<'_>, BasicBlock)> = Vec::new();
         for (mut loc, stmt) in new_statements {
             if loc.block != last_bb {
                 delta = 0;
@@ -156,27 +158,11 @@ impl<'tcx> MirPatch<'tcx> {
             debug!("MirPatch: adding statement {:?} at loc {:?}+{}", stmt, loc, delta);
             loc.statement_index += delta;
             let source_info = Self::source_info_for_index(&body[loc.block], loc);
-
-            // For mir-opt `Derefer` to work in all cases we need to
-            // get terminator's targets and apply the statement to all of them.
-            if loc.statement_index > body[loc.block].statements.len() {
-                let term = body[loc.block].terminator();
-                for i in term.successors() {
-                    stmts_and_targets.push((Statement { source_info, kind: stmt.clone() }, i));
-                }
-                delta += 1;
-                continue;
-            }
-
             body[loc.block]
                 .statements
                 .insert(loc.statement_index, Statement { source_info, kind: stmt });
             delta += 1;
         }
-
-        for (stmt, target) in stmts_and_targets.into_iter().rev() {
-            body[target].statements.insert(0, stmt);
-        }
     }
 
     pub fn source_info_for_index(data: &BasicBlockData<'_>, loc: Location) -> SourceInfo {
@@ -187,7 +173,7 @@ impl<'tcx> MirPatch<'tcx> {
     }
 
     pub fn source_info_for_location(&self, body: &Body<'tcx>, loc: Location) -> SourceInfo {
-        let data = match loc.block.index().checked_sub(body.basic_blocks().len()) {
+        let data = match loc.block.index().checked_sub(body.basic_blocks.len()) {
             Some(new) => &self.new_blocks[new],
             None => &body[loc.block],
         };
diff --git a/compiler/rustc_middle/src/mir/pretty.rs b/compiler/rustc_middle/src/mir/pretty.rs
index 0ce41337b..0b42137d4 100644
--- a/compiler/rustc_middle/src/mir/pretty.rs
+++ b/compiler/rustc_middle/src/mir/pretty.rs
@@ -318,10 +318,10 @@ where
     F: FnMut(PassWhere, &mut dyn Write) -> io::Result<()>,
 {
     write_mir_intro(tcx, body, w)?;
-    for block in body.basic_blocks().indices() {
+    for block in body.basic_blocks.indices() {
         extra_data(PassWhere::BeforeBlock(block), w)?;
         write_basic_block(tcx, block, body, extra_data, w)?;
-        if block.index() + 1 != body.basic_blocks().len() {
+        if block.index() + 1 != body.basic_blocks.len() {
             writeln!(w)?;
         }
     }
@@ -464,12 +464,14 @@ impl<'tcx> Visitor<'tcx> for ExtraComments<'tcx> {
             let val = match literal {
                 ConstantKind::Ty(ct) => match ct.kind() {
                     ty::ConstKind::Param(p) => format!("Param({})", p),
-                    ty::ConstKind::Unevaluated(uv) => format!(
-                        "Unevaluated({}, {:?}, {:?})",
-                        self.tcx.def_path_str(uv.def.did),
-                        uv.substs,
-                        uv.promoted,
-                    ),
+                    ty::ConstKind::Unevaluated(uv) => {
+                        format!(
+                            "Unevaluated({}, {:?}, {:?})",
+                            self.tcx.def_path_str(uv.def.did),
+                            uv.substs,
+                            uv.promoted,
+                        )
+                    }
                     ty::ConstKind::Value(val) => format!("Value({})", fmt_valtree(&val)),
                     ty::ConstKind::Error(_) => "Error".to_string(),
                     // These variants shouldn't exist in the MIR.
@@ -477,6 +479,14 @@ impl<'tcx> Visitor<'tcx> for ExtraComments<'tcx> {
                     | ty::ConstKind::Infer(_)
                     | ty::ConstKind::Bound(..) => bug!("unexpected MIR constant: {:?}", literal),
                 },
+                ConstantKind::Unevaluated(uv, _) => {
+                    format!(
+                        "Unevaluated({}, {:?}, {:?})",
+                        self.tcx.def_path_str(uv.def.did),
+                        uv.substs,
+                        uv.promoted,
+                    )
+                }
                 // To keep the diffs small, we render this like we render `ty::Const::Value`.
                 //
                 // This changes once `ty::Const::Value` is represented using valtrees.
@@ -676,7 +686,7 @@ pub fn write_allocations<'tcx>(
     fn alloc_ids_from_alloc(
         alloc: ConstAllocation<'_>,
     ) -> impl DoubleEndedIterator<Item = AllocId> + '_ {
-        alloc.inner().relocations().values().map(|id| *id)
+        alloc.inner().provenance().values().map(|id| *id)
     }
 
     fn alloc_ids_from_const_val(val: ConstValue<'_>) -> impl Iterator<Item = AllocId> + '_ {
@@ -696,9 +706,9 @@ pub fn write_allocations<'tcx>(
     struct CollectAllocIds(BTreeSet<AllocId>);
 
     impl<'tcx> Visitor<'tcx> for CollectAllocIds {
-        fn visit_constant(&mut self, c: &Constant<'tcx>, loc: Location) {
+        fn visit_constant(&mut self, c: &Constant<'tcx>, _: Location) {
             match c.literal {
-                ConstantKind::Ty(c) => self.visit_const(c, loc),
+                ConstantKind::Ty(_) | ConstantKind::Unevaluated(..) => {}
                 ConstantKind::Val(val, _) => {
                     self.0.extend(alloc_ids_from_const_val(val));
                 }
@@ -778,7 +788,7 @@ pub fn write_allocations<'tcx>(
 /// If the allocation is small enough to fit into a single line, no start address is given.
 /// After the hex dump, an ascii dump follows, replacing all unprintable characters (control
 /// characters or characters whose value is larger than 127) with a `.`
-/// This also prints relocations adequately.
+/// This also prints provenance adequately.
 pub fn display_allocation<'a, 'tcx, Prov, Extra>(
     tcx: TyCtxt<'tcx>,
     alloc: &'a Allocation<Prov, Extra>,
@@ -873,34 +883,34 @@ fn write_allocation_bytes<'tcx, Prov: Provenance, Extra>(
         if i != line_start {
             write!(w, " ")?;
         }
-        if let Some(&prov) = alloc.relocations().get(&i) {
-            // Memory with a relocation must be defined
+        if let Some(&prov) = alloc.provenance().get(&i) {
+            // Memory with provenance must be defined
             assert!(alloc.init_mask().is_range_initialized(i, i + ptr_size).is_ok());
             let j = i.bytes_usize();
             let offset = alloc
                 .inspect_with_uninit_and_ptr_outside_interpreter(j..j + ptr_size.bytes_usize());
             let offset = read_target_uint(tcx.data_layout.endian, offset).unwrap();
             let offset = Size::from_bytes(offset);
-            let relocation_width = |bytes| bytes * 3;
+            let provenance_width = |bytes| bytes * 3;
             let ptr = Pointer::new(prov, offset);
             let mut target = format!("{:?}", ptr);
-            if target.len() > relocation_width(ptr_size.bytes_usize() - 1) {
+            if target.len() > provenance_width(ptr_size.bytes_usize() - 1) {
                 // This is too long, try to save some space.
                 target = format!("{:#?}", ptr);
             }
             if ((i - line_start) + ptr_size).bytes_usize() > BYTES_PER_LINE {
-                // This branch handles the situation where a relocation starts in the current line
+                // This branch handles the situation where a provenance starts in the current line
                 // but ends in the next one.
                 let remainder = Size::from_bytes(BYTES_PER_LINE) - (i - line_start);
                 let overflow = ptr_size - remainder;
-                let remainder_width = relocation_width(remainder.bytes_usize()) - 2;
-                let overflow_width = relocation_width(overflow.bytes_usize() - 1) + 1;
+                let remainder_width = provenance_width(remainder.bytes_usize()) - 2;
+                let overflow_width = provenance_width(overflow.bytes_usize() - 1) + 1;
                 ascii.push('╾');
                 for _ in 0..remainder.bytes() - 1 {
                     ascii.push('─');
                 }
                 if overflow_width > remainder_width && overflow_width >= target.len() {
-                    // The case where the relocation fits into the part in the next line
+                    // The case where the provenance fits into the part in the next line
                     write!(w, "╾{0:─^1$}", "", remainder_width)?;
                     line_start =
                         write_allocation_newline(w, line_start, &ascii, pos_width, prefix)?;
@@ -921,11 +931,11 @@ fn write_allocation_bytes<'tcx, Prov: Provenance, Extra>(
                 i += ptr_size;
                 continue;
             } else {
-                // This branch handles a relocation that starts and ends in the current line.
-                let relocation_width = relocation_width(ptr_size.bytes_usize() - 1);
-                oversized_ptr(&mut target, relocation_width);
+                // This branch handles a provenance that starts and ends in the current line.
+                let provenance_width = provenance_width(ptr_size.bytes_usize() - 1);
+                oversized_ptr(&mut target, provenance_width);
                 ascii.push('╾');
-                write!(w, "╾{0:─^1$}╼", target, relocation_width)?;
+                write!(w, "╾{0:─^1$}╼", target, provenance_width)?;
                 for _ in 0..ptr_size.bytes() - 2 {
                     ascii.push('─');
                 }
@@ -935,7 +945,7 @@ fn write_allocation_bytes<'tcx, Prov: Provenance, Extra>(
         } else if alloc.init_mask().is_range_initialized(i, i + Size::from_bytes(1)).is_ok() {
             let j = i.bytes_usize();
 
-            // Checked definedness (and thus range) and relocations. This access also doesn't
+            // Checked definedness (and thus range) and provenance. This access also doesn't
             // influence interpreter execution but is only for debugging.
             let c = alloc.inspect_with_uninit_and_ptr_outside_interpreter(j..j + 1)[0];
             write!(w, "{:02x}", c)?;
diff --git a/compiler/rustc_middle/src/mir/query.rs b/compiler/rustc_middle/src/mir/query.rs
index dd9f8795f..d89efe2b3 100644
--- a/compiler/rustc_middle/src/mir/query.rs
+++ b/compiler/rustc_middle/src/mir/query.rs
@@ -2,7 +2,7 @@
 
 use crate::mir::{Body, ConstantKind, Promoted};
 use crate::ty::{self, OpaqueHiddenType, Ty, TyCtxt};
-use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::fx::FxHashSet;
 use rustc_data_structures::vec_map::VecMap;
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir as hir;
@@ -115,21 +115,6 @@ pub enum UnusedUnsafe {
     /// `unsafe` block nested under another (used) `unsafe` block
     /// > ``… because it's nested under this `unsafe` block``
     InUnsafeBlock(hir::HirId),
-    /// `unsafe` block nested under `unsafe fn`
-    /// > ``… because it's nested under this `unsafe fn` ``
-    ///
-    /// the second HirId here indicates the first usage of the `unsafe` block,
-    /// which allows retrieval of the LintLevelSource for why that operation would
-    /// have been permitted without the block
-    InUnsafeFn(hir::HirId, hir::HirId),
-}
-
-#[derive(Copy, Clone, PartialEq, TyEncodable, TyDecodable, HashStable, Debug)]
-pub enum UsedUnsafeBlockData {
-    SomeDisallowedInUnsafeFn,
-    // the HirId here indicates the first usage of the `unsafe` block
-    // (i.e. the one that's first encountered in the MIR traversal of the unsafety check)
-    AllAllowedInUnsafeFn(hir::HirId),
 }
 
 #[derive(TyEncodable, TyDecodable, HashStable, Debug)]
@@ -138,10 +123,7 @@ pub struct UnsafetyCheckResult {
     pub violations: Vec<UnsafetyViolation>,
 
     /// Used `unsafe` blocks in this function. This is used for the "unused_unsafe" lint.
-    ///
-    /// The keys are the used `unsafe` blocks, the UnusedUnsafeKind indicates whether
-    /// or not any of the usages happen at a place that doesn't allow `unsafe_op_in_unsafe_fn`.
-    pub used_unsafe_blocks: FxHashMap<hir::HirId, UsedUnsafeBlockData>,
+    pub used_unsafe_blocks: FxHashSet<hir::HirId>,
 
     /// This is `Some` iff the item is not a closure.
     pub unused_unsafes: Option<Vec<(hir::HirId, UnusedUnsafe)>>,
@@ -345,7 +327,7 @@ rustc_data_structures::static_assert_size!(ConstraintCategory<'_>, 16);
 ///
 /// See also `rustc_const_eval::borrow_check::constraints`.
 #[derive(Copy, Clone, Debug, Eq, PartialEq, PartialOrd, Ord, Hash)]
-#[derive(TyEncodable, TyDecodable, HashStable)]
+#[derive(TyEncodable, TyDecodable, HashStable, Lift, TypeVisitable, TypeFoldable)]
 pub enum ConstraintCategory<'tcx> {
     Return(ReturnConstraint),
     Yield,
@@ -387,7 +369,7 @@ pub enum ConstraintCategory<'tcx> {
 }
 
 #[derive(Copy, Clone, Debug, Eq, PartialEq, PartialOrd, Ord, Hash)]
-#[derive(TyEncodable, TyDecodable, HashStable)]
+#[derive(TyEncodable, TyDecodable, HashStable, TypeVisitable, TypeFoldable)]
 pub enum ReturnConstraint {
     Normal,
     ClosureUpvar(Field),
diff --git a/compiler/rustc_middle/src/mir/spanview.rs b/compiler/rustc_middle/src/mir/spanview.rs
index 4418b848e..4e06d9101 100644
--- a/compiler/rustc_middle/src/mir/spanview.rs
+++ b/compiler/rustc_middle/src/mir/spanview.rs
@@ -105,7 +105,7 @@ where
     }
     let body_span = hir_body.unwrap().value.span;
     let mut span_viewables = Vec::new();
-    for (bb, data) in body.basic_blocks().iter_enumerated() {
+    for (bb, data) in body.basic_blocks.iter_enumerated() {
         match spanview {
             MirSpanview::Statement => {
                 for (i, statement) in data.statements.iter().enumerate() {
@@ -249,7 +249,7 @@ pub fn statement_kind_name(statement: &Statement<'_>) -> &'static str {
         Retag(..) => "Retag",
         AscribeUserType(..) => "AscribeUserType",
         Coverage(..) => "Coverage",
-        CopyNonOverlapping(..) => "CopyNonOverlapping",
+        Intrinsic(..) => "Intrinsic",
         Nop => "Nop",
     }
 }
diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs
index eb90169d0..c7d0283aa 100644
--- a/compiler/rustc_middle/src/mir/syntax.rs
+++ b/compiler/rustc_middle/src/mir/syntax.rs
@@ -23,75 +23,110 @@ use rustc_span::symbol::Symbol;
 use rustc_span::Span;
 use rustc_target::asm::InlineAsmRegOrRegClass;
 
-/// The various "big phases" that MIR goes through.
+/// Represents the "flavors" of MIR.
 ///
-/// These phases all describe dialects of MIR. Since all MIR uses the same datastructures, the
-/// dialects forbid certain variants or values in certain phases. The sections below summarize the
-/// changes, but do not document them thoroughly. The full documentation is found in the appropriate
-/// documentation for the thing the change is affecting.
+/// All flavors of MIR use the same data structure, but there are some important differences. These
+/// differences come in two forms: Dialects and phases.
 ///
-/// Warning: ordering of variants is significant.
+/// Dialects represent a stronger distinction than phases. This is because the transitions between
+/// dialects are semantic changes, and therefore technically *lowerings* between distinct IRs. In
+/// other words, the same [`Body`](crate::mir::Body) might be well-formed for multiple dialects, but
+/// have different semantic meaning and different behavior at runtime.
+///
+/// Each dialect additionally has a number of phases. However, phase changes never involve semantic
+/// changes. If some MIR is well-formed both before and after a phase change, it is also guaranteed
+/// that it has the same semantic meaning. In this sense, phase changes can only add additional
+/// restrictions on what MIR is well-formed.
+///
+/// When adding phases, remember to update [`MirPhase::phase_index`].
 #[derive(Copy, Clone, TyEncodable, TyDecodable, Debug, PartialEq, Eq, PartialOrd, Ord)]
 #[derive(HashStable)]
 pub enum MirPhase {
-    /// The dialect of MIR used during all phases before `DropsLowered` is the same. This is also
-    /// the MIR that analysis such as borrowck uses.
-    ///
-    /// One important thing to remember about the behavior of this section of MIR is that drop terminators
-    /// (including drop and replace) are *conditional*. The elaborate drops pass will then replace each
-    /// instance of a drop terminator with a nop, an unconditional drop, or a drop conditioned on a drop
-    /// flag. Of course, this means that it is important that the drop elaboration can accurately recognize
-    /// when things are initialized and when things are de-initialized. That means any code running on this
-    /// version of MIR must be sure to produce output that drop elaboration can reason about. See the
-    /// section on the drop terminatorss for more details.
-    Built = 0,
-    // FIXME(oli-obk): it's unclear whether we still need this phase (and its corresponding query).
-    // We used to have this for pre-miri MIR based const eval.
-    Const = 1,
-    /// This phase checks the MIR for promotable elements and takes them out of the main MIR body
-    /// by creating a new MIR body per promoted element. After this phase (and thus the termination
-    /// of the `mir_promoted` query), these promoted elements are available in the `promoted_mir`
-    /// query.
-    ConstsPromoted = 2,
-    /// After this projections may only contain deref projections as the first element.
-    Derefered = 3,
-    /// Beginning with this phase, the following variants are disallowed:
-    /// * [`TerminatorKind::DropAndReplace`]
+    /// The MIR that is generated by MIR building.
+    ///
+    /// The only things that operate on this dialect are unsafeck, the various MIR lints, and const
+    /// qualifs.
+    ///
+    /// This has no distinct phases.
+    Built,
+    /// The MIR used for most analysis.
+    ///
+    /// The only semantic change between analysis and built MIR is constant promotion. In built MIR,
+    /// sequences of statements that would generally be subject to constant promotion are
+    /// semantically constants, while in analysis MIR all constants are explicit.
+    ///
+    /// The result of const promotion is available from the `mir_promoted` and `promoted_mir` queries.
+    ///
+    /// This is the version of MIR used by borrowck and friends.
+    Analysis(AnalysisPhase),
+    /// The MIR used for CTFE, optimizations, and codegen.
+    ///
+    /// The semantic changes that occur in the lowering from analysis to runtime MIR are as follows:
+    ///
+    ///  - Drops: In analysis MIR, `Drop` terminators represent *conditional* drops; roughly speaking,
+    ///    if dataflow analysis determines that the place being dropped is uninitialized, the drop will
+    ///    not be executed. The exact semantics of this aren't written down anywhere, which means they
+    ///    are essentially "what drop elaboration does." In runtime MIR, the drops are unconditional;
+    ///    when a `Drop` terminator is reached, if the type has drop glue that drop glue is always
+    ///    executed. This may be UB if the underlying place is not initialized.
+    ///  - Packed drops: Places might in general be misaligned - in most cases this is UB, the exception
+    ///    is fields of packed structs. In analysis MIR, `Drop(P)` for a `P` that might be misaligned
+    ///    for this reason implicitly moves `P` to a temporary before dropping. Runtime MIR has no such
+    ///    rules, and dropping a misaligned place is simply UB.
+    ///  - Unwinding: in analysis MIR, unwinding from a function which may not unwind aborts. In runtime
+    ///    MIR, this is UB.
+    ///  - Retags: If `-Zmir-emit-retag` is enabled, analysis MIR has "implicit" retags in the same way
+    ///    that Rust itself has them. Where exactly these are is generally subject to change, and so we
+    ///    don't document this here. Runtime MIR has all retags explicit.
+    ///  - Generator bodies: In analysis MIR, locals may actually be behind a pointer that user code has
+    ///    access to. This occurs in generator bodies. Such locals do not behave like other locals,
+    ///    because they eg may be aliased in surprising ways. Runtime MIR has no such special locals -
+    ///    all generator bodies are lowered and so all places that look like locals really are locals.
+    ///  - Const prop lints: The lint pass which reports eg `200_u8 + 200_u8` as an error is run as a
+    ///    part of analysis to runtime MIR lowering. This means that transformations which may supress
+    ///    such errors may not run on analysis MIR.
+    Runtime(RuntimePhase),
+}
+
+/// See [`MirPhase::Analysis`].
+#[derive(Copy, Clone, TyEncodable, TyDecodable, Debug, PartialEq, Eq, PartialOrd, Ord)]
+#[derive(HashStable)]
+pub enum AnalysisPhase {
+    Initial = 0,
+    /// Beginning in this phase, the following variants are disallowed:
     /// * [`TerminatorKind::FalseUnwind`]
     /// * [`TerminatorKind::FalseEdge`]
     /// * [`StatementKind::FakeRead`]
     /// * [`StatementKind::AscribeUserType`]
     /// * [`Rvalue::Ref`] with `BorrowKind::Shallow`
     ///
-    /// And the following variant is allowed:
-    /// * [`StatementKind::Retag`]
-    ///
-    /// Furthermore, `Drop` now uses explicit drop flags visible in the MIR and reaching a `Drop`
-    /// terminator means that the auto-generated drop glue will be invoked. Also, `Copy` operands
-    /// are allowed for non-`Copy` types.
-    DropsLowered = 4,
-    /// Beginning with this phase, the following variant is disallowed:
+    /// Furthermore, `Deref` projections must be the first projection within any place (if they
+    /// appear at all)
+    PostCleanup = 1,
+}
+
+/// See [`MirPhase::Runtime`].
+#[derive(Copy, Clone, TyEncodable, TyDecodable, Debug, PartialEq, Eq, PartialOrd, Ord)]
+#[derive(HashStable)]
+pub enum RuntimePhase {
+    /// In addition to the semantic changes, beginning with this phase, the following variants are
+    /// disallowed:
+    /// * [`TerminatorKind::DropAndReplace`]
+    /// * [`TerminatorKind::Yield`]
+    /// * [`TerminatorKind::GeneratorDrop`]
     /// * [`Rvalue::Aggregate`] for any `AggregateKind` except `Array`
     ///
-    /// And the following variant is allowed:
+    /// And the following variants are allowed:
+    /// * [`StatementKind::Retag`]
     /// * [`StatementKind::SetDiscriminant`]
-    Deaggregated = 5,
-    /// Before this phase, generators are in the "source code" form, featuring `yield` statements
-    /// and such. With this phase change, they are transformed into a proper state machine. Running
-    /// optimizations before this change can be potentially dangerous because the source code is to
-    /// some extent a "lie." In particular, `yield` terminators effectively make the value of all
-    /// locals visible to the caller. This means that dead store elimination before them, or code
-    /// motion across them, is not correct in general. This is also exasperated by type checking
-    /// having pre-computed a list of the types that it thinks are ok to be live across a yield
-    /// point - this is necessary to decide eg whether autotraits are implemented. Introducing new
-    /// types across a yield point will lead to ICEs becaues of this.
-    ///
-    /// Beginning with this phase, the following variants are disallowed:
-    /// * [`TerminatorKind::Yield`]
-    /// * [`TerminatorKind::GeneratorDrop`]
+    /// * [`StatementKind::Deinit`]
+    ///
+    /// Furthermore, `Copy` operands are allowed for non-`Copy` types.
+    Initial = 0,
+    /// Beginning with this phase, the following variant is disallowed:
     /// * [`ProjectionElem::Deref`] of `Box`
-    GeneratorsLowered = 6,
-    Optimized = 7,
+    PostCleanup = 1,
+    Optimized = 2,
 }
 
 ///////////////////////////////////////////////////////////////////////////
@@ -292,12 +327,40 @@ pub enum StatementKind<'tcx> {
     /// executed.
     Coverage(Box<Coverage>),
 
+    /// Denotes a call to an intrinsic that does not require an unwind path and always returns.
+    /// This avoids adding a new block and a terminator for simple intrinsics.
+    Intrinsic(Box<NonDivergingIntrinsic<'tcx>>),
+
+    /// No-op. Useful for deleting instructions without affecting statement indices.
+    Nop,
+}
+
+#[derive(
+    Clone,
+    TyEncodable,
+    TyDecodable,
+    Debug,
+    PartialEq,
+    Hash,
+    HashStable,
+    TypeFoldable,
+    TypeVisitable
+)]
+pub enum NonDivergingIntrinsic<'tcx> {
+    /// Denotes a call to the intrinsic function `assume`.
+    ///
+    /// The operand must be a boolean. Optimizers may use the value of the boolean to backtrack its
+    /// computation to infer information about other variables. So if the boolean came from a
+    /// `x < y` operation, subsequent operations on `x` and `y` could elide various bound checks.
+    /// If the argument is `false`, this operation is equivalent to `TerminatorKind::Unreachable`.
+    Assume(Operand<'tcx>),
+
     /// Denotes a call to the intrinsic function `copy_nonoverlapping`.
     ///
     /// First, all three operands are evaluated. `src` and `dest` must each be a reference, pointer,
     /// or `Box` pointing to the same type `T`. `count` must evaluate to a `usize`. Then, `src` and
     /// `dest` are dereferenced, and `count * size_of::<T>()` bytes beginning with the first byte of
-    /// the `src` place are copied to the continguous range of bytes beginning with the first byte
+    /// the `src` place are copied to the contiguous range of bytes beginning with the first byte
     /// of `dest`.
     ///
     /// **Needs clarification**: In what order are operands computed and dereferenced? It should
@@ -305,10 +368,18 @@ pub enum StatementKind<'tcx> {
     ///
     /// **Needs clarification**: Is this typed or not, ie is there a typed load and store involved?
     /// I vaguely remember Ralf saying somewhere that he thought it should not be.
-    CopyNonOverlapping(Box<CopyNonOverlapping<'tcx>>),
+    CopyNonOverlapping(CopyNonOverlapping<'tcx>),
+}
 
-    /// No-op. Useful for deleting instructions without affecting statement indices.
-    Nop,
+impl std::fmt::Display for NonDivergingIntrinsic<'_> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Assume(op) => write!(f, "assume({op:?})"),
+            Self::CopyNonOverlapping(CopyNonOverlapping { src, dst, count }) => {
+                write!(f, "copy_nonoverlapping(dst = {dst:?}, src = {src:?}, count = {count:?})")
+            }
+        }
+    }
 }
 
 /// Describes what kind of retag is to be performed.
@@ -343,7 +414,7 @@ pub enum FakeReadCause {
     /// Some(closure_def_id).
     /// Otherwise, the value of the optional LocalDefId will be None.
     //
-    // We can use LocaDefId here since fake read statements are removed
+    // We can use LocalDefId here since fake read statements are removed
     // before codegen in the `CleanupNonCodegenStatements` pass.
     ForMatchedPlace(Option<LocalDefId>),
 
@@ -417,7 +488,7 @@ pub struct CopyNonOverlapping<'tcx> {
 ///     must also be `cleanup`. This is a part of the type system and checked statically, so it is
 ///     still an error to have such an edge in the CFG even if it's known that it won't be taken at
 ///     runtime.
-#[derive(Clone, TyEncodable, TyDecodable, Hash, HashStable, PartialEq)]
+#[derive(Clone, TyEncodable, TyDecodable, Hash, HashStable, PartialEq, TypeFoldable, TypeVisitable)]
 pub enum TerminatorKind<'tcx> {
     /// Block has one successor; we continue execution there.
     Goto { target: BasicBlock },
@@ -670,7 +741,7 @@ pub enum TerminatorKind<'tcx> {
 }
 
 /// Information about an assertion failure.
-#[derive(Clone, TyEncodable, TyDecodable, Hash, HashStable, PartialEq, PartialOrd)]
+#[derive(Clone, TyEncodable, TyDecodable, Hash, HashStable, PartialEq, TypeFoldable, TypeVisitable)]
 pub enum AssertKind<O> {
     BoundsCheck { len: O, index: O },
     Overflow(BinOp, O, O),
@@ -792,7 +863,7 @@ pub type AssertMessage<'tcx> = AssertKind<Operand<'tcx>>;
 ///
 /// Rust currently requires that every place obey those two rules. This is checked by MIRI and taken
 /// advantage of by codegen (via `gep inbounds`). That is possibly subject to change.
-#[derive(Copy, Clone, PartialEq, Eq, Hash, TyEncodable, HashStable)]
+#[derive(Copy, Clone, PartialEq, Eq, Hash, TyEncodable, HashStable, TypeFoldable, TypeVisitable)]
 pub struct Place<'tcx> {
     pub local: Local,
 
@@ -801,7 +872,7 @@ pub struct Place<'tcx> {
 }
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
-#[derive(TyEncodable, TyDecodable, HashStable)]
+#[derive(TyEncodable, TyDecodable, HashStable, TypeFoldable, TypeVisitable)]
 pub enum ProjectionElem<V, T> {
     Deref,
     Field(Field, T),
@@ -884,7 +955,7 @@ pub type PlaceElem<'tcx> = ProjectionElem<Local, Ty<'tcx>>;
 /// **Needs clarifiation:** Is loading a place that has its variant index set well-formed? Miri
 /// currently implements it, but it seems like this may be something to check against in the
 /// validator.
-#[derive(Clone, PartialEq, TyEncodable, TyDecodable, Hash, HashStable)]
+#[derive(Clone, PartialEq, TyEncodable, TyDecodable, Hash, HashStable, TypeFoldable, TypeVisitable)]
 pub enum Operand<'tcx> {
     /// Creates a value by loading the given place.
     ///
@@ -915,7 +986,7 @@ pub enum Operand<'tcx> {
 /// Computing any rvalue begins by evaluating the places and operands in some order (**Needs
 /// clarification**: Which order?). These are then used to produce a "value" - the same kind of
 /// value that an [`Operand`] produces.
-#[derive(Clone, TyEncodable, TyDecodable, Hash, HashStable, PartialEq)]
+#[derive(Clone, TyEncodable, TyDecodable, Hash, HashStable, PartialEq, TypeFoldable, TypeVisitable)]
 pub enum Rvalue<'tcx> {
     /// Yields the operand unchanged
     Use(Operand<'tcx>),
@@ -1068,11 +1139,14 @@ pub enum CastKind {
     /// All sorts of pointer-to-pointer casts. Note that reference-to-raw-ptr casts are
     /// translated into `&raw mut/const *r`, i.e., they are not actually casts.
     Pointer(PointerCast),
+    /// Cast into a dyn* object.
+    DynStar,
     /// Remaining unclassified casts.
     Misc,
 }
 
 #[derive(Clone, Debug, PartialEq, Eq, TyEncodable, TyDecodable, Hash, HashStable)]
+#[derive(TypeFoldable, TypeVisitable)]
 pub enum AggregateKind<'tcx> {
     /// The type is of the element
     Array(Ty<'tcx>),
@@ -1160,7 +1234,8 @@ pub enum BinOp {
 mod size_asserts {
     use super::*;
     // These are in alphabetical order, which is easy to maintain.
-    static_assert_size!(AggregateKind<'_>, 48);
+    #[cfg(not(bootstrap))]
+    static_assert_size!(AggregateKind<'_>, 40);
     static_assert_size!(Operand<'_>, 24);
     static_assert_size!(Place<'_>, 16);
     static_assert_size!(PlaceElem<'_>, 24);
diff --git a/compiler/rustc_middle/src/mir/terminator.rs b/compiler/rustc_middle/src/mir/terminator.rs
index 9ccf5aea6..4ea333cff 100644
--- a/compiler/rustc_middle/src/mir/terminator.rs
+++ b/compiler/rustc_middle/src/mir/terminator.rs
@@ -14,7 +14,7 @@ use std::slice;
 
 pub use super::query::*;
 
-#[derive(Debug, Clone, TyEncodable, TyDecodable, Hash, HashStable, PartialEq, PartialOrd)]
+#[derive(Debug, Clone, TyEncodable, TyDecodable, Hash, HashStable, PartialEq)]
 pub struct SwitchTargets {
     /// Possible values. The locations to branch to in each case
     /// are found in the corresponding indices from the `targets` vector.
@@ -102,7 +102,7 @@ impl<'a> Iterator for SwitchTargetsIter<'a> {
 
 impl<'a> ExactSizeIterator for SwitchTargetsIter<'a> {}
 
-#[derive(Clone, Debug, TyEncodable, TyDecodable, HashStable)]
+#[derive(Clone, Debug, TyEncodable, TyDecodable, HashStable, TypeFoldable, TypeVisitable)]
 pub struct Terminator<'tcx> {
     pub source_info: SourceInfo,
     pub kind: TerminatorKind<'tcx>,
diff --git a/compiler/rustc_middle/src/mir/traversal.rs b/compiler/rustc_middle/src/mir/traversal.rs
index 627dc32f3..55b2c5927 100644
--- a/compiler/rustc_middle/src/mir/traversal.rs
+++ b/compiler/rustc_middle/src/mir/traversal.rs
@@ -37,7 +37,7 @@ impl<'a, 'tcx> Preorder<'a, 'tcx> {
 
         Preorder {
             body,
-            visited: BitSet::new_empty(body.basic_blocks().len()),
+            visited: BitSet::new_empty(body.basic_blocks.len()),
             worklist,
             root_is_start_block: root == START_BLOCK,
         }
@@ -71,7 +71,7 @@ impl<'a, 'tcx> Iterator for Preorder<'a, 'tcx> {
 
     fn size_hint(&self) -> (usize, Option<usize>) {
         // All the blocks, minus the number of blocks we've visited.
-        let upper = self.body.basic_blocks().len() - self.visited.count();
+        let upper = self.body.basic_blocks.len() - self.visited.count();
 
         let lower = if self.root_is_start_block {
             // We will visit all remaining blocks exactly once.
diff --git a/compiler/rustc_middle/src/mir/type_foldable.rs b/compiler/rustc_middle/src/mir/type_foldable.rs
index 82a6b0c50..9d098c808 100644
--- a/compiler/rustc_middle/src/mir/type_foldable.rs
+++ b/compiler/rustc_middle/src/mir/type_foldable.rs
@@ -1,8 +1,9 @@
 //! `TypeFoldable` implementations for MIR types
 
+use rustc_ast::InlineAsmTemplatePiece;
+
 use super::*;
 use crate::ty;
-use rustc_data_structures::functor::IdFunctor;
 
 TrivialTypeTraversalAndLiftImpls! {
     BlockTailInfo,
@@ -13,96 +14,27 @@ TrivialTypeTraversalAndLiftImpls! {
     SourceScope,
     SourceScopeLocalData,
     UserTypeAnnotationIndex,
+    BorrowKind,
+    CastKind,
+    BinOp,
+    NullOp,
+    UnOp,
+    hir::Movability,
+    BasicBlock,
+    SwitchTargets,
+    GeneratorKind,
+    GeneratorSavedLocal,
 }
 
-impl<'tcx> TypeFoldable<'tcx> for Terminator<'tcx> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
-        use crate::mir::TerminatorKind::*;
-
-        let kind = match self.kind {
-            Goto { target } => Goto { target },
-            SwitchInt { discr, switch_ty, targets } => SwitchInt {
-                discr: discr.try_fold_with(folder)?,
-                switch_ty: switch_ty.try_fold_with(folder)?,
-                targets,
-            },
-            Drop { place, target, unwind } => {
-                Drop { place: place.try_fold_with(folder)?, target, unwind }
-            }
-            DropAndReplace { place, value, target, unwind } => DropAndReplace {
-                place: place.try_fold_with(folder)?,
-                value: value.try_fold_with(folder)?,
-                target,
-                unwind,
-            },
-            Yield { value, resume, resume_arg, drop } => Yield {
-                value: value.try_fold_with(folder)?,
-                resume,
-                resume_arg: resume_arg.try_fold_with(folder)?,
-                drop,
-            },
-            Call { func, args, destination, target, cleanup, from_hir_call, fn_span } => Call {
-                func: func.try_fold_with(folder)?,
-                args: args.try_fold_with(folder)?,
-                destination: destination.try_fold_with(folder)?,
-                target,
-                cleanup,
-                from_hir_call,
-                fn_span,
-            },
-            Assert { cond, expected, msg, target, cleanup } => {
-                use AssertKind::*;
-                let msg = match msg {
-                    BoundsCheck { len, index } => BoundsCheck {
-                        len: len.try_fold_with(folder)?,
-                        index: index.try_fold_with(folder)?,
-                    },
-                    Overflow(op, l, r) => {
-                        Overflow(op, l.try_fold_with(folder)?, r.try_fold_with(folder)?)
-                    }
-                    OverflowNeg(op) => OverflowNeg(op.try_fold_with(folder)?),
-                    DivisionByZero(op) => DivisionByZero(op.try_fold_with(folder)?),
-                    RemainderByZero(op) => RemainderByZero(op.try_fold_with(folder)?),
-                    ResumedAfterReturn(_) | ResumedAfterPanic(_) => msg,
-                };
-                Assert { cond: cond.try_fold_with(folder)?, expected, msg, target, cleanup }
-            }
-            GeneratorDrop => GeneratorDrop,
-            Resume => Resume,
-            Abort => Abort,
-            Return => Return,
-            Unreachable => Unreachable,
-            FalseEdge { real_target, imaginary_target } => {
-                FalseEdge { real_target, imaginary_target }
-            }
-            FalseUnwind { real_target, unwind } => FalseUnwind { real_target, unwind },
-            InlineAsm { template, operands, options, line_spans, destination, cleanup } => {
-                InlineAsm {
-                    template,
-                    operands: operands.try_fold_with(folder)?,
-                    options,
-                    line_spans,
-                    destination,
-                    cleanup,
-                }
-            }
-        };
-        Ok(Terminator { source_info: self.source_info, kind })
-    }
-}
-
-impl<'tcx> TypeFoldable<'tcx> for GeneratorKind {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, _: &mut F) -> Result<Self, F::Error> {
+impl<'tcx> TypeFoldable<'tcx> for &'tcx [InlineAsmTemplatePiece] {
+    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, _folder: &mut F) -> Result<Self, F::Error> {
         Ok(self)
     }
 }
 
-impl<'tcx> TypeFoldable<'tcx> for Place<'tcx> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
-        Ok(Place {
-            local: self.local.try_fold_with(folder)?,
-            projection: self.projection.try_fold_with(folder)?,
-        })
+impl<'tcx> TypeFoldable<'tcx> for &'tcx [Span] {
+    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, _folder: &mut F) -> Result<Self, F::Error> {
+        Ok(self)
     }
 }
 
@@ -112,114 +44,12 @@ impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::List<PlaceElem<'tcx>> {
     }
 }
 
-impl<'tcx> TypeFoldable<'tcx> for Rvalue<'tcx> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
-        use crate::mir::Rvalue::*;
-        Ok(match self {
-            Use(op) => Use(op.try_fold_with(folder)?),
-            Repeat(op, len) => Repeat(op.try_fold_with(folder)?, len.try_fold_with(folder)?),
-            ThreadLocalRef(did) => ThreadLocalRef(did.try_fold_with(folder)?),
-            Ref(region, bk, place) => {
-                Ref(region.try_fold_with(folder)?, bk, place.try_fold_with(folder)?)
-            }
-            CopyForDeref(place) => CopyForDeref(place.try_fold_with(folder)?),
-            AddressOf(mutability, place) => AddressOf(mutability, place.try_fold_with(folder)?),
-            Len(place) => Len(place.try_fold_with(folder)?),
-            Cast(kind, op, ty) => Cast(kind, op.try_fold_with(folder)?, ty.try_fold_with(folder)?),
-            BinaryOp(op, box (rhs, lhs)) => {
-                BinaryOp(op, Box::new((rhs.try_fold_with(folder)?, lhs.try_fold_with(folder)?)))
-            }
-            CheckedBinaryOp(op, box (rhs, lhs)) => CheckedBinaryOp(
-                op,
-                Box::new((rhs.try_fold_with(folder)?, lhs.try_fold_with(folder)?)),
-            ),
-            UnaryOp(op, val) => UnaryOp(op, val.try_fold_with(folder)?),
-            Discriminant(place) => Discriminant(place.try_fold_with(folder)?),
-            NullaryOp(op, ty) => NullaryOp(op, ty.try_fold_with(folder)?),
-            Aggregate(kind, fields) => {
-                let kind = kind.try_map_id(|kind| {
-                    Ok(match kind {
-                        AggregateKind::Array(ty) => AggregateKind::Array(ty.try_fold_with(folder)?),
-                        AggregateKind::Tuple => AggregateKind::Tuple,
-                        AggregateKind::Adt(def, v, substs, user_ty, n) => AggregateKind::Adt(
-                            def,
-                            v,
-                            substs.try_fold_with(folder)?,
-                            user_ty.try_fold_with(folder)?,
-                            n,
-                        ),
-                        AggregateKind::Closure(id, substs) => {
-                            AggregateKind::Closure(id, substs.try_fold_with(folder)?)
-                        }
-                        AggregateKind::Generator(id, substs, movablity) => {
-                            AggregateKind::Generator(id, substs.try_fold_with(folder)?, movablity)
-                        }
-                    })
-                })?;
-                Aggregate(kind, fields.try_fold_with(folder)?)
-            }
-            ShallowInitBox(op, ty) => {
-                ShallowInitBox(op.try_fold_with(folder)?, ty.try_fold_with(folder)?)
-            }
-        })
-    }
-}
-
-impl<'tcx> TypeFoldable<'tcx> for Operand<'tcx> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
-        Ok(match self {
-            Operand::Copy(place) => Operand::Copy(place.try_fold_with(folder)?),
-            Operand::Move(place) => Operand::Move(place.try_fold_with(folder)?),
-            Operand::Constant(c) => Operand::Constant(c.try_fold_with(folder)?),
-        })
-    }
-}
-
-impl<'tcx> TypeFoldable<'tcx> for PlaceElem<'tcx> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
-        use crate::mir::ProjectionElem::*;
-
-        Ok(match self {
-            Deref => Deref,
-            Field(f, ty) => Field(f, ty.try_fold_with(folder)?),
-            Index(v) => Index(v.try_fold_with(folder)?),
-            Downcast(symbol, variantidx) => Downcast(symbol, variantidx),
-            ConstantIndex { offset, min_length, from_end } => {
-                ConstantIndex { offset, min_length, from_end }
-            }
-            Subslice { from, to, from_end } => Subslice { from, to, from_end },
-        })
-    }
-}
-
-impl<'tcx> TypeFoldable<'tcx> for Field {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, _: &mut F) -> Result<Self, F::Error> {
-        Ok(self)
-    }
-}
-
-impl<'tcx> TypeFoldable<'tcx> for GeneratorSavedLocal {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, _: &mut F) -> Result<Self, F::Error> {
-        Ok(self)
-    }
-}
-
 impl<'tcx, R: Idx, C: Idx> TypeFoldable<'tcx> for BitMatrix<R, C> {
     fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, _: &mut F) -> Result<Self, F::Error> {
         Ok(self)
     }
 }
 
-impl<'tcx> TypeFoldable<'tcx> for Constant<'tcx> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
-        Ok(Constant {
-            span: self.span,
-            user_ty: self.user_ty.try_fold_with(folder)?,
-            literal: self.literal.try_fold_with(folder)?,
-        })
-    }
-}
-
 impl<'tcx> TypeFoldable<'tcx> for ConstantKind<'tcx> {
     #[inline(always)]
     fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
@@ -235,6 +65,9 @@ impl<'tcx> TypeSuperFoldable<'tcx> for ConstantKind<'tcx> {
         match self {
             ConstantKind::Ty(c) => Ok(ConstantKind::Ty(c.try_fold_with(folder)?)),
             ConstantKind::Val(v, t) => Ok(ConstantKind::Val(v, t.try_fold_with(folder)?)),
+            ConstantKind::Unevaluated(uv, t) => {
+                Ok(ConstantKind::Unevaluated(uv.try_fold_with(folder)?, t.try_fold_with(folder)?))
+            }
         }
     }
 }
diff --git a/compiler/rustc_middle/src/mir/type_visitable.rs b/compiler/rustc_middle/src/mir/type_visitable.rs
index 6a0801cb0..be19bb486 100644
--- a/compiler/rustc_middle/src/mir/type_visitable.rs
+++ b/compiler/rustc_middle/src/mir/type_visitable.rs
@@ -1,165 +1,6 @@
 //! `TypeVisitable` implementations for MIR types
 
 use super::*;
-use crate::ty;
-
-impl<'tcx> TypeVisitable<'tcx> for Terminator<'tcx> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
-        use crate::mir::TerminatorKind::*;
-
-        match self.kind {
-            SwitchInt { ref discr, switch_ty, .. } => {
-                discr.visit_with(visitor)?;
-                switch_ty.visit_with(visitor)
-            }
-            Drop { ref place, .. } => place.visit_with(visitor),
-            DropAndReplace { ref place, ref value, .. } => {
-                place.visit_with(visitor)?;
-                value.visit_with(visitor)
-            }
-            Yield { ref value, .. } => value.visit_with(visitor),
-            Call { ref func, ref args, ref destination, .. } => {
-                destination.visit_with(visitor)?;
-                func.visit_with(visitor)?;
-                args.visit_with(visitor)
-            }
-            Assert { ref cond, ref msg, .. } => {
-                cond.visit_with(visitor)?;
-                use AssertKind::*;
-                match msg {
-                    BoundsCheck { ref len, ref index } => {
-                        len.visit_with(visitor)?;
-                        index.visit_with(visitor)
-                    }
-                    Overflow(_, l, r) => {
-                        l.visit_with(visitor)?;
-                        r.visit_with(visitor)
-                    }
-                    OverflowNeg(op) | DivisionByZero(op) | RemainderByZero(op) => {
-                        op.visit_with(visitor)
-                    }
-                    ResumedAfterReturn(_) | ResumedAfterPanic(_) => ControlFlow::CONTINUE,
-                }
-            }
-            InlineAsm { ref operands, .. } => operands.visit_with(visitor),
-            Goto { .. }
-            | Resume
-            | Abort
-            | Return
-            | GeneratorDrop
-            | Unreachable
-            | FalseEdge { .. }
-            | FalseUnwind { .. } => ControlFlow::CONTINUE,
-        }
-    }
-}
-
-impl<'tcx> TypeVisitable<'tcx> for GeneratorKind {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, _: &mut V) -> ControlFlow<V::BreakTy> {
-        ControlFlow::CONTINUE
-    }
-}
-
-impl<'tcx> TypeVisitable<'tcx> for Place<'tcx> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
-        self.local.visit_with(visitor)?;
-        self.projection.visit_with(visitor)
-    }
-}
-
-impl<'tcx> TypeVisitable<'tcx> for &'tcx ty::List<PlaceElem<'tcx>> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
-        self.iter().try_for_each(|t| t.visit_with(visitor))
-    }
-}
-
-impl<'tcx> TypeVisitable<'tcx> for Rvalue<'tcx> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
-        use crate::mir::Rvalue::*;
-        match *self {
-            Use(ref op) => op.visit_with(visitor),
-            CopyForDeref(ref place) => {
-                let op = &Operand::Copy(*place);
-                op.visit_with(visitor)
-            }
-            Repeat(ref op, _) => op.visit_with(visitor),
-            ThreadLocalRef(did) => did.visit_with(visitor),
-            Ref(region, _, ref place) => {
-                region.visit_with(visitor)?;
-                place.visit_with(visitor)
-            }
-            AddressOf(_, ref place) => place.visit_with(visitor),
-            Len(ref place) => place.visit_with(visitor),
-            Cast(_, ref op, ty) => {
-                op.visit_with(visitor)?;
-                ty.visit_with(visitor)
-            }
-            BinaryOp(_, box (ref rhs, ref lhs)) | CheckedBinaryOp(_, box (ref rhs, ref lhs)) => {
-                rhs.visit_with(visitor)?;
-                lhs.visit_with(visitor)
-            }
-            UnaryOp(_, ref val) => val.visit_with(visitor),
-            Discriminant(ref place) => place.visit_with(visitor),
-            NullaryOp(_, ty) => ty.visit_with(visitor),
-            Aggregate(ref kind, ref fields) => {
-                match **kind {
-                    AggregateKind::Array(ty) => {
-                        ty.visit_with(visitor)?;
-                    }
-                    AggregateKind::Tuple => {}
-                    AggregateKind::Adt(_, _, substs, user_ty, _) => {
-                        substs.visit_with(visitor)?;
-                        user_ty.visit_with(visitor)?;
-                    }
-                    AggregateKind::Closure(_, substs) => {
-                        substs.visit_with(visitor)?;
-                    }
-                    AggregateKind::Generator(_, substs, _) => {
-                        substs.visit_with(visitor)?;
-                    }
-                }
-                fields.visit_with(visitor)
-            }
-            ShallowInitBox(ref op, ty) => {
-                op.visit_with(visitor)?;
-                ty.visit_with(visitor)
-            }
-        }
-    }
-}
-
-impl<'tcx> TypeVisitable<'tcx> for Operand<'tcx> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
-        match *self {
-            Operand::Copy(ref place) | Operand::Move(ref place) => place.visit_with(visitor),
-            Operand::Constant(ref c) => c.visit_with(visitor),
-        }
-    }
-}
-
-impl<'tcx> TypeVisitable<'tcx> for PlaceElem<'tcx> {
-    fn visit_with<Vs: TypeVisitor<'tcx>>(&self, visitor: &mut Vs) -> ControlFlow<Vs::BreakTy> {
-        use crate::mir::ProjectionElem::*;
-
-        match self {
-            Field(_, ty) => ty.visit_with(visitor),
-            Index(v) => v.visit_with(visitor),
-            _ => ControlFlow::CONTINUE,
-        }
-    }
-}
-
-impl<'tcx> TypeVisitable<'tcx> for Field {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, _: &mut V) -> ControlFlow<V::BreakTy> {
-        ControlFlow::CONTINUE
-    }
-}
-
-impl<'tcx> TypeVisitable<'tcx> for GeneratorSavedLocal {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, _: &mut V) -> ControlFlow<V::BreakTy> {
-        ControlFlow::CONTINUE
-    }
-}
 
 impl<'tcx, R: Idx, C: Idx> TypeVisitable<'tcx> for BitMatrix<R, C> {
     fn visit_with<V: TypeVisitor<'tcx>>(&self, _: &mut V) -> ControlFlow<V::BreakTy> {
@@ -167,13 +8,6 @@ impl<'tcx, R: Idx, C: Idx> TypeVisitable<'tcx> for BitMatrix<R, C> {
     }
 }
 
-impl<'tcx> TypeVisitable<'tcx> for Constant<'tcx> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
-        self.literal.visit_with(visitor)?;
-        self.user_ty.visit_with(visitor)
-    }
-}
-
 impl<'tcx> TypeVisitable<'tcx> for ConstantKind<'tcx> {
     fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
         visitor.visit_mir_const(*self)
@@ -185,6 +19,10 @@ impl<'tcx> TypeSuperVisitable<'tcx> for ConstantKind<'tcx> {
         match *self {
             ConstantKind::Ty(c) => c.visit_with(visitor),
             ConstantKind::Val(_, t) => t.visit_with(visitor),
+            ConstantKind::Unevaluated(uv, t) => {
+                uv.visit_with(visitor)?;
+                t.visit_with(visitor)
+            }
         }
     }
 }
diff --git a/compiler/rustc_middle/src/mir/visit.rs b/compiler/rustc_middle/src/mir/visit.rs
index 891608764..d9b24566b 100644
--- a/compiler/rustc_middle/src/mir/visit.rs
+++ b/compiler/rustc_middle/src/mir/visit.rs
@@ -80,6 +80,8 @@ macro_rules! make_mir_visitor {
                 self.super_body(body);
             }
 
+            extra_body_methods!($($mutability)?);
+
             fn visit_basic_block_data(
                 &mut self,
                 block: BasicBlock,
@@ -235,14 +237,6 @@ macro_rules! make_mir_visitor {
                 self.super_region(region);
             }
 
-            fn visit_const(
-                &mut self,
-                constant: $(& $mutability)? ty::Const<'tcx>,
-                _: Location,
-            ) {
-                self.super_const(constant);
-            }
-
             fn visit_substs(
                 &mut self,
                 substs: & $($mutability)? SubstsRef<'tcx>,
@@ -287,63 +281,7 @@ macro_rules! make_mir_visitor {
                 &mut self,
                 body: &$($mutability)? Body<'tcx>,
             ) {
-                let span = body.span;
-                if let Some(gen) = &$($mutability)? body.generator {
-                    if let Some(yield_ty) = $(& $mutability)? gen.yield_ty {
-                        self.visit_ty(
-                            yield_ty,
-                            TyContext::YieldTy(SourceInfo::outermost(span))
-                        );
-                    }
-                }
-
-                // for best performance, we want to use an iterator rather
-                // than a for-loop, to avoid calling `body::Body::invalidate` for
-                // each basic block.
-                #[allow(unused_macro_rules)]
-                macro_rules! basic_blocks {
-                    (mut) => (body.basic_blocks_mut().iter_enumerated_mut());
-                    () => (body.basic_blocks().iter_enumerated());
-                }
-                for (bb, data) in basic_blocks!($($mutability)?) {
-                    self.visit_basic_block_data(bb, data);
-                }
-
-                for scope in &$($mutability)? body.source_scopes {
-                    self.visit_source_scope_data(scope);
-                }
-
-                self.visit_ty(
-                    $(& $mutability)? body.return_ty(),
-                    TyContext::ReturnTy(SourceInfo::outermost(body.span))
-                );
-
-                for local in body.local_decls.indices() {
-                    self.visit_local_decl(local, & $($mutability)? body.local_decls[local]);
-                }
-
-                #[allow(unused_macro_rules)]
-                macro_rules! type_annotations {
-                    (mut) => (body.user_type_annotations.iter_enumerated_mut());
-                    () => (body.user_type_annotations.iter_enumerated());
-                }
-
-                for (index, annotation) in type_annotations!($($mutability)?) {
-                    self.visit_user_type_annotation(
-                        index, annotation
-                    );
-                }
-
-                for var_debug_info in &$($mutability)? body.var_debug_info {
-                    self.visit_var_debug_info(var_debug_info);
-                }
-
-                self.visit_span($(& $mutability)? body.span);
-
-                for const_ in &$($mutability)? body.required_consts {
-                    let location = START_BLOCK.start_location();
-                    self.visit_constant(const_, location);
-                }
+                super_body!(self, body, $($mutability, true)?);
             }
 
             fn super_basic_block_data(&mut self,
@@ -479,14 +417,15 @@ macro_rules! make_mir_visitor {
                             location
                         )
                     }
-                    StatementKind::CopyNonOverlapping(box crate::mir::CopyNonOverlapping{
-                        src,
-                        dst,
-                        count,
-                    }) => {
-                      self.visit_operand(src, location);
-                      self.visit_operand(dst, location);
-                      self.visit_operand(count, location)
+                    StatementKind::Intrinsic(box ref $($mutability)? intrinsic) => {
+                        match intrinsic {
+                            NonDivergingIntrinsic::Assume(op) => self.visit_operand(op, location),
+                            NonDivergingIntrinsic::CopyNonOverlapping(CopyNonOverlapping { src, dst, count }) => {
+                                self.visit_operand(src, location);
+                                self.visit_operand(dst, location);
+                                self.visit_operand(count, location);
+                            }
+                        }
                     }
                     StatementKind::Nop => {}
                 }
@@ -930,8 +869,9 @@ macro_rules! make_mir_visitor {
                 self.visit_span($(& $mutability)? *span);
                 drop(user_ty); // no visit method for this
                 match literal {
-                    ConstantKind::Ty(ct) => self.visit_const($(& $mutability)? *ct, location),
+                    ConstantKind::Ty(_) => {}
                     ConstantKind::Val(_, ty) => self.visit_ty($(& $mutability)? *ty, TyContext::Location(location)),
+                    ConstantKind::Unevaluated(_, ty) => self.visit_ty($(& $mutability)? *ty, TyContext::Location(location)),
                 }
             }
 
@@ -969,9 +909,6 @@ macro_rules! make_mir_visitor {
             fn super_region(&mut self, _region: $(& $mutability)? ty::Region<'tcx>) {
             }
 
-            fn super_const(&mut self, _const: $(& $mutability)? ty::Const<'tcx>) {
-            }
-
             fn super_substs(&mut self, _substs: & $($mutability)? SubstsRef<'tcx>) {
             }
 
@@ -982,12 +919,7 @@ macro_rules! make_mir_visitor {
                 body: &$($mutability)? Body<'tcx>,
                 location: Location
             ) {
-                #[allow(unused_macro_rules)]
-                macro_rules! basic_blocks {
-                    (mut) => (body.basic_blocks_mut());
-                    () => (body.basic_blocks());
-                }
-                let basic_block = & $($mutability)? basic_blocks!($($mutability)?)[location.block];
+                let basic_block = & $($mutability)? basic_blocks!(body, $($mutability, true)?)[location.block];
                 if basic_block.statements.len() == location.statement_index {
                     if let Some(ref $($mutability)? terminator) = basic_block.terminator {
                         self.visit_terminator(terminator, location)
@@ -1002,6 +934,94 @@ macro_rules! make_mir_visitor {
     }
 }
 
+macro_rules! basic_blocks {
+    ($body:ident, mut, true) => {
+        $body.basic_blocks.as_mut()
+    };
+    ($body:ident, mut, false) => {
+        $body.basic_blocks.as_mut_preserves_cfg()
+    };
+    ($body:ident,) => {
+        $body.basic_blocks
+    };
+}
+
+macro_rules! basic_blocks_iter {
+    ($body:ident, mut, $invalidate:tt) => {
+        basic_blocks!($body, mut, $invalidate).iter_enumerated_mut()
+    };
+    ($body:ident,) => {
+        basic_blocks!($body,).iter_enumerated()
+    };
+}
+
+macro_rules! extra_body_methods {
+    (mut) => {
+        fn visit_body_preserves_cfg(&mut self, body: &mut Body<'tcx>) {
+            self.super_body_preserves_cfg(body);
+        }
+
+        fn super_body_preserves_cfg(&mut self, body: &mut Body<'tcx>) {
+            super_body!(self, body, mut, false);
+        }
+    };
+    () => {};
+}
+
+macro_rules! super_body {
+    ($self:ident, $body:ident, $($mutability:ident, $invalidate:tt)?) => {
+        let span = $body.span;
+        if let Some(gen) = &$($mutability)? $body.generator {
+            if let Some(yield_ty) = $(& $mutability)? gen.yield_ty {
+                $self.visit_ty(
+                    yield_ty,
+                    TyContext::YieldTy(SourceInfo::outermost(span))
+                );
+            }
+        }
+
+        for (bb, data) in basic_blocks_iter!($body, $($mutability, $invalidate)?) {
+            $self.visit_basic_block_data(bb, data);
+        }
+
+        for scope in &$($mutability)? $body.source_scopes {
+            $self.visit_source_scope_data(scope);
+        }
+
+        $self.visit_ty(
+            $(& $mutability)? $body.return_ty(),
+            TyContext::ReturnTy(SourceInfo::outermost($body.span))
+        );
+
+        for local in $body.local_decls.indices() {
+            $self.visit_local_decl(local, & $($mutability)? $body.local_decls[local]);
+        }
+
+        #[allow(unused_macro_rules)]
+        macro_rules! type_annotations {
+            (mut) => ($body.user_type_annotations.iter_enumerated_mut());
+            () => ($body.user_type_annotations.iter_enumerated());
+        }
+
+        for (index, annotation) in type_annotations!($($mutability)?) {
+            $self.visit_user_type_annotation(
+                index, annotation
+            );
+        }
+
+        for var_debug_info in &$($mutability)? $body.var_debug_info {
+            $self.visit_var_debug_info(var_debug_info);
+        }
+
+        $self.visit_span($(& $mutability)? $body.span);
+
+        for const_ in &$($mutability)? $body.required_consts {
+            let location = START_BLOCK.start_location();
+            $self.visit_constant(const_, location);
+        }
+    }
+}
+
 macro_rules! visit_place_fns {
     (mut) => {
         fn tcx<'a>(&'a self) -> TyCtxt<'tcx>;
diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs
index d8483e7e4..4b336ea62 100644
--- a/compiler/rustc_middle/src/query/mod.rs
+++ b/compiler/rustc_middle/src/query/mod.rs
@@ -47,14 +47,14 @@ rustc_queries! {
     /// To avoid this fate, do not call `tcx.hir().krate()`; instead,
     /// prefer wrappers like `tcx.visit_all_items_in_krate()`.
     query hir_crate(key: ()) -> Crate<'tcx> {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         eval_always
         desc { "get the crate HIR" }
     }
 
     /// All items in the crate.
     query hir_crate_items(_: ()) -> rustc_middle::hir::ModuleItems {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         eval_always
         desc { "get HIR crate items" }
     }
@@ -64,7 +64,7 @@ rustc_queries! {
     /// This can be conveniently accessed by `tcx.hir().visit_item_likes_in_module`.
     /// Avoid calling this query directly.
     query hir_module_items(key: LocalDefId) -> rustc_middle::hir::ModuleItems {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         desc { |tcx| "HIR module items in `{}`", tcx.def_path_str(key.to_def_id()) }
         cache_on_disk_if { true }
     }
@@ -161,6 +161,14 @@ rustc_queries! {
         separate_provide_extern
     }
 
+    query collect_trait_impl_trait_tys(key: DefId)
+        -> Result<&'tcx FxHashMap<DefId, Ty<'tcx>>, ErrorGuaranteed>
+    {
+        desc { "compare an impl and trait method signature, inferring any hidden `impl Trait` types in the process" }
+        cache_on_disk_if { key.is_local() }
+        separate_provide_extern
+    }
+
     query analysis(key: ()) -> Result<(), ErrorGuaranteed> {
         eval_always
         desc { "running analysis passes on this crate" }
@@ -189,7 +197,7 @@ rustc_queries! {
     /// associated generics.
     query generics_of(key: DefId) -> ty::Generics {
         desc { |tcx| "computing generics of `{}`", tcx.def_path_str(key) }
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         cache_on_disk_if { key.is_local() }
         separate_provide_extern
     }
@@ -261,13 +269,13 @@ rustc_queries! {
     }
 
     query native_libraries(_: CrateNum) -> Vec<NativeLib> {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         desc { "looking up the native libraries of a linked crate" }
         separate_provide_extern
     }
 
     query lint_levels(_: ()) -> LintLevelMap {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         eval_always
         desc { "computing the lint levels for items in this crate" }
     }
@@ -300,7 +308,7 @@ rustc_queries! {
     /// Create a THIR tree for debugging.
     query thir_tree(key: ty::WithOptConstParam<LocalDefId>) -> String {
         no_hash
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         desc { |tcx| "constructing THIR tree for `{}`", tcx.def_path_str(key.did.to_def_id()) }
     }
 
@@ -308,7 +316,7 @@ rustc_queries! {
     /// them. This includes all the body owners, but also things like struct
     /// constructors.
     query mir_keys(_: ()) -> rustc_data_structures::fx::FxIndexSet<LocalDefId> {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         desc { "getting a list of all mir_keys" }
     }
 
@@ -415,7 +423,7 @@ rustc_queries! {
     query symbols_for_closure_captures(
         key: (LocalDefId, LocalDefId)
     ) -> Vec<rustc_span::Symbol> {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         desc {
             |tcx| "symbols for captures of closure `{}` in `{}`",
             tcx.def_path_str(key.1.to_def_id()),
@@ -435,7 +443,7 @@ rustc_queries! {
     /// MIR pass (assuming the -Cinstrument-coverage option is enabled).
     query coverageinfo(key: ty::InstanceDef<'tcx>) -> mir::CoverageInfo {
         desc { |tcx| "retrieving coverage info from MIR for `{}`", tcx.def_path_str(key.def_id()) }
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
     }
 
     /// Returns the `CodeRegions` for a function that has instrumented coverage, in case the
@@ -445,7 +453,7 @@ rustc_queries! {
             |tcx| "retrieving the covered `CodeRegion`s, if instrumented, for `{}`",
             tcx.def_path_str(key)
         }
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         cache_on_disk_if { key.is_local() }
     }
 
@@ -483,7 +491,7 @@ rustc_queries! {
     }
 
     query wasm_import_module_map(_: CrateNum) -> FxHashMap<DefId, String> {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         desc { "wasm import module map" }
     }
 
@@ -559,7 +567,7 @@ rustc_queries! {
 
     query trait_def(key: DefId) -> ty::TraitDef {
         desc { |tcx| "computing trait definition for `{}`", tcx.def_path_str(key) }
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         cache_on_disk_if { key.is_local() }
         separate_provide_extern
     }
@@ -637,7 +645,7 @@ rustc_queries! {
 
     /// Gets a map with the variance of every item; use `item_variance` instead.
     query crate_variances(_: ()) -> ty::CrateVariancesMap<'tcx> {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         desc { "computing the variances for items in this crate" }
     }
 
@@ -650,7 +658,7 @@ rustc_queries! {
 
     /// Maps from thee `DefId` of a type to its (inferred) outlives.
     query inferred_outlives_crate(_: ()) -> ty::CratePredicatesMap<'tcx> {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         desc { "computing the inferred outlives predicates for items in this crate" }
     }
 
@@ -664,14 +672,14 @@ rustc_queries! {
     /// Maps from a trait item to the trait item "descriptor".
     query associated_item(key: DefId) -> ty::AssocItem {
         desc { |tcx| "computing associated item data for `{}`", tcx.def_path_str(key) }
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         cache_on_disk_if { key.is_local() }
         separate_provide_extern
     }
 
     /// Collects the associated items defined on a trait or impl.
     query associated_items(key: DefId) -> ty::AssocItems<'tcx> {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         desc { |tcx| "collecting associated items of {}", tcx.def_path_str(key) }
     }
 
@@ -697,7 +705,7 @@ rustc_queries! {
     /// The map returned for `tcx.impl_item_implementor_ids(impl_id)` would be
     ///`{ trait_f: impl_f, trait_g: impl_g }`
     query impl_item_implementor_ids(impl_id: DefId) -> FxHashMap<DefId, DefId> {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         desc { |tcx| "comparing impl items against trait for {}", tcx.def_path_str(impl_id) }
     }
 
@@ -765,11 +773,20 @@ rustc_queries! {
         desc { |tcx| "processing `{}`", tcx.def_path_str(key.to_def_id()) }
     }
 
+    /// Returns the types assumed to be well formed while "inside" of the given item.
+    ///
+    /// Note that we've liberated the late bound regions of function signatures, so
+    /// this can not be used to check whether these types are well formed.
+    query assumed_wf_types(key: DefId) -> &'tcx ty::List<Ty<'tcx>> {
+        desc { |tcx| "computing the implied bounds of {}", tcx.def_path_str(key) }
+    }
+
     /// Computes the signature of the function.
     query fn_sig(key: DefId) -> ty::PolyFnSig<'tcx> {
         desc { |tcx| "computing function signature of `{}`", tcx.def_path_str(key) }
         cache_on_disk_if { key.is_local() }
         separate_provide_extern
+        cycle_delay_bug
     }
 
     /// Performs lint checking for the module.
@@ -809,8 +826,8 @@ rustc_queries! {
         desc { |tcx| "checking privacy in {}", describe_as_module(key, tcx) }
     }
 
-    query check_mod_liveness(key: LocalDefId) -> () {
-        desc { |tcx| "checking liveness of variables in {}", describe_as_module(key, tcx) }
+    query check_liveness(key: DefId) {
+        desc { |tcx| "checking liveness of variables in {}", tcx.def_path_str(key) }
     }
 
     /// Return the live symbols in the crate for dead code check.
@@ -821,7 +838,7 @@ rustc_queries! {
         FxHashSet<LocalDefId>,
         FxHashMap<LocalDefId, Vec<(DefId, DefId)>>
     ) {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         desc { "find live symbols in crate" }
     }
 
@@ -867,13 +884,6 @@ rustc_queries! {
     query diagnostic_only_typeck(key: LocalDefId) -> &'tcx ty::TypeckResults<'tcx> {
         desc { |tcx| "type-checking `{}`", tcx.def_path_str(key.to_def_id()) }
         cache_on_disk_if { true }
-        load_cached(tcx, id) {
-            let typeck_results: Option<ty::TypeckResults<'tcx>> = tcx
-                .on_disk_cache().as_ref()
-                .and_then(|c| c.try_load_query_result(*tcx, id));
-
-            typeck_results.map(|x| &*tcx.arena.alloc(x))
-        }
     }
 
     query used_trait_imports(key: LocalDefId) -> &'tcx FxHashSet<LocalDefId> {
@@ -905,7 +915,7 @@ rustc_queries! {
     /// Gets a complete map from all types to their inherent impls.
     /// Not meant to be used directly outside of coherence.
     query crate_inherent_impls(k: ()) -> CrateInherentImpls {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         desc { "all inherent impls defined in crate" }
     }
 
@@ -1038,7 +1048,7 @@ rustc_queries! {
     }
 
     query reachable_set(_: ()) -> FxHashSet<LocalDefId> {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         desc { "reachability" }
     }
 
@@ -1050,7 +1060,7 @@ rustc_queries! {
 
     /// Generates a MIR body for the shim.
     query mir_shims(key: ty::InstanceDef<'tcx>) -> mir::Body<'tcx> {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         desc { |tcx| "generating MIR shim for `{}`", tcx.def_path_str(key.def_id()) }
     }
 
@@ -1094,6 +1104,11 @@ rustc_queries! {
         separate_provide_extern
     }
 
+    query lookup_default_body_stability(def_id: DefId) -> Option<attr::DefaultBodyStability> {
+        desc { |tcx| "looking up default body stability of `{}`", tcx.def_path_str(def_id) }
+        separate_provide_extern
+    }
+
     query should_inherit_track_caller(def_id: DefId) -> bool {
         desc { |tcx| "computing should_inherit_track_caller of `{}`", tcx.def_path_str(def_id) }
     }
@@ -1119,7 +1134,7 @@ rustc_queries! {
 
     query codegen_fn_attrs(def_id: DefId) -> CodegenFnAttrs {
         desc { |tcx| "computing codegen attributes of `{}`", tcx.def_path_str(def_id) }
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         cache_on_disk_if { def_id.is_local() }
         separate_provide_extern
     }
@@ -1136,8 +1151,8 @@ rustc_queries! {
     /// Gets the rendered value of the specified constant or associated constant.
     /// Used by rustdoc.
     query rendered_const(def_id: DefId) -> String {
-        storage(ArenaCacheSelector<'tcx>)
-        desc { |tcx| "rendering constant intializer of `{}`", tcx.def_path_str(def_id) }
+        arena_cache
+        desc { |tcx| "rendering constant initializer of `{}`", tcx.def_path_str(def_id) }
         cache_on_disk_if { def_id.is_local() }
         separate_provide_extern
     }
@@ -1181,14 +1196,11 @@ rustc_queries! {
         }
     }
 
-    query codegen_fulfill_obligation(
+    query codegen_select_candidate(
         key: (ty::ParamEnv<'tcx>, ty::PolyTraitRef<'tcx>)
     ) -> Result<&'tcx ImplSource<'tcx, ()>, traits::CodegenObligationError> {
         cache_on_disk_if { true }
-        desc { |tcx|
-            "checking if `{}` fulfills its obligations",
-            tcx.def_path_str(key.1.def_id())
-        }
+        desc { |tcx| "computing candidate for `{}`", key.1 }
     }
 
     /// Return all `impl` blocks in the current crate.
@@ -1198,12 +1210,12 @@ rustc_queries! {
 
     /// Given a trait `trait_id`, return all known `impl` blocks.
     query trait_impls_of(trait_id: DefId) -> ty::trait_def::TraitImpls {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         desc { |tcx| "trait impls of `{}`", tcx.def_path_str(trait_id) }
     }
 
     query specialization_graph_of(trait_id: DefId) -> specialization_graph::Graph {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         desc { |tcx| "building specialization graph of trait `{}`", tcx.def_path_str(trait_id) }
         cache_on_disk_if { true }
     }
@@ -1295,6 +1307,7 @@ rustc_queries! {
     query layout_of(
         key: ty::ParamEnvAnd<'tcx, Ty<'tcx>>
     ) -> Result<ty::layout::TyAndLayout<'tcx>, ty::layout::LayoutError<'tcx>> {
+        depth_limit
         desc { "computing layout of `{}`", key.value }
         remap_env_constness
     }
@@ -1329,7 +1342,7 @@ rustc_queries! {
     }
 
     query dependency_formats(_: ()) -> Lrc<crate::middle::dependency_format::Dependencies> {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         desc { "get the linkage format of all dependencies" }
     }
 
@@ -1422,7 +1435,7 @@ rustc_queries! {
     // like the compiler-generated `main` function and so on.
     query reachable_non_generics(_: CrateNum)
         -> DefIdMap<SymbolExportInfo> {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         desc { "looking up the exported symbols of a crate" }
         separate_provide_extern
     }
@@ -1445,7 +1458,7 @@ rustc_queries! {
     /// `upstream_monomorphizations_for`, `upstream_drop_glue_for`, or, even
     /// better, `Instance::upstream_monomorphization()`.
     query upstream_monomorphizations(_: ()) -> DefIdMap<FxHashMap<SubstsRef<'tcx>, CrateNum>> {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         desc { "collecting available upstream monomorphizations" }
     }
 
@@ -1459,7 +1472,7 @@ rustc_queries! {
     query upstream_monomorphizations_for(def_id: DefId)
         -> Option<&'tcx FxHashMap<SubstsRef<'tcx>, CrateNum>>
     {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         desc { |tcx|
             "collecting available upstream monomorphizations for `{}`",
             tcx.def_path_str(def_id),
@@ -1487,7 +1500,7 @@ rustc_queries! {
     }
 
     query foreign_modules(_: CrateNum) -> FxHashMap<DefId, ForeignModule> {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         desc { "looking up the foreign modules of a linked crate" }
         separate_provide_extern
     }
@@ -1513,13 +1526,13 @@ rustc_queries! {
         separate_provide_extern
     }
     query extra_filename(_: CrateNum) -> String {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         eval_always
         desc { "looking up the extra filename for a crate" }
         separate_provide_extern
     }
     query crate_extern_paths(_: CrateNum) -> Vec<PathBuf> {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         eval_always
         desc { "looking up the paths for extern crates" }
         separate_provide_extern
@@ -1551,6 +1564,9 @@ rustc_queries! {
         -> Option<NativeLibKind> {
         desc { |tcx| "native_library_kind({})", tcx.def_path_str(def_id) }
     }
+    query native_library(def_id: DefId) -> Option<&'tcx NativeLib> {
+        desc { |tcx| "native_library({})", tcx.def_path_str(def_id) }
+    }
 
     /// Does lifetime resolution, but does not descend into trait items. This
     /// should only be used for resolving lifetimes of on trait definitions,
@@ -1558,14 +1574,14 @@ rustc_queries! {
     /// the same lifetimes and is responsible for diagnostics.
     /// See `rustc_resolve::late::lifetimes for details.
     query resolve_lifetimes_trait_definition(_: LocalDefId) -> ResolveLifetimes {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         desc { "resolving lifetimes for a trait definition" }
     }
     /// Does lifetime resolution on items. Importantly, we can't resolve
     /// lifetimes directly on things like trait methods, because of trait params.
     /// See `rustc_resolve::late::lifetimes for details.
     query resolve_lifetimes(_: LocalDefId) -> ResolveLifetimes {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         desc { "resolving lifetimes" }
     }
     query named_region_map(_: LocalDefId) ->
@@ -1575,19 +1591,31 @@ rustc_queries! {
     query is_late_bound_map(_: LocalDefId) -> Option<&'tcx FxIndexSet<LocalDefId>> {
         desc { "testing if a region is late bound" }
     }
-    /// For a given item (like a struct), gets the default lifetimes to be used
+    /// For a given item's generic parameter, gets the default lifetimes to be used
     /// for each parameter if a trait object were to be passed for that parameter.
-    /// For example, for `struct Foo<'a, T, U>`, this would be `['static, 'static]`.
-    /// For `struct Foo<'a, T: 'a, U>`, this would instead be `['a, 'static]`.
-    query object_lifetime_defaults(_: LocalDefId) -> Option<&'tcx [ObjectLifetimeDefault]> {
-        desc { "looking up lifetime defaults for a region on an item" }
+    /// For example, for `T` in `struct Foo<'a, T>`, this would be `'static`.
+    /// For `T` in `struct Foo<'a, T: 'a>`, this would instead be `'a`.
+    /// This query will panic if passed something that is not a type parameter.
+    query object_lifetime_default(key: DefId) -> ObjectLifetimeDefault {
+        desc { "looking up lifetime defaults for generic parameter `{}`", tcx.def_path_str(key) }
+        separate_provide_extern
     }
     query late_bound_vars_map(_: LocalDefId)
         -> Option<&'tcx FxHashMap<ItemLocalId, Vec<ty::BoundVariableKind>>> {
         desc { "looking up late bound vars" }
     }
 
-    query visibility(def_id: DefId) -> ty::Visibility {
+    /// Computes the visibility of the provided `def_id`.
+    ///
+    /// If the item from the `def_id` doesn't have a visibility, it will panic. For example
+    /// a generic type parameter will panic if you call this method on it:
+    ///
+    /// ```
+    /// pub trait Foo<T: Debug> {}
+    /// ```
+    ///
+    /// In here, if you call `visibility` on `T`, it'll panic.
+    query visibility(def_id: DefId) -> ty::Visibility<DefId> {
         desc { |tcx| "computing visibility of `{}`", tcx.def_path_str(def_id) }
         separate_provide_extern
     }
@@ -1623,7 +1651,7 @@ rustc_queries! {
     }
 
     query lib_features(_: ()) -> LibFeatures {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         desc { "calculating the lib features map" }
     }
     query defined_lib_features(_: CrateNum) -> &'tcx [(Symbol, Option<Symbol>)] {
@@ -1631,7 +1659,7 @@ rustc_queries! {
         separate_provide_extern
     }
     query stability_implications(_: CrateNum) -> FxHashMap<Symbol, Symbol> {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         desc { "calculating the implications between `#[unstable]` features defined in a crate" }
         separate_provide_extern
     }
@@ -1642,14 +1670,14 @@ rustc_queries! {
     }
     /// Returns the lang items defined in another crate by loading it from metadata.
     query get_lang_items(_: ()) -> LanguageItems {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         eval_always
         desc { "calculating the lang items map" }
     }
 
     /// Returns all diagnostic items defined in all crates.
     query all_diagnostic_items(_: ()) -> rustc_hir::diagnostic_items::DiagnosticItems {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         eval_always
         desc { "calculating the diagnostic items map" }
     }
@@ -1662,7 +1690,7 @@ rustc_queries! {
 
     /// Returns the diagnostic items defined in a crate.
     query diagnostic_items(_: CrateNum) -> rustc_hir::diagnostic_items::DiagnosticItems {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         desc { "calculating the diagnostic items map in a crate" }
         separate_provide_extern
     }
@@ -1672,11 +1700,11 @@ rustc_queries! {
         separate_provide_extern
     }
     query visible_parent_map(_: ()) -> DefIdMap<DefId> {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         desc { "calculating the visible parent map" }
     }
     query trimmed_def_paths(_: ()) -> FxHashMap<DefId, Symbol> {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         desc { "calculating trimmed def paths" }
     }
     query missing_extern_crate_item(_: CrateNum) -> bool {
@@ -1685,14 +1713,14 @@ rustc_queries! {
         separate_provide_extern
     }
     query used_crate_source(_: CrateNum) -> Lrc<CrateSource> {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         eval_always
         desc { "looking at the source for a crate" }
         separate_provide_extern
     }
     /// Returns the debugger visualizers defined for this crate.
     query debugger_visualizers(_: CrateNum) -> Vec<rustc_span::DebuggerVisualizerFile> {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         desc { "looking up the debugger visualizers for this crate" }
         separate_provide_extern
     }
@@ -1726,7 +1754,7 @@ rustc_queries! {
     }
 
     query stability_index(_: ()) -> stability::Index {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         eval_always
         desc { "calculating the stability index for the local crate" }
     }
@@ -1951,6 +1979,14 @@ rustc_queries! {
         }
     }
 
+    query is_impossible_method(key: (DefId, DefId)) -> bool {
+        desc { |tcx|
+            "checking if {} is impossible to call within {}",
+            tcx.def_path_str(key.1),
+            tcx.def_path_str(key.0),
+        }
+    }
+
     query method_autoderef_steps(
         goal: CanonicalTyGoal<'tcx>
     ) -> MethodAutoderefStepsResult<'tcx> {
@@ -1959,7 +1995,7 @@ rustc_queries! {
     }
 
     query supported_target_features(_: CrateNum) -> FxHashMap<String, Option<Symbol>> {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         eval_always
         desc { "looking up supported target features" }
     }
@@ -2029,7 +2065,7 @@ rustc_queries! {
     /// all of the cases that the normal `ty::Ty`-based wfcheck does. This is fine,
     /// because the `ty::Ty`-based wfcheck is always run.
     query diagnostic_hir_wf_check(key: (ty::Predicate<'tcx>, traits::WellFormedLoc)) -> Option<traits::ObligationCause<'tcx>> {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         eval_always
         no_hash
         desc { "performing HIR wf-checking for predicate {:?} at item {:?}", key.0, key.1 }
@@ -2039,13 +2075,13 @@ rustc_queries! {
     /// The list of backend features computed from CLI flags (`-Ctarget-cpu`, `-Ctarget-feature`,
     /// `--target` and similar).
     query global_backend_features(_: ()) -> Vec<String> {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         eval_always
         desc { "computing the backend features for CLI flags" }
     }
 
     query generator_diagnostic_data(key: DefId) -> Option<GeneratorDiagnosticData<'tcx>> {
-        storage(ArenaCacheSelector<'tcx>)
+        arena_cache
         desc { |tcx| "looking up generator diagnostic data of `{}`", tcx.def_path_str(key) }
         separate_provide_extern
     }
diff --git a/compiler/rustc_middle/src/thir.rs b/compiler/rustc_middle/src/thir.rs
index b856af1d8..86b415050 100644
--- a/compiler/rustc_middle/src/thir.rs
+++ b/compiler/rustc_middle/src/thir.rs
@@ -15,50 +15,33 @@ use rustc_hir::def_id::DefId;
 use rustc_hir::RangeEnd;
 use rustc_index::newtype_index;
 use rustc_index::vec::IndexVec;
-use rustc_middle::infer::canonical::Canonical;
 use rustc_middle::middle::region;
 use rustc_middle::mir::interpret::AllocId;
 use rustc_middle::mir::{self, BinOp, BorrowKind, FakeReadCause, Field, Mutability, UnOp};
 use rustc_middle::ty::adjustment::PointerCast;
 use rustc_middle::ty::subst::SubstsRef;
-use rustc_middle::ty::CanonicalUserTypeAnnotation;
-use rustc_middle::ty::{self, AdtDef, Ty, UpvarSubsts, UserType};
-use rustc_span::{Span, Symbol, DUMMY_SP};
+use rustc_middle::ty::{self, AdtDef, Ty, UpvarSubsts};
+use rustc_middle::ty::{CanonicalUserType, CanonicalUserTypeAnnotation};
+use rustc_span::def_id::LocalDefId;
+use rustc_span::{sym, Span, Symbol, DUMMY_SP};
 use rustc_target::abi::VariantIdx;
 use rustc_target::asm::InlineAsmRegOrRegClass;
-
-use rustc_span::def_id::LocalDefId;
 use std::fmt;
 use std::ops::Index;
 
 pub mod visit;
 
-newtype_index! {
-    /// An index to an [`Arm`] stored in [`Thir::arms`]
-    #[derive(HashStable)]
-    pub struct ArmId {
-        DEBUG_FORMAT = "a{}"
-    }
-}
-
-newtype_index! {
-    /// An index to an [`Expr`] stored in [`Thir::exprs`]
-    #[derive(HashStable)]
-    pub struct ExprId {
-        DEBUG_FORMAT = "e{}"
-    }
-}
-
-newtype_index! {
-    #[derive(HashStable)]
-    /// An index to a [`Stmt`] stored in [`Thir::stmts`]
-    pub struct StmtId {
-        DEBUG_FORMAT = "s{}"
-    }
-}
-
 macro_rules! thir_with_elements {
-    ($($name:ident: $id:ty => $value:ty,)*) => {
+    ($($name:ident: $id:ty => $value:ty => $format:literal,)*) => {
+        $(
+            newtype_index! {
+                #[derive(HashStable)]
+                pub struct $id {
+                    DEBUG_FORMAT = $format
+                }
+            }
+        )*
+
         /// A container for a THIR body.
         ///
         /// This can be indexed directly by any THIR index (e.g. [`ExprId`]).
@@ -90,10 +73,29 @@ macro_rules! thir_with_elements {
     }
 }
 
+pub const UPVAR_ENV_PARAM: ParamId = ParamId::from_u32(0);
+
 thir_with_elements! {
-    arms: ArmId => Arm<'tcx>,
-    exprs: ExprId => Expr<'tcx>,
-    stmts: StmtId => Stmt<'tcx>,
+    arms: ArmId => Arm<'tcx> => "a{}",
+    blocks: BlockId => Block => "b{}",
+    exprs: ExprId => Expr<'tcx> => "e{}",
+    stmts: StmtId => Stmt<'tcx> => "s{}",
+    params: ParamId => Param<'tcx> => "p{}",
+}
+
+/// Description of a type-checked function parameter.
+#[derive(Clone, Debug, HashStable)]
+pub struct Param<'tcx> {
+    /// The pattern that appears in the parameter list, or None for implicit parameters.
+    pub pat: Option<Box<Pat<'tcx>>>,
+    /// The possibly inferred type.
+    pub ty: Ty<'tcx>,
+    /// Span of the explicitly provided type, or None if inferred for closures.
+    pub ty_span: Option<Span>,
+    /// Whether this param is `self`, and how it is bound.
+    pub self_kind: Option<hir::ImplicitSelfKind>,
+    /// HirId for lints.
+    pub hir_id: Option<hir::HirId>,
 }
 
 #[derive(Copy, Clone, Debug, HashStable)]
@@ -121,8 +123,10 @@ pub struct Block {
     pub safety_mode: BlockSafety,
 }
 
+type UserTy<'tcx> = Option<Box<CanonicalUserType<'tcx>>>;
+
 #[derive(Clone, Debug, HashStable)]
-pub struct Adt<'tcx> {
+pub struct AdtExpr<'tcx> {
     /// The ADT we're constructing.
     pub adt_def: AdtDef<'tcx>,
     /// The variant of the ADT.
@@ -131,13 +135,30 @@ pub struct Adt<'tcx> {
 
     /// Optional user-given substs: for something like `let x =
     /// Bar::<T> { ... }`.
-    pub user_ty: Option<Canonical<'tcx, UserType<'tcx>>>,
+    pub user_ty: UserTy<'tcx>,
 
     pub fields: Box<[FieldExpr]>,
     /// The base, e.g. `Foo {x: 1, .. base}`.
     pub base: Option<FruInfo<'tcx>>,
 }
 
+#[derive(Clone, Debug, HashStable)]
+pub struct ClosureExpr<'tcx> {
+    pub closure_id: LocalDefId,
+    pub substs: UpvarSubsts<'tcx>,
+    pub upvars: Box<[ExprId]>,
+    pub movability: Option<hir::Movability>,
+    pub fake_reads: Vec<(ExprId, FakeReadCause, hir::HirId)>,
+}
+
+#[derive(Clone, Debug, HashStable)]
+pub struct InlineAsmExpr<'tcx> {
+    pub template: &'tcx [InlineAsmTemplatePiece],
+    pub operands: Box<[InlineAsmOperand<'tcx>]>,
+    pub options: InlineAsmOptions,
+    pub line_spans: &'tcx [Span],
+}
+
 #[derive(Copy, Clone, Debug, HashStable)]
 pub enum BlockSafety {
     Safe,
@@ -177,13 +198,13 @@ pub enum StmtKind<'tcx> {
         /// `let <PAT> = ...`
         ///
         /// If a type annotation is included, it is added as an ascription pattern.
-        pattern: Pat<'tcx>,
+        pattern: Box<Pat<'tcx>>,
 
         /// `let pat: ty = <INIT>`
         initializer: Option<ExprId>,
 
         /// `let pat: ty = <INIT> else { <ELSE> }
-        else_block: Option<Block>,
+        else_block: Option<BlockId>,
 
         /// The lint level for this `let` statement.
         lint_level: LintLevel,
@@ -298,7 +319,7 @@ pub enum ExprKind<'tcx> {
     },
     Let {
         expr: ExprId,
-        pat: Pat<'tcx>,
+        pat: Box<Pat<'tcx>>,
     },
     /// A `match` expression.
     Match {
@@ -307,7 +328,7 @@ pub enum ExprKind<'tcx> {
     },
     /// A block.
     Block {
-        body: Block,
+        block: BlockId,
     },
     /// An assignment: `lhs = rhs`.
     Assign {
@@ -387,27 +408,21 @@ pub enum ExprKind<'tcx> {
         fields: Box<[ExprId]>,
     },
     /// An ADT constructor, e.g. `Foo {x: 1, y: 2}`.
-    Adt(Box<Adt<'tcx>>),
+    Adt(Box<AdtExpr<'tcx>>),
     /// A type ascription on a place.
     PlaceTypeAscription {
         source: ExprId,
         /// Type that the user gave to this expression
-        user_ty: Option<Canonical<'tcx, UserType<'tcx>>>,
+        user_ty: UserTy<'tcx>,
     },
     /// A type ascription on a value, e.g. `42: i32`.
     ValueTypeAscription {
         source: ExprId,
         /// Type that the user gave to this expression
-        user_ty: Option<Canonical<'tcx, UserType<'tcx>>>,
+        user_ty: UserTy<'tcx>,
     },
     /// A closure definition.
-    Closure {
-        closure_id: LocalDefId,
-        substs: UpvarSubsts<'tcx>,
-        upvars: Box<[ExprId]>,
-        movability: Option<hir::Movability>,
-        fake_reads: Vec<(ExprId, FakeReadCause, hir::HirId)>,
-    },
+    Closure(Box<ClosureExpr<'tcx>>),
     /// A literal.
     Literal {
         lit: &'tcx hir::Lit,
@@ -416,17 +431,17 @@ pub enum ExprKind<'tcx> {
     /// For literals that don't correspond to anything in the HIR
     NonHirLiteral {
         lit: ty::ScalarInt,
-        user_ty: Option<Canonical<'tcx, UserType<'tcx>>>,
+        user_ty: UserTy<'tcx>,
     },
     /// A literal of a ZST type.
     ZstLiteral {
-        user_ty: Option<Canonical<'tcx, UserType<'tcx>>>,
+        user_ty: UserTy<'tcx>,
     },
     /// Associated constants and named constants
     NamedConst {
         def_id: DefId,
         substs: SubstsRef<'tcx>,
-        user_ty: Option<Canonical<'tcx, UserType<'tcx>>>,
+        user_ty: UserTy<'tcx>,
     },
     ConstParam {
         param: ty::ParamConst,
@@ -443,12 +458,7 @@ pub enum ExprKind<'tcx> {
         def_id: DefId,
     },
     /// Inline assembly, i.e. `asm!()`.
-    InlineAsm {
-        template: &'tcx [InlineAsmTemplatePiece],
-        operands: Box<[InlineAsmOperand<'tcx>]>,
-        options: InlineAsmOptions,
-        line_spans: &'tcx [Span],
-    },
+    InlineAsm(Box<InlineAsmExpr<'tcx>>),
     /// An expression taking a reference to a thread local.
     ThreadLocalRef(DefId),
     /// A `yield` expression.
@@ -475,7 +485,7 @@ pub struct FruInfo<'tcx> {
 /// A `match` arm.
 #[derive(Clone, Debug, HashStable)]
 pub struct Arm<'tcx> {
-    pub pattern: Pat<'tcx>,
+    pub pattern: Box<Pat<'tcx>>,
     pub guard: Option<Guard<'tcx>>,
     pub body: ExprId,
     pub lint_level: LintLevel,
@@ -487,7 +497,7 @@ pub struct Arm<'tcx> {
 #[derive(Clone, Debug, HashStable)]
 pub enum Guard<'tcx> {
     If(ExprId),
-    IfLet(Pat<'tcx>, ExprId),
+    IfLet(Box<Pat<'tcx>>, ExprId),
 }
 
 #[derive(Copy, Clone, Debug, HashStable)]
@@ -542,19 +552,28 @@ pub enum BindingMode {
 #[derive(Clone, Debug, HashStable)]
 pub struct FieldPat<'tcx> {
     pub field: Field,
-    pub pattern: Pat<'tcx>,
+    pub pattern: Box<Pat<'tcx>>,
 }
 
 #[derive(Clone, Debug, HashStable)]
 pub struct Pat<'tcx> {
     pub ty: Ty<'tcx>,
     pub span: Span,
-    pub kind: Box<PatKind<'tcx>>,
+    pub kind: PatKind<'tcx>,
 }
 
 impl<'tcx> Pat<'tcx> {
     pub fn wildcard_from_ty(ty: Ty<'tcx>) -> Self {
-        Pat { ty, span: DUMMY_SP, kind: Box::new(PatKind::Wild) }
+        Pat { ty, span: DUMMY_SP, kind: PatKind::Wild }
+    }
+
+    pub fn simple_ident(&self) -> Option<Symbol> {
+        match self.kind {
+            PatKind::Binding { name, mode: BindingMode::ByValue, subpattern: None, .. } => {
+                Some(name)
+            }
+            _ => None,
+        }
     }
 }
 
@@ -589,7 +608,7 @@ pub enum PatKind<'tcx> {
 
     AscribeUserType {
         ascription: Ascription<'tcx>,
-        subpattern: Pat<'tcx>,
+        subpattern: Box<Pat<'tcx>>,
     },
 
     /// `x`, `ref x`, `x @ P`, etc.
@@ -599,7 +618,7 @@ pub enum PatKind<'tcx> {
         mode: BindingMode,
         var: LocalVarId,
         ty: Ty<'tcx>,
-        subpattern: Option<Pat<'tcx>>,
+        subpattern: Option<Box<Pat<'tcx>>>,
         /// Is this the leftmost occurrence of the binding, i.e., is `var` the
         /// `HirId` of this pattern?
         is_primary: bool,
@@ -622,7 +641,7 @@ pub enum PatKind<'tcx> {
 
     /// `box P`, `&P`, `&mut P`, etc.
     Deref {
-        subpattern: Pat<'tcx>,
+        subpattern: Box<Pat<'tcx>>,
     },
 
     /// One of the following:
@@ -636,32 +655,32 @@ pub enum PatKind<'tcx> {
         value: mir::ConstantKind<'tcx>,
     },
 
-    Range(PatRange<'tcx>),
+    Range(Box<PatRange<'tcx>>),
 
     /// Matches against a slice, checking the length and extracting elements.
     /// irrefutable when there is a slice pattern and both `prefix` and `suffix` are empty.
     /// e.g., `&[ref xs @ ..]`.
     Slice {
-        prefix: Vec<Pat<'tcx>>,
-        slice: Option<Pat<'tcx>>,
-        suffix: Vec<Pat<'tcx>>,
+        prefix: Box<[Box<Pat<'tcx>>]>,
+        slice: Option<Box<Pat<'tcx>>>,
+        suffix: Box<[Box<Pat<'tcx>>]>,
     },
 
     /// Fixed match against an array; irrefutable.
     Array {
-        prefix: Vec<Pat<'tcx>>,
-        slice: Option<Pat<'tcx>>,
-        suffix: Vec<Pat<'tcx>>,
+        prefix: Box<[Box<Pat<'tcx>>]>,
+        slice: Option<Box<Pat<'tcx>>>,
+        suffix: Box<[Box<Pat<'tcx>>]>,
     },
 
     /// An or-pattern, e.g. `p | q`.
     /// Invariant: `pats.len() >= 2`.
     Or {
-        pats: Vec<Pat<'tcx>>,
+        pats: Box<[Box<Pat<'tcx>>]>,
     },
 }
 
-#[derive(Copy, Clone, Debug, PartialEq, HashStable)]
+#[derive(Clone, Debug, PartialEq, HashStable)]
 pub struct PatRange<'tcx> {
     pub lo: mir::ConstantKind<'tcx>,
     pub hi: mir::ConstantKind<'tcx>,
@@ -682,7 +701,7 @@ impl<'tcx> fmt::Display for Pat<'tcx> {
         };
         let mut start_or_comma = || start_or_continue(", ");
 
-        match *self.kind {
+        match self.kind {
             PatKind::Wild => write!(f, "_"),
             PatKind::AscribeUserType { ref subpattern, .. } => write!(f, "{}: _", subpattern),
             PatKind::Binding { mutability, name, mode, ref subpattern, .. } => {
@@ -703,17 +722,32 @@ impl<'tcx> fmt::Display for Pat<'tcx> {
                 Ok(())
             }
             PatKind::Variant { ref subpatterns, .. } | PatKind::Leaf { ref subpatterns } => {
-                let variant = match *self.kind {
-                    PatKind::Variant { adt_def, variant_index, .. } => {
-                        Some(adt_def.variant(variant_index))
-                    }
-                    _ => self.ty.ty_adt_def().and_then(|adt| {
-                        if !adt.is_enum() { Some(adt.non_enum_variant()) } else { None }
+                let variant_and_name = match self.kind {
+                    PatKind::Variant { adt_def, variant_index, .. } => ty::tls::with(|tcx| {
+                        let variant = adt_def.variant(variant_index);
+                        let adt_did = adt_def.did();
+                        let name = if tcx.get_diagnostic_item(sym::Option) == Some(adt_did)
+                            || tcx.get_diagnostic_item(sym::Result) == Some(adt_did)
+                        {
+                            variant.name.to_string()
+                        } else {
+                            format!("{}::{}", tcx.def_path_str(adt_def.did()), variant.name)
+                        };
+                        Some((variant, name))
+                    }),
+                    _ => self.ty.ty_adt_def().and_then(|adt_def| {
+                        if !adt_def.is_enum() {
+                            ty::tls::with(|tcx| {
+                                Some((adt_def.non_enum_variant(), tcx.def_path_str(adt_def.did())))
+                            })
+                        } else {
+                            None
+                        }
                     }),
                 };
 
-                if let Some(variant) = variant {
-                    write!(f, "{}", variant.name)?;
+                if let Some((variant, name)) = &variant_and_name {
+                    write!(f, "{}", name)?;
 
                     // Only for Adt we can have `S {...}`,
                     // which we handle separately here.
@@ -722,7 +756,7 @@ impl<'tcx> fmt::Display for Pat<'tcx> {
 
                         let mut printed = 0;
                         for p in subpatterns {
-                            if let PatKind::Wild = *p.pattern.kind {
+                            if let PatKind::Wild = p.pattern.kind {
                                 continue;
                             }
                             let name = variant.fields[p.field.index()].name;
@@ -738,8 +772,9 @@ impl<'tcx> fmt::Display for Pat<'tcx> {
                     }
                 }
 
-                let num_fields = variant.map_or(subpatterns.len(), |v| v.fields.len());
-                if num_fields != 0 || variant.is_none() {
+                let num_fields =
+                    variant_and_name.as_ref().map_or(subpatterns.len(), |(v, _)| v.fields.len());
+                if num_fields != 0 || variant_and_name.is_none() {
                     write!(f, "(")?;
                     for i in 0..num_fields {
                         write!(f, "{}", start_or_comma())?;
@@ -775,7 +810,7 @@ impl<'tcx> fmt::Display for Pat<'tcx> {
                 write!(f, "{}", subpattern)
             }
             PatKind::Constant { value } => write!(f, "{}", value),
-            PatKind::Range(PatRange { lo, hi, end }) => {
+            PatKind::Range(box PatRange { lo, hi, end }) => {
                 write!(f, "{}", lo)?;
                 write!(f, "{}", end)?;
                 write!(f, "{}", hi)
@@ -783,24 +818,24 @@ impl<'tcx> fmt::Display for Pat<'tcx> {
             PatKind::Slice { ref prefix, ref slice, ref suffix }
             | PatKind::Array { ref prefix, ref slice, ref suffix } => {
                 write!(f, "[")?;
-                for p in prefix {
+                for p in prefix.iter() {
                     write!(f, "{}{}", start_or_comma(), p)?;
                 }
                 if let Some(ref slice) = *slice {
                     write!(f, "{}", start_or_comma())?;
-                    match *slice.kind {
+                    match slice.kind {
                         PatKind::Wild => {}
                         _ => write!(f, "{}", slice)?,
                     }
                     write!(f, "..")?;
                 }
-                for p in suffix {
+                for p in suffix.iter() {
                     write!(f, "{}{}", start_or_comma(), p)?;
                 }
                 write!(f, "]")
             }
             PatKind::Or { ref pats } => {
-                for pat in pats {
+                for pat in pats.iter() {
                     write!(f, "{}{}", start_or_continue(" | "), pat)?;
                 }
                 Ok(())
@@ -814,8 +849,15 @@ impl<'tcx> fmt::Display for Pat<'tcx> {
 mod size_asserts {
     use super::*;
     // These are in alphabetical order, which is easy to maintain.
-    rustc_data_structures::static_assert_size!(Block, 56);
-    rustc_data_structures::static_assert_size!(Expr<'_>, 104);
-    rustc_data_structures::static_assert_size!(Pat<'_>, 24);
-    rustc_data_structures::static_assert_size!(Stmt<'_>, 120);
+    static_assert_size!(Block, 56);
+    static_assert_size!(Expr<'_>, 64);
+    static_assert_size!(ExprKind<'_>, 40);
+    #[cfg(not(bootstrap))]
+    static_assert_size!(Pat<'_>, 72);
+    #[cfg(not(bootstrap))]
+    static_assert_size!(PatKind<'_>, 56);
+    #[cfg(not(bootstrap))]
+    static_assert_size!(Stmt<'_>, 48);
+    #[cfg(not(bootstrap))]
+    static_assert_size!(StmtKind<'_>, 40);
 }
diff --git a/compiler/rustc_middle/src/thir/visit.rs b/compiler/rustc_middle/src/thir/visit.rs
index 97249fdd1..79a0e75aa 100644
--- a/compiler/rustc_middle/src/thir/visit.rs
+++ b/compiler/rustc_middle/src/thir/visit.rs
@@ -1,5 +1,6 @@
 use super::{
-    Arm, Block, Expr, ExprKind, Guard, InlineAsmOperand, Pat, PatKind, Stmt, StmtKind, Thir,
+    AdtExpr, Arm, Block, ClosureExpr, Expr, ExprKind, Guard, InlineAsmExpr, InlineAsmOperand, Pat,
+    PatKind, Stmt, StmtKind, Thir,
 };
 
 pub trait Visitor<'a, 'tcx: 'a>: Sized {
@@ -75,7 +76,7 @@ pub fn walk_expr<'a, 'tcx: 'a, V: Visitor<'a, 'tcx>>(visitor: &mut V, expr: &Exp
                 visitor.visit_arm(&visitor.thir()[arm]);
             }
         }
-        Block { ref body } => visitor.visit_block(body),
+        Block { block } => visitor.visit_block(&visitor.thir()[block]),
         Assign { lhs, rhs } | AssignOp { lhs, rhs, op: _ } => {
             visitor.visit_expr(&visitor.thir()[lhs]);
             visitor.visit_expr(&visitor.thir()[rhs]);
@@ -108,7 +109,7 @@ pub fn walk_expr<'a, 'tcx: 'a, V: Visitor<'a, 'tcx>>(visitor: &mut V, expr: &Exp
                 visitor.visit_expr(&visitor.thir()[field]);
             }
         }
-        Adt(box crate::thir::Adt {
+        Adt(box AdtExpr {
             ref fields,
             ref base,
             adt_def: _,
@@ -126,14 +127,20 @@ pub fn walk_expr<'a, 'tcx: 'a, V: Visitor<'a, 'tcx>>(visitor: &mut V, expr: &Exp
         PlaceTypeAscription { source, user_ty: _ } | ValueTypeAscription { source, user_ty: _ } => {
             visitor.visit_expr(&visitor.thir()[source])
         }
-        Closure { closure_id: _, substs: _, upvars: _, movability: _, fake_reads: _ } => {}
+        Closure(box ClosureExpr {
+            closure_id: _,
+            substs: _,
+            upvars: _,
+            movability: _,
+            fake_reads: _,
+        }) => {}
         Literal { lit: _, neg: _ } => {}
         NonHirLiteral { lit: _, user_ty: _ } => {}
         ZstLiteral { user_ty: _ } => {}
         NamedConst { def_id: _, substs: _, user_ty: _ } => {}
         ConstParam { param: _, def_id: _ } => {}
         StaticRef { alloc_id: _, ty: _, def_id: _ } => {}
-        InlineAsm { ref operands, template: _, options: _, line_spans: _ } => {
+        InlineAsm(box InlineAsmExpr { ref operands, template: _, options: _, line_spans: _ }) => {
             for op in &**operands {
                 use InlineAsmOperand::*;
                 match op {
@@ -174,7 +181,7 @@ pub fn walk_stmt<'a, 'tcx: 'a, V: Visitor<'a, 'tcx>>(visitor: &mut V, stmt: &Stm
             }
             visitor.visit_pat(pattern);
             if let Some(block) = else_block {
-                visitor.visit_block(block)
+                visitor.visit_block(&visitor.thir()[*block])
             }
         }
     }
@@ -204,7 +211,7 @@ pub fn walk_arm<'a, 'tcx: 'a, V: Visitor<'a, 'tcx>>(visitor: &mut V, arm: &Arm<'
 
 pub fn walk_pat<'a, 'tcx: 'a, V: Visitor<'a, 'tcx>>(visitor: &mut V, pat: &Pat<'tcx>) {
     use PatKind::*;
-    match pat.kind.as_ref() {
+    match &pat.kind {
         AscribeUserType { subpattern, ascription: _ }
         | Deref { subpattern }
         | Binding {
@@ -225,18 +232,18 @@ pub fn walk_pat<'a, 'tcx: 'a, V: Visitor<'a, 'tcx>>(visitor: &mut V, pat: &Pat<'
         Constant { value: _ } => {}
         Range(_) => {}
         Slice { prefix, slice, suffix } | Array { prefix, slice, suffix } => {
-            for subpattern in prefix {
+            for subpattern in prefix.iter() {
                 visitor.visit_pat(&subpattern);
             }
             if let Some(pat) = slice {
-                visitor.visit_pat(pat);
+                visitor.visit_pat(&pat);
             }
-            for subpattern in suffix {
+            for subpattern in suffix.iter() {
                 visitor.visit_pat(&subpattern);
             }
         }
         Or { pats } => {
-            for pat in pats {
+            for pat in pats.iter() {
                 visitor.visit_pat(&pat);
             }
         }
diff --git a/compiler/rustc_middle/src/traits/mod.rs b/compiler/rustc_middle/src/traits/mod.rs
index 72b848c3e..68a7af0b8 100644
--- a/compiler/rustc_middle/src/traits/mod.rs
+++ b/compiler/rustc_middle/src/traits/mod.rs
@@ -10,9 +10,10 @@ mod structural_impls;
 pub mod util;
 
 use crate::infer::canonical::Canonical;
+use crate::mir::ConstraintCategory;
 use crate::ty::abstract_const::NotConstEvaluatable;
 use crate::ty::subst::SubstsRef;
-use crate::ty::{self, AdtKind, Predicate, Ty, TyCtxt};
+use crate::ty::{self, AdtKind, Ty, TyCtxt};
 
 use rustc_data_structures::sync::Lrc;
 use rustc_errors::{Applicability, Diagnostic};
@@ -183,6 +184,16 @@ impl<'tcx> ObligationCause<'tcx> {
             variant(DerivedObligationCause { parent_trait_pred, parent_code: self.code }).into();
         self
     }
+
+    pub fn to_constraint_category(&self) -> ConstraintCategory<'tcx> {
+        match self.code() {
+            MatchImpl(cause, _) => cause.to_constraint_category(),
+            AscribeUserTypeProvePredicate(predicate_span) => {
+                ConstraintCategory::Predicate(*predicate_span)
+            }
+            _ => ConstraintCategory::BoringNoLocation,
+        }
+    }
 }
 
 #[derive(Clone, Debug, PartialEq, Eq, Hash, Lift)]
@@ -234,13 +245,23 @@ pub enum ObligationCauseCode<'tcx> {
     /// This is the trait reference from the given projection.
     ProjectionWf(ty::ProjectionTy<'tcx>),
 
-    /// In an impl of trait `X` for type `Y`, type `Y` must
-    /// also implement all supertraits of `X`.
+    /// Must satisfy all of the where-clause predicates of the
+    /// given item.
     ItemObligation(DefId),
 
-    /// Like `ItemObligation`, but with extra detail on the source of the obligation.
+    /// Like `ItemObligation`, but carries the span of the
+    /// predicate when it can be identified.
     BindingObligation(DefId, Span),
 
+    /// Like `ItemObligation`, but carries the `HirId` of the
+    /// expression that caused the obligation, and the `usize`
+    /// indicates exactly which predicate it is in the list of
+    /// instantiated predicates.
+    ExprItemObligation(DefId, rustc_hir::HirId, usize),
+
+    /// Combines `ExprItemObligation` and `BindingObligation`.
+    ExprBindingObligation(DefId, Span, rustc_hir::HirId, usize),
+
     /// A type like `&'a T` is WF only if `T: 'a`.
     ReferenceOutlivesReferent(Ty<'tcx>),
 
@@ -406,8 +427,10 @@ pub enum ObligationCauseCode<'tcx> {
     BinOp {
         rhs_span: Option<Span>,
         is_lit: bool,
-        output_pred: Option<Predicate<'tcx>>,
+        output_ty: Option<Ty<'tcx>>,
     },
+
+    AscribeUserTypeProvePredicate(Span),
 }
 
 /// The 'location' at which we try to perform HIR-based wf checking.
@@ -459,6 +482,13 @@ impl<'tcx> ObligationCauseCode<'tcx> {
             _ => None,
         }
     }
+
+    pub fn peel_match_impls(&self) -> &Self {
+        match self {
+            MatchImpl(cause, _) => cause.code(),
+            _ => self,
+        }
+    }
 }
 
 // `ObligationCauseCode` is used a lot. Make sure it doesn't unintentionally get bigger.
@@ -634,6 +664,10 @@ pub enum ImplSource<'tcx, N> {
 
     /// ImplSource for a `const Drop` implementation.
     ConstDestruct(ImplSourceConstDestructData<N>),
+
+    /// ImplSource for a `std::marker::Tuple` implementation.
+    /// This has no nested predicates ever, so no data.
+    Tuple,
 }
 
 impl<'tcx, N> ImplSource<'tcx, N> {
@@ -648,7 +682,8 @@ impl<'tcx, N> ImplSource<'tcx, N> {
             ImplSource::Object(d) => d.nested,
             ImplSource::FnPointer(d) => d.nested,
             ImplSource::DiscriminantKind(ImplSourceDiscriminantKindData)
-            | ImplSource::Pointee(ImplSourcePointeeData) => Vec::new(),
+            | ImplSource::Pointee(ImplSourcePointeeData)
+            | ImplSource::Tuple => Vec::new(),
             ImplSource::TraitAlias(d) => d.nested,
             ImplSource::TraitUpcasting(d) => d.nested,
             ImplSource::ConstDestruct(i) => i.nested,
@@ -666,7 +701,8 @@ impl<'tcx, N> ImplSource<'tcx, N> {
             ImplSource::Object(d) => &d.nested,
             ImplSource::FnPointer(d) => &d.nested,
             ImplSource::DiscriminantKind(ImplSourceDiscriminantKindData)
-            | ImplSource::Pointee(ImplSourcePointeeData) => &[],
+            | ImplSource::Pointee(ImplSourcePointeeData)
+            | ImplSource::Tuple => &[],
             ImplSource::TraitAlias(d) => &d.nested,
             ImplSource::TraitUpcasting(d) => &d.nested,
             ImplSource::ConstDestruct(i) => &i.nested,
@@ -733,6 +769,7 @@ impl<'tcx, N> ImplSource<'tcx, N> {
                     nested: i.nested.into_iter().map(f).collect(),
                 })
             }
+            ImplSource::Tuple => ImplSource::Tuple,
         }
     }
 }
@@ -893,6 +930,12 @@ impl ObjectSafetyViolation {
             }
             ObjectSafetyViolation::Method(
                 name,
+                MethodViolationCode::ReferencesImplTraitInTrait,
+                _,
+            ) => format!("method `{}` references an `impl Trait` type in its return type", name)
+                .into(),
+            ObjectSafetyViolation::Method(
+                name,
                 MethodViolationCode::WhereClauseReferencesSelf,
                 _,
             ) => {
@@ -997,6 +1040,9 @@ pub enum MethodViolationCode {
     /// e.g., `fn foo(&self) -> Self`
     ReferencesSelfOutput,
 
+    /// e.g., `fn foo(&self) -> impl Sized`
+    ReferencesImplTraitInTrait,
+
     /// e.g., `fn foo(&self) where Self: Clone`
     WhereClauseReferencesSelf,
 
@@ -1007,7 +1053,7 @@ pub enum MethodViolationCode {
     UndispatchableReceiver(Option<Span>),
 }
 
-/// These are the error cases for `codegen_fulfill_obligation`.
+/// These are the error cases for `codegen_select_candidate`.
 #[derive(Copy, Clone, Debug, Hash, HashStable, Encodable, Decodable)]
 pub enum CodegenObligationError {
     /// Ambiguity can happen when monomorphizing during trans
diff --git a/compiler/rustc_middle/src/traits/query.rs b/compiler/rustc_middle/src/traits/query.rs
index 1f9b474ad..0e6cacb9f 100644
--- a/compiler/rustc_middle/src/traits/query.rs
+++ b/compiler/rustc_middle/src/traits/query.rs
@@ -5,11 +5,11 @@
 //! The providers for the queries defined here can be found in
 //! `rustc_traits`.
 
+use crate::error::DropCheckOverflow;
 use crate::infer::canonical::{Canonical, QueryResponse};
 use crate::ty::error::TypeError;
 use crate::ty::subst::GenericArg;
 use crate::ty::{self, Ty, TyCtxt};
-use rustc_errors::struct_span_err;
 use rustc_span::source_map::Span;
 use std::iter::FromIterator;
 
@@ -117,15 +117,7 @@ pub struct DropckOutlivesResult<'tcx> {
 impl<'tcx> DropckOutlivesResult<'tcx> {
     pub fn report_overflows(&self, tcx: TyCtxt<'tcx>, span: Span, ty: Ty<'tcx>) {
         if let Some(overflow_ty) = self.overflows.get(0) {
-            let mut err = struct_span_err!(
-                tcx.sess,
-                span,
-                E0320,
-                "overflow while adding drop-check rules for {}",
-                ty,
-            );
-            err.note(&format!("overflowed on {}", overflow_ty));
-            err.emit();
+            tcx.sess.emit_err(DropCheckOverflow { span, ty, overflow_ty: *overflow_ty });
         }
     }
 
diff --git a/compiler/rustc_middle/src/traits/select.rs b/compiler/rustc_middle/src/traits/select.rs
index e836ba47e..53af3e905 100644
--- a/compiler/rustc_middle/src/traits/select.rs
+++ b/compiler/rustc_middle/src/traits/select.rs
@@ -160,6 +160,9 @@ pub enum SelectionCandidate<'tcx> {
 
     /// Implementation of `const Destruct`, optionally from a custom `impl const Drop`.
     ConstDestructCandidate(Option<DefId>),
+
+    /// Witnesses the fact that a type is a tuple.
+    TupleCandidate,
 }
 
 /// The result of trait evaluation. The order is important
diff --git a/compiler/rustc_middle/src/traits/specialization_graph.rs b/compiler/rustc_middle/src/traits/specialization_graph.rs
index 2465f8e25..0a2819fee 100644
--- a/compiler/rustc_middle/src/traits/specialization_graph.rs
+++ b/compiler/rustc_middle/src/traits/specialization_graph.rs
@@ -115,7 +115,7 @@ impl Node {
         matches!(self, Node::Trait(..))
     }
 
-    /// Trys to find the associated item that implements `trait_item_def_id`
+    /// Tries to find the associated item that implements `trait_item_def_id`
     /// defined in this node.
     ///
     /// If this returns `None`, the item can potentially still be found in
diff --git a/compiler/rustc_middle/src/traits/structural_impls.rs b/compiler/rustc_middle/src/traits/structural_impls.rs
index 7fbd57ac7..c526344e1 100644
--- a/compiler/rustc_middle/src/traits/structural_impls.rs
+++ b/compiler/rustc_middle/src/traits/structural_impls.rs
@@ -34,6 +34,8 @@ impl<'tcx, N: fmt::Debug> fmt::Debug for traits::ImplSource<'tcx, N> {
             super::ImplSource::TraitUpcasting(ref d) => write!(f, "{:?}", d),
 
             super::ImplSource::ConstDestruct(ref d) => write!(f, "{:?}", d),
+
+            super::ImplSource::Tuple => write!(f, "ImplSource::Tuple"),
         }
     }
 }
diff --git a/compiler/rustc_middle/src/ty/abstract_const.rs b/compiler/rustc_middle/src/ty/abstract_const.rs
index bed809930..8f79b4705 100644
--- a/compiler/rustc_middle/src/ty/abstract_const.rs
+++ b/compiler/rustc_middle/src/ty/abstract_const.rs
@@ -42,7 +42,7 @@ impl<'tcx> AbstractConst<'tcx> {
         ct: ty::Const<'tcx>,
     ) -> Result<Option<AbstractConst<'tcx>>, ErrorGuaranteed> {
         match ct.kind() {
-            ty::ConstKind::Unevaluated(uv) => AbstractConst::new(tcx, uv.shrink()),
+            ty::ConstKind::Unevaluated(uv) => AbstractConst::new(tcx, uv),
             ty::ConstKind::Error(DelaySpanBugEmitted { reported, .. }) => Err(reported),
             _ => Ok(None),
         }
diff --git a/compiler/rustc_middle/src/ty/adjustment.rs b/compiler/rustc_middle/src/ty/adjustment.rs
index d36cf2fe3..b809f1767 100644
--- a/compiler/rustc_middle/src/ty/adjustment.rs
+++ b/compiler/rustc_middle/src/ty/adjustment.rs
@@ -77,7 +77,7 @@ pub enum PointerCast {
 ///    At some point, of course, `Box` should move out of the compiler, in which
 ///    case this is analogous to transforming a struct. E.g., `Box<[i32; 4]>` ->
 ///    `Box<[i32]>` is an `Adjust::Unsize` with the target `Box<[i32]>`.
-#[derive(Clone, TyEncodable, TyDecodable, HashStable, TypeFoldable, TypeVisitable)]
+#[derive(Clone, TyEncodable, TyDecodable, HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub struct Adjustment<'tcx> {
     pub kind: Adjust<'tcx>,
     pub target: Ty<'tcx>,
@@ -89,7 +89,7 @@ impl<'tcx> Adjustment<'tcx> {
     }
 }
 
-#[derive(Clone, Debug, TyEncodable, TyDecodable, HashStable, TypeFoldable, TypeVisitable)]
+#[derive(Clone, Debug, TyEncodable, TyDecodable, HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub enum Adjust<'tcx> {
     /// Go from ! to any type.
     NeverToAny,
@@ -108,7 +108,7 @@ pub enum Adjust<'tcx> {
 /// The target type is `U` in both cases, with the region and mutability
 /// being those shared by both the receiver and the returned reference.
 #[derive(Copy, Clone, PartialEq, Debug, TyEncodable, TyDecodable, HashStable)]
-#[derive(TypeFoldable, TypeVisitable)]
+#[derive(TypeFoldable, TypeVisitable, Lift)]
 pub struct OverloadedDeref<'tcx> {
     pub region: ty::Region<'tcx>,
     pub mutbl: hir::Mutability,
@@ -167,7 +167,7 @@ impl From<AutoBorrowMutability> for hir::Mutability {
 }
 
 #[derive(Copy, Clone, PartialEq, Debug, TyEncodable, TyDecodable, HashStable)]
-#[derive(TypeFoldable, TypeVisitable)]
+#[derive(TypeFoldable, TypeVisitable, Lift)]
 pub enum AutoBorrow<'tcx> {
     /// Converts from T to &T.
     Ref(ty::Region<'tcx>, AutoBorrowMutability),
diff --git a/compiler/rustc_middle/src/ty/assoc.rs b/compiler/rustc_middle/src/ty/assoc.rs
index c97156ac1..55ee5bd2f 100644
--- a/compiler/rustc_middle/src/ty/assoc.rs
+++ b/compiler/rustc_middle/src/ty/assoc.rs
@@ -42,7 +42,7 @@ impl AssocItem {
     }
 
     #[inline]
-    pub fn visibility(&self, tcx: TyCtxt<'_>) -> Visibility {
+    pub fn visibility(&self, tcx: TyCtxt<'_>) -> Visibility<DefId> {
         tcx.visibility(self.def_id)
     }
 
diff --git a/compiler/rustc_middle/src/ty/binding.rs b/compiler/rustc_middle/src/ty/binding.rs
index 3d65429f2..a5b05a4f9 100644
--- a/compiler/rustc_middle/src/ty/binding.rs
+++ b/compiler/rustc_middle/src/ty/binding.rs
@@ -1,6 +1,4 @@
-use rustc_hir::BindingAnnotation;
-use rustc_hir::BindingAnnotation::*;
-use rustc_hir::Mutability;
+use rustc_hir::{BindingAnnotation, ByRef, Mutability};
 
 #[derive(Clone, PartialEq, TyEncodable, TyDecodable, Debug, Copy, HashStable)]
 pub enum BindingMode {
@@ -11,12 +9,10 @@ pub enum BindingMode {
 TrivialTypeTraversalAndLiftImpls! { BindingMode, }
 
 impl BindingMode {
-    pub fn convert(ba: BindingAnnotation) -> BindingMode {
-        match ba {
-            Unannotated => BindingMode::BindByValue(Mutability::Not),
-            Mutable => BindingMode::BindByValue(Mutability::Mut),
-            Ref => BindingMode::BindByReference(Mutability::Not),
-            RefMut => BindingMode::BindByReference(Mutability::Mut),
+    pub fn convert(BindingAnnotation(by_ref, mutbl): BindingAnnotation) -> BindingMode {
+        match by_ref {
+            ByRef::No => BindingMode::BindByValue(mutbl),
+            ByRef::Yes => BindingMode::BindByReference(mutbl),
         }
     }
 }
diff --git a/compiler/rustc_middle/src/ty/cast.rs b/compiler/rustc_middle/src/ty/cast.rs
index c4b743dd4..981e2d3b6 100644
--- a/compiler/rustc_middle/src/ty/cast.rs
+++ b/compiler/rustc_middle/src/ty/cast.rs
@@ -33,6 +33,8 @@ pub enum CastTy<'tcx> {
     FnPtr,
     /// Raw pointers.
     Ptr(ty::TypeAndMut<'tcx>),
+    /// Casting into a `dyn*` value.
+    DynStar,
 }
 
 /// Cast Kind. See [RFC 401](https://rust-lang.github.io/rfcs/0401-coercions.html)
@@ -50,6 +52,7 @@ pub enum CastKind {
     ArrayPtrCast,
     FnPtrPtrCast,
     FnPtrAddrCast,
+    DynStarCast,
 }
 
 impl<'tcx> CastTy<'tcx> {
@@ -67,6 +70,7 @@ impl<'tcx> CastTy<'tcx> {
             ty::Adt(d, _) if d.is_enum() && d.is_payloadfree() => Some(CastTy::Int(IntTy::CEnum)),
             ty::RawPtr(mt) => Some(CastTy::Ptr(mt)),
             ty::FnPtr(..) => Some(CastTy::FnPtr),
+            ty::Dynamic(_, _, ty::DynStar) => Some(CastTy::DynStar),
             _ => None,
         }
     }
diff --git a/compiler/rustc_middle/src/ty/consts.rs b/compiler/rustc_middle/src/ty/consts.rs
index f8792edc0..339ff4d35 100644
--- a/compiler/rustc_middle/src/ty/consts.rs
+++ b/compiler/rustc_middle/src/ty/consts.rs
@@ -41,7 +41,7 @@ pub struct ConstS<'tcx> {
 }
 
 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
-static_assert_size!(ConstS<'_>, 48);
+static_assert_size!(ConstS<'_>, 40);
 
 impl<'tcx> Const<'tcx> {
     #[inline]
@@ -65,8 +65,6 @@ impl<'tcx> Const<'tcx> {
         tcx: TyCtxt<'tcx>,
         def: ty::WithOptConstParam<LocalDefId>,
     ) -> Self {
-        debug!("Const::from_anon_const(def={:?})", def);
-
         let body_id = match tcx.hir().get_by_def_id(def.did) {
             hir::Node::AnonConst(ac) => ac.body,
             _ => span_bug!(
@@ -86,7 +84,7 @@ impl<'tcx> Const<'tcx> {
                 kind: ty::ConstKind::Unevaluated(ty::Unevaluated {
                     def: def.to_global(),
                     substs: InternalSubsts::identity_for_item(tcx, def.did.to_def_id()),
-                    promoted: None,
+                    promoted: (),
                 }),
                 ty,
             }),
@@ -183,7 +181,7 @@ impl<'tcx> Const<'tcx> {
                     kind: ty::ConstKind::Unevaluated(ty::Unevaluated {
                         def: ty::WithOptConstParam::unknown(def_id).to_global(),
                         substs,
-                        promoted: None,
+                        promoted: (),
                     }),
                     ty,
                 })
diff --git a/compiler/rustc_middle/src/ty/consts/kind.rs b/compiler/rustc_middle/src/ty/consts/kind.rs
index cb0137d2e..455015280 100644
--- a/compiler/rustc_middle/src/ty/consts/kind.rs
+++ b/compiler/rustc_middle/src/ty/consts/kind.rs
@@ -11,6 +11,7 @@ use rustc_macros::HashStable;
 use rustc_target::abi::Size;
 
 use super::ScalarInt;
+
 /// An unevaluated, potentially generic, constant.
 #[derive(Copy, Clone, Debug, Eq, PartialEq, PartialOrd, Ord, TyEncodable, TyDecodable, Lift)]
 #[derive(Hash, HashStable)]
@@ -20,6 +21,12 @@ pub struct Unevaluated<'tcx, P = Option<Promoted>> {
     pub promoted: P,
 }
 
+impl rustc_errors::IntoDiagnosticArg for Unevaluated<'_> {
+    fn into_diagnostic_arg(self) -> rustc_errors::DiagnosticArgValue<'static> {
+        format!("{:?}", self).into_diagnostic_arg()
+    }
+}
+
 impl<'tcx> Unevaluated<'tcx> {
     #[inline]
     pub fn shrink(self) -> Unevaluated<'tcx, ()> {
@@ -44,7 +51,7 @@ impl<'tcx, P: Default> Unevaluated<'tcx, P> {
 
 /// Represents a constant in Rust.
 #[derive(Copy, Clone, Debug, Eq, PartialEq, PartialOrd, Ord, TyEncodable, TyDecodable)]
-#[derive(Hash, HashStable)]
+#[derive(Hash, HashStable, TypeFoldable, TypeVisitable)]
 pub enum ConstKind<'tcx> {
     /// A const generic parameter.
     Param(ty::ParamConst),
@@ -60,7 +67,7 @@ pub enum ConstKind<'tcx> {
 
     /// Used in the HIR by using `Unevaluated` everywhere and later normalizing to one of the other
     /// variants when the code is monomorphic enough for that.
-    Unevaluated(Unevaluated<'tcx>),
+    Unevaluated(Unevaluated<'tcx, ()>),
 
     /// Used to hold computed value.
     Value(ty::ValTree<'tcx>),
@@ -71,7 +78,7 @@ pub enum ConstKind<'tcx> {
 }
 
 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
-static_assert_size!(ConstKind<'_>, 40);
+static_assert_size!(ConstKind<'_>, 32);
 
 impl<'tcx> ConstKind<'tcx> {
     #[inline]
@@ -174,9 +181,12 @@ impl<'tcx> ConstKind<'tcx> {
         param_env: ParamEnv<'tcx>,
         eval_mode: EvalMode,
     ) -> Option<Result<EvalResult<'tcx>, ErrorGuaranteed>> {
+        assert!(!self.has_escaping_bound_vars(), "escaping vars in {self:?}");
         if let ConstKind::Unevaluated(unevaluated) = self {
             use crate::mir::interpret::ErrorHandled;
 
+            assert_eq!(unevaluated.promoted, ());
+
             // HACK(eddyb) this erases lifetimes even though `const_eval_resolve`
             // also does later, but we want to do it before checking for
             // inference variables.
@@ -197,7 +207,7 @@ impl<'tcx> ConstKind<'tcx> {
                 tcx.param_env(unevaluated.def.did).and(ty::Unevaluated {
                     def: unevaluated.def,
                     substs: InternalSubsts::identity_for_item(tcx, unevaluated.def.did),
-                    promoted: unevaluated.promoted,
+                    promoted: (),
                 })
             } else {
                 param_env_and
@@ -221,7 +231,7 @@ impl<'tcx> ConstKind<'tcx> {
                     }
                 }
                 EvalMode::Mir => {
-                    match tcx.const_eval_resolve(param_env, unevaluated, None) {
+                    match tcx.const_eval_resolve(param_env, unevaluated.expand(), None) {
                         // NOTE(eddyb) `val` contains no lifetimes/types/consts,
                         // and we use the original type, so nothing from `substs`
                         // (which may be identity substs, see above),
diff --git a/compiler/rustc_middle/src/ty/consts/valtree.rs b/compiler/rustc_middle/src/ty/consts/valtree.rs
index 93707bb18..a803fca0d 100644
--- a/compiler/rustc_middle/src/ty/consts/valtree.rs
+++ b/compiler/rustc_middle/src/ty/consts/valtree.rs
@@ -18,7 +18,7 @@ use rustc_macros::{HashStable, TyDecodable, TyEncodable};
 /// `ValTree` does not have this problem with representation, as it only contains integers or
 /// lists of (nested) `ValTree`.
 pub enum ValTree<'tcx> {
-    /// ZSTs, integers, `bool`, `char` are represented as scalars.
+    /// integers, `bool`, `char` are represented as scalars.
     /// See the `ScalarInt` documentation for how `ScalarInt` guarantees that equal values
     /// of these types have the same representation.
     Leaf(ScalarInt),
@@ -27,8 +27,11 @@ pub enum ValTree<'tcx> {
     // dont use SliceOrStr for now
     /// The fields of any kind of aggregate. Structs, tuples and arrays are represented by
     /// listing their fields' values in order.
+    ///
     /// Enums are represented by storing their discriminant as a field, followed by all
     /// the fields of the variant.
+    ///
+    /// ZST types are represented as an empty slice.
     Branch(&'tcx [ValTree<'tcx>]),
 }
 
diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index 0a0f45ce1..0b497fa4a 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -22,6 +22,7 @@ use crate::ty::{
     FloatVar, FloatVid, GenericParamDefKind, InferConst, InferTy, IntTy, IntVar, IntVid, List,
     ParamConst, ParamTy, PolyFnSig, Predicate, PredicateKind, PredicateS, ProjectionTy, Region,
     RegionKind, ReprOptions, TraitObjectVisitor, Ty, TyKind, TyS, TyVar, TyVid, TypeAndMut, UintTy,
+    Visibility,
 };
 use rustc_ast as ast;
 use rustc_data_structures::fingerprint::Fingerprint;
@@ -62,7 +63,7 @@ use rustc_span::{Span, DUMMY_SP};
 use rustc_target::abi::{Layout, LayoutS, TargetDataLayout, VariantIdx};
 use rustc_target::spec::abi;
 use rustc_type_ir::sty::TyKind::*;
-use rustc_type_ir::{InternAs, InternIteratorElement, Interner, TypeFlags};
+use rustc_type_ir::{DynKind, InternAs, InternIteratorElement, Interner, TypeFlags};
 
 use std::any::Any;
 use std::borrow::Borrow;
@@ -275,9 +276,6 @@ pub struct CommonTypes<'tcx> {
 }
 
 pub struct CommonLifetimes<'tcx> {
-    /// `ReEmpty` in the root universe.
-    pub re_root_empty: Region<'tcx>,
-
     /// `ReStatic`
     pub re_static: Region<'tcx>,
 
@@ -874,7 +872,7 @@ pub type CanonicalUserTypeAnnotations<'tcx> =
 
 #[derive(Clone, Debug, TyEncodable, TyDecodable, HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub struct CanonicalUserTypeAnnotation<'tcx> {
-    pub user_ty: CanonicalUserType<'tcx>,
+    pub user_ty: Box<CanonicalUserType<'tcx>>,
     pub span: Span,
     pub inferred_ty: Ty<'tcx>,
 }
@@ -986,11 +984,7 @@ impl<'tcx> CommonLifetimes<'tcx> {
             ))
         };
 
-        CommonLifetimes {
-            re_root_empty: mk(ty::ReEmpty(ty::UniverseIndex::ROOT)),
-            re_static: mk(ty::ReStatic),
-            re_erased: mk(ty::ReErased),
-        }
+        CommonLifetimes { re_static: mk(ty::ReStatic), re_erased: mk(ty::ReErased) }
     }
 }
 
@@ -1089,7 +1083,7 @@ pub struct GlobalCtxt<'tcx> {
 
     pub queries: &'tcx dyn query::QueryEngine<'tcx>,
     pub query_caches: query::QueryCaches<'tcx>,
-    query_kinds: &'tcx [DepKindStruct],
+    query_kinds: &'tcx [DepKindStruct<'tcx>],
 
     // Internal caches for metadata decoding. No need to track deps on this.
     pub ty_rcache: Lock<FxHashMap<ty::CReaderCacheKey, Ty<'tcx>>>,
@@ -1246,12 +1240,12 @@ impl<'tcx> TyCtxt<'tcx> {
         dep_graph: DepGraph,
         on_disk_cache: Option<&'tcx dyn OnDiskCache<'tcx>>,
         queries: &'tcx dyn query::QueryEngine<'tcx>,
-        query_kinds: &'tcx [DepKindStruct],
+        query_kinds: &'tcx [DepKindStruct<'tcx>],
         crate_name: &str,
         output_filenames: OutputFilenames,
     ) -> GlobalCtxt<'tcx> {
         let data_layout = TargetDataLayout::parse(&s.target).unwrap_or_else(|err| {
-            s.fatal(&err);
+            s.emit_fatal(err);
         });
         let interners = CtxtInterners::new(arena);
         let common_types = CommonTypes::new(
@@ -1296,7 +1290,7 @@ impl<'tcx> TyCtxt<'tcx> {
         }
     }
 
-    pub(crate) fn query_kind(self, k: DepKind) -> &'tcx DepKindStruct {
+    pub(crate) fn query_kind(self, k: DepKind) -> &'tcx DepKindStruct<'tcx> {
         &self.query_kinds[k as usize]
     }
 
@@ -1498,17 +1492,17 @@ impl<'tcx> TyCtxt<'tcx> {
         // Create a dependency to the crate to be sure we re-execute this when the amount of
         // definitions change.
         self.ensure().hir_crate(());
-        // Leak a read lock once we start iterating on definitions, to prevent adding new onces
+        // Leak a read lock once we start iterating on definitions, to prevent adding new ones
         // while iterating.  If some query needs to add definitions, it should be `ensure`d above.
         let definitions = self.definitions.leak();
         definitions.iter_local_def_id()
     }
 
     pub fn def_path_table(self) -> &'tcx rustc_hir::definitions::DefPathTable {
-        // Create a dependency to the crate to be sure we reexcute this when the amount of
+        // Create a dependency to the crate to be sure we re-execute this when the amount of
         // definitions change.
         self.ensure().hir_crate(());
-        // Leak a read lock once we start iterating on definitions, to prevent adding new onces
+        // Leak a read lock once we start iterating on definitions, to prevent adding new ones
         // while iterating.  If some query needs to add definitions, it should be `ensure`d above.
         let definitions = self.definitions.leak();
         definitions.def_path_table()
@@ -1517,10 +1511,10 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn def_path_hash_to_def_index_map(
         self,
     ) -> &'tcx rustc_hir::def_path_hash_map::DefPathHashMap {
-        // Create a dependency to the crate to be sure we reexcute this when the amount of
+        // Create a dependency to the crate to be sure we re-execute this when the amount of
         // definitions change.
         self.ensure().hir_crate(());
-        // Leak a read lock once we start iterating on definitions, to prevent adding new onces
+        // Leak a read lock once we start iterating on definitions, to prevent adding new ones
         // while iterating.  If some query needs to add definitions, it should be `ensure`d above.
         let definitions = self.definitions.leak();
         definitions.def_path_hash_to_def_index_map()
@@ -1596,7 +1590,7 @@ impl<'tcx> TyCtxt<'tcx> {
         })
     }
 
-    // Returns the `DefId` and the `BoundRegionKind` corresponding to the given region.
+    /// Returns the `DefId` and the `BoundRegionKind` corresponding to the given region.
     pub fn is_suitable_region(self, region: Region<'tcx>) -> Option<FreeRegionInfo> {
         let (suitable_region_binding_scope, bound_region) = match *region {
             ty::ReFree(ref free_region) => {
@@ -1728,6 +1722,11 @@ impl<'tcx> TyCtxt<'tcx> {
             .chain(self.crates(()).iter().copied())
             .flat_map(move |cnum| self.traits_in_crate(cnum).iter().copied())
     }
+
+    #[inline]
+    pub fn local_visibility(self, def_id: LocalDefId) -> Visibility {
+        self.visibility(def_id.to_def_id()).expect_local()
+    }
 }
 
 /// A trait implemented for all `X<'a>` types that can be safely and
@@ -1821,7 +1820,9 @@ nop_list_lift! {bound_variable_kinds; ty::BoundVariableKind => ty::BoundVariable
 // This is the impl for `&'a InternalSubsts<'a>`.
 nop_list_lift! {substs; GenericArg<'a> => GenericArg<'tcx>}
 
-CloneLiftImpls! { for<'tcx> { Constness, traits::WellFormedLoc, } }
+CloneLiftImpls! { for<'tcx> {
+    Constness, traits::WellFormedLoc, ImplPolarity, crate::mir::ReturnConstraint,
+} }
 
 pub mod tls {
     use super::{ptr_eq, GlobalCtxt, TyCtxt};
@@ -1829,9 +1830,9 @@ pub mod tls {
     use crate::dep_graph::TaskDepsRef;
     use crate::ty::query;
     use rustc_data_structures::sync::{self, Lock};
-    use rustc_data_structures::thin_vec::ThinVec;
     use rustc_errors::Diagnostic;
     use std::mem;
+    use thin_vec::ThinVec;
 
     #[cfg(not(parallel_compiler))]
     use std::cell::Cell;
@@ -1857,8 +1858,8 @@ pub mod tls {
         /// This is updated by `JobOwner::start` in `ty::query::plumbing` when executing a query.
         pub diagnostics: Option<&'a Lock<ThinVec<Diagnostic>>>,
 
-        /// Used to prevent layout from recursing too deeply.
-        pub layout_depth: usize,
+        /// Used to prevent queries from calling too deeply.
+        pub query_depth: usize,
 
         /// The current dep graph task. This is used to add dependencies to queries
         /// when executing them.
@@ -1872,7 +1873,7 @@ pub mod tls {
                 tcx,
                 query: None,
                 diagnostics: None,
-                layout_depth: 0,
+                query_depth: 0,
                 task_deps: TaskDepsRef::Ignore,
             }
         }
@@ -2546,8 +2547,9 @@ impl<'tcx> TyCtxt<'tcx> {
         self,
         obj: &'tcx List<ty::Binder<'tcx, ExistentialPredicate<'tcx>>>,
         reg: ty::Region<'tcx>,
+        repr: DynKind,
     ) -> Ty<'tcx> {
-        self.mk_ty(Dynamic(obj, reg))
+        self.mk_ty(Dynamic(obj, reg, repr))
     }
 
     #[inline]
diff --git a/compiler/rustc_middle/src/ty/diagnostics.rs b/compiler/rustc_middle/src/ty/diagnostics.rs
index dd2f43210..855917fb8 100644
--- a/compiler/rustc_middle/src/ty/diagnostics.rs
+++ b/compiler/rustc_middle/src/ty/diagnostics.rs
@@ -102,13 +102,25 @@ pub fn suggest_arbitrary_trait_bound<'tcx>(
     generics: &hir::Generics<'_>,
     err: &mut Diagnostic,
     trait_pred: PolyTraitPredicate<'tcx>,
+    associated_ty: Option<(&'static str, Ty<'tcx>)>,
 ) -> bool {
     if !trait_pred.is_suggestable(tcx, false) {
         return false;
     }
 
     let param_name = trait_pred.skip_binder().self_ty().to_string();
-    let constraint = trait_pred.print_modifiers_and_trait_path().to_string();
+    let mut constraint = trait_pred.print_modifiers_and_trait_path().to_string();
+
+    if let Some((name, term)) = associated_ty {
+        // FIXME: this case overlaps with code in TyCtxt::note_and_explain_type_err.
+        // That should be extracted into a helper function.
+        if constraint.ends_with('>') {
+            constraint = format!("{}, {} = {}>", &constraint[..constraint.len() - 1], name, term);
+        } else {
+            constraint.push_str(&format!("<{} = {}>", name, term));
+        }
+    }
+
     let param = generics.params.iter().find(|p| p.name.ident().as_str() == param_name);
 
     // Skip, there is a param named Self
@@ -396,7 +408,7 @@ impl<'v> hir::intravisit::Visitor<'v> for TraitObjectVisitor<'v> {
             ) => {
                 self.0.push(ty);
             }
-            hir::TyKind::OpaqueDef(item_id, _) => {
+            hir::TyKind::OpaqueDef(item_id, _, _) => {
                 self.0.push(ty);
                 let item = self.1.item(item_id);
                 hir::intravisit::walk_item(self, item);
@@ -455,7 +467,7 @@ impl<'tcx> TypeVisitor<'tcx> for IsSuggestableVisitor<'tcx> {
                 }
             }
 
-            Dynamic(dty, _) => {
+            Dynamic(dty, _, _) => {
                 for pred in *dty {
                     match pred.skip_binder() {
                         ExistentialPredicate::Trait(_) | ExistentialPredicate::Projection(_) => {
diff --git a/compiler/rustc_middle/src/ty/error.rs b/compiler/rustc_middle/src/ty/error.rs
index 4b0bc3c11..01e1e97b2 100644
--- a/compiler/rustc_middle/src/ty/error.rs
+++ b/compiler/rustc_middle/src/ty/error.rs
@@ -2,6 +2,7 @@ use crate::traits::{ObligationCause, ObligationCauseCode};
 use crate::ty::diagnostics::suggest_constraining_type_param;
 use crate::ty::print::{FmtPrinter, Printer};
 use crate::ty::{self, BoundRegionKind, Region, Ty, TyCtxt};
+use hir::def::DefKind;
 use rustc_errors::Applicability::{MachineApplicable, MaybeIncorrect};
 use rustc_errors::{pluralize, Diagnostic, MultiSpan};
 use rustc_hir as hir;
@@ -13,7 +14,7 @@ use rustc_target::spec::abi;
 use std::borrow::Cow;
 use std::fmt;
 
-#[derive(Clone, Copy, Debug, PartialEq, Eq, TypeFoldable, TypeVisitable)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, TypeFoldable, TypeVisitable, Lift)]
 pub struct ExpectedFound<T> {
     pub expected: T,
     pub found: T,
@@ -30,7 +31,8 @@ impl<T> ExpectedFound<T> {
 }
 
 // Data structures used in type unification
-#[derive(Clone, Debug, TypeFoldable, TypeVisitable)]
+#[derive(Copy, Clone, Debug, TypeFoldable, TypeVisitable, Lift)]
+#[rustc_pass_by_value]
 pub enum TypeError<'tcx> {
     Mismatch,
     ConstnessMismatch(ExpectedFound<ty::BoundConstness>),
@@ -73,6 +75,18 @@ pub enum TypeError<'tcx> {
     TargetFeatureCast(DefId),
 }
 
+impl TypeError<'_> {
+    pub fn involves_regions(self) -> bool {
+        match self {
+            TypeError::RegionsDoesNotOutlive(_, _)
+            | TypeError::RegionsInsufficientlyPolymorphic(_, _)
+            | TypeError::RegionsOverlyPolymorphic(_, _)
+            | TypeError::RegionsPlaceholderMismatch => true,
+            _ => false,
+        }
+    }
+}
+
 /// Explains the source of a type err in a short, human readable way. This is meant to be placed
 /// in parentheses after some larger message. You should also invoke `note_and_explain_type_err()`
 /// afterwards to present additional details, particularly when it comes to lifetime-related
@@ -211,7 +225,7 @@ impl<'tcx> fmt::Display for TypeError<'tcx> {
 }
 
 impl<'tcx> TypeError<'tcx> {
-    pub fn must_include_note(&self) -> bool {
+    pub fn must_include_note(self) -> bool {
         use self::TypeError::*;
         match self {
             CyclicTy(_) | CyclicConst(_) | UnsafetyMismatch(_) | ConstnessMismatch(_)
@@ -263,10 +277,23 @@ impl<'tcx> Ty<'tcx> {
             }
             ty::Slice(ty) if ty.is_simple_ty() => format!("slice `{}`", self).into(),
             ty::Slice(_) => "slice".into(),
-            ty::RawPtr(_) => "*-ptr".into(),
+            ty::RawPtr(tymut) => {
+                let tymut_string = match tymut.mutbl {
+                    hir::Mutability::Mut => tymut.to_string(),
+                    hir::Mutability::Not => format!("const {}", tymut.ty),
+                };
+
+                if tymut_string != "_" && (tymut.ty.is_simple_text() || tymut_string.len() < "const raw pointer".len()) {
+                    format!("`*{}`", tymut_string).into()
+                } else {
+                    // Unknown type name, it's long or has type arguments
+                    "raw pointer".into()
+                }
+            },
             ty::Ref(_, ty, mutbl) => {
                 let tymut = ty::TypeAndMut { ty, mutbl };
                 let tymut_string = tymut.to_string();
+
                 if tymut_string != "_"
                     && (ty.is_simple_text() || tymut_string.len() < "mutable reference".len())
                 {
@@ -347,7 +374,7 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn note_and_explain_type_err(
         self,
         diag: &mut Diagnostic,
-        err: &TypeError<'tcx>,
+        err: TypeError<'tcx>,
         cause: &ObligationCause<'tcx>,
         sp: Span,
         body_owner_def_id: DefId,
@@ -512,7 +539,7 @@ impl<T> Trait<T> for X {
                             diag.span_label(p_span, "this type parameter");
                         }
                     }
-                    (ty::Projection(proj_ty), _) => {
+                    (ty::Projection(proj_ty), _) if self.def_kind(proj_ty.item_def_id) != DefKind::ImplTraitPlaceholder => {
                         self.expected_projection(
                             diag,
                             proj_ty,
@@ -521,7 +548,7 @@ impl<T> Trait<T> for X {
                             cause.code(),
                         );
                     }
-                    (_, ty::Projection(proj_ty)) => {
+                    (_, ty::Projection(proj_ty)) if self.def_kind(proj_ty.item_def_id) != DefKind::ImplTraitPlaceholder => {
                         let msg = format!(
                             "consider constraining the associated type `{}` to `{}`",
                             values.found, values.expected,
@@ -568,7 +595,7 @@ impl<T> Trait<T> for X {
             }
             TargetFeatureCast(def_id) => {
                 let target_spans =
-                    self.get_attrs(*def_id, sym::target_feature).map(|attr| attr.span);
+                    self.get_attrs(def_id, sym::target_feature).map(|attr| attr.span);
                 diag.note(
                     "functions with `#[target_feature]` can only be coerced to `unsafe` function pointers"
                 );
@@ -640,7 +667,7 @@ impl<T> Trait<T> for X {
         self,
         diag: &mut Diagnostic,
         proj_ty: &ty::ProjectionTy<'tcx>,
-        values: &ExpectedFound<Ty<'tcx>>,
+        values: ExpectedFound<Ty<'tcx>>,
         body_owner_def_id: DefId,
         cause_code: &ObligationCauseCode<'_>,
     ) {
diff --git a/compiler/rustc_middle/src/ty/flags.rs b/compiler/rustc_middle/src/ty/flags.rs
index ea6bb8a7a..98b8a7386 100644
--- a/compiler/rustc_middle/src/ty/flags.rs
+++ b/compiler/rustc_middle/src/ty/flags.rs
@@ -1,5 +1,5 @@
 use crate::ty::subst::{GenericArg, GenericArgKind};
-use crate::ty::{self, InferConst, Term, Ty, TypeFlags};
+use crate::ty::{self, InferConst, Ty, TypeFlags};
 use std::slice;
 
 #[derive(Debug)]
@@ -171,7 +171,7 @@ impl FlagComputation {
                 self.add_substs(substs);
             }
 
-            &ty::Dynamic(obj, r) => {
+            &ty::Dynamic(obj, r, _) => {
                 for predicate in obj.iter() {
                     self.bound_computation(predicate, |computation, predicate| match predicate {
                         ty::ExistentialPredicate::Trait(tr) => computation.add_substs(tr.substs),
@@ -243,9 +243,9 @@ impl FlagComputation {
             }
             ty::PredicateKind::Projection(ty::ProjectionPredicate { projection_ty, term }) => {
                 self.add_projection_ty(projection_ty);
-                match term {
-                    Term::Ty(ty) => self.add_ty(ty),
-                    Term::Const(c) => self.add_const(c),
+                match term.unpack() {
+                    ty::TermKind::Ty(ty) => self.add_ty(ty),
+                    ty::TermKind::Const(c) => self.add_const(c),
                 }
             }
             ty::PredicateKind::WellFormed(arg) => {
@@ -320,9 +320,9 @@ impl FlagComputation {
 
     fn add_existential_projection(&mut self, projection: &ty::ExistentialProjection<'_>) {
         self.add_substs(projection.substs);
-        match projection.term {
-            ty::Term::Ty(ty) => self.add_ty(ty),
-            ty::Term::Const(ct) => self.add_const(ct),
+        match projection.term.unpack() {
+            ty::TermKind::Ty(ty) => self.add_ty(ty),
+            ty::TermKind::Const(ct) => self.add_const(ct),
         }
     }
 
diff --git a/compiler/rustc_middle/src/ty/fold.rs b/compiler/rustc_middle/src/ty/fold.rs
index 5e96e278b..cac95e14a 100644
--- a/compiler/rustc_middle/src/ty/fold.rs
+++ b/compiler/rustc_middle/src/ty/fold.rs
@@ -302,6 +302,17 @@ impl<'tcx> TyCtxt<'tcx> {
     {
         value.fold_with(&mut RegionFolder::new(self, &mut f))
     }
+
+    pub fn super_fold_regions<T>(
+        self,
+        value: T,
+        mut f: impl FnMut(ty::Region<'tcx>, ty::DebruijnIndex) -> ty::Region<'tcx>,
+    ) -> T
+    where
+        T: TypeSuperFoldable<'tcx>,
+    {
+        value.super_fold_with(&mut RegionFolder::new(self, &mut f))
+    }
 }
 
 /// Folds over the substructure of a type, visiting its component
@@ -353,7 +364,7 @@ impl<'a, 'tcx> TypeFolder<'tcx> for RegionFolder<'a, 'tcx> {
         t
     }
 
-    #[instrument(skip(self), level = "debug")]
+    #[instrument(skip(self), level = "debug", ret)]
     fn fold_region(&mut self, r: ty::Region<'tcx>) -> ty::Region<'tcx> {
         match *r {
             ty::ReLateBound(debruijn, _) if debruijn < self.current_index => {
@@ -377,17 +388,13 @@ pub trait BoundVarReplacerDelegate<'tcx> {
     fn replace_const(&mut self, bv: ty::BoundVar, ty: Ty<'tcx>) -> ty::Const<'tcx>;
 }
 
-pub struct FnMutDelegate<R, T, C> {
-    pub regions: R,
-    pub types: T,
-    pub consts: C,
+pub struct FnMutDelegate<'a, 'tcx> {
+    pub regions: &'a mut (dyn FnMut(ty::BoundRegion) -> ty::Region<'tcx> + 'a),
+    pub types: &'a mut (dyn FnMut(ty::BoundTy) -> Ty<'tcx> + 'a),
+    pub consts: &'a mut (dyn FnMut(ty::BoundVar, Ty<'tcx>) -> ty::Const<'tcx> + 'a),
 }
-impl<'tcx, R, T, C> BoundVarReplacerDelegate<'tcx> for FnMutDelegate<R, T, C>
-where
-    R: FnMut(ty::BoundRegion) -> ty::Region<'tcx>,
-    T: FnMut(ty::BoundTy) -> Ty<'tcx>,
-    C: FnMut(ty::BoundVar, Ty<'tcx>) -> ty::Const<'tcx>,
-{
+
+impl<'a, 'tcx> BoundVarReplacerDelegate<'tcx> for FnMutDelegate<'a, 'tcx> {
     fn replace_region(&mut self, br: ty::BoundRegion) -> ty::Region<'tcx> {
         (self.regions)(br)
     }
@@ -511,7 +518,7 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn replace_late_bound_regions_uncached<T, F>(
         self,
         value: Binder<'tcx, T>,
-        replace_regions: F,
+        mut replace_regions: F,
     ) -> T
     where
         F: FnMut(ty::BoundRegion) -> ty::Region<'tcx>,
@@ -522,9 +529,9 @@ impl<'tcx> TyCtxt<'tcx> {
             value
         } else {
             let delegate = FnMutDelegate {
-                regions: replace_regions,
-                types: |b| bug!("unexpected bound ty in binder: {b:?}"),
-                consts: |b, ty| bug!("unexpected bound ct in binder: {b:?} {ty}"),
+                regions: &mut replace_regions,
+                types: &mut |b| bug!("unexpected bound ty in binder: {b:?}"),
+                consts: &mut |b, ty| bug!("unexpected bound ct in binder: {b:?} {ty}"),
             };
             let mut replacer = BoundVarReplacer::new(self, delegate);
             value.fold_with(&mut replacer)
@@ -584,19 +591,19 @@ impl<'tcx> TyCtxt<'tcx> {
         self.replace_escaping_bound_vars_uncached(
             value,
             FnMutDelegate {
-                regions: |r: ty::BoundRegion| {
+                regions: &mut |r: ty::BoundRegion| {
                     self.mk_region(ty::ReLateBound(
                         ty::INNERMOST,
                         ty::BoundRegion { var: shift_bv(r.var), kind: r.kind },
                     ))
                 },
-                types: |t: ty::BoundTy| {
+                types: &mut |t: ty::BoundTy| {
                     self.mk_ty(ty::Bound(
                         ty::INNERMOST,
                         ty::BoundTy { var: shift_bv(t.var), kind: t.kind },
                     ))
                 },
-                consts: |c, ty: Ty<'tcx>| {
+                consts: &mut |c, ty: Ty<'tcx>| {
                     self.mk_const(ty::ConstS {
                         kind: ty::ConstKind::Bound(ty::INNERMOST, shift_bv(c)),
                         ty,
diff --git a/compiler/rustc_middle/src/ty/generics.rs b/compiler/rustc_middle/src/ty/generics.rs
index add2df258..0c8bdde9c 100644
--- a/compiler/rustc_middle/src/ty/generics.rs
+++ b/compiler/rustc_middle/src/ty/generics.rs
@@ -1,4 +1,3 @@
-use crate::middle::resolve_lifetime::ObjectLifetimeDefault;
 use crate::ty;
 use crate::ty::subst::{Subst, SubstsRef};
 use crate::ty::EarlyBinder;
@@ -13,7 +12,7 @@ use super::{EarlyBoundRegion, InstantiatedPredicates, ParamConst, ParamTy, Predi
 #[derive(Clone, Debug, TyEncodable, TyDecodable, HashStable)]
 pub enum GenericParamDefKind {
     Lifetime,
-    Type { has_default: bool, object_lifetime_default: ObjectLifetimeDefault, synthetic: bool },
+    Type { has_default: bool, synthetic: bool },
     Const { has_default: bool },
 }
 
@@ -28,8 +27,9 @@ impl GenericParamDefKind {
     pub fn to_ord(&self) -> ast::ParamKindOrd {
         match self {
             GenericParamDefKind::Lifetime => ast::ParamKindOrd::Lifetime,
-            GenericParamDefKind::Type { .. } => ast::ParamKindOrd::Type,
-            GenericParamDefKind::Const { .. } => ast::ParamKindOrd::Const,
+            GenericParamDefKind::Type { .. } | GenericParamDefKind::Const { .. } => {
+                ast::ParamKindOrd::TypeOrConst
+            }
         }
     }
 
@@ -122,6 +122,21 @@ pub struct Generics {
 }
 
 impl<'tcx> Generics {
+    /// Looks through the generics and all parents to find the index of the
+    /// given param def-id. This is in comparison to the `param_def_id_to_index`
+    /// struct member, which only stores information about this item's own
+    /// generics.
+    pub fn param_def_id_to_index(&self, tcx: TyCtxt<'tcx>, def_id: DefId) -> Option<u32> {
+        if let Some(idx) = self.param_def_id_to_index.get(&def_id) {
+            Some(*idx)
+        } else if let Some(parent) = self.parent {
+            let parent = tcx.generics_of(parent);
+            parent.param_def_id_to_index(tcx, def_id)
+        } else {
+            None
+        }
+    }
+
     #[inline]
     pub fn count(&self) -> usize {
         self.parent_count + self.params.len()
@@ -252,7 +267,7 @@ impl<'tcx> Generics {
         // Filter the default arguments.
         //
         // This currently uses structural equality instead
-        // of semantic equivalance. While not ideal, that's
+        // of semantic equivalence. While not ideal, that's
         // good enough for now as this should only be used
         // for diagnostics anyways.
         own_params.end -= self
@@ -314,6 +329,7 @@ impl<'tcx> GenericPredicates<'tcx> {
         }
     }
 
+    #[instrument(level = "debug", skip(self, tcx))]
     fn instantiate_into(
         &self,
         tcx: TyCtxt<'tcx>,
diff --git a/compiler/rustc_middle/src/ty/impls_ty.rs b/compiler/rustc_middle/src/ty/impls_ty.rs
index cd00b26b8..d1c0d62ac 100644
--- a/compiler/rustc_middle/src/ty/impls_ty.rs
+++ b/compiler/rustc_middle/src/ty/impls_ty.rs
@@ -113,7 +113,7 @@ impl<'a> HashStable<StableHashingContext<'a>> for mir::interpret::AllocId {
 }
 
 // `Relocations` with default type parameters is a sorted map.
-impl<'a, Prov> HashStable<StableHashingContext<'a>> for mir::interpret::Relocations<Prov>
+impl<'a, Prov> HashStable<StableHashingContext<'a>> for mir::interpret::ProvenanceMap<Prov>
 where
     Prov: HashStable<StableHashingContext<'a>>,
 {
diff --git a/compiler/rustc_middle/src/ty/inhabitedness/mod.rs b/compiler/rustc_middle/src/ty/inhabitedness/mod.rs
index 3d22f5a04..aaa66deb2 100644
--- a/compiler/rustc_middle/src/ty/inhabitedness/mod.rs
+++ b/compiler/rustc_middle/src/ty/inhabitedness/mod.rs
@@ -169,14 +169,10 @@ impl<'tcx> FieldDef {
         param_env: ty::ParamEnv<'tcx>,
     ) -> DefIdForest<'tcx> {
         let data_uninhabitedness = move || self.ty(tcx, substs).uninhabited_from(tcx, param_env);
-        // FIXME(canndrew): Currently enum fields are (incorrectly) stored with
-        // `Visibility::Invisible` so we need to override `self.vis` if we're
-        // dealing with an enum.
         if is_enum {
             data_uninhabitedness()
         } else {
             match self.vis {
-                Visibility::Invisible => DefIdForest::empty(),
                 Visibility::Restricted(from) => {
                     let forest = DefIdForest::from_id(from);
                     let iter = Some(forest).into_iter().chain(Some(data_uninhabitedness()));
diff --git a/compiler/rustc_middle/src/ty/instance.rs b/compiler/rustc_middle/src/ty/instance.rs
index 53218225d..9afd66207 100644
--- a/compiler/rustc_middle/src/ty/instance.rs
+++ b/compiler/rustc_middle/src/ty/instance.rs
@@ -20,14 +20,14 @@ use std::fmt;
 /// simply couples a potentially generic `InstanceDef` with some substs, and codegen and const eval
 /// will do all required substitution as they run.
 #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, TyEncodable, TyDecodable)]
-#[derive(HashStable, Lift)]
+#[derive(HashStable, Lift, TypeFoldable, TypeVisitable)]
 pub struct Instance<'tcx> {
     pub def: InstanceDef<'tcx>,
     pub substs: SubstsRef<'tcx>,
 }
 
 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
-#[derive(TyEncodable, TyDecodable, HashStable, TypeFoldable, TypeVisitable)]
+#[derive(TyEncodable, TyDecodable, HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub enum InstanceDef<'tcx> {
     /// A user-defined callable item.
     ///
diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs
index ad78d24e9..042eeec3f 100644
--- a/compiler/rustc_middle/src/ty/layout.rs
+++ b/compiler/rustc_middle/src/ty/layout.rs
@@ -2,7 +2,10 @@ use crate::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use crate::mir::{GeneratorLayout, GeneratorSavedLocal};
 use crate::ty::normalize_erasing_regions::NormalizationError;
 use crate::ty::subst::Subst;
-use crate::ty::{self, subst::SubstsRef, EarlyBinder, ReprOptions, Ty, TyCtxt, TypeVisitable};
+use crate::ty::{
+    self, layout_sanity_check::sanity_check_layout, subst::SubstsRef, EarlyBinder, ReprOptions, Ty,
+    TyCtxt, TypeVisitable,
+};
 use rustc_ast as ast;
 use rustc_attr as attr;
 use rustc_hir as hir;
@@ -19,7 +22,7 @@ use rustc_target::abi::call::{
 use rustc_target::abi::*;
 use rustc_target::spec::{abi::Abi as SpecAbi, HasTargetSpec, PanicStrategy, Target};
 
-use std::cmp;
+use std::cmp::{self, Ordering};
 use std::fmt;
 use std::iter;
 use std::num::NonZeroUsize;
@@ -221,164 +224,46 @@ impl<'tcx> fmt::Display for LayoutError<'tcx> {
     }
 }
 
-/// Enforce some basic invariants on layouts.
-fn sanity_check_layout<'tcx>(
-    tcx: TyCtxt<'tcx>,
-    param_env: ty::ParamEnv<'tcx>,
-    layout: &TyAndLayout<'tcx>,
-) {
-    // Type-level uninhabitedness should always imply ABI uninhabitedness.
-    if tcx.conservative_is_privately_uninhabited(param_env.and(layout.ty)) {
-        assert!(layout.abi.is_uninhabited());
-    }
-
-    if layout.size.bytes() % layout.align.abi.bytes() != 0 {
-        bug!("size is not a multiple of align, in the following layout:\n{layout:#?}");
-    }
-
-    if cfg!(debug_assertions) {
-        fn check_layout_abi<'tcx>(tcx: TyCtxt<'tcx>, layout: Layout<'tcx>) {
-            match layout.abi() {
-                Abi::Scalar(scalar) => {
-                    // No padding in scalars.
-                    assert_eq!(
-                        layout.align().abi,
-                        scalar.align(&tcx).abi,
-                        "alignment mismatch between ABI and layout in {layout:#?}"
-                    );
-                    assert_eq!(
-                        layout.size(),
-                        scalar.size(&tcx),
-                        "size mismatch between ABI and layout in {layout:#?}"
-                    );
-                }
-                Abi::Vector { count, element } => {
-                    // No padding in vectors. Alignment can be strengthened, though.
-                    assert!(
-                        layout.align().abi >= element.align(&tcx).abi,
-                        "alignment mismatch between ABI and layout in {layout:#?}"
-                    );
-                    let size = element.size(&tcx) * count;
-                    assert_eq!(
-                        layout.size(),
-                        size.align_to(tcx.data_layout().vector_align(size).abi),
-                        "size mismatch between ABI and layout in {layout:#?}"
-                    );
-                }
-                Abi::ScalarPair(scalar1, scalar2) => {
-                    // Sanity-check scalar pairs. These are a bit more flexible and support
-                    // padding, but we can at least ensure both fields actually fit into the layout
-                    // and the alignment requirement has not been weakened.
-                    let align1 = scalar1.align(&tcx).abi;
-                    let align2 = scalar2.align(&tcx).abi;
-                    assert!(
-                        layout.align().abi >= cmp::max(align1, align2),
-                        "alignment mismatch between ABI and layout in {layout:#?}",
-                    );
-                    let field2_offset = scalar1.size(&tcx).align_to(align2);
-                    assert!(
-                        layout.size() >= field2_offset + scalar2.size(&tcx),
-                        "size mismatch between ABI and layout in {layout:#?}"
-                    );
-                }
-                Abi::Uninhabited | Abi::Aggregate { .. } => {} // Nothing to check.
-            }
-        }
-
-        check_layout_abi(tcx, layout.layout);
-
-        if let Variants::Multiple { variants, .. } = &layout.variants {
-            for variant in variants {
-                check_layout_abi(tcx, *variant);
-                // No nested "multiple".
-                assert!(matches!(variant.variants(), Variants::Single { .. }));
-                // Skip empty variants.
-                if variant.size() == Size::ZERO
-                    || variant.fields().count() == 0
-                    || variant.abi().is_uninhabited()
-                {
-                    // These are never actually accessed anyway, so we can skip them. (Note that
-                    // sometimes, variants with fields have size 0, and sometimes, variants without
-                    // fields have non-0 size.)
-                    continue;
-                }
-                // Variants should have the same or a smaller size as the full thing.
-                if variant.size() > layout.size {
-                    bug!(
-                        "Type with size {} bytes has variant with size {} bytes: {layout:#?}",
-                        layout.size.bytes(),
-                        variant.size().bytes(),
-                    )
-                }
-                // The top-level ABI and the ABI of the variants should be coherent.
-                let abi_coherent = match (layout.abi, variant.abi()) {
-                    (Abi::Scalar(..), Abi::Scalar(..)) => true,
-                    (Abi::ScalarPair(..), Abi::ScalarPair(..)) => true,
-                    (Abi::Uninhabited, _) => true,
-                    (Abi::Aggregate { .. }, _) => true,
-                    _ => false,
-                };
-                if !abi_coherent {
-                    bug!(
-                        "Variant ABI is incompatible with top-level ABI:\nvariant={:#?}\nTop-level: {layout:#?}",
-                        variant
-                    );
-                }
-            }
-        }
-    }
-}
-
 #[instrument(skip(tcx, query), level = "debug")]
 fn layout_of<'tcx>(
     tcx: TyCtxt<'tcx>,
     query: ty::ParamEnvAnd<'tcx, Ty<'tcx>>,
 ) -> Result<TyAndLayout<'tcx>, LayoutError<'tcx>> {
-    ty::tls::with_related_context(tcx, move |icx| {
-        let (param_env, ty) = query.into_parts();
-        debug!(?ty);
-
-        if !tcx.recursion_limit().value_within_limit(icx.layout_depth) {
-            tcx.sess.fatal(&format!("overflow representing the type `{}`", ty));
+    let (param_env, ty) = query.into_parts();
+    debug!(?ty);
+
+    let param_env = param_env.with_reveal_all_normalized(tcx);
+    let unnormalized_ty = ty;
+
+    // FIXME: We might want to have two different versions of `layout_of`:
+    // One that can be called after typecheck has completed and can use
+    // `normalize_erasing_regions` here and another one that can be called
+    // before typecheck has completed and uses `try_normalize_erasing_regions`.
+    let ty = match tcx.try_normalize_erasing_regions(param_env, ty) {
+        Ok(t) => t,
+        Err(normalization_error) => {
+            return Err(LayoutError::NormalizationFailure(ty, normalization_error));
         }
+    };
 
-        // Update the ImplicitCtxt to increase the layout_depth
-        let icx = ty::tls::ImplicitCtxt { layout_depth: icx.layout_depth + 1, ..icx.clone() };
-
-        ty::tls::enter_context(&icx, |_| {
-            let param_env = param_env.with_reveal_all_normalized(tcx);
-            let unnormalized_ty = ty;
-
-            // FIXME: We might want to have two different versions of `layout_of`:
-            // One that can be called after typecheck has completed and can use
-            // `normalize_erasing_regions` here and another one that can be called
-            // before typecheck has completed and uses `try_normalize_erasing_regions`.
-            let ty = match tcx.try_normalize_erasing_regions(param_env, ty) {
-                Ok(t) => t,
-                Err(normalization_error) => {
-                    return Err(LayoutError::NormalizationFailure(ty, normalization_error));
-                }
-            };
-
-            if ty != unnormalized_ty {
-                // Ensure this layout is also cached for the normalized type.
-                return tcx.layout_of(param_env.and(ty));
-            }
+    if ty != unnormalized_ty {
+        // Ensure this layout is also cached for the normalized type.
+        return tcx.layout_of(param_env.and(ty));
+    }
 
-            let cx = LayoutCx { tcx, param_env };
+    let cx = LayoutCx { tcx, param_env };
 
-            let layout = cx.layout_of_uncached(ty)?;
-            let layout = TyAndLayout { ty, layout };
+    let layout = cx.layout_of_uncached(ty)?;
+    let layout = TyAndLayout { ty, layout };
 
-            cx.record_layout_for_printing(layout);
+    cx.record_layout_for_printing(layout);
 
-            sanity_check_layout(tcx, param_env, &layout);
+    sanity_check_layout(&cx, &layout);
 
-            Ok(layout)
-        })
-    })
+    Ok(layout)
 }
 
+#[derive(Clone, Copy)]
 pub struct LayoutCx<'tcx, C> {
     pub tcx: C,
     pub param_env: ty::ParamEnv<'tcx>,
@@ -740,6 +625,14 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
                 tcx.intern_layout(self.scalar_pair(data_ptr, metadata))
             }
 
+            ty::Dynamic(_, _, ty::DynStar) => {
+                let mut data = scalar_unit(Int(dl.ptr_sized_integer(), false));
+                data.valid_range_mut().start = 0;
+                let mut vtable = scalar_unit(Pointer);
+                vtable.valid_range_mut().start = 1;
+                tcx.intern_layout(self.scalar_pair(data, vtable))
+            }
+
             // Arrays and slices.
             ty::Array(element, mut count) => {
                 if count.has_projections() {
@@ -794,7 +687,7 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
 
             // Odd unit types.
             ty::FnDef(..) => univariant(&[], &ReprOptions::default(), StructKind::AlwaysSized)?,
-            ty::Dynamic(..) | ty::Foreign(..) => {
+            ty::Dynamic(_, _, ty::Dyn) | ty::Foreign(..) => {
                 let mut unit = self.univariant_uninterned(
                     ty,
                     &[],
@@ -872,7 +765,7 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
                 // * the element type and length of the single array field, if
                 // the first field is of array type, or
                 //
-                // * the homogenous field type and the number of fields.
+                // * the homogeneous field type and the number of fields.
                 let (e_ty, e_len, is_array) = if let ty::Array(e_ty, _) = f0_ty.kind() {
                     // First ADT field is an array:
 
@@ -1161,131 +1054,191 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
                 // that allow representation optimization.)
                 assert!(def.is_enum());
 
-                // The current code for niche-filling relies on variant indices
-                // instead of actual discriminants, so dataful enums with
-                // explicit discriminants (RFC #2363) would misbehave.
-                let no_explicit_discriminants = def
-                    .variants()
-                    .iter_enumerated()
-                    .all(|(i, v)| v.discr == ty::VariantDiscr::Relative(i.as_u32()));
-
-                let mut niche_filling_layout = None;
-
-                // Niche-filling enum optimization.
-                if !def.repr().inhibit_enum_layout_opt() && no_explicit_discriminants {
-                    let mut dataful_variant = None;
-                    let mut niche_variants = VariantIdx::MAX..=VariantIdx::new(0);
+                // Until we've decided whether to use the tagged or
+                // niche filling LayoutS, we don't want to intern the
+                // variant layouts, so we can't store them in the
+                // overall LayoutS. Store the overall LayoutS
+                // and the variant LayoutSs here until then.
+                struct TmpLayout<'tcx> {
+                    layout: LayoutS<'tcx>,
+                    variants: IndexVec<VariantIdx, LayoutS<'tcx>>,
+                }
 
-                    // Find one non-ZST variant.
-                    'variants: for (v, fields) in variants.iter_enumerated() {
-                        if absent(fields) {
-                            continue 'variants;
+                let calculate_niche_filling_layout =
+                    || -> Result<Option<TmpLayout<'tcx>>, LayoutError<'tcx>> {
+                        // The current code for niche-filling relies on variant indices
+                        // instead of actual discriminants, so enums with
+                        // explicit discriminants (RFC #2363) would misbehave.
+                        if def.repr().inhibit_enum_layout_opt()
+                            || def
+                                .variants()
+                                .iter_enumerated()
+                                .any(|(i, v)| v.discr != ty::VariantDiscr::Relative(i.as_u32()))
+                        {
+                            return Ok(None);
                         }
-                        for f in fields {
-                            if !f.is_zst() {
-                                if dataful_variant.is_none() {
-                                    dataful_variant = Some(v);
-                                    continue 'variants;
-                                } else {
-                                    dataful_variant = None;
-                                    break 'variants;
-                                }
-                            }
+
+                        if variants.len() < 2 {
+                            return Ok(None);
                         }
-                        niche_variants = *niche_variants.start().min(&v)..=v;
-                    }
 
-                    if niche_variants.start() > niche_variants.end() {
-                        dataful_variant = None;
-                    }
+                        let mut align = dl.aggregate_align;
+                        let mut variant_layouts = variants
+                            .iter_enumerated()
+                            .map(|(j, v)| {
+                                let mut st = self.univariant_uninterned(
+                                    ty,
+                                    v,
+                                    &def.repr(),
+                                    StructKind::AlwaysSized,
+                                )?;
+                                st.variants = Variants::Single { index: j };
+
+                                align = align.max(st.align);
+
+                                Ok(st)
+                            })
+                            .collect::<Result<IndexVec<VariantIdx, _>, _>>()?;
+
+                        let largest_variant_index = match variant_layouts
+                            .iter_enumerated()
+                            .max_by_key(|(_i, layout)| layout.size.bytes())
+                            .map(|(i, _layout)| i)
+                        {
+                            None => return Ok(None),
+                            Some(i) => i,
+                        };
+
+                        let all_indices = VariantIdx::new(0)..=VariantIdx::new(variants.len() - 1);
+                        let needs_disc = |index: VariantIdx| {
+                            index != largest_variant_index && !absent(&variants[index])
+                        };
+                        let niche_variants = all_indices.clone().find(|v| needs_disc(*v)).unwrap()
+                            ..=all_indices.rev().find(|v| needs_disc(*v)).unwrap();
 
-                    if let Some(i) = dataful_variant {
-                        let count = (niche_variants.end().as_u32()
-                            - niche_variants.start().as_u32()
-                            + 1) as u128;
+                        let count = niche_variants.size_hint().1.unwrap() as u128;
 
                         // Find the field with the largest niche
-                        let niche_candidate = variants[i]
+                        let (field_index, niche, (niche_start, niche_scalar)) = match variants
+                            [largest_variant_index]
                             .iter()
                             .enumerate()
                             .filter_map(|(j, field)| Some((j, field.largest_niche?)))
-                            .max_by_key(|(_, niche)| niche.available(dl));
-
-                        if let Some((field_index, niche, (niche_start, niche_scalar))) =
-                            niche_candidate.and_then(|(field_index, niche)| {
-                                Some((field_index, niche, niche.reserve(self, count)?))
-                            })
+                            .max_by_key(|(_, niche)| niche.available(dl))
+                            .and_then(|(j, niche)| Some((j, niche, niche.reserve(self, count)?)))
                         {
-                            let mut align = dl.aggregate_align;
-                            let st = variants
-                                .iter_enumerated()
-                                .map(|(j, v)| {
-                                    let mut st = self.univariant_uninterned(
-                                        ty,
-                                        v,
-                                        &def.repr(),
-                                        StructKind::AlwaysSized,
-                                    )?;
-                                    st.variants = Variants::Single { index: j };
+                            None => return Ok(None),
+                            Some(x) => x,
+                        };
+
+                        let niche_offset = niche.offset
+                            + variant_layouts[largest_variant_index].fields.offset(field_index);
+                        let niche_size = niche.value.size(dl);
+                        let size = variant_layouts[largest_variant_index].size.align_to(align.abi);
 
-                                    align = align.max(st.align);
+                        let all_variants_fit =
+                            variant_layouts.iter_enumerated_mut().all(|(i, layout)| {
+                                if i == largest_variant_index {
+                                    return true;
+                                }
 
-                                    Ok(tcx.intern_layout(st))
-                                })
-                                .collect::<Result<IndexVec<VariantIdx, _>, _>>()?;
+                                layout.largest_niche = None;
 
-                            let offset = st[i].fields().offset(field_index) + niche.offset;
+                                if layout.size <= niche_offset {
+                                    // This variant will fit before the niche.
+                                    return true;
+                                }
 
-                            // Align the total size to the largest alignment.
-                            let size = st[i].size().align_to(align.abi);
+                                // Determine if it'll fit after the niche.
+                                let this_align = layout.align.abi;
+                                let this_offset = (niche_offset + niche_size).align_to(this_align);
 
-                            let abi = if st.iter().all(|v| v.abi().is_uninhabited()) {
-                                Abi::Uninhabited
-                            } else if align == st[i].align() && size == st[i].size() {
-                                // When the total alignment and size match, we can use the
-                                // same ABI as the scalar variant with the reserved niche.
-                                match st[i].abi() {
-                                    Abi::Scalar(_) => Abi::Scalar(niche_scalar),
-                                    Abi::ScalarPair(first, second) => {
-                                        // Only the niche is guaranteed to be initialised,
-                                        // so use union layout for the other primitive.
-                                        if offset.bytes() == 0 {
-                                            Abi::ScalarPair(niche_scalar, second.to_union())
-                                        } else {
-                                            Abi::ScalarPair(first.to_union(), niche_scalar)
+                                if this_offset + layout.size > size {
+                                    return false;
+                                }
+
+                                // It'll fit, but we need to make some adjustments.
+                                match layout.fields {
+                                    FieldsShape::Arbitrary { ref mut offsets, .. } => {
+                                        for (j, offset) in offsets.iter_mut().enumerate() {
+                                            if !variants[i][j].is_zst() {
+                                                *offset += this_offset;
+                                            }
                                         }
                                     }
-                                    _ => Abi::Aggregate { sized: true },
+                                    _ => {
+                                        panic!("Layout of fields should be Arbitrary for variants")
+                                    }
                                 }
-                            } else {
-                                Abi::Aggregate { sized: true }
-                            };
 
-                            let largest_niche = Niche::from_scalar(dl, offset, niche_scalar);
-
-                            niche_filling_layout = Some(LayoutS {
-                                variants: Variants::Multiple {
-                                    tag: niche_scalar,
-                                    tag_encoding: TagEncoding::Niche {
-                                        dataful_variant: i,
-                                        niche_variants,
-                                        niche_start,
-                                    },
-                                    tag_field: 0,
-                                    variants: st,
-                                },
-                                fields: FieldsShape::Arbitrary {
-                                    offsets: vec![offset],
-                                    memory_index: vec![0],
-                                },
-                                abi,
-                                largest_niche,
-                                size,
-                                align,
+                                // It can't be a Scalar or ScalarPair because the offset isn't 0.
+                                if !layout.abi.is_uninhabited() {
+                                    layout.abi = Abi::Aggregate { sized: true };
+                                }
+                                layout.size += this_offset;
+
+                                true
                             });
+
+                        if !all_variants_fit {
+                            return Ok(None);
                         }
-                    }
-                }
+
+                        let largest_niche = Niche::from_scalar(dl, niche_offset, niche_scalar);
+
+                        let others_zst = variant_layouts.iter_enumerated().all(|(i, layout)| {
+                            i == largest_variant_index || layout.size == Size::ZERO
+                        });
+                        let same_size = size == variant_layouts[largest_variant_index].size;
+                        let same_align = align == variant_layouts[largest_variant_index].align;
+
+                        let abi = if variant_layouts.iter().all(|v| v.abi.is_uninhabited()) {
+                            Abi::Uninhabited
+                        } else if same_size && same_align && others_zst {
+                            match variant_layouts[largest_variant_index].abi {
+                                // When the total alignment and size match, we can use the
+                                // same ABI as the scalar variant with the reserved niche.
+                                Abi::Scalar(_) => Abi::Scalar(niche_scalar),
+                                Abi::ScalarPair(first, second) => {
+                                    // Only the niche is guaranteed to be initialised,
+                                    // so use union layouts for the other primitive.
+                                    if niche_offset == Size::ZERO {
+                                        Abi::ScalarPair(niche_scalar, second.to_union())
+                                    } else {
+                                        Abi::ScalarPair(first.to_union(), niche_scalar)
+                                    }
+                                }
+                                _ => Abi::Aggregate { sized: true },
+                            }
+                        } else {
+                            Abi::Aggregate { sized: true }
+                        };
+
+                        let layout = LayoutS {
+                            variants: Variants::Multiple {
+                                tag: niche_scalar,
+                                tag_encoding: TagEncoding::Niche {
+                                    untagged_variant: largest_variant_index,
+                                    niche_variants,
+                                    niche_start,
+                                },
+                                tag_field: 0,
+                                variants: IndexVec::new(),
+                            },
+                            fields: FieldsShape::Arbitrary {
+                                offsets: vec![niche_offset],
+                                memory_index: vec![0],
+                            },
+                            abi,
+                            largest_niche,
+                            size,
+                            align,
+                        };
+
+                        Ok(Some(TmpLayout { layout, variants: variant_layouts }))
+                    };
+
+                let niche_filling_layout = calculate_niche_filling_layout()?;
 
                 let (mut min, mut max) = (i128::MAX, i128::MIN);
                 let discr_type = def.repr().discr_type();
@@ -1540,15 +1493,12 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
 
                 let largest_niche = Niche::from_scalar(dl, Size::ZERO, tag);
 
-                let layout_variants =
-                    layout_variants.into_iter().map(|v| tcx.intern_layout(v)).collect();
-
                 let tagged_layout = LayoutS {
                     variants: Variants::Multiple {
                         tag,
                         tag_encoding: TagEncoding::Direct,
                         tag_field: 0,
-                        variants: layout_variants,
+                        variants: IndexVec::new(),
                     },
                     fields: FieldsShape::Arbitrary {
                         offsets: vec![Size::ZERO],
@@ -1560,20 +1510,45 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
                     size,
                 };
 
-                let best_layout = match (tagged_layout, niche_filling_layout) {
-                    (tagged_layout, Some(niche_filling_layout)) => {
+                let tagged_layout = TmpLayout { layout: tagged_layout, variants: layout_variants };
+
+                let mut best_layout = match (tagged_layout, niche_filling_layout) {
+                    (tl, Some(nl)) => {
                         // Pick the smaller layout; otherwise,
                         // pick the layout with the larger niche; otherwise,
                         // pick tagged as it has simpler codegen.
-                        cmp::min_by_key(tagged_layout, niche_filling_layout, |layout| {
-                            let niche_size = layout.largest_niche.map_or(0, |n| n.available(dl));
-                            (layout.size, cmp::Reverse(niche_size))
-                        })
+                        use Ordering::*;
+                        let niche_size = |tmp_l: &TmpLayout<'_>| {
+                            tmp_l.layout.largest_niche.map_or(0, |n| n.available(dl))
+                        };
+                        match (
+                            tl.layout.size.cmp(&nl.layout.size),
+                            niche_size(&tl).cmp(&niche_size(&nl)),
+                        ) {
+                            (Greater, _) => nl,
+                            (Equal, Less) => nl,
+                            _ => tl,
+                        }
                     }
-                    (tagged_layout, None) => tagged_layout,
+                    (tl, None) => tl,
                 };
 
-                tcx.intern_layout(best_layout)
+                // Now we can intern the variant layouts and store them in the enum layout.
+                best_layout.layout.variants = match best_layout.layout.variants {
+                    Variants::Multiple { tag, tag_encoding, tag_field, .. } => Variants::Multiple {
+                        tag,
+                        tag_encoding,
+                        tag_field,
+                        variants: best_layout
+                            .variants
+                            .into_iter()
+                            .map(|layout| tcx.intern_layout(layout))
+                            .collect(),
+                    },
+                    _ => bug!(),
+                };
+
+                tcx.intern_layout(best_layout.layout)
             }
 
             // Types with no meaningful known layout.
@@ -2468,7 +2443,9 @@ where
                 | ty::FnDef(..)
                 | ty::GeneratorWitness(..)
                 | ty::Foreign(..)
-                | ty::Dynamic(..) => bug!("TyAndLayout::field({:?}): not applicable", this),
+                | ty::Dynamic(_, _, ty::Dyn) => {
+                    bug!("TyAndLayout::field({:?}): not applicable", this)
+                }
 
                 // Potentially-fat pointers.
                 ty::Ref(_, pointee, _) | ty::RawPtr(ty::TypeAndMut { ty: pointee, .. }) => {
@@ -2497,7 +2474,7 @@ where
 
                     match tcx.struct_tail_erasing_lifetimes(pointee, cx.param_env()).kind() {
                         ty::Slice(_) | ty::Str => TyMaybeWithLayout::Ty(tcx.types.usize),
-                        ty::Dynamic(_, _) => {
+                        ty::Dynamic(_, _, ty::Dyn) => {
                             TyMaybeWithLayout::Ty(tcx.mk_imm_ref(
                                 tcx.lifetimes.re_static,
                                 tcx.mk_array(tcx.types.usize, 3),
@@ -2566,6 +2543,22 @@ where
                     }
                 }
 
+                ty::Dynamic(_, _, ty::DynStar) => {
+                    if i == 0 {
+                        TyMaybeWithLayout::Ty(tcx.types.usize)
+                    } else if i == 1 {
+                        // FIXME(dyn-star) same FIXME as above applies here too
+                        TyMaybeWithLayout::Ty(
+                            tcx.mk_imm_ref(
+                                tcx.lifetimes.re_static,
+                                tcx.mk_array(tcx.types.usize, 3),
+                            ),
+                        )
+                    } else {
+                        bug!("no field {i} on dyn*")
+                    }
+                }
+
                 ty::Projection(_)
                 | ty::Bound(..)
                 | ty::Placeholder(..)
@@ -2674,11 +2667,11 @@ where
                     // using more niches than just null (e.g., the first page of
                     // the address space, or unaligned pointers).
                     Variants::Multiple {
-                        tag_encoding: TagEncoding::Niche { dataful_variant, .. },
+                        tag_encoding: TagEncoding::Niche { untagged_variant, .. },
                         tag_field,
                         ..
                     } if this.fields.offset(tag_field) == offset => {
-                        Some(this.for_variant(cx, dataful_variant))
+                        Some(this.for_variant(cx, untagged_variant))
                     }
                     _ => Some(this),
                 };
@@ -2761,6 +2754,7 @@ impl<'tcx> ty::Instance<'tcx> {
     // for `Instance` (e.g. typeck would use `Ty::fn_sig` instead),
     // or should go through `FnAbi` instead, to avoid losing any
     // adjustments `fn_abi_of_instance` might be performing.
+    #[tracing::instrument(level = "debug", skip(tcx, param_env))]
     fn fn_sig_for_fn_abi(
         &self,
         tcx: TyCtxt<'tcx>,
@@ -2907,6 +2901,7 @@ impl<'tcx> ty::Instance<'tcx> {
 /// with `-Cpanic=abort` will look like they can't unwind when in fact they
 /// might (from a foreign exception or similar).
 #[inline]
+#[tracing::instrument(level = "debug", skip(tcx))]
 pub fn fn_can_unwind<'tcx>(tcx: TyCtxt<'tcx>, fn_def_id: Option<DefId>, abi: SpecAbi) -> bool {
     if let Some(did) = fn_def_id {
         // Special attribute for functions which can't unwind.
@@ -3123,6 +3118,7 @@ pub trait FnAbiOf<'tcx>: FnAbiOfHelpers<'tcx> {
     /// NB: that includes virtual calls, which are represented by "direct calls"
     /// to an `InstanceDef::Virtual` instance (of `<dyn Trait as Trait>::fn`).
     #[inline]
+    #[tracing::instrument(level = "debug", skip(self))]
     fn fn_abi_of_instance(
         &self,
         instance: ty::Instance<'tcx>,
@@ -3179,9 +3175,100 @@ fn fn_abi_of_instance<'tcx>(
     )
 }
 
+// Handle safe Rust thin and fat pointers.
+pub fn adjust_for_rust_scalar<'tcx>(
+    cx: LayoutCx<'tcx, TyCtxt<'tcx>>,
+    attrs: &mut ArgAttributes,
+    scalar: Scalar,
+    layout: TyAndLayout<'tcx>,
+    offset: Size,
+    is_return: bool,
+) {
+    // Booleans are always a noundef i1 that needs to be zero-extended.
+    if scalar.is_bool() {
+        attrs.ext(ArgExtension::Zext);
+        attrs.set(ArgAttribute::NoUndef);
+        return;
+    }
+
+    // Scalars which have invalid values cannot be undef.
+    if !scalar.is_always_valid(&cx) {
+        attrs.set(ArgAttribute::NoUndef);
+    }
+
+    // Only pointer types handled below.
+    let Scalar::Initialized { value: Pointer, valid_range} = scalar else { return };
+
+    if !valid_range.contains(0) {
+        attrs.set(ArgAttribute::NonNull);
+    }
+
+    if let Some(pointee) = layout.pointee_info_at(&cx, offset) {
+        if let Some(kind) = pointee.safe {
+            attrs.pointee_align = Some(pointee.align);
+
+            // `Box` (`UniqueBorrowed`) are not necessarily dereferenceable
+            // for the entire duration of the function as they can be deallocated
+            // at any time. Same for shared mutable references. If LLVM had a
+            // way to say "dereferenceable on entry" we could use it here.
+            attrs.pointee_size = match kind {
+                PointerKind::UniqueBorrowed
+                | PointerKind::UniqueBorrowedPinned
+                | PointerKind::Frozen => pointee.size,
+                PointerKind::SharedMutable | PointerKind::UniqueOwned => Size::ZERO,
+            };
+
+            // `Box`, `&T`, and `&mut T` cannot be undef.
+            // Note that this only applies to the value of the pointer itself;
+            // this attribute doesn't make it UB for the pointed-to data to be undef.
+            attrs.set(ArgAttribute::NoUndef);
+
+            // The aliasing rules for `Box<T>` are still not decided, but currently we emit
+            // `noalias` for it. This can be turned off using an unstable flag.
+            // See https://github.com/rust-lang/unsafe-code-guidelines/issues/326
+            let noalias_for_box = cx.tcx.sess.opts.unstable_opts.box_noalias.unwrap_or(true);
+
+            // `&mut` pointer parameters never alias other parameters,
+            // or mutable global data
+            //
+            // `&T` where `T` contains no `UnsafeCell<U>` is immutable,
+            // and can be marked as both `readonly` and `noalias`, as
+            // LLVM's definition of `noalias` is based solely on memory
+            // dependencies rather than pointer equality
+            //
+            // Due to past miscompiles in LLVM, we apply a separate NoAliasMutRef attribute
+            // for UniqueBorrowed arguments, so that the codegen backend can decide whether
+            // or not to actually emit the attribute. It can also be controlled with the
+            // `-Zmutable-noalias` debugging option.
+            let no_alias = match kind {
+                PointerKind::SharedMutable
+                | PointerKind::UniqueBorrowed
+                | PointerKind::UniqueBorrowedPinned => false,
+                PointerKind::UniqueOwned => noalias_for_box,
+                PointerKind::Frozen => !is_return,
+            };
+            if no_alias {
+                attrs.set(ArgAttribute::NoAlias);
+            }
+
+            if kind == PointerKind::Frozen && !is_return {
+                attrs.set(ArgAttribute::ReadOnly);
+            }
+
+            if kind == PointerKind::UniqueBorrowed && !is_return {
+                attrs.set(ArgAttribute::NoAliasMutRef);
+            }
+        }
+    }
+}
+
 impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
     // FIXME(eddyb) perhaps group the signature/type-containing (or all of them?)
     // arguments of this method, into a separate `struct`.
+    #[tracing::instrument(
+        level = "debug",
+        skip(self, caller_location, fn_def_id, force_thin_self_ptr)
+    )]
     fn fn_abi_new_uncached(
         &self,
         sig: ty::PolyFnSig<'tcx>,
@@ -3191,8 +3278,6 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
         // FIXME(eddyb) replace this with something typed, like an `enum`.
         force_thin_self_ptr: bool,
     ) -> Result<&'tcx FnAbi<'tcx, Ty<'tcx>>, FnAbiError<'tcx>> {
-        debug!("fn_abi_new_uncached({:?}, {:?})", sig, extra_args);
-
         let sig = self.tcx.normalize_erasing_late_bound_regions(self.param_env, sig);
 
         let conv = conv_from_spec_abi(self.tcx(), sig.abi);
@@ -3234,92 +3319,9 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
         use SpecAbi::*;
         let rust_abi = matches!(sig.abi, RustIntrinsic | PlatformIntrinsic | Rust | RustCall);
 
-        // Handle safe Rust thin and fat pointers.
-        let adjust_for_rust_scalar = |attrs: &mut ArgAttributes,
-                                      scalar: Scalar,
-                                      layout: TyAndLayout<'tcx>,
-                                      offset: Size,
-                                      is_return: bool| {
-            // Booleans are always a noundef i1 that needs to be zero-extended.
-            if scalar.is_bool() {
-                attrs.ext(ArgExtension::Zext);
-                attrs.set(ArgAttribute::NoUndef);
-                return;
-            }
-
-            // Scalars which have invalid values cannot be undef.
-            if !scalar.is_always_valid(self) {
-                attrs.set(ArgAttribute::NoUndef);
-            }
-
-            // Only pointer types handled below.
-            let Scalar::Initialized { value: Pointer, valid_range} = scalar else { return };
-
-            if !valid_range.contains(0) {
-                attrs.set(ArgAttribute::NonNull);
-            }
-
-            if let Some(pointee) = layout.pointee_info_at(self, offset) {
-                if let Some(kind) = pointee.safe {
-                    attrs.pointee_align = Some(pointee.align);
-
-                    // `Box` (`UniqueBorrowed`) are not necessarily dereferenceable
-                    // for the entire duration of the function as they can be deallocated
-                    // at any time. Same for shared mutable references. If LLVM had a
-                    // way to say "dereferenceable on entry" we could use it here.
-                    attrs.pointee_size = match kind {
-                        PointerKind::UniqueBorrowed
-                        | PointerKind::UniqueBorrowedPinned
-                        | PointerKind::Frozen => pointee.size,
-                        PointerKind::SharedMutable | PointerKind::UniqueOwned => Size::ZERO,
-                    };
-
-                    // `Box`, `&T`, and `&mut T` cannot be undef.
-                    // Note that this only applies to the value of the pointer itself;
-                    // this attribute doesn't make it UB for the pointed-to data to be undef.
-                    attrs.set(ArgAttribute::NoUndef);
-
-                    // The aliasing rules for `Box<T>` are still not decided, but currently we emit
-                    // `noalias` for it. This can be turned off using an unstable flag.
-                    // See https://github.com/rust-lang/unsafe-code-guidelines/issues/326
-                    let noalias_for_box =
-                        self.tcx().sess.opts.unstable_opts.box_noalias.unwrap_or(true);
-
-                    // `&mut` pointer parameters never alias other parameters,
-                    // or mutable global data
-                    //
-                    // `&T` where `T` contains no `UnsafeCell<U>` is immutable,
-                    // and can be marked as both `readonly` and `noalias`, as
-                    // LLVM's definition of `noalias` is based solely on memory
-                    // dependencies rather than pointer equality
-                    //
-                    // Due to past miscompiles in LLVM, we apply a separate NoAliasMutRef attribute
-                    // for UniqueBorrowed arguments, so that the codegen backend can decide whether
-                    // or not to actually emit the attribute. It can also be controlled with the
-                    // `-Zmutable-noalias` debugging option.
-                    let no_alias = match kind {
-                        PointerKind::SharedMutable
-                        | PointerKind::UniqueBorrowed
-                        | PointerKind::UniqueBorrowedPinned => false,
-                        PointerKind::UniqueOwned => noalias_for_box,
-                        PointerKind::Frozen => !is_return,
-                    };
-                    if no_alias {
-                        attrs.set(ArgAttribute::NoAlias);
-                    }
-
-                    if kind == PointerKind::Frozen && !is_return {
-                        attrs.set(ArgAttribute::ReadOnly);
-                    }
-
-                    if kind == PointerKind::UniqueBorrowed && !is_return {
-                        attrs.set(ArgAttribute::NoAliasMutRef);
-                    }
-                }
-            }
-        };
-
         let arg_of = |ty: Ty<'tcx>, arg_idx: Option<usize>| -> Result<_, FnAbiError<'tcx>> {
+            let span = tracing::debug_span!("arg_of");
+            let _entered = span.enter();
             let is_return = arg_idx.is_none();
 
             let layout = self.layout_of(ty)?;
@@ -3334,7 +3336,7 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
 
             let mut arg = ArgAbi::new(self, layout, |layout, scalar, offset| {
                 let mut attrs = ArgAttributes::new();
-                adjust_for_rust_scalar(&mut attrs, scalar, *layout, offset, is_return);
+                adjust_for_rust_scalar(*self, &mut attrs, scalar, *layout, offset, is_return);
                 attrs
             });
 
@@ -3367,7 +3369,7 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
                 .map(|(i, ty)| arg_of(ty, Some(i)))
                 .collect::<Result<_, _>>()?,
             c_variadic: sig.c_variadic,
-            fixed_count: inputs.len(),
+            fixed_count: inputs.len() as u32,
             conv,
             can_unwind: fn_can_unwind(self.tcx(), fn_def_id, sig.abi),
         };
@@ -3376,6 +3378,7 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
         Ok(self.tcx.arena.alloc(fn_abi))
     }
 
+    #[tracing::instrument(level = "trace", skip(self))]
     fn fn_abi_adjust_for_abi(
         &self,
         fn_abi: &mut FnAbi<'tcx, Ty<'tcx>>,
@@ -3439,7 +3442,7 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
                 }
             };
             fixup(&mut fn_abi.ret);
-            for arg in &mut fn_abi.args {
+            for arg in fn_abi.args.iter_mut() {
                 fixup(arg);
             }
         } else {
@@ -3450,6 +3453,7 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
     }
 }
 
+#[tracing::instrument(level = "debug", skip(cx))]
 fn make_thin_self_ptr<'tcx>(
     cx: &(impl HasTyCtxt<'tcx> + HasParamEnv<'tcx>),
     layout: TyAndLayout<'tcx>,
@@ -3461,7 +3465,7 @@ fn make_thin_self_ptr<'tcx>(
         tcx.mk_mut_ptr(layout.ty)
     } else {
         match layout.abi {
-            Abi::ScalarPair(..) => (),
+            Abi::ScalarPair(..) | Abi::Scalar(..) => (),
             _ => bug!("receiver type has unsupported layout: {:?}", layout),
         }
 
diff --git a/compiler/rustc_middle/src/ty/layout_sanity_check.rs b/compiler/rustc_middle/src/ty/layout_sanity_check.rs
new file mode 100644
index 000000000..87c85dcff
--- /dev/null
+++ b/compiler/rustc_middle/src/ty/layout_sanity_check.rs
@@ -0,0 +1,303 @@
+use crate::ty::{
+    layout::{LayoutCx, TyAndLayout},
+    TyCtxt,
+};
+use rustc_target::abi::*;
+
+use std::cmp;
+
+/// Enforce some basic invariants on layouts.
+pub(super) fn sanity_check_layout<'tcx>(
+    cx: &LayoutCx<'tcx, TyCtxt<'tcx>>,
+    layout: &TyAndLayout<'tcx>,
+) {
+    // Type-level uninhabitedness should always imply ABI uninhabitedness.
+    if cx.tcx.conservative_is_privately_uninhabited(cx.param_env.and(layout.ty)) {
+        assert!(layout.abi.is_uninhabited());
+    }
+
+    if layout.size.bytes() % layout.align.abi.bytes() != 0 {
+        bug!("size is not a multiple of align, in the following layout:\n{layout:#?}");
+    }
+
+    if cfg!(debug_assertions) {
+        /// Yields non-ZST fields of the type
+        fn non_zst_fields<'tcx, 'a>(
+            cx: &'a LayoutCx<'tcx, TyCtxt<'tcx>>,
+            layout: &'a TyAndLayout<'tcx>,
+        ) -> impl Iterator<Item = (Size, TyAndLayout<'tcx>)> + 'a {
+            (0..layout.layout.fields().count()).filter_map(|i| {
+                let field = layout.field(cx, i);
+                // Also checking `align == 1` here leads to test failures in
+                // `layout/zero-sized-array-union.rs`, where a type has a zero-size field with
+                // alignment 4 that still gets ignored during layout computation (which is okay
+                // since other fields already force alignment 4).
+                let zst = field.is_zst();
+                (!zst).then(|| (layout.fields.offset(i), field))
+            })
+        }
+
+        fn skip_newtypes<'tcx>(
+            cx: &LayoutCx<'tcx, TyCtxt<'tcx>>,
+            layout: &TyAndLayout<'tcx>,
+        ) -> TyAndLayout<'tcx> {
+            if matches!(layout.layout.variants(), Variants::Multiple { .. }) {
+                // Definitely not a newtype of anything.
+                return *layout;
+            }
+            let mut fields = non_zst_fields(cx, layout);
+            let Some(first) = fields.next() else {
+                // No fields here, so this could be a primitive or enum -- either way it's not a newtype around a thing
+                return *layout
+            };
+            if fields.next().is_none() {
+                let (offset, first) = first;
+                if offset == Size::ZERO && first.layout.size() == layout.size {
+                    // This is a newtype, so keep recursing.
+                    // FIXME(RalfJung): I don't think it would be correct to do any checks for
+                    // alignment here, so we don't. Is that correct?
+                    return skip_newtypes(cx, &first);
+                }
+            }
+            // No more newtypes here.
+            *layout
+        }
+
+        fn check_layout_abi<'tcx>(cx: &LayoutCx<'tcx, TyCtxt<'tcx>>, layout: &TyAndLayout<'tcx>) {
+            match layout.layout.abi() {
+                Abi::Scalar(scalar) => {
+                    // No padding in scalars.
+                    let size = scalar.size(cx);
+                    let align = scalar.align(cx).abi;
+                    assert_eq!(
+                        layout.layout.size(),
+                        size,
+                        "size mismatch between ABI and layout in {layout:#?}"
+                    );
+                    assert_eq!(
+                        layout.layout.align().abi,
+                        align,
+                        "alignment mismatch between ABI and layout in {layout:#?}"
+                    );
+                    // Check that this matches the underlying field.
+                    let inner = skip_newtypes(cx, layout);
+                    assert!(
+                        matches!(inner.layout.abi(), Abi::Scalar(_)),
+                        "`Scalar` type {} is newtype around non-`Scalar` type {}",
+                        layout.ty,
+                        inner.ty
+                    );
+                    match inner.layout.fields() {
+                        FieldsShape::Primitive => {
+                            // Fine.
+                        }
+                        FieldsShape::Union(..) => {
+                            // FIXME: I guess we could also check something here? Like, look at all fields?
+                            return;
+                        }
+                        FieldsShape::Arbitrary { .. } => {
+                            // Should be an enum, the only field is the discriminant.
+                            assert!(
+                                inner.ty.is_enum(),
+                                "`Scalar` layout for non-primitive non-enum type {}",
+                                inner.ty
+                            );
+                            assert_eq!(
+                                inner.layout.fields().count(),
+                                1,
+                                "`Scalar` layout for multiple-field type in {inner:#?}",
+                            );
+                            let offset = inner.layout.fields().offset(0);
+                            let field = inner.field(cx, 0);
+                            // The field should be at the right offset, and match the `scalar` layout.
+                            assert_eq!(
+                                offset,
+                                Size::ZERO,
+                                "`Scalar` field at non-0 offset in {inner:#?}",
+                            );
+                            assert_eq!(
+                                field.size, size,
+                                "`Scalar` field with bad size in {inner:#?}",
+                            );
+                            assert_eq!(
+                                field.align.abi, align,
+                                "`Scalar` field with bad align in {inner:#?}",
+                            );
+                            assert!(
+                                matches!(field.abi, Abi::Scalar(_)),
+                                "`Scalar` field with bad ABI in {inner:#?}",
+                            );
+                        }
+                        _ => {
+                            panic!("`Scalar` layout for non-primitive non-enum type {}", inner.ty);
+                        }
+                    }
+                }
+                Abi::ScalarPair(scalar1, scalar2) => {
+                    // Sanity-check scalar pairs. These are a bit more flexible and support
+                    // padding, but we can at least ensure both fields actually fit into the layout
+                    // and the alignment requirement has not been weakened.
+                    let size1 = scalar1.size(cx);
+                    let align1 = scalar1.align(cx).abi;
+                    let size2 = scalar2.size(cx);
+                    let align2 = scalar2.align(cx).abi;
+                    assert!(
+                        layout.layout.align().abi >= cmp::max(align1, align2),
+                        "alignment mismatch between ABI and layout in {layout:#?}",
+                    );
+                    let field2_offset = size1.align_to(align2);
+                    assert!(
+                        layout.layout.size() >= field2_offset + size2,
+                        "size mismatch between ABI and layout in {layout:#?}"
+                    );
+                    // Check that the underlying pair of fields matches.
+                    let inner = skip_newtypes(cx, layout);
+                    assert!(
+                        matches!(inner.layout.abi(), Abi::ScalarPair(..)),
+                        "`ScalarPair` type {} is newtype around non-`ScalarPair` type {}",
+                        layout.ty,
+                        inner.ty
+                    );
+                    if matches!(inner.layout.variants(), Variants::Multiple { .. }) {
+                        // FIXME: ScalarPair for enums is enormously complicated and it is very hard
+                        // to check anything about them.
+                        return;
+                    }
+                    match inner.layout.fields() {
+                        FieldsShape::Arbitrary { .. } => {
+                            // Checked below.
+                        }
+                        FieldsShape::Union(..) => {
+                            // FIXME: I guess we could also check something here? Like, look at all fields?
+                            return;
+                        }
+                        _ => {
+                            panic!("`ScalarPair` layout with unexpected field shape in {inner:#?}");
+                        }
+                    }
+                    let mut fields = non_zst_fields(cx, &inner);
+                    let (offset1, field1) = fields.next().unwrap_or_else(|| {
+                        panic!("`ScalarPair` layout for type with not even one non-ZST field: {inner:#?}")
+                    });
+                    let (offset2, field2) = fields.next().unwrap_or_else(|| {
+                        panic!("`ScalarPair` layout for type with less than two non-ZST fields: {inner:#?}")
+                    });
+                    assert!(
+                        fields.next().is_none(),
+                        "`ScalarPair` layout for type with at least three non-ZST fields: {inner:#?}"
+                    );
+                    // The fields might be in opposite order.
+                    let (offset1, field1, offset2, field2) = if offset1 <= offset2 {
+                        (offset1, field1, offset2, field2)
+                    } else {
+                        (offset2, field2, offset1, field1)
+                    };
+                    // The fields should be at the right offset, and match the `scalar` layout.
+                    assert_eq!(
+                        offset1,
+                        Size::ZERO,
+                        "`ScalarPair` first field at non-0 offset in {inner:#?}",
+                    );
+                    assert_eq!(
+                        field1.size, size1,
+                        "`ScalarPair` first field with bad size in {inner:#?}",
+                    );
+                    assert_eq!(
+                        field1.align.abi, align1,
+                        "`ScalarPair` first field with bad align in {inner:#?}",
+                    );
+                    assert!(
+                        matches!(field1.abi, Abi::Scalar(_)),
+                        "`ScalarPair` first field with bad ABI in {inner:#?}",
+                    );
+                    assert_eq!(
+                        offset2, field2_offset,
+                        "`ScalarPair` second field at bad offset in {inner:#?}",
+                    );
+                    assert_eq!(
+                        field2.size, size2,
+                        "`ScalarPair` second field with bad size in {inner:#?}",
+                    );
+                    assert_eq!(
+                        field2.align.abi, align2,
+                        "`ScalarPair` second field with bad align in {inner:#?}",
+                    );
+                    assert!(
+                        matches!(field2.abi, Abi::Scalar(_)),
+                        "`ScalarPair` second field with bad ABI in {inner:#?}",
+                    );
+                }
+                Abi::Vector { count, element } => {
+                    // No padding in vectors. Alignment can be strengthened, though.
+                    assert!(
+                        layout.layout.align().abi >= element.align(cx).abi,
+                        "alignment mismatch between ABI and layout in {layout:#?}"
+                    );
+                    let size = element.size(cx) * count;
+                    assert_eq!(
+                        layout.layout.size(),
+                        size.align_to(cx.data_layout().vector_align(size).abi),
+                        "size mismatch between ABI and layout in {layout:#?}"
+                    );
+                }
+                Abi::Uninhabited | Abi::Aggregate { .. } => {} // Nothing to check.
+            }
+        }
+
+        check_layout_abi(cx, layout);
+
+        if let Variants::Multiple { variants, .. } = &layout.variants {
+            for variant in variants.iter() {
+                // No nested "multiple".
+                assert!(matches!(variant.variants(), Variants::Single { .. }));
+                // Variants should have the same or a smaller size as the full thing,
+                // and same for alignment.
+                if variant.size() > layout.size {
+                    bug!(
+                        "Type with size {} bytes has variant with size {} bytes: {layout:#?}",
+                        layout.size.bytes(),
+                        variant.size().bytes(),
+                    )
+                }
+                if variant.align().abi > layout.align.abi {
+                    bug!(
+                        "Type with alignment {} bytes has variant with alignment {} bytes: {layout:#?}",
+                        layout.align.abi.bytes(),
+                        variant.align().abi.bytes(),
+                    )
+                }
+                // Skip empty variants.
+                if variant.size() == Size::ZERO
+                    || variant.fields().count() == 0
+                    || variant.abi().is_uninhabited()
+                {
+                    // These are never actually accessed anyway, so we can skip the coherence check
+                    // for them. They also fail that check, since they have
+                    // `Aggregate`/`Uninhbaited` ABI even when the main type is
+                    // `Scalar`/`ScalarPair`. (Note that sometimes, variants with fields have size
+                    // 0, and sometimes, variants without fields have non-0 size.)
+                    continue;
+                }
+                // The top-level ABI and the ABI of the variants should be coherent.
+                let scalar_coherent = |s1: Scalar, s2: Scalar| {
+                    s1.size(cx) == s2.size(cx) && s1.align(cx) == s2.align(cx)
+                };
+                let abi_coherent = match (layout.abi, variant.abi()) {
+                    (Abi::Scalar(s1), Abi::Scalar(s2)) => scalar_coherent(s1, s2),
+                    (Abi::ScalarPair(a1, b1), Abi::ScalarPair(a2, b2)) => {
+                        scalar_coherent(a1, a2) && scalar_coherent(b1, b2)
+                    }
+                    (Abi::Uninhabited, _) => true,
+                    (Abi::Aggregate { .. }, _) => true,
+                    _ => false,
+                };
+                if !abi_coherent {
+                    bug!(
+                        "Variant ABI is incompatible with top-level ABI:\nvariant={:#?}\nTop-level: {layout:#?}",
+                        variant
+                    );
+                }
+            }
+        }
+    }
+}
diff --git a/compiler/rustc_middle/src/ty/list.rs b/compiler/rustc_middle/src/ty/list.rs
index db3b5cfd1..79365ef28 100644
--- a/compiler/rustc_middle/src/ty/list.rs
+++ b/compiler/rustc_middle/src/ty/list.rs
@@ -65,6 +65,10 @@ impl<T> List<T> {
     pub fn len(&self) -> usize {
         self.len
     }
+
+    pub fn as_slice(&self) -> &[T] {
+        self
+    }
 }
 
 impl<T: Copy> List<T> {
diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs
index 02da02568..3f9871190 100644
--- a/compiler/rustc_middle/src/ty/mod.rs
+++ b/compiler/rustc_middle/src/ty/mod.rs
@@ -15,6 +15,7 @@ pub use self::AssocItemContainer::*;
 pub use self::BorrowKind::*;
 pub use self::IntVarValue::*;
 pub use self::Variance::*;
+use crate::error::{OpaqueHiddenTypeMismatch, TypeMismatchReason};
 use crate::metadata::ModChild;
 use crate::middle::privacy::AccessLevels;
 use crate::mir::{Body, GeneratorLayout};
@@ -40,6 +41,7 @@ use rustc_hir::Node;
 use rustc_index::vec::IndexVec;
 use rustc_macros::HashStable;
 use rustc_query_system::ich::StableHashingContext;
+use rustc_serialize::{Decodable, Encodable};
 use rustc_span::hygiene::MacroKind;
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::{ExpnId, Span};
@@ -49,10 +51,14 @@ pub use vtable::*;
 
 use std::fmt::Debug;
 use std::hash::{Hash, Hasher};
+use std::marker::PhantomData;
+use std::mem;
+use std::num::NonZeroUsize;
 use std::ops::ControlFlow;
 use std::{fmt, str};
 
 pub use crate::ty::diagnostics::*;
+pub use rustc_type_ir::DynKind::*;
 pub use rustc_type_ir::InferTy::*;
 pub use rustc_type_ir::RegionKind::*;
 pub use rustc_type_ir::TyKind::*;
@@ -124,6 +130,7 @@ mod erase_regions;
 mod generics;
 mod impls_ty;
 mod instance;
+mod layout_sanity_check;
 mod list;
 mod parameterized;
 mod rvalue_scopes;
@@ -177,11 +184,6 @@ pub struct ResolverAstLowering {
     pub label_res_map: NodeMap<ast::NodeId>,
     /// Resolutions for lifetimes.
     pub lifetimes_res_map: NodeMap<LifetimeRes>,
-    /// Mapping from generics `def_id`s to TAIT generics `def_id`s.
-    /// For each captured lifetime (e.g., 'a), we create a new lifetime parameter that is a generic
-    /// defined on the TAIT, so we have type Foo<'a1> = ... and we establish a mapping in this
-    /// field from the original parameter 'a to the new parameter 'a1.
-    pub generics_def_id_map: Vec<FxHashMap<LocalDefId, LocalDefId>>,
     /// Lifetime parameters that lowering will have to introduce.
     pub extra_lifetime_params_map: NodeMap<Vec<(Ident, ast::NodeId, LifetimeRes)>>,
 
@@ -262,13 +264,11 @@ impl fmt::Display for ImplPolarity {
 }
 
 #[derive(Clone, Debug, PartialEq, Eq, Copy, Hash, Encodable, Decodable, HashStable)]
-pub enum Visibility {
+pub enum Visibility<Id = LocalDefId> {
     /// Visible everywhere (including in other crates).
     Public,
     /// Visible only in the given crate-local module.
-    Restricted(DefId),
-    /// Not visible anywhere in the local crate. This is the visibility of private external items.
-    Invisible,
+    Restricted(Id),
 }
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, HashStable, TyEncodable, TyDecodable)]
@@ -359,31 +359,45 @@ impl<'tcx> DefIdTree for TyCtxt<'tcx> {
     }
 }
 
-impl Visibility {
-    /// Returns `true` if an item with this visibility is accessible from the given block.
-    pub fn is_accessible_from<T: DefIdTree>(self, module: DefId, tree: T) -> bool {
-        let restriction = match self {
-            // Public items are visible everywhere.
-            Visibility::Public => return true,
-            // Private items from other crates are visible nowhere.
-            Visibility::Invisible => return false,
-            // Restricted items are visible in an arbitrary local module.
-            Visibility::Restricted(other) if other.krate != module.krate => return false,
-            Visibility::Restricted(module) => module,
-        };
+impl<Id> Visibility<Id> {
+    pub fn is_public(self) -> bool {
+        matches!(self, Visibility::Public)
+    }
+
+    pub fn map_id<OutId>(self, f: impl FnOnce(Id) -> OutId) -> Visibility<OutId> {
+        match self {
+            Visibility::Public => Visibility::Public,
+            Visibility::Restricted(id) => Visibility::Restricted(f(id)),
+        }
+    }
+}
 
-        tree.is_descendant_of(module, restriction)
+impl<Id: Into<DefId>> Visibility<Id> {
+    pub fn to_def_id(self) -> Visibility<DefId> {
+        self.map_id(Into::into)
+    }
+
+    /// Returns `true` if an item with this visibility is accessible from the given module.
+    pub fn is_accessible_from(self, module: impl Into<DefId>, tree: impl DefIdTree) -> bool {
+        match self {
+            // Public items are visible everywhere.
+            Visibility::Public => true,
+            Visibility::Restricted(id) => tree.is_descendant_of(module.into(), id.into()),
+        }
     }
 
     /// Returns `true` if this visibility is at least as accessible as the given visibility
-    pub fn is_at_least<T: DefIdTree>(self, vis: Visibility, tree: T) -> bool {
-        let vis_restriction = match vis {
-            Visibility::Public => return self == Visibility::Public,
-            Visibility::Invisible => return true,
-            Visibility::Restricted(module) => module,
-        };
+    pub fn is_at_least(self, vis: Visibility<impl Into<DefId>>, tree: impl DefIdTree) -> bool {
+        match vis {
+            Visibility::Public => self.is_public(),
+            Visibility::Restricted(id) => self.is_accessible_from(id, tree),
+        }
+    }
+}
 
-        self.is_accessible_from(vis_restriction, tree)
+impl Visibility<DefId> {
+    pub fn expect_local(self) -> Visibility {
+        self.map_id(|id| id.expect_local())
     }
 
     // Returns `true` if this item is visible anywhere in the local crate.
@@ -391,13 +405,8 @@ impl Visibility {
         match self {
             Visibility::Public => true,
             Visibility::Restricted(def_id) => def_id.is_local(),
-            Visibility::Invisible => false,
         }
     }
-
-    pub fn is_public(self) -> bool {
-        matches!(self, Visibility::Public)
-    }
 }
 
 /// The crate variances map is computed during typeck and contains the
@@ -468,15 +477,6 @@ pub(crate) struct TyS<'tcx> {
     outer_exclusive_binder: ty::DebruijnIndex,
 }
 
-// `TyS` is used a lot. Make sure it doesn't unintentionally get bigger.
-#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
-static_assert_size!(TyS<'_>, 40);
-
-// We are actually storing a stable hash cache next to the type, so let's
-// also check the full size
-#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
-static_assert_size!(WithStableHash<TyS<'_>>, 56);
-
 /// Use this rather than `TyS`, whenever possible.
 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, HashStable)]
 #[rustc_diagnostic_item = "Ty"]
@@ -533,10 +533,6 @@ pub(crate) struct PredicateS<'tcx> {
     outer_exclusive_binder: ty::DebruijnIndex,
 }
 
-// This type is used a lot. Make sure it doesn't unintentionally get bigger.
-#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
-static_assert_size!(PredicateS<'_>, 56);
-
 /// Use this rather than `PredicateS`, whenever possible.
 #[derive(Clone, Copy, PartialEq, Eq, Hash)]
 #[rustc_pass_by_value]
@@ -593,6 +589,29 @@ impl<'tcx> Predicate<'tcx> {
         }
         self
     }
+
+    /// Whether this projection can be soundly normalized.
+    ///
+    /// Wf predicates must not be normalized, as normalization
+    /// can remove required bounds which would cause us to
+    /// unsoundly accept some programs. See #91068.
+    #[inline]
+    pub fn allow_normalization(self) -> bool {
+        match self.kind().skip_binder() {
+            PredicateKind::WellFormed(_) => false,
+            PredicateKind::Trait(_)
+            | PredicateKind::RegionOutlives(_)
+            | PredicateKind::TypeOutlives(_)
+            | PredicateKind::Projection(_)
+            | PredicateKind::ObjectSafe(_)
+            | PredicateKind::ClosureKind(_, _, _)
+            | PredicateKind::Subtype(_)
+            | PredicateKind::Coerce(_)
+            | PredicateKind::ConstEvaluatable(_)
+            | PredicateKind::ConstEquate(_, _)
+            | PredicateKind::TypeWellFormedFromEnv(_) => true,
+        }
+    }
 }
 
 impl<'a, 'tcx> HashStable<StableHashingContext<'a>> for Predicate<'tcx> {
@@ -617,7 +636,7 @@ impl rustc_errors::IntoDiagnosticArg for Predicate<'_> {
 }
 
 #[derive(Clone, Copy, PartialEq, Eq, Hash, TyEncodable, TyDecodable)]
-#[derive(HashStable, TypeFoldable, TypeVisitable)]
+#[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub enum PredicateKind<'tcx> {
     /// Corresponds to `where Foo: Bar<A, B, C>`. `Foo` here would be
     /// the `Self` type of the trait reference and `A`, `B`, and `C`
@@ -789,7 +808,7 @@ impl<'tcx> Predicate<'tcx> {
 }
 
 #[derive(Clone, Copy, PartialEq, Eq, Hash, TyEncodable, TyDecodable)]
-#[derive(HashStable, TypeFoldable, TypeVisitable)]
+#[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub struct TraitPredicate<'tcx> {
     pub trait_ref: TraitRef<'tcx>,
 
@@ -869,7 +888,7 @@ impl<'tcx> PolyTraitPredicate<'tcx> {
 }
 
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, TyEncodable, TyDecodable)]
-#[derive(HashStable, TypeFoldable, TypeVisitable)]
+#[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub struct OutlivesPredicate<A, B>(pub A, pub B); // `A: B`
 pub type RegionOutlivesPredicate<'tcx> = OutlivesPredicate<ty::Region<'tcx>, ty::Region<'tcx>>;
 pub type TypeOutlivesPredicate<'tcx> = OutlivesPredicate<Ty<'tcx>, ty::Region<'tcx>>;
@@ -880,7 +899,7 @@ pub type PolyTypeOutlivesPredicate<'tcx> = ty::Binder<'tcx, TypeOutlivesPredicat
 /// whether the `a` type is the type that we should label as "expected" when
 /// presenting user diagnostics.
 #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, TyEncodable, TyDecodable)]
-#[derive(HashStable, TypeFoldable, TypeVisitable)]
+#[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub struct SubtypePredicate<'tcx> {
     pub a_is_expected: bool,
     pub a: Ty<'tcx>,
@@ -890,49 +909,142 @@ pub type PolySubtypePredicate<'tcx> = ty::Binder<'tcx, SubtypePredicate<'tcx>>;
 
 /// Encodes that we have to coerce *from* the `a` type to the `b` type.
 #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, TyEncodable, TyDecodable)]
-#[derive(HashStable, TypeFoldable, TypeVisitable)]
+#[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub struct CoercePredicate<'tcx> {
     pub a: Ty<'tcx>,
     pub b: Ty<'tcx>,
 }
 pub type PolyCoercePredicate<'tcx> = ty::Binder<'tcx, CoercePredicate<'tcx>>;
 
-#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, TyEncodable, TyDecodable)]
-#[derive(HashStable, TypeFoldable, TypeVisitable)]
-pub enum Term<'tcx> {
-    Ty(Ty<'tcx>),
-    Const(Const<'tcx>),
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct Term<'tcx> {
+    ptr: NonZeroUsize,
+    marker: PhantomData<(Ty<'tcx>, Const<'tcx>)>,
+}
+
+impl Debug for Term<'_> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let data = if let Some(ty) = self.ty() {
+            format!("Term::Ty({:?})", ty)
+        } else if let Some(ct) = self.ct() {
+            format!("Term::Ct({:?})", ct)
+        } else {
+            unreachable!()
+        };
+        f.write_str(&data)
+    }
 }
 
 impl<'tcx> From<Ty<'tcx>> for Term<'tcx> {
     fn from(ty: Ty<'tcx>) -> Self {
-        Term::Ty(ty)
+        TermKind::Ty(ty).pack()
     }
 }
 
 impl<'tcx> From<Const<'tcx>> for Term<'tcx> {
     fn from(c: Const<'tcx>) -> Self {
-        Term::Const(c)
+        TermKind::Const(c).pack()
+    }
+}
+
+impl<'a, 'tcx> HashStable<StableHashingContext<'a>> for Term<'tcx> {
+    fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
+        self.unpack().hash_stable(hcx, hasher);
+    }
+}
+
+impl<'tcx> TypeFoldable<'tcx> for Term<'tcx> {
+    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
+        Ok(self.unpack().try_fold_with(folder)?.pack())
+    }
+}
+
+impl<'tcx> TypeVisitable<'tcx> for Term<'tcx> {
+    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
+        self.unpack().visit_with(visitor)
+    }
+}
+
+impl<'tcx, E: TyEncoder<I = TyCtxt<'tcx>>> Encodable<E> for Term<'tcx> {
+    fn encode(&self, e: &mut E) {
+        self.unpack().encode(e)
+    }
+}
+
+impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> Decodable<D> for Term<'tcx> {
+    fn decode(d: &mut D) -> Self {
+        let res: TermKind<'tcx> = Decodable::decode(d);
+        res.pack()
     }
 }
 
 impl<'tcx> Term<'tcx> {
+    #[inline]
+    pub fn unpack(self) -> TermKind<'tcx> {
+        let ptr = self.ptr.get();
+        // SAFETY: use of `Interned::new_unchecked` here is ok because these
+        // pointers were originally created from `Interned` types in `pack()`,
+        // and this is just going in the other direction.
+        unsafe {
+            match ptr & TAG_MASK {
+                TYPE_TAG => TermKind::Ty(Ty(Interned::new_unchecked(
+                    &*((ptr & !TAG_MASK) as *const WithStableHash<ty::TyS<'tcx>>),
+                ))),
+                CONST_TAG => TermKind::Const(ty::Const(Interned::new_unchecked(
+                    &*((ptr & !TAG_MASK) as *const ty::ConstS<'tcx>),
+                ))),
+                _ => core::intrinsics::unreachable(),
+            }
+        }
+    }
+
     pub fn ty(&self) -> Option<Ty<'tcx>> {
-        if let Term::Ty(ty) = self { Some(*ty) } else { None }
+        if let TermKind::Ty(ty) = self.unpack() { Some(ty) } else { None }
     }
 
     pub fn ct(&self) -> Option<Const<'tcx>> {
-        if let Term::Const(c) = self { Some(*c) } else { None }
+        if let TermKind::Const(c) = self.unpack() { Some(c) } else { None }
     }
 
     pub fn into_arg(self) -> GenericArg<'tcx> {
-        match self {
-            Term::Ty(ty) => ty.into(),
-            Term::Const(c) => c.into(),
+        match self.unpack() {
+            TermKind::Ty(ty) => ty.into(),
+            TermKind::Const(c) => c.into(),
         }
     }
 }
 
+const TAG_MASK: usize = 0b11;
+const TYPE_TAG: usize = 0b00;
+const CONST_TAG: usize = 0b01;
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, TyEncodable, TyDecodable)]
+#[derive(HashStable, TypeFoldable, TypeVisitable)]
+pub enum TermKind<'tcx> {
+    Ty(Ty<'tcx>),
+    Const(Const<'tcx>),
+}
+
+impl<'tcx> TermKind<'tcx> {
+    #[inline]
+    fn pack(self) -> Term<'tcx> {
+        let (tag, ptr) = match self {
+            TermKind::Ty(ty) => {
+                // Ensure we can use the tag bits.
+                assert_eq!(mem::align_of_val(&*ty.0.0) & TAG_MASK, 0);
+                (TYPE_TAG, ty.0.0 as *const WithStableHash<ty::TyS<'tcx>> as usize)
+            }
+            TermKind::Const(ct) => {
+                // Ensure we can use the tag bits.
+                assert_eq!(mem::align_of_val(&*ct.0.0) & TAG_MASK, 0);
+                (CONST_TAG, ct.0.0 as *const ty::ConstS<'tcx> as usize)
+            }
+        };
+
+        Term { ptr: unsafe { NonZeroUsize::new_unchecked(ptr | tag) }, marker: PhantomData }
+    }
+}
+
 /// This kind of predicate has no *direct* correspondent in the
 /// syntax, but it roughly corresponds to the syntactic forms:
 ///
@@ -946,7 +1058,7 @@ impl<'tcx> Term<'tcx> {
 /// Form #2 eventually yields one of these `ProjectionPredicate`
 /// instances to normalize the LHS.
 #[derive(Copy, Clone, PartialEq, Eq, Hash, TyEncodable, TyDecodable)]
-#[derive(HashStable, TypeFoldable, TypeVisitable)]
+#[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub struct ProjectionPredicate<'tcx> {
     pub projection_ty: ProjectionTy<'tcx>,
     pub term: Term<'tcx>,
@@ -1002,6 +1114,12 @@ pub trait ToPredicate<'tcx> {
     fn to_predicate(self, tcx: TyCtxt<'tcx>) -> Predicate<'tcx>;
 }
 
+impl<'tcx> ToPredicate<'tcx> for Predicate<'tcx> {
+    fn to_predicate(self, _tcx: TyCtxt<'tcx>) -> Predicate<'tcx> {
+        self
+    }
+}
+
 impl<'tcx> ToPredicate<'tcx> for Binder<'tcx, PredicateKind<'tcx>> {
     #[inline(always)]
     fn to_predicate(self, tcx: TyCtxt<'tcx>) -> Predicate<'tcx> {
@@ -1166,20 +1284,17 @@ pub struct OpaqueHiddenType<'tcx> {
 impl<'tcx> OpaqueHiddenType<'tcx> {
     pub fn report_mismatch(&self, other: &Self, tcx: TyCtxt<'tcx>) {
         // Found different concrete types for the opaque type.
-        let mut err = tcx.sess.struct_span_err(
-            other.span,
-            "concrete type differs from previous defining opaque type use",
-        );
-        err.span_label(other.span, format!("expected `{}`, got `{}`", self.ty, other.ty));
-        if self.span == other.span {
-            err.span_label(
-                self.span,
-                "this expression supplies two conflicting concrete types for the same opaque type",
-            );
+        let sub_diag = if self.span == other.span {
+            TypeMismatchReason::ConflictType { span: self.span }
         } else {
-            err.span_note(self.span, "previous use here");
-        }
-        err.emit();
+            TypeMismatchReason::PreviousUse { span: self.span }
+        };
+        tcx.sess.emit_err(OpaqueHiddenTypeMismatch {
+            self_ty: self.ty,
+            other_ty: other.ty,
+            other_span: other.span,
+            sub: sub_diag,
+        });
     }
 }
 
@@ -1411,7 +1526,7 @@ impl<'tcx> TypeFoldable<'tcx> for ParamEnv<'tcx> {
         Ok(ParamEnv::new(
             self.caller_bounds().try_fold_with(folder)?,
             self.reveal().try_fold_with(folder)?,
-            self.constness().try_fold_with(folder)?,
+            self.constness(),
         ))
     }
 }
@@ -1419,8 +1534,7 @@ impl<'tcx> TypeFoldable<'tcx> for ParamEnv<'tcx> {
 impl<'tcx> TypeVisitable<'tcx> for ParamEnv<'tcx> {
     fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
         self.caller_bounds().visit_with(visitor)?;
-        self.reveal().visit_with(visitor)?;
-        self.constness().visit_with(visitor)
+        self.reveal().visit_with(visitor)
     }
 }
 
@@ -1577,7 +1691,7 @@ impl<'tcx> PolyTraitRef<'tcx> {
 }
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, TypeFoldable, TypeVisitable)]
-#[derive(HashStable)]
+#[derive(HashStable, Lift)]
 pub struct ParamEnvAnd<'tcx, T> {
     pub param_env: ParamEnv<'tcx>,
     pub value: T,
@@ -1779,7 +1893,7 @@ pub enum VariantDiscr {
 pub struct FieldDef {
     pub did: DefId,
     pub name: Symbol,
-    pub vis: Visibility,
+    pub vis: Visibility<DefId>,
 }
 
 impl PartialEq for FieldDef {
@@ -2256,7 +2370,11 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     pub fn get_attr(self, did: DefId, attr: Symbol) -> Option<&'tcx ast::Attribute> {
-        self.get_attrs(did, attr).next()
+        if cfg!(debug_assertions) && !rustc_feature::is_valid_for_get_attr(attr) {
+            bug!("get_attr: unexpected called with DefId `{:?}`, attr `{:?}`", did, attr);
+        } else {
+            self.get_attrs(did, attr).next()
+        }
     }
 
     /// Determines whether an item is annotated with an attribute.
@@ -2358,6 +2476,25 @@ impl<'tcx> TyCtxt<'tcx> {
         (ident, scope)
     }
 
+    /// Returns `true` if the debuginfo for `span` should be collapsed to the outermost expansion
+    /// site. Only applies when `Span` is the result of macro expansion.
+    ///
+    /// - If the `collapse_debuginfo` feature is enabled then debuginfo is not collapsed by default
+    ///   and only when a macro definition is annotated with `#[collapse_debuginfo]`.
+    /// - If `collapse_debuginfo` is not enabled, then debuginfo is collapsed by default.
+    ///
+    /// When `-Zdebug-macros` is provided then debuginfo will never be collapsed.
+    pub fn should_collapse_debuginfo(self, span: Span) -> bool {
+        !self.sess.opts.unstable_opts.debug_macros
+            && if self.features().collapse_debuginfo {
+                span.in_macro_expansion_with_collapse_debuginfo()
+            } else {
+                // Inlined spans should not be collapsed as that leads to all of the
+                // inlined code being attributed to the inline callsite.
+                span.from_expansion() && !span.is_inlined()
+            }
+    }
+
     pub fn is_object_safe(self, key: DefId) -> bool {
         self.object_safety_violations(key).is_empty()
     }
@@ -2372,6 +2509,14 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn is_const_default_method(self, def_id: DefId) -> bool {
         matches!(self.trait_of_item(def_id), Some(trait_id) if self.has_attr(trait_id, sym::const_trait))
     }
+
+    pub fn impl_trait_in_trait_parent(self, mut def_id: DefId) -> DefId {
+        while let def_kind = self.def_kind(def_id) && def_kind != DefKind::AssocFn {
+            debug_assert_eq!(def_kind, DefKind::ImplTraitPlaceholder);
+            def_id = self.parent(def_id);
+        }
+        def_id
+    }
 }
 
 /// Yields the parent function's `LocalDefId` if `def_id` is an `impl Trait` definition.
@@ -2516,3 +2661,14 @@ pub struct DestructuredConst<'tcx> {
     pub variant: Option<VariantIdx>,
     pub fields: &'tcx [ty::Const<'tcx>],
 }
+
+// Some types are used a lot. Make sure they don't unintentionally get bigger.
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
+mod size_asserts {
+    use super::*;
+    use rustc_data_structures::static_assert_size;
+    // These are in alphabetical order, which is easy to maintain.
+    static_assert_size!(PredicateS<'_>, 48);
+    static_assert_size!(TyS<'_>, 40);
+    static_assert_size!(WithStableHash<TyS<'_>>, 56);
+}
diff --git a/compiler/rustc_middle/src/ty/normalize_erasing_regions.rs b/compiler/rustc_middle/src/ty/normalize_erasing_regions.rs
index 9d8a81165..9db5a2894 100644
--- a/compiler/rustc_middle/src/ty/normalize_erasing_regions.rs
+++ b/compiler/rustc_middle/src/ty/normalize_erasing_regions.rs
@@ -36,6 +36,7 @@ impl<'tcx> TyCtxt<'tcx> {
     ///
     /// This should only be used outside of type inference. For example,
     /// it assumes that normalization will succeed.
+    #[tracing::instrument(level = "debug", skip(self, param_env))]
     pub fn normalize_erasing_regions<T>(self, param_env: ty::ParamEnv<'tcx>, value: T) -> T
     where
         T: TypeFoldable<'tcx>,
@@ -100,6 +101,7 @@ impl<'tcx> TyCtxt<'tcx> {
     /// N.B., currently, higher-ranked type bounds inhibit
     /// normalization. Therefore, each time we erase them in
     /// codegen, we need to normalize the contents.
+    #[tracing::instrument(level = "debug", skip(self, param_env))]
     pub fn normalize_erasing_late_bound_regions<T>(
         self,
         param_env: ty::ParamEnv<'tcx>,
@@ -188,13 +190,11 @@ struct NormalizeAfterErasingRegionsFolder<'tcx> {
 }
 
 impl<'tcx> NormalizeAfterErasingRegionsFolder<'tcx> {
-    #[instrument(skip(self), level = "debug")]
     fn normalize_generic_arg_after_erasing_regions(
         &self,
         arg: ty::GenericArg<'tcx>,
     ) -> ty::GenericArg<'tcx> {
         let arg = self.param_env.and(arg);
-        debug!(?arg);
 
         self.tcx.try_normalize_generic_arg_after_erasing_regions(arg).unwrap_or_else(|_| bug!(
                 "Failed to normalize {:?}, maybe try to call `try_normalize_erasing_regions` instead",
diff --git a/compiler/rustc_middle/src/ty/parameterized.rs b/compiler/rustc_middle/src/ty/parameterized.rs
index e189ee2fc..9c8dc30e2 100644
--- a/compiler/rustc_middle/src/ty/parameterized.rs
+++ b/compiler/rustc_middle/src/ty/parameterized.rs
@@ -1,4 +1,4 @@
-use rustc_hir::def_id::DefId;
+use rustc_hir::def_id::{DefId, DefIndex};
 use rustc_index::vec::{Idx, IndexVec};
 
 use crate::middle::exported_symbols::ExportedSymbol;
@@ -53,17 +53,20 @@ trivially_parameterized_over_tcx! {
     crate::metadata::ModChild,
     crate::middle::codegen_fn_attrs::CodegenFnAttrs,
     crate::middle::exported_symbols::SymbolExportInfo,
+    crate::middle::resolve_lifetime::ObjectLifetimeDefault,
     crate::mir::ConstQualifs,
+    ty::AssocItemContainer,
     ty::Generics,
     ty::ImplPolarity,
     ty::ReprOptions,
     ty::TraitDef,
-    ty::Visibility,
+    ty::Visibility<DefIndex>,
     ty::adjustment::CoerceUnsizedInfo,
     ty::fast_reject::SimplifiedTypeGen<DefId>,
     rustc_ast::Attribute,
     rustc_ast::MacArgs,
     rustc_attr::ConstStability,
+    rustc_attr::DefaultBodyStability,
     rustc_attr::Deprecation,
     rustc_attr::Stability,
     rustc_hir::Constness,
diff --git a/compiler/rustc_middle/src/ty/print/pretty.rs b/compiler/rustc_middle/src/ty/print/pretty.rs
index 7f2e81a71..97bddb93e 100644
--- a/compiler/rustc_middle/src/ty/print/pretty.rs
+++ b/compiler/rustc_middle/src/ty/print/pretty.rs
@@ -1,7 +1,7 @@
 use crate::mir::interpret::{AllocRange, GlobalAlloc, Pointer, Provenance, Scalar};
 use crate::ty::subst::{GenericArg, GenericArgKind, Subst};
 use crate::ty::{
-    self, ConstInt, DefIdTree, ParamConst, ScalarInt, Term, Ty, TyCtxt, TypeFoldable,
+    self, ConstInt, DefIdTree, ParamConst, ScalarInt, Term, TermKind, Ty, TyCtxt, TypeFoldable,
     TypeSuperFoldable, TypeSuperVisitable, TypeVisitable,
 };
 use rustc_apfloat::ieee::{Double, Single};
@@ -619,12 +619,16 @@ pub trait PrettyPrinter<'tcx>:
             ty::Adt(def, substs) => {
                 p!(print_def_path(def.did(), substs));
             }
-            ty::Dynamic(data, r) => {
+            ty::Dynamic(data, r, repr) => {
                 let print_r = self.should_print_region(r);
                 if print_r {
                     p!("(");
                 }
-                p!("dyn ", print(data));
+                match repr {
+                    ty::Dyn => p!("dyn "),
+                    ty::DynStar => p!("dyn* "),
+                }
+                p!(print(data));
                 if print_r {
                     p!(" + ", print(r), ")");
                 }
@@ -632,7 +636,13 @@ pub trait PrettyPrinter<'tcx>:
             ty::Foreign(def_id) => {
                 p!(print_def_path(def_id, &[]));
             }
-            ty::Projection(ref data) => p!(print(data)),
+            ty::Projection(ref data) => {
+                if self.tcx().def_kind(data.item_def_id) == DefKind::ImplTraitPlaceholder {
+                    return self.pretty_print_opaque_impl_type(data.item_def_id, data.substs);
+                } else {
+                    p!(print(data))
+                }
+            }
             ty::Placeholder(placeholder) => p!(write("Placeholder({:?})", placeholder)),
             ty::Opaque(def_id, substs) => {
                 // FIXME(eddyb) print this with `print_def_path`.
@@ -855,7 +865,7 @@ pub trait PrettyPrinter<'tcx>:
                         }
 
                         p!(")");
-                        if let Term::Ty(ty) = return_ty.skip_binder() {
+                        if let Some(ty) = return_ty.skip_binder().ty() {
                             if !ty.is_unit() {
                                 p!(" -> ", print(return_ty));
                             }
@@ -916,12 +926,14 @@ pub trait PrettyPrinter<'tcx>:
                         // Skip printing `<[generator@] as Generator<_>>::Return` from async blocks,
                         // unless we can find out what generator return type it comes from.
                         let term = if let Some(ty) = term.skip_binder().ty()
-                            && let ty::Projection(ty::ProjectionTy { item_def_id, substs }) = ty.kind()
-                            && Some(*item_def_id) == tcx.lang_items().generator_return()
+                            && let ty::Projection(proj) = ty.kind()
+                            && let assoc = tcx.associated_item(proj.item_def_id)
+                            && assoc.trait_container(tcx) == tcx.lang_items().gen_trait()
+                            && assoc.name == rustc_span::sym::Return
                         {
                             if let ty::Generator(_, substs, _) = substs.type_at(0).kind() {
                                 let return_ty = substs.as_generator().return_ty();
-                                if !return_ty.is_ty_infer() {
+                                if !return_ty.is_ty_var() {
                                     return_ty.into()
                                 } else {
                                     continue;
@@ -942,13 +954,9 @@ pub trait PrettyPrinter<'tcx>:
 
                         p!(write("{} = ", tcx.associated_item(assoc_item_def_id).name));
 
-                        match term {
-                            Term::Ty(ty) => {
-                                p!(print(ty))
-                            }
-                            Term::Const(c) => {
-                                p!(print(c));
-                            }
+                        match term.unpack() {
+                            TermKind::Ty(ty) => p!(print(ty)),
+                            TermKind::Const(c) => p!(print(c)),
                         };
                     }
 
@@ -1193,15 +1201,9 @@ pub trait PrettyPrinter<'tcx>:
         }
 
         match ct.kind() {
-            ty::ConstKind::Unevaluated(ty::Unevaluated {
-                def,
-                substs,
-                promoted: Some(promoted),
-            }) => {
-                p!(print_value_path(def.did, substs));
-                p!(write("::{:?}", promoted));
-            }
-            ty::ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted: None }) => {
+            ty::ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted }) => {
+                assert_eq!(promoted, ());
+
                 match self.tcx().def_kind(def.did) {
                     DefKind::Static(..) | DefKind::Const | DefKind::AssocConst => {
                         p!(print_value_path(def.did, substs))
@@ -1275,7 +1277,7 @@ pub trait PrettyPrinter<'tcx>:
                                     let range =
                                         AllocRange { start: offset, size: Size::from_bytes(len) };
                                     if let Ok(byte_str) =
-                                        alloc.inner().get_bytes(&self.tcx(), range)
+                                        alloc.inner().get_bytes_strip_provenance(&self.tcx(), range)
                                     {
                                         p!(pretty_print_byte_str(byte_str))
                                     } else {
@@ -1401,14 +1403,7 @@ pub trait PrettyPrinter<'tcx>:
     }
 
     fn pretty_print_byte_str(mut self, byte_str: &'tcx [u8]) -> Result<Self::Const, Self::Error> {
-        define_scoped_cx!(self);
-        p!("b\"");
-        for &c in byte_str {
-            for e in std::ascii::escape_default(c) {
-                self.write_char(e as char)?;
-            }
-        }
-        p!("\"");
+        write!(self, "b\"{}\"", byte_str.escape_ascii())?;
         Ok(self)
     }
 
@@ -1513,6 +1508,10 @@ pub trait PrettyPrinter<'tcx>:
                 }
                 return Ok(self);
             }
+            (ty::ValTree::Leaf(leaf), ty::Ref(_, inner_ty, _)) => {
+                p!(write("&"));
+                return self.pretty_print_const_scalar_int(leaf, *inner_ty, print_ty);
+            }
             (ty::ValTree::Leaf(leaf), _) => {
                 return self.pretty_print_const_scalar_int(leaf, ty, print_ty);
             }
@@ -1532,6 +1531,34 @@ pub trait PrettyPrinter<'tcx>:
         }
         Ok(self)
     }
+
+    fn pretty_closure_as_impl(
+        mut self,
+        closure: ty::ClosureSubsts<'tcx>,
+    ) -> Result<Self::Const, Self::Error> {
+        let sig = closure.sig();
+        let kind = closure.kind_ty().to_opt_closure_kind().unwrap_or(ty::ClosureKind::Fn);
+
+        write!(self, "impl ")?;
+        self.wrap_binder(&sig, |sig, mut cx| {
+            define_scoped_cx!(cx);
+
+            p!(print(kind), "(");
+            for (i, arg) in sig.inputs()[0].tuple_fields().iter().enumerate() {
+                if i > 0 {
+                    p!(", ");
+                }
+                p!(print(arg));
+            }
+            p!(")");
+
+            if !sig.output().is_unit() {
+                p!(" -> ", print(sig.output()));
+            }
+
+            Ok(cx)
+        })
+    }
 }
 
 // HACK(eddyb) boxed to avoid moving around a large struct by-value.
@@ -1950,7 +1977,7 @@ impl<'tcx> PrettyPrinter<'tcx> for FmtPrinter<'_, 'tcx> {
 
             ty::ReVar(_) | ty::ReErased => false,
 
-            ty::ReStatic | ty::ReEmpty(_) => true,
+            ty::ReStatic => true,
         }
     }
 
@@ -2034,14 +2061,6 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
                 p!("'static");
                 return Ok(self);
             }
-            ty::ReEmpty(ty::UniverseIndex::ROOT) => {
-                p!("'<empty>");
-                return Ok(self);
-            }
-            ty::ReEmpty(ui) => {
-                p!(write("'<empty:{:?}>", ui));
-                return Ok(self);
-            }
         }
 
         p!("'_");
@@ -2446,6 +2465,11 @@ impl<'tcx> ty::PolyTraitPredicate<'tcx> {
     }
 }
 
+#[derive(Debug, Copy, Clone, TypeFoldable, TypeVisitable, Lift)]
+pub struct PrintClosureAsImpl<'tcx> {
+    pub closure: ty::ClosureSubsts<'tcx>,
+}
+
 forward_display_to_print! {
     ty::Region<'tcx>,
     Ty<'tcx>,
@@ -2538,6 +2562,10 @@ define_print_and_forward_display! {
         p!(print(self.0.trait_ref.print_only_trait_path()));
     }
 
+    PrintClosureAsImpl<'tcx> {
+        p!(pretty_closure_as_impl(self.closure))
+    }
+
     ty::ParamTy {
         p!(write("{}", self.name))
     }
@@ -2567,9 +2595,9 @@ define_print_and_forward_display! {
     }
 
     ty::Term<'tcx> {
-      match self {
-        ty::Term::Ty(ty) => p!(print(ty)),
-        ty::Term::Const(c) => p!(print(c)),
+      match self.unpack() {
+        ty::TermKind::Ty(ty) => p!(print(ty)),
+        ty::TermKind::Const(c) => p!(print(c)),
       }
     }
 
diff --git a/compiler/rustc_middle/src/ty/query.rs b/compiler/rustc_middle/src/ty/query.rs
index 2452bcf6a..a300a8df2 100644
--- a/compiler/rustc_middle/src/ty/query.rs
+++ b/compiler/rustc_middle/src/ty/query.rs
@@ -121,8 +121,8 @@ macro_rules! query_storage {
     ([][$K:ty, $V:ty]) => {
         <DefaultCacheSelector as CacheSelector<$K, $V>>::Cache
     };
-    ([(storage $ty:ty) $($rest:tt)*][$K:ty, $V:ty]) => {
-        <$ty as CacheSelector<$K, $V>>::Cache
+    ([(arena_cache) $($rest:tt)*][$K:ty, $V:ty]) => {
+        <ArenaCacheSelector<'tcx> as CacheSelector<$K, $V>>::Cache
     };
     ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
         query_storage!([$($modifiers)*][$($args)*])
@@ -173,7 +173,7 @@ macro_rules! opt_remap_env_constness {
 }
 
 macro_rules! define_callbacks {
-    (<$tcx:tt>
+    (
      $($(#[$attr:meta])*
         [$($modifiers:tt)*] fn $name:ident($($K:tt)*) -> $V:ty,)*) => {
 
@@ -187,33 +187,33 @@ macro_rules! define_callbacks {
         pub mod query_keys {
             use super::*;
 
-            $(pub type $name<$tcx> = $($K)*;)*
+            $(pub type $name<'tcx> = $($K)*;)*
         }
         #[allow(nonstandard_style, unused_lifetimes)]
         pub mod query_values {
             use super::*;
 
-            $(pub type $name<$tcx> = $V;)*
+            $(pub type $name<'tcx> = $V;)*
         }
         #[allow(nonstandard_style, unused_lifetimes)]
         pub mod query_storage {
             use super::*;
 
-            $(pub type $name<$tcx> = query_storage!([$($modifiers)*][$($K)*, $V]);)*
+            $(pub type $name<'tcx> = query_storage!([$($modifiers)*][$($K)*, $V]);)*
         }
         #[allow(nonstandard_style, unused_lifetimes)]
         pub mod query_stored {
             use super::*;
 
-            $(pub type $name<$tcx> = <query_storage::$name<$tcx> as QueryStorage>::Stored;)*
+            $(pub type $name<'tcx> = <query_storage::$name<'tcx> as QueryStorage>::Stored;)*
         }
 
         #[derive(Default)]
-        pub struct QueryCaches<$tcx> {
-            $($(#[$attr])* pub $name: query_storage::$name<$tcx>,)*
+        pub struct QueryCaches<'tcx> {
+            $($(#[$attr])* pub $name: query_storage::$name<'tcx>,)*
         }
 
-        impl<$tcx> TyCtxtEnsure<$tcx> {
+        impl<'tcx> TyCtxtEnsure<'tcx> {
             $($(#[$attr])*
             #[inline(always)]
             pub fn $name(self, key: query_helper_param_ty!($($K)*)) {
@@ -231,20 +231,20 @@ macro_rules! define_callbacks {
             })*
         }
 
-        impl<$tcx> TyCtxt<$tcx> {
+        impl<'tcx> TyCtxt<'tcx> {
             $($(#[$attr])*
             #[inline(always)]
             #[must_use]
-            pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> query_stored::$name<$tcx>
+            pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> query_stored::$name<'tcx>
             {
                 self.at(DUMMY_SP).$name(key)
             })*
         }
 
-        impl<$tcx> TyCtxtAt<$tcx> {
+        impl<'tcx> TyCtxtAt<'tcx> {
             $($(#[$attr])*
             #[inline(always)]
-            pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> query_stored::$name<$tcx>
+            pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> query_stored::$name<'tcx>
             {
                 let key = key.into_query_param();
                 opt_remap_env_constness!([$($modifiers)*][key]);
@@ -311,11 +311,11 @@ macro_rules! define_callbacks {
             $($(#[$attr])*
             fn $name(
                 &'tcx self,
-                tcx: TyCtxt<$tcx>,
+                tcx: TyCtxt<'tcx>,
                 span: Span,
-                key: query_keys::$name<$tcx>,
+                key: query_keys::$name<'tcx>,
                 mode: QueryMode,
-            ) -> Option<query_stored::$name<$tcx>>;)*
+            ) -> Option<query_stored::$name<'tcx>>;)*
         }
     };
 }
@@ -332,7 +332,7 @@ macro_rules! define_callbacks {
 // Queries marked with `fatal_cycle` do not need the latter implementation,
 // as they will raise an fatal error on query cycles instead.
 
-rustc_query_append! { [define_callbacks!][<'tcx>] }
+rustc_query_append! { define_callbacks! }
 
 mod sealed {
     use super::{DefId, LocalDefId};
diff --git a/compiler/rustc_middle/src/ty/relate.rs b/compiler/rustc_middle/src/ty/relate.rs
index 818affa71..61c34730d 100644
--- a/compiler/rustc_middle/src/ty/relate.rs
+++ b/compiler/rustc_middle/src/ty/relate.rs
@@ -6,7 +6,7 @@
 
 use crate::ty::error::{ExpectedFound, TypeError};
 use crate::ty::subst::{GenericArg, GenericArgKind, Subst, SubstsRef};
-use crate::ty::{self, ImplSubject, Term, Ty, TyCtxt, TypeFoldable};
+use crate::ty::{self, ImplSubject, Term, TermKind, Ty, TyCtxt, TypeFoldable};
 use rustc_hir as ast;
 use rustc_hir::def_id::DefId;
 use rustc_span::DUMMY_SP;
@@ -441,7 +441,9 @@ pub fn super_relate_tys<'tcx, R: TypeRelation<'tcx>>(
 
         (&ty::Foreign(a_id), &ty::Foreign(b_id)) if a_id == b_id => Ok(tcx.mk_foreign(a_id)),
 
-        (&ty::Dynamic(a_obj, a_region), &ty::Dynamic(b_obj, b_region)) => {
+        (&ty::Dynamic(a_obj, a_region, a_repr), &ty::Dynamic(b_obj, b_region, b_repr))
+            if a_repr == b_repr =>
+        {
             let region_bound = relation.with_cause(Cause::ExistentialRegionBound, |relation| {
                 relation.relate_with_variance(
                     ty::Contravariant,
@@ -450,7 +452,7 @@ pub fn super_relate_tys<'tcx, R: TypeRelation<'tcx>>(
                     b_region,
                 )
             })?;
-            Ok(tcx.mk_dynamic(relation.relate(a_obj, b_obj)?, region_bound))
+            Ok(tcx.mk_dynamic(relation.relate(a_obj, b_obj)?, region_bound, a_repr))
         }
 
         (&ty::Generator(a_id, a_substs, movability), &ty::Generator(b_id, b_substs, _))
@@ -572,8 +574,8 @@ pub fn super_relate_tys<'tcx, R: TypeRelation<'tcx>>(
 /// it.
 pub fn super_relate_consts<'tcx, R: TypeRelation<'tcx>>(
     relation: &mut R,
-    a: ty::Const<'tcx>,
-    b: ty::Const<'tcx>,
+    mut a: ty::Const<'tcx>,
+    mut b: ty::Const<'tcx>,
 ) -> RelateResult<'tcx, ty::Const<'tcx>> {
     debug!("{}.super_relate_consts(a = {:?}, b = {:?})", relation.tag(), a, b);
     let tcx = relation.tcx();
@@ -594,9 +596,16 @@ pub fn super_relate_consts<'tcx, R: TypeRelation<'tcx>>(
         );
     }
 
-    let eagerly_eval = |x: ty::Const<'tcx>| x.eval(tcx, relation.param_env());
-    let a = eagerly_eval(a);
-    let b = eagerly_eval(b);
+    // HACK(const_generics): We still need to eagerly evaluate consts when
+    // relating them because during `normalize_param_env_or_error`,
+    // we may relate an evaluated constant in a obligation against
+    // an unnormalized (i.e. unevaluated) const in the param-env.
+    // FIXME(generic_const_exprs): Once we always lazily unify unevaluated constants
+    // these `eval` calls can be removed.
+    if !relation.tcx().features().generic_const_exprs {
+        a = a.eval(tcx, relation.param_env());
+        b = b.eval(tcx, relation.param_env());
+    }
 
     // Currently, the values that can be unified are primitive types,
     // and those that derive both `PartialEq` and `Eq`, corresponding
@@ -617,7 +626,7 @@ pub fn super_relate_consts<'tcx, R: TypeRelation<'tcx>>(
         (ty::ConstKind::Unevaluated(au), ty::ConstKind::Unevaluated(bu))
             if tcx.features().generic_const_exprs =>
         {
-            tcx.try_unify_abstract_consts(relation.param_env().and((au.shrink(), bu.shrink())))
+            tcx.try_unify_abstract_consts(relation.param_env().and((au, bu)))
         }
 
         // While this is slightly incorrect, it shouldn't matter for `min_const_generics`
@@ -626,6 +635,8 @@ pub fn super_relate_consts<'tcx, R: TypeRelation<'tcx>>(
         (ty::ConstKind::Unevaluated(au), ty::ConstKind::Unevaluated(bu))
             if au.def == bu.def && au.promoted == bu.promoted =>
         {
+            assert_eq!(au.promoted, ());
+
             let substs = relation.relate_with_variance(
                 ty::Variance::Invariant,
                 ty::VarianceDiagInfo::default(),
@@ -636,7 +647,7 @@ pub fn super_relate_consts<'tcx, R: TypeRelation<'tcx>>(
                 kind: ty::ConstKind::Unevaluated(ty::Unevaluated {
                     def: au.def,
                     substs,
-                    promoted: au.promoted,
+                    promoted: (),
                 }),
                 ty: a.ty(),
             }));
@@ -803,15 +814,15 @@ impl<'tcx> Relate<'tcx> for ty::TraitPredicate<'tcx> {
     }
 }
 
-impl<'tcx> Relate<'tcx> for ty::Term<'tcx> {
+impl<'tcx> Relate<'tcx> for Term<'tcx> {
     fn relate<R: TypeRelation<'tcx>>(
         relation: &mut R,
         a: Self,
         b: Self,
     ) -> RelateResult<'tcx, Self> {
-        Ok(match (a, b) {
-            (Term::Ty(a), Term::Ty(b)) => relation.relate(a, b)?.into(),
-            (Term::Const(a), Term::Const(b)) => relation.relate(a, b)?.into(),
+        Ok(match (a.unpack(), b.unpack()) {
+            (TermKind::Ty(a), TermKind::Ty(b)) => relation.relate(a, b)?.into(),
+            (TermKind::Const(a), TermKind::Const(b)) => relation.relate(a, b)?.into(),
             _ => return Err(TypeError::Mismatch),
         })
     }
diff --git a/compiler/rustc_middle/src/ty/structural_impls.rs b/compiler/rustc_middle/src/ty/structural_impls.rs
index 7660a2f3a..84d6a8b97 100644
--- a/compiler/rustc_middle/src/ty/structural_impls.rs
+++ b/compiler/rustc_middle/src/ty/structural_impls.rs
@@ -3,13 +3,12 @@
 //! hand, though we've recently added some macros and proc-macros to help with the tedium.
 
 use crate::mir::interpret;
-use crate::mir::ProjectionKind;
+use crate::mir::{Field, ProjectionKind};
 use crate::ty::fold::{FallibleTypeFolder, TypeFoldable, TypeSuperFoldable};
 use crate::ty::print::{with_no_trimmed_paths, FmtPrinter, Printer};
 use crate::ty::visit::{TypeSuperVisitable, TypeVisitable, TypeVisitor};
-use crate::ty::{self, InferConst, Lift, Term, Ty, TyCtxt};
+use crate::ty::{self, InferConst, Lift, Term, TermKind, Ty, TyCtxt};
 use rustc_data_structures::functor::IdFunctor;
-use rustc_hir as hir;
 use rustc_hir::def::Namespace;
 use rustc_index::vec::{Idx, IndexVec};
 
@@ -238,12 +237,24 @@ TrivialTypeTraversalAndLiftImpls! {
     crate::ty::Variance,
     ::rustc_span::Span,
     ::rustc_errors::ErrorGuaranteed,
+    Field,
+    interpret::Scalar,
+    rustc_target::abi::Size,
+    ty::DelaySpanBugEmitted,
+    rustc_type_ir::DebruijnIndex,
+    ty::BoundVar,
+    ty::Placeholder<ty::BoundVar>,
+}
+
+TrivialTypeTraversalAndLiftImpls! {
+    for<'tcx> {
+        ty::ValTree<'tcx>,
+    }
 }
 
 ///////////////////////////////////////////////////////////////////////////
 // Lift implementations
 
-// FIXME(eddyb) replace all the uses of `Option::map` with `?`.
 impl<'tcx, A: Lift<'tcx>, B: Lift<'tcx>> Lift<'tcx> for (A, B) {
     type Lifted = (A::Lifted, B::Lifted);
     fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
@@ -261,10 +272,10 @@ impl<'tcx, A: Lift<'tcx>, B: Lift<'tcx>, C: Lift<'tcx>> Lift<'tcx> for (A, B, C)
 impl<'tcx, T: Lift<'tcx>> Lift<'tcx> for Option<T> {
     type Lifted = Option<T::Lifted>;
     fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        match self {
-            Some(x) => tcx.lift(x).map(Some),
-            None => Some(None),
-        }
+        Some(match self {
+            Some(x) => Some(tcx.lift(x)?),
+            None => None,
+        })
     }
 }
 
@@ -281,21 +292,21 @@ impl<'tcx, T: Lift<'tcx>, E: Lift<'tcx>> Lift<'tcx> for Result<T, E> {
 impl<'tcx, T: Lift<'tcx>> Lift<'tcx> for Box<T> {
     type Lifted = Box<T::Lifted>;
     fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        tcx.lift(*self).map(Box::new)
+        Some(Box::new(tcx.lift(*self)?))
     }
 }
 
 impl<'tcx, T: Lift<'tcx> + Clone> Lift<'tcx> for Rc<T> {
     type Lifted = Rc<T::Lifted>;
     fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        tcx.lift(self.as_ref().clone()).map(Rc::new)
+        Some(Rc::new(tcx.lift(self.as_ref().clone())?))
     }
 }
 
 impl<'tcx, T: Lift<'tcx> + Clone> Lift<'tcx> for Arc<T> {
     type Lifted = Arc<T::Lifted>;
     fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        tcx.lift(self.as_ref().clone()).map(Arc::new)
+        Some(Arc::new(tcx.lift(self.as_ref().clone())?))
     }
 }
 impl<'tcx, T: Lift<'tcx>> Lift<'tcx> for Vec<T> {
@@ -312,159 +323,18 @@ impl<'tcx, I: Idx, T: Lift<'tcx>> Lift<'tcx> for IndexVec<I, T> {
     }
 }
 
-impl<'a, 'tcx> Lift<'tcx> for ty::TraitRef<'a> {
-    type Lifted = ty::TraitRef<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        tcx.lift(self.substs).map(|substs| ty::TraitRef { def_id: self.def_id, substs })
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::ExistentialTraitRef<'a> {
-    type Lifted = ty::ExistentialTraitRef<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        tcx.lift(self.substs).map(|substs| ty::ExistentialTraitRef { def_id: self.def_id, substs })
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::ExistentialPredicate<'a> {
-    type Lifted = ty::ExistentialPredicate<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        match self {
-            ty::ExistentialPredicate::Trait(x) => tcx.lift(x).map(ty::ExistentialPredicate::Trait),
-            ty::ExistentialPredicate::Projection(x) => {
-                tcx.lift(x).map(ty::ExistentialPredicate::Projection)
-            }
-            ty::ExistentialPredicate::AutoTrait(def_id) => {
-                Some(ty::ExistentialPredicate::AutoTrait(def_id))
-            }
-        }
-    }
-}
-
 impl<'a, 'tcx> Lift<'tcx> for Term<'a> {
     type Lifted = ty::Term<'tcx>;
     fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        Some(match self {
-            Term::Ty(ty) => Term::Ty(tcx.lift(ty)?),
-            Term::Const(c) => Term::Const(tcx.lift(c)?),
-        })
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::TraitPredicate<'a> {
-    type Lifted = ty::TraitPredicate<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<ty::TraitPredicate<'tcx>> {
-        tcx.lift(self.trait_ref).map(|trait_ref| ty::TraitPredicate {
-            trait_ref,
-            constness: self.constness,
-            polarity: self.polarity,
-        })
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::SubtypePredicate<'a> {
-    type Lifted = ty::SubtypePredicate<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<ty::SubtypePredicate<'tcx>> {
-        tcx.lift((self.a, self.b)).map(|(a, b)| ty::SubtypePredicate {
-            a_is_expected: self.a_is_expected,
-            a,
-            b,
-        })
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::CoercePredicate<'a> {
-    type Lifted = ty::CoercePredicate<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<ty::CoercePredicate<'tcx>> {
-        tcx.lift((self.a, self.b)).map(|(a, b)| ty::CoercePredicate { a, b })
-    }
-}
-
-impl<'tcx, A: Copy + Lift<'tcx>, B: Copy + Lift<'tcx>> Lift<'tcx> for ty::OutlivesPredicate<A, B> {
-    type Lifted = ty::OutlivesPredicate<A::Lifted, B::Lifted>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        tcx.lift((self.0, self.1)).map(|(a, b)| ty::OutlivesPredicate(a, b))
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::ProjectionTy<'a> {
-    type Lifted = ty::ProjectionTy<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<ty::ProjectionTy<'tcx>> {
-        tcx.lift(self.substs)
-            .map(|substs| ty::ProjectionTy { item_def_id: self.item_def_id, substs })
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::ProjectionPredicate<'a> {
-    type Lifted = ty::ProjectionPredicate<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<ty::ProjectionPredicate<'tcx>> {
-        tcx.lift((self.projection_ty, self.term))
-            .map(|(projection_ty, term)| ty::ProjectionPredicate { projection_ty, term })
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::ExistentialProjection<'a> {
-    type Lifted = ty::ExistentialProjection<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        tcx.lift(self.substs).map(|substs| ty::ExistentialProjection {
-            substs,
-            term: tcx.lift(self.term).expect("type must lift when substs do"),
-            item_def_id: self.item_def_id,
-        })
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::PredicateKind<'a> {
-    type Lifted = ty::PredicateKind<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        match self {
-            ty::PredicateKind::Trait(data) => tcx.lift(data).map(ty::PredicateKind::Trait),
-            ty::PredicateKind::Subtype(data) => tcx.lift(data).map(ty::PredicateKind::Subtype),
-            ty::PredicateKind::Coerce(data) => tcx.lift(data).map(ty::PredicateKind::Coerce),
-            ty::PredicateKind::RegionOutlives(data) => {
-                tcx.lift(data).map(ty::PredicateKind::RegionOutlives)
-            }
-            ty::PredicateKind::TypeOutlives(data) => {
-                tcx.lift(data).map(ty::PredicateKind::TypeOutlives)
-            }
-            ty::PredicateKind::Projection(data) => {
-                tcx.lift(data).map(ty::PredicateKind::Projection)
-            }
-            ty::PredicateKind::WellFormed(ty) => tcx.lift(ty).map(ty::PredicateKind::WellFormed),
-            ty::PredicateKind::ClosureKind(closure_def_id, closure_substs, kind) => {
-                tcx.lift(closure_substs).map(|closure_substs| {
-                    ty::PredicateKind::ClosureKind(closure_def_id, closure_substs, kind)
-                })
-            }
-            ty::PredicateKind::ObjectSafe(trait_def_id) => {
-                Some(ty::PredicateKind::ObjectSafe(trait_def_id))
-            }
-            ty::PredicateKind::ConstEvaluatable(uv) => {
-                tcx.lift(uv).map(|uv| ty::PredicateKind::ConstEvaluatable(uv))
-            }
-            ty::PredicateKind::ConstEquate(c1, c2) => {
-                tcx.lift((c1, c2)).map(|(c1, c2)| ty::PredicateKind::ConstEquate(c1, c2))
-            }
-            ty::PredicateKind::TypeWellFormedFromEnv(ty) => {
-                tcx.lift(ty).map(ty::PredicateKind::TypeWellFormedFromEnv)
+        Some(
+            match self.unpack() {
+                TermKind::Ty(ty) => TermKind::Ty(tcx.lift(ty)?),
+                TermKind::Const(c) => TermKind::Const(tcx.lift(c)?),
             }
-        }
+            .pack(),
+        )
     }
 }
-
-impl<'a, 'tcx, T: Lift<'tcx>> Lift<'tcx> for ty::Binder<'a, T>
-where
-    <T as Lift<'tcx>>::Lifted: TypeVisitable<'tcx>,
-{
-    type Lifted = ty::Binder<'tcx, T::Lifted>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        let bound_vars = tcx.lift(self.bound_vars());
-        tcx.lift(self.skip_binder())
-            .zip(bound_vars)
-            .map(|(value, vars)| ty::Binder::bind_with_vars(value, vars))
-    }
-}
-
 impl<'a, 'tcx> Lift<'tcx> for ty::ParamEnv<'a> {
     type Lifted = ty::ParamEnv<'tcx>;
     fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
@@ -473,178 +343,6 @@ impl<'a, 'tcx> Lift<'tcx> for ty::ParamEnv<'a> {
     }
 }
 
-impl<'a, 'tcx, T: Lift<'tcx>> Lift<'tcx> for ty::ParamEnvAnd<'a, T> {
-    type Lifted = ty::ParamEnvAnd<'tcx, T::Lifted>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        tcx.lift(self.param_env).and_then(|param_env| {
-            tcx.lift(self.value).map(|value| ty::ParamEnvAnd { param_env, value })
-        })
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::ClosureSubsts<'a> {
-    type Lifted = ty::ClosureSubsts<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        tcx.lift(self.substs).map(|substs| ty::ClosureSubsts { substs })
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::GeneratorSubsts<'a> {
-    type Lifted = ty::GeneratorSubsts<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        tcx.lift(self.substs).map(|substs| ty::GeneratorSubsts { substs })
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::adjustment::Adjustment<'a> {
-    type Lifted = ty::adjustment::Adjustment<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        let ty::adjustment::Adjustment { kind, target } = self;
-        tcx.lift(kind).and_then(|kind| {
-            tcx.lift(target).map(|target| ty::adjustment::Adjustment { kind, target })
-        })
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::adjustment::Adjust<'a> {
-    type Lifted = ty::adjustment::Adjust<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        match self {
-            ty::adjustment::Adjust::NeverToAny => Some(ty::adjustment::Adjust::NeverToAny),
-            ty::adjustment::Adjust::Pointer(ptr) => Some(ty::adjustment::Adjust::Pointer(ptr)),
-            ty::adjustment::Adjust::Deref(overloaded) => {
-                tcx.lift(overloaded).map(ty::adjustment::Adjust::Deref)
-            }
-            ty::adjustment::Adjust::Borrow(autoref) => {
-                tcx.lift(autoref).map(ty::adjustment::Adjust::Borrow)
-            }
-        }
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::adjustment::OverloadedDeref<'a> {
-    type Lifted = ty::adjustment::OverloadedDeref<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        tcx.lift(self.region).map(|region| ty::adjustment::OverloadedDeref {
-            region,
-            mutbl: self.mutbl,
-            span: self.span,
-        })
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::adjustment::AutoBorrow<'a> {
-    type Lifted = ty::adjustment::AutoBorrow<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        match self {
-            ty::adjustment::AutoBorrow::Ref(r, m) => {
-                tcx.lift(r).map(|r| ty::adjustment::AutoBorrow::Ref(r, m))
-            }
-            ty::adjustment::AutoBorrow::RawPtr(m) => Some(ty::adjustment::AutoBorrow::RawPtr(m)),
-        }
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::GenSig<'a> {
-    type Lifted = ty::GenSig<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        tcx.lift((self.resume_ty, self.yield_ty, self.return_ty))
-            .map(|(resume_ty, yield_ty, return_ty)| ty::GenSig { resume_ty, yield_ty, return_ty })
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::FnSig<'a> {
-    type Lifted = ty::FnSig<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        tcx.lift(self.inputs_and_output).map(|x| ty::FnSig {
-            inputs_and_output: x,
-            c_variadic: self.c_variadic,
-            unsafety: self.unsafety,
-            abi: self.abi,
-        })
-    }
-}
-
-impl<'tcx, T: Lift<'tcx>> Lift<'tcx> for ty::error::ExpectedFound<T> {
-    type Lifted = ty::error::ExpectedFound<T::Lifted>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        let ty::error::ExpectedFound { expected, found } = self;
-        tcx.lift(expected).and_then(|expected| {
-            tcx.lift(found).map(|found| ty::error::ExpectedFound { expected, found })
-        })
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::error::TypeError<'a> {
-    type Lifted = ty::error::TypeError<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        use crate::ty::error::TypeError::*;
-
-        Some(match self {
-            Mismatch => Mismatch,
-            ConstnessMismatch(x) => ConstnessMismatch(x),
-            PolarityMismatch(x) => PolarityMismatch(x),
-            UnsafetyMismatch(x) => UnsafetyMismatch(x),
-            AbiMismatch(x) => AbiMismatch(x),
-            Mutability => Mutability,
-            ArgumentMutability(i) => ArgumentMutability(i),
-            TupleSize(x) => TupleSize(x),
-            FixedArraySize(x) => FixedArraySize(x),
-            ArgCount => ArgCount,
-            FieldMisMatch(x, y) => FieldMisMatch(x, y),
-            RegionsDoesNotOutlive(a, b) => {
-                return tcx.lift((a, b)).map(|(a, b)| RegionsDoesNotOutlive(a, b));
-            }
-            RegionsInsufficientlyPolymorphic(a, b) => {
-                return tcx.lift(b).map(|b| RegionsInsufficientlyPolymorphic(a, b));
-            }
-            RegionsOverlyPolymorphic(a, b) => {
-                return tcx.lift(b).map(|b| RegionsOverlyPolymorphic(a, b));
-            }
-            RegionsPlaceholderMismatch => RegionsPlaceholderMismatch,
-            IntMismatch(x) => IntMismatch(x),
-            FloatMismatch(x) => FloatMismatch(x),
-            Traits(x) => Traits(x),
-            VariadicMismatch(x) => VariadicMismatch(x),
-            CyclicTy(t) => return tcx.lift(t).map(|t| CyclicTy(t)),
-            CyclicConst(ct) => return tcx.lift(ct).map(|ct| CyclicConst(ct)),
-            ProjectionMismatched(x) => ProjectionMismatched(x),
-            ArgumentSorts(x, i) => return tcx.lift(x).map(|x| ArgumentSorts(x, i)),
-            Sorts(x) => return tcx.lift(x).map(Sorts),
-            ExistentialMismatch(x) => return tcx.lift(x).map(ExistentialMismatch),
-            ConstMismatch(x) => return tcx.lift(x).map(ConstMismatch),
-            IntrinsicCast => IntrinsicCast,
-            TargetFeatureCast(x) => TargetFeatureCast(x),
-            ObjectUnsafeCoercion(x) => return tcx.lift(x).map(ObjectUnsafeCoercion),
-        })
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::InstanceDef<'a> {
-    type Lifted = ty::InstanceDef<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        match self {
-            ty::InstanceDef::Item(def_id) => Some(ty::InstanceDef::Item(def_id)),
-            ty::InstanceDef::VTableShim(def_id) => Some(ty::InstanceDef::VTableShim(def_id)),
-            ty::InstanceDef::ReifyShim(def_id) => Some(ty::InstanceDef::ReifyShim(def_id)),
-            ty::InstanceDef::Intrinsic(def_id) => Some(ty::InstanceDef::Intrinsic(def_id)),
-            ty::InstanceDef::FnPtrShim(def_id, ty) => {
-                Some(ty::InstanceDef::FnPtrShim(def_id, tcx.lift(ty)?))
-            }
-            ty::InstanceDef::Virtual(def_id, n) => Some(ty::InstanceDef::Virtual(def_id, n)),
-            ty::InstanceDef::ClosureOnceShim { call_once, track_caller } => {
-                Some(ty::InstanceDef::ClosureOnceShim { call_once, track_caller })
-            }
-            ty::InstanceDef::DropGlue(def_id, ty) => {
-                Some(ty::InstanceDef::DropGlue(def_id, tcx.lift(ty)?))
-            }
-            ty::InstanceDef::CloneShim(def_id, ty) => {
-                Some(ty::InstanceDef::CloneShim(def_id, tcx.lift(ty)?))
-            }
-        }
-    }
-}
-
 ///////////////////////////////////////////////////////////////////////////
 // TypeFoldable implementations.
 
@@ -844,6 +542,12 @@ impl<'tcx, T: TypeVisitable<'tcx>> TypeVisitable<'tcx> for Vec<T> {
     }
 }
 
+impl<'tcx, T: TypeVisitable<'tcx>> TypeVisitable<'tcx> for &[T] {
+    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
+        self.iter().try_for_each(|t| t.visit_with(visitor))
+    }
+}
+
 impl<'tcx, T: TypeFoldable<'tcx>> TypeFoldable<'tcx> for Box<[T]> {
     fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
         self.try_map_id(|t| t.try_fold_with(folder))
@@ -901,88 +605,12 @@ impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::List<ty::Binder<'tcx, ty::Existentia
     }
 }
 
-impl<'tcx> TypeVisitable<'tcx>
-    for &'tcx ty::List<ty::Binder<'tcx, ty::ExistentialPredicate<'tcx>>>
-{
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
-        self.iter().try_for_each(|p| p.visit_with(visitor))
-    }
-}
-
 impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::List<ProjectionKind> {
     fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
         ty::util::fold_list(self, folder, |tcx, v| tcx.intern_projs(v))
     }
 }
 
-impl<'tcx> TypeVisitable<'tcx> for &'tcx ty::List<ProjectionKind> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
-        self.iter().try_for_each(|t| t.visit_with(visitor))
-    }
-}
-
-impl<'tcx> TypeFoldable<'tcx> for ty::instance::Instance<'tcx> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
-        use crate::ty::InstanceDef::*;
-        Ok(Self {
-            substs: self.substs.try_fold_with(folder)?,
-            def: match self.def {
-                Item(def) => Item(def.try_fold_with(folder)?),
-                VTableShim(did) => VTableShim(did.try_fold_with(folder)?),
-                ReifyShim(did) => ReifyShim(did.try_fold_with(folder)?),
-                Intrinsic(did) => Intrinsic(did.try_fold_with(folder)?),
-                FnPtrShim(did, ty) => {
-                    FnPtrShim(did.try_fold_with(folder)?, ty.try_fold_with(folder)?)
-                }
-                Virtual(did, i) => Virtual(did.try_fold_with(folder)?, i),
-                ClosureOnceShim { call_once, track_caller } => {
-                    ClosureOnceShim { call_once: call_once.try_fold_with(folder)?, track_caller }
-                }
-                DropGlue(did, ty) => {
-                    DropGlue(did.try_fold_with(folder)?, ty.try_fold_with(folder)?)
-                }
-                CloneShim(did, ty) => {
-                    CloneShim(did.try_fold_with(folder)?, ty.try_fold_with(folder)?)
-                }
-            },
-        })
-    }
-}
-
-impl<'tcx> TypeVisitable<'tcx> for ty::instance::Instance<'tcx> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
-        use crate::ty::InstanceDef::*;
-        self.substs.visit_with(visitor)?;
-        match self.def {
-            Item(def) => def.visit_with(visitor),
-            VTableShim(did) | ReifyShim(did) | Intrinsic(did) | Virtual(did, _) => {
-                did.visit_with(visitor)
-            }
-            FnPtrShim(did, ty) | CloneShim(did, ty) => {
-                did.visit_with(visitor)?;
-                ty.visit_with(visitor)
-            }
-            DropGlue(did, ty) => {
-                did.visit_with(visitor)?;
-                ty.visit_with(visitor)
-            }
-            ClosureOnceShim { call_once, track_caller: _ } => call_once.visit_with(visitor),
-        }
-    }
-}
-
-impl<'tcx> TypeFoldable<'tcx> for interpret::GlobalId<'tcx> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
-        Ok(Self { instance: self.instance.try_fold_with(folder)?, promoted: self.promoted })
-    }
-}
-
-impl<'tcx> TypeVisitable<'tcx> for interpret::GlobalId<'tcx> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
-        self.instance.visit_with(visitor)
-    }
-}
-
 impl<'tcx> TypeFoldable<'tcx> for Ty<'tcx> {
     fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
         folder.try_fold_ty(self)
@@ -1005,9 +633,11 @@ impl<'tcx> TypeSuperFoldable<'tcx> for Ty<'tcx> {
             ty::Array(typ, sz) => ty::Array(typ.try_fold_with(folder)?, sz.try_fold_with(folder)?),
             ty::Slice(typ) => ty::Slice(typ.try_fold_with(folder)?),
             ty::Adt(tid, substs) => ty::Adt(tid, substs.try_fold_with(folder)?),
-            ty::Dynamic(trait_ty, region) => {
-                ty::Dynamic(trait_ty.try_fold_with(folder)?, region.try_fold_with(folder)?)
-            }
+            ty::Dynamic(trait_ty, region, representation) => ty::Dynamic(
+                trait_ty.try_fold_with(folder)?,
+                region.try_fold_with(folder)?,
+                representation,
+            ),
             ty::Tuple(ts) => ty::Tuple(ts.try_fold_with(folder)?),
             ty::FnDef(def_id, substs) => ty::FnDef(def_id, substs.try_fold_with(folder)?),
             ty::FnPtr(f) => ty::FnPtr(f.try_fold_with(folder)?),
@@ -1051,7 +681,7 @@ impl<'tcx> TypeSuperVisitable<'tcx> for Ty<'tcx> {
             }
             ty::Slice(typ) => typ.visit_with(visitor),
             ty::Adt(_, substs) => substs.visit_with(visitor),
-            ty::Dynamic(ref trait_ty, ref reg) => {
+            ty::Dynamic(ref trait_ty, ref reg, _) => {
                 trait_ty.visit_with(visitor)?;
                 reg.visit_with(visitor)
             }
@@ -1156,12 +786,6 @@ impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::List<ty::Predicate<'tcx>> {
     }
 }
 
-impl<'tcx> TypeVisitable<'tcx> for &'tcx ty::List<ty::Predicate<'tcx>> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
-        self.iter().try_for_each(|p| p.visit_with(visitor))
-    }
-}
-
 impl<'tcx, T: TypeFoldable<'tcx>, I: Idx> TypeFoldable<'tcx> for IndexVec<I, T> {
     fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
         self.try_map_id(|x| x.try_fold_with(folder))
@@ -1208,34 +832,6 @@ impl<'tcx> TypeSuperVisitable<'tcx> for ty::Const<'tcx> {
     }
 }
 
-impl<'tcx> TypeFoldable<'tcx> for ty::ConstKind<'tcx> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
-        Ok(match self {
-            ty::ConstKind::Infer(ic) => ty::ConstKind::Infer(ic.try_fold_with(folder)?),
-            ty::ConstKind::Param(p) => ty::ConstKind::Param(p.try_fold_with(folder)?),
-            ty::ConstKind::Unevaluated(uv) => ty::ConstKind::Unevaluated(uv.try_fold_with(folder)?),
-            ty::ConstKind::Value(_)
-            | ty::ConstKind::Bound(..)
-            | ty::ConstKind::Placeholder(..)
-            | ty::ConstKind::Error(_) => self,
-        })
-    }
-}
-
-impl<'tcx> TypeVisitable<'tcx> for ty::ConstKind<'tcx> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
-        match *self {
-            ty::ConstKind::Infer(ic) => ic.visit_with(visitor),
-            ty::ConstKind::Param(p) => p.visit_with(visitor),
-            ty::ConstKind::Unevaluated(uv) => uv.visit_with(visitor),
-            ty::ConstKind::Value(_)
-            | ty::ConstKind::Bound(..)
-            | ty::ConstKind::Placeholder(_)
-            | ty::ConstKind::Error(_) => ControlFlow::CONTINUE,
-        }
-    }
-}
-
 impl<'tcx> TypeFoldable<'tcx> for InferConst<'tcx> {
     fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, _folder: &mut F) -> Result<Self, F::Error> {
         Ok(self)
@@ -1290,15 +886,3 @@ impl<'tcx> TypeVisitable<'tcx> for ty::Unevaluated<'tcx, ()> {
         self.expand().visit_with(visitor)
     }
 }
-
-impl<'tcx> TypeFoldable<'tcx> for hir::Constness {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, _: &mut F) -> Result<Self, F::Error> {
-        Ok(self)
-    }
-}
-
-impl<'tcx> TypeVisitable<'tcx> for hir::Constness {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, _: &mut V) -> ControlFlow<V::BreakTy> {
-        ControlFlow::CONTINUE
-    }
-}
diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs
index 52c3a3886..36e560850 100644
--- a/compiler/rustc_middle/src/ty/sty.rs
+++ b/compiler/rustc_middle/src/ty/sty.rs
@@ -11,6 +11,7 @@ use crate::ty::{
     TypeVisitor,
 };
 use crate::ty::{List, ParamEnv};
+use hir::def::DefKind;
 use polonius_engine::Atom;
 use rustc_data_structures::captures::Captures;
 use rustc_data_structures::intern::Interned;
@@ -201,7 +202,7 @@ static_assert_size!(TyKind<'_>, 32);
 /// * `GR`: The "return type", which is the type of value returned upon
 ///   completion of the generator.
 /// * `GW`: The "generator witness".
-#[derive(Copy, Clone, Debug, TypeFoldable, TypeVisitable)]
+#[derive(Copy, Clone, Debug, TypeFoldable, TypeVisitable, Lift)]
 pub struct ClosureSubsts<'tcx> {
     /// Lifetime and type parameters from the enclosing function,
     /// concatenated with a tuple containing the types of the upvars.
@@ -325,10 +326,14 @@ impl<'tcx> ClosureSubsts<'tcx> {
             _ => bug!("closure_sig_as_fn_ptr_ty is not a fn-ptr: {:?}", ty.kind()),
         }
     }
+
+    pub fn print_as_impl_trait(self) -> ty::print::PrintClosureAsImpl<'tcx> {
+        ty::print::PrintClosureAsImpl { closure: self }
+    }
 }
 
 /// Similar to `ClosureSubsts`; see the above documentation for more.
-#[derive(Copy, Clone, Debug, TypeFoldable, TypeVisitable)]
+#[derive(Copy, Clone, Debug, TypeFoldable, TypeVisitable, Lift)]
 pub struct GeneratorSubsts<'tcx> {
     pub substs: SubstsRef<'tcx>,
 }
@@ -655,7 +660,7 @@ impl<'tcx> InlineConstSubsts<'tcx> {
 }
 
 #[derive(Debug, Copy, Clone, PartialEq, PartialOrd, Ord, Eq, Hash, TyEncodable, TyDecodable)]
-#[derive(HashStable, TypeFoldable, TypeVisitable)]
+#[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub enum ExistentialPredicate<'tcx> {
     /// E.g., `Iterator`.
     Trait(ExistentialTraitRef<'tcx>),
@@ -687,6 +692,9 @@ impl<'tcx> ExistentialPredicate<'tcx> {
 }
 
 impl<'tcx> Binder<'tcx, ExistentialPredicate<'tcx>> {
+    /// Given an existential predicate like `?Self: PartialEq<u32>` (e.g., derived from `dyn PartialEq<u32>`),
+    /// and a concrete type `self_ty`, returns a full predicate where the existentially quantified variable `?Self`
+    /// has been replaced with `self_ty` (e.g., `self_ty: PartialEq<u32>`, in our example).
     pub fn with_self_ty(&self, tcx: TyCtxt<'tcx>, self_ty: Ty<'tcx>) -> ty::Predicate<'tcx> {
         use crate::ty::ToPredicate;
         match self.skip_binder() {
@@ -781,7 +789,7 @@ impl<'tcx> List<ty::Binder<'tcx, ExistentialPredicate<'tcx>>> {
 /// Trait references also appear in object types like `Foo<U>`, but in
 /// that case the `Self` parameter is absent from the substitutions.
 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, TyEncodable, TyDecodable)]
-#[derive(HashStable, TypeFoldable, TypeVisitable)]
+#[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub struct TraitRef<'tcx> {
     pub def_id: DefId,
     pub substs: SubstsRef<'tcx>,
@@ -845,6 +853,12 @@ impl<'tcx> PolyTraitRef<'tcx> {
     }
 }
 
+impl rustc_errors::IntoDiagnosticArg for PolyTraitRef<'_> {
+    fn into_diagnostic_arg(self) -> rustc_errors::DiagnosticArgValue<'static> {
+        self.to_string().into_diagnostic_arg()
+    }
+}
+
 /// An existential reference to a trait, where `Self` is erased.
 /// For example, the trait object `Trait<'a, 'b, X, Y>` is:
 /// ```ignore (illustrative)
@@ -853,7 +867,7 @@ impl<'tcx> PolyTraitRef<'tcx> {
 /// The substitutions don't include the erased `Self`, only trait
 /// type and lifetime parameters (`[X, Y]` and `['a, 'b]` above).
 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, TyEncodable, TyDecodable)]
-#[derive(HashStable, TypeFoldable, TypeVisitable)]
+#[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub struct ExistentialTraitRef<'tcx> {
     pub def_id: DefId,
     pub substs: SubstsRef<'tcx>,
@@ -1009,7 +1023,7 @@ impl BoundVariableKind {
 ///
 /// `Decodable` and `Encodable` are implemented for `Binder<T>` using the `impl_binder_encode_decode!` macro.
 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
-#[derive(HashStable)]
+#[derive(HashStable, Lift)]
 pub struct Binder<'tcx, T>(T, &'tcx List<BoundVariableKind>);
 
 impl<'tcx, T> Binder<'tcx, T>
@@ -1171,7 +1185,7 @@ impl<'tcx, T> Binder<'tcx, Option<T>> {
 /// Represents the projection of an associated type. In explicit UFCS
 /// form this would be written `<T as Trait<..>>::N`.
 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, TyEncodable, TyDecodable)]
-#[derive(HashStable, TypeFoldable, TypeVisitable)]
+#[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub struct ProjectionTy<'tcx> {
     /// The parameters of the associated item.
     pub substs: SubstsRef<'tcx>,
@@ -1186,7 +1200,9 @@ pub struct ProjectionTy<'tcx> {
 
 impl<'tcx> ProjectionTy<'tcx> {
     pub fn trait_def_id(&self, tcx: TyCtxt<'tcx>) -> DefId {
-        tcx.parent(self.item_def_id)
+        let parent = tcx.parent(self.item_def_id);
+        assert_eq!(tcx.def_kind(parent), DefKind::Trait);
+        parent
     }
 
     /// Extracts the underlying trait reference and own substs from this projection.
@@ -1221,7 +1237,7 @@ impl<'tcx> ProjectionTy<'tcx> {
     }
 }
 
-#[derive(Copy, Clone, Debug, TypeFoldable, TypeVisitable)]
+#[derive(Copy, Clone, Debug, TypeFoldable, TypeVisitable, Lift)]
 pub struct GenSig<'tcx> {
     pub resume_ty: Ty<'tcx>,
     pub yield_ty: Ty<'tcx>,
@@ -1237,7 +1253,7 @@ pub type PolyGenSig<'tcx> = Binder<'tcx, GenSig<'tcx>>;
 /// - `output`: is the return type.
 /// - `c_variadic`: indicates whether this is a C-variadic function.
 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, TyEncodable, TyDecodable)]
-#[derive(HashStable, TypeFoldable, TypeVisitable)]
+#[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub struct FnSig<'tcx> {
     pub inputs_and_output: &'tcx List<Ty<'tcx>>,
     pub c_variadic: bool,
@@ -1419,7 +1435,7 @@ impl From<BoundVar> for BoundTy {
 
 /// A `ProjectionPredicate` for an `ExistentialTraitRef`.
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, TyEncodable, TyDecodable)]
-#[derive(HashStable, TypeFoldable, TypeVisitable)]
+#[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub struct ExistentialProjection<'tcx> {
     pub item_def_id: DefId,
     pub substs: SubstsRef<'tcx>,
@@ -1501,7 +1517,6 @@ impl<'tcx> Region<'tcx> {
             ty::ReStatic => true,
             ty::ReVar(..) => false,
             ty::RePlaceholder(placeholder) => placeholder.name.is_named(),
-            ty::ReEmpty(_) => false,
             ty::ReErased => false,
         }
     }
@@ -1527,11 +1542,6 @@ impl<'tcx> Region<'tcx> {
     }
 
     #[inline]
-    pub fn is_empty(self) -> bool {
-        matches!(*self, ty::ReEmpty(..))
-    }
-
-    #[inline]
     pub fn bound_at_or_above_binder(self, index: ty::DebruijnIndex) -> bool {
         match *self {
             ty::ReLateBound(debruijn, _) => debruijn >= index,
@@ -1562,7 +1572,7 @@ impl<'tcx> Region<'tcx> {
                 flags = flags | TypeFlags::HAS_FREE_REGIONS;
                 flags = flags | TypeFlags::HAS_FREE_LOCAL_REGIONS;
             }
-            ty::ReEmpty(_) | ty::ReStatic => {
+            ty::ReStatic => {
                 flags = flags | TypeFlags::HAS_FREE_REGIONS;
             }
             ty::ReLateBound(..) => {
@@ -1617,6 +1627,10 @@ impl<'tcx> Region<'tcx> {
             _ => self.is_free(),
         }
     }
+
+    pub fn is_var(self) -> bool {
+        matches!(self.kind(), ty::ReVar(_))
+    }
 }
 
 /// Type utilities
@@ -1838,7 +1852,12 @@ impl<'tcx> Ty<'tcx> {
 
     #[inline]
     pub fn is_trait(self) -> bool {
-        matches!(self.kind(), Dynamic(..))
+        matches!(self.kind(), Dynamic(_, _, ty::Dyn))
+    }
+
+    #[inline]
+    pub fn is_dyn_star(self) -> bool {
+        matches!(self.kind(), Dynamic(_, _, ty::DynStar))
     }
 
     #[inline]
diff --git a/compiler/rustc_middle/src/ty/subst.rs b/compiler/rustc_middle/src/ty/subst.rs
index 6262aa180..8e69bf067 100644
--- a/compiler/rustc_middle/src/ty/subst.rs
+++ b/compiler/rustc_middle/src/ty/subst.rs
@@ -459,12 +459,6 @@ impl<'tcx> TypeFoldable<'tcx> for SubstsRef<'tcx> {
     }
 }
 
-impl<'tcx> TypeVisitable<'tcx> for SubstsRef<'tcx> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
-        self.iter().try_for_each(|t| t.visit_with(visitor))
-    }
-}
-
 impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::List<Ty<'tcx>> {
     fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
         // This code is fairly hot, though not as hot as `SubstsRef`.
@@ -497,7 +491,7 @@ impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::List<Ty<'tcx>> {
     }
 }
 
-impl<'tcx> TypeVisitable<'tcx> for &'tcx ty::List<Ty<'tcx>> {
+impl<'tcx, T: TypeVisitable<'tcx>> TypeVisitable<'tcx> for &'tcx ty::List<T> {
     fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
         self.iter().try_for_each(|t| t.visit_with(visitor))
     }
diff --git a/compiler/rustc_middle/src/ty/trait_def.rs b/compiler/rustc_middle/src/ty/trait_def.rs
index 541dace5c..ac79949fc 100644
--- a/compiler/rustc_middle/src/ty/trait_def.rs
+++ b/compiler/rustc_middle/src/ty/trait_def.rs
@@ -256,7 +256,6 @@ pub(super) fn trait_impls_of_provider(tcx: TyCtxt<'_>, trait_id: DefId) -> Trait
 }
 
 // Query provider for `incoherent_impls`.
-#[instrument(level = "debug", skip(tcx))]
 pub(super) fn incoherent_impls_provider(tcx: TyCtxt<'_>, simp: SimplifiedType) -> &[DefId] {
     let mut impls = Vec::new();
 
diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs
index 591bb7831..23ad4d27d 100644
--- a/compiler/rustc_middle/src/ty/util.rs
+++ b/compiler/rustc_middle/src/ty/util.rs
@@ -627,7 +627,7 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     /// Expands the given impl trait type, stopping if the type is recursive.
-    #[instrument(skip(self), level = "debug")]
+    #[instrument(skip(self), level = "debug", ret)]
     pub fn try_expand_impl_trait_type(
         self,
         def_id: DefId,
@@ -644,7 +644,6 @@ impl<'tcx> TyCtxt<'tcx> {
         };
 
         let expanded_type = visitor.expand_opaque_ty(def_id, substs).unwrap();
-        trace!(?expanded_type);
         if visitor.found_recursion { Err(expanded_type) } else { Ok(expanded_type) }
     }
 
@@ -652,6 +651,13 @@ impl<'tcx> TyCtxt<'tcx> {
         ty::EarlyBinder(self.type_of(def_id))
     }
 
+    pub fn bound_trait_impl_trait_tys(
+        self,
+        def_id: DefId,
+    ) -> ty::EarlyBinder<Result<&'tcx FxHashMap<DefId, Ty<'tcx>>, ErrorGuaranteed>> {
+        ty::EarlyBinder(self.collect_trait_impl_trait_tys(def_id))
+    }
+
     pub fn bound_fn_sig(self, def_id: DefId) -> ty::EarlyBinder<ty::PolyFnSig<'tcx>> {
         ty::EarlyBinder(self.fn_sig(def_id))
     }
diff --git a/compiler/rustc_middle/src/ty/visit.rs b/compiler/rustc_middle/src/ty/visit.rs
index 536506720..5f8cb5782 100644
--- a/compiler/rustc_middle/src/ty/visit.rs
+++ b/compiler/rustc_middle/src/ty/visit.rs
@@ -84,7 +84,7 @@ pub trait TypeVisitable<'tcx>: fmt::Debug + Clone {
         self.has_vars_bound_at_or_above(ty::INNERMOST)
     }
 
-    #[instrument(level = "trace")]
+    #[instrument(level = "trace", ret)]
     fn has_type_flags(&self, flags: TypeFlags) -> bool {
         self.visit_with(&mut HasTypeFlagsVisitor { flags }).break_value() == Some(FoundFlags)
     }
@@ -560,7 +560,7 @@ impl<'tcx> TypeVisitor<'tcx> for HasTypeFlagsVisitor {
     type BreakTy = FoundFlags;
 
     #[inline]
-    #[instrument(skip(self), level = "trace")]
+    #[instrument(skip(self), level = "trace", ret)]
     fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
         let flags = t.flags();
         trace!(t.flags=?t.flags());
@@ -572,7 +572,7 @@ impl<'tcx> TypeVisitor<'tcx> for HasTypeFlagsVisitor {
     }
 
     #[inline]
-    #[instrument(skip(self), level = "trace")]
+    #[instrument(skip(self), level = "trace", ret)]
     fn visit_region(&mut self, r: ty::Region<'tcx>) -> ControlFlow<Self::BreakTy> {
         let flags = r.type_flags();
         trace!(r.flags=?flags);
@@ -584,7 +584,7 @@ impl<'tcx> TypeVisitor<'tcx> for HasTypeFlagsVisitor {
     }
 
     #[inline]
-    #[instrument(level = "trace")]
+    #[instrument(level = "trace", ret)]
     fn visit_const(&mut self, c: ty::Const<'tcx>) -> ControlFlow<Self::BreakTy> {
         let flags = FlagComputation::for_const(c);
         trace!(r.flags=?flags);
@@ -596,7 +596,7 @@ impl<'tcx> TypeVisitor<'tcx> for HasTypeFlagsVisitor {
     }
 
     #[inline]
-    #[instrument(level = "trace")]
+    #[instrument(level = "trace", ret)]
     fn visit_unevaluated(&mut self, uv: ty::Unevaluated<'tcx>) -> ControlFlow<Self::BreakTy> {
         let flags = FlagComputation::for_unevaluated_const(uv);
         trace!(r.flags=?flags);
@@ -608,7 +608,7 @@ impl<'tcx> TypeVisitor<'tcx> for HasTypeFlagsVisitor {
     }
 
     #[inline]
-    #[instrument(level = "trace")]
+    #[instrument(level = "trace", ret)]
     fn visit_predicate(&mut self, predicate: ty::Predicate<'tcx>) -> ControlFlow<Self::BreakTy> {
         debug!(
             "HasTypeFlagsVisitor: predicate={:?} predicate.flags={:?} self.flags={:?}",
@@ -666,7 +666,7 @@ impl<'tcx> TypeVisitor<'tcx> for LateBoundRegionsCollector {
         // ignore the inputs to a projection, as they may not appear
         // in the normalized form
         if self.just_constrained {
-            if let ty::Projection(..) = t.kind() {
+            if let ty::Projection(..) | ty::Opaque(..) = t.kind() {
                 return ControlFlow::CONTINUE;
             }
         }
diff --git a/compiler/rustc_middle/src/ty/vtable.rs b/compiler/rustc_middle/src/ty/vtable.rs
index 04a9fd1f7..5ca51c25a 100644
--- a/compiler/rustc_middle/src/ty/vtable.rs
+++ b/compiler/rustc_middle/src/ty/vtable.rs
@@ -1,7 +1,7 @@
 use std::convert::TryFrom;
 use std::fmt;
 
-use crate::mir::interpret::{alloc_range, AllocId, Allocation, Pointer, Scalar, ScalarMaybeUninit};
+use crate::mir::interpret::{alloc_range, AllocId, Allocation, Pointer, Scalar};
 use crate::ty::{self, Instance, PolyTraitRef, Ty, TyCtxt};
 use rustc_ast::Mutability;
 
@@ -87,7 +87,7 @@ pub(super) fn vtable_allocation_provider<'tcx>(
                 let instance = ty::Instance::resolve_drop_in_place(tcx, ty);
                 let fn_alloc_id = tcx.create_fn_alloc(instance);
                 let fn_ptr = Pointer::from(fn_alloc_id);
-                ScalarMaybeUninit::from_pointer(fn_ptr, &tcx)
+                Scalar::from_pointer(fn_ptr, &tcx)
             }
             VtblEntry::MetadataSize => Scalar::from_uint(size, ptr_size).into(),
             VtblEntry::MetadataAlign => Scalar::from_uint(align, ptr_size).into(),
@@ -97,14 +97,14 @@ pub(super) fn vtable_allocation_provider<'tcx>(
                 let instance = instance.polymorphize(tcx);
                 let fn_alloc_id = tcx.create_fn_alloc(instance);
                 let fn_ptr = Pointer::from(fn_alloc_id);
-                ScalarMaybeUninit::from_pointer(fn_ptr, &tcx)
+                Scalar::from_pointer(fn_ptr, &tcx)
             }
             VtblEntry::TraitVPtr(trait_ref) => {
                 let super_trait_ref = trait_ref
                     .map_bound(|trait_ref| ty::ExistentialTraitRef::erase_self_ty(tcx, trait_ref));
                 let supertrait_alloc_id = tcx.vtable_allocation((ty, Some(super_trait_ref)));
                 let vptr = Pointer::from(supertrait_alloc_id);
-                ScalarMaybeUninit::from_pointer(vptr, &tcx)
+                Scalar::from_pointer(vptr, &tcx)
             }
         };
         vtable
diff --git a/compiler/rustc_middle/src/ty/walk.rs b/compiler/rustc_middle/src/ty/walk.rs
index 02fe1f3a7..a3e11bbf0 100644
--- a/compiler/rustc_middle/src/ty/walk.rs
+++ b/compiler/rustc_middle/src/ty/walk.rs
@@ -152,7 +152,7 @@ fn push_inner<'tcx>(stack: &mut TypeWalkerStack<'tcx>, parent: GenericArg<'tcx>)
             ty::Projection(data) => {
                 stack.extend(data.substs.iter().rev());
             }
-            ty::Dynamic(obj, lt) => {
+            ty::Dynamic(obj, lt, _) => {
                 stack.push(lt.into());
                 stack.extend(obj.iter().rev().flat_map(|predicate| {
                     let (substs, opt_ty) = match predicate.skip_binder() {
@@ -165,9 +165,9 @@ fn push_inner<'tcx>(stack: &mut TypeWalkerStack<'tcx>, parent: GenericArg<'tcx>)
                         }
                     };
 
-                    substs.iter().rev().chain(opt_ty.map(|term| match term {
-                        ty::Term::Ty(ty) => ty.into(),
-                        ty::Term::Const(ct) => ct.into(),
+                    substs.iter().rev().chain(opt_ty.map(|term| match term.unpack() {
+                        ty::TermKind::Ty(ty) => ty.into(),
+                        ty::TermKind::Const(ct) => ct.into(),
                     }))
                 }));
             }
diff --git a/compiler/rustc_middle/src/values.rs b/compiler/rustc_middle/src/values.rs
new file mode 100644
index 000000000..7fbe9ae2a
--- /dev/null
+++ b/compiler/rustc_middle/src/values.rs
@@ -0,0 +1,54 @@
+use rustc_middle::ty::{self, AdtSizedConstraint, Ty, TyCtxt};
+use rustc_query_system::Value;
+
+impl<'tcx> Value<TyCtxt<'tcx>> for Ty<'_> {
+    fn from_cycle_error(tcx: TyCtxt<'tcx>) -> Self {
+        // SAFETY: This is never called when `Self` is not `Ty<'tcx>`.
+        // FIXME: Represent the above fact in the trait system somehow.
+        unsafe { std::mem::transmute::<Ty<'tcx>, Ty<'_>>(tcx.ty_error()) }
+    }
+}
+
+impl<'tcx> Value<TyCtxt<'tcx>> for ty::SymbolName<'_> {
+    fn from_cycle_error(tcx: TyCtxt<'tcx>) -> Self {
+        // SAFETY: This is never called when `Self` is not `SymbolName<'tcx>`.
+        // FIXME: Represent the above fact in the trait system somehow.
+        unsafe {
+            std::mem::transmute::<ty::SymbolName<'tcx>, ty::SymbolName<'_>>(ty::SymbolName::new(
+                tcx, "<error>",
+            ))
+        }
+    }
+}
+
+impl<'tcx> Value<TyCtxt<'tcx>> for AdtSizedConstraint<'_> {
+    fn from_cycle_error(tcx: TyCtxt<'tcx>) -> Self {
+        // SAFETY: This is never called when `Self` is not `AdtSizedConstraint<'tcx>`.
+        // FIXME: Represent the above fact in the trait system somehow.
+        unsafe {
+            std::mem::transmute::<AdtSizedConstraint<'tcx>, AdtSizedConstraint<'_>>(
+                AdtSizedConstraint(tcx.intern_type_list(&[tcx.ty_error()])),
+            )
+        }
+    }
+}
+
+impl<'tcx> Value<TyCtxt<'tcx>> for ty::Binder<'_, ty::FnSig<'_>> {
+    fn from_cycle_error(tcx: TyCtxt<'tcx>) -> Self {
+        let err = tcx.ty_error();
+        // FIXME(compiler-errors): It would be nice if we could get the
+        // query key, so we could at least generate a fn signature that
+        // has the right arity.
+        let fn_sig = ty::Binder::dummy(tcx.mk_fn_sig(
+            [].into_iter(),
+            err,
+            false,
+            rustc_hir::Unsafety::Normal,
+            rustc_target::spec::abi::Abi::Rust,
+        ));
+
+        // SAFETY: This is never called when `Self` is not `ty::Binder<'tcx, ty::FnSig<'tcx>>`.
+        // FIXME: Represent the above fact in the trait system somehow.
+        unsafe { std::mem::transmute::<ty::PolyFnSig<'tcx>, ty::Binder<'_, ty::FnSig<'_>>>(fn_sig) }
+    }
+}