Merging upstream version 1.67.1+dfsg1.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

70 files changed, 3623 insertions, 2440 deletions

diff --git a/compiler/rustc_middle/Cargo.toml b/compiler/rustc_middle/Cargo.toml
index de916ea8c..cf1ab47de 100644
--- a/compiler/rustc_middle/Cargo.toml
+++ b/compiler/rustc_middle/Cargo.toml
@@ -4,11 +4,11 @@ version = "0.0.0"
 edition = "2021"
 
 [lib]
-doctest = false
 
 [dependencies]
 bitflags = "1.2.1"
-chalk-ir = "0.80.0"
+chalk-ir = "0.87.0"
+derive_more = "0.99.17"
 either = "1.5.0"
 gsgdt = "0.1.2"
 polonius-engine = "0.13.0"
@@ -32,7 +32,7 @@ rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
 rustc_type_ir = { path = "../rustc_type_ir" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
-thin-vec = "0.2.8"
+thin-vec = "0.2.9"
 tracing = "0.1"
 
 [features]
diff --git a/compiler/rustc_middle/src/arena.rs b/compiler/rustc_middle/src/arena.rs
index f8aae86fe..6de68841f 100644
--- a/compiler/rustc_middle/src/arena.rs
+++ b/compiler/rustc_middle/src/arena.rs
@@ -1,3 +1,5 @@
+#![allow(rustc::usage_of_ty_tykind)]
+
 /// This higher-order macro declares a list of types which can be allocated by `Arena`.
 ///
 /// Specifying the `decode` modifier will add decode impls for `&T` and `&[T]` where `T` is the type
@@ -6,7 +8,7 @@
 macro_rules! arena_types {
     ($macro:path) => (
         $macro!([
-            [] layout: rustc_target::abi::LayoutS<'tcx>,
+            [] layout: rustc_target::abi::LayoutS<rustc_target::abi::VariantIdx>,
             [] fn_abi: rustc_target::abi::call::FnAbi<'tcx, rustc_middle::ty::Ty<'tcx>>,
             // AdtDef are interned and compared by address
             [decode] adt_def: rustc_middle::ty::AdtDefData,
@@ -28,6 +30,7 @@ macro_rules! arena_types {
             [decode] typeck_results: rustc_middle::ty::TypeckResults<'tcx>,
             [decode] borrowck_result:
                 rustc_middle::mir::BorrowCheckResult<'tcx>,
+            [] resolver: rustc_data_structures::steal::Steal<rustc_middle::ty::ResolverAstLowering>,
             [decode] unsafety_check_result: rustc_middle::mir::UnsafetyCheckResult,
             [decode] code_region: rustc_middle::mir::coverage::CodeRegion,
             [] const_allocs: rustc_middle::mir::interpret::Allocation,
@@ -88,8 +91,8 @@ macro_rules! arena_types {
             [] hir_id_set: rustc_hir::HirIdSet,
 
             // Interned types
-            [] tys: rustc_data_structures::intern::WithStableHash<rustc_middle::ty::TyS<'tcx>>,
-            [] predicates: rustc_middle::ty::PredicateS<'tcx>,
+            [] tys: rustc_type_ir::WithCachedTypeInfo<rustc_middle::ty::TyKind<'tcx>>,
+            [] predicates: rustc_type_ir::WithCachedTypeInfo<rustc_middle::ty::PredicateKind<'tcx>>,
             [] consts: rustc_middle::ty::ConstS<'tcx>,
 
             // Note that this deliberately duplicates items in the `rustc_hir::arena`,
diff --git a/compiler/rustc_middle/src/error.rs b/compiler/rustc_middle/src/error.rs
index a7a7ac059..5e94da8cb 100644
--- a/compiler/rustc_middle/src/error.rs
+++ b/compiler/rustc_middle/src/error.rs
@@ -55,3 +55,20 @@ pub struct ConstEvalNonIntError {
     #[primary_span]
     pub span: Span,
 }
+
+#[derive(Diagnostic)]
+#[diag(middle_strict_coherence_needs_negative_coherence)]
+pub(crate) struct StrictCoherenceNeedsNegativeCoherence {
+    #[primary_span]
+    pub span: Span,
+    #[label]
+    pub attr_span: Option<Span>,
+}
+
+#[derive(Diagnostic)]
+#[diag(middle_const_not_used_in_type_alias)]
+pub(super) struct ConstNotUsedTraitAlias {
+    pub ct: String,
+    #[primary_span]
+    pub span: Span,
+}
diff --git a/compiler/rustc_middle/src/hir/map/mod.rs b/compiler/rustc_middle/src/hir/map/mod.rs
index 83a4d16d7..1bd8f9535 100644
--- a/compiler/rustc_middle/src/hir/map/mod.rs
+++ b/compiler/rustc_middle/src/hir/map/mod.rs
@@ -245,15 +245,15 @@ impl<'hir> Map<'hir> {
             },
             Node::Variant(_) => DefKind::Variant,
             Node::Ctor(variant_data) => {
-                // FIXME(eddyb) is this even possible, if we have a `Node::Ctor`?
-                assert_ne!(variant_data.ctor_hir_id(), None);
-
                 let ctor_of = match self.find(self.get_parent_node(hir_id)) {
                     Some(Node::Item(..)) => def::CtorOf::Struct,
                     Some(Node::Variant(..)) => def::CtorOf::Variant,
                     _ => unreachable!(),
                 };
-                DefKind::Ctor(ctor_of, def::CtorKind::from_hir(variant_data))
+                match variant_data.ctor_kind() {
+                    Some(kind) => DefKind::Ctor(ctor_of, kind),
+                    None => bug!("constructor node without a constructor"),
+                }
             }
             Node::AnonConst(_) => {
                 let inline = match self.find(self.get_parent_node(hir_id)) {
@@ -353,6 +353,10 @@ impl<'hir> Map<'hir> {
         node.node.generics()
     }
 
+    pub fn owner(self, id: OwnerId) -> OwnerNode<'hir> {
+        self.tcx.hir_owner(id).unwrap_or_else(|| bug!("expected owner for {:?}", id)).node
+    }
+
     pub fn item(self, id: ItemId) -> &'hir Item<'hir> {
         self.tcx.hir_owner(id.owner_id).unwrap().node.expect_item()
     }
@@ -822,8 +826,11 @@ impl<'hir> Map<'hir> {
         )
     }
 
-    pub fn expect_owner(self, id: OwnerId) -> OwnerNode<'hir> {
-        self.tcx.hir_owner(id).unwrap_or_else(|| bug!("expected owner for {:?}", id)).node
+    pub fn expect_owner(self, def_id: LocalDefId) -> OwnerNode<'hir> {
+        self.tcx
+            .hir_owner(OwnerId { def_id })
+            .unwrap_or_else(|| bug!("expected owner for {:?}", def_id))
+            .node
     }
 
     pub fn expect_item(self, id: LocalDefId) -> &'hir Item<'hir> {
@@ -1015,7 +1022,7 @@ impl<'hir> Map<'hir> {
                 ..
             }) => {
                 // Ensure that the returned span has the item's SyntaxContext.
-                fn_decl_span.find_ancestor_in_same_ctxt(*span).unwrap_or(*span)
+                fn_decl_span.find_ancestor_inside(*span).unwrap_or(*span)
             }
             _ => self.span_with_body(hir_id),
         };
@@ -1051,7 +1058,7 @@ impl<'hir> Map<'hir> {
             Node::Arm(arm) => arm.span,
             Node::Block(block) => block.span,
             Node::Ctor(..) => self.span_with_body(self.get_parent_node(hir_id)),
-            Node::Lifetime(lifetime) => lifetime.span,
+            Node::Lifetime(lifetime) => lifetime.ident.span,
             Node::GenericParam(param) => param.span,
             Node::Infer(i) => i.span,
             Node::Local(local) => local.span,
@@ -1079,10 +1086,10 @@ impl<'hir> Map<'hir> {
 
     /// Returns the HirId of `N` in `struct Foo<const N: usize = { ... }>` when
     /// called with the HirId for the `{ ... }` anon const
-    pub fn opt_const_param_default_param_hir_id(self, anon_const: HirId) -> Option<HirId> {
+    pub fn opt_const_param_default_param_def_id(self, anon_const: HirId) -> Option<LocalDefId> {
         match self.get(self.get_parent_node(anon_const)) {
             Node::GenericParam(GenericParam {
-                hir_id: param_id,
+                def_id: param_id,
                 kind: GenericParamKind::Const { .. },
                 ..
             }) => Some(*param_id),
@@ -1191,20 +1198,7 @@ fn upstream_crates(tcx: TyCtxt<'_>) -> Vec<(StableCrateId, Svh)> {
 fn hir_id_to_string(map: Map<'_>, id: HirId) -> String {
     let id_str = format!(" (hir_id={})", id);
 
-    let path_str = || {
-        // This functionality is used for debugging, try to use `TyCtxt` to get
-        // the user-friendly path, otherwise fall back to stringifying `DefPath`.
-        crate::ty::tls::with_opt(|tcx| {
-            if let Some(tcx) = tcx {
-                let def_id = map.local_def_id(id);
-                tcx.def_path_str(def_id.to_def_id())
-            } else if let Some(path) = map.def_path_from_hir_id(id) {
-                path.data.into_iter().map(|elem| elem.to_string()).collect::<Vec<_>>().join("::")
-            } else {
-                String::from("<missing path>")
-            }
-        })
-    };
+    let path_str = |def_id: LocalDefId| map.tcx.def_path_str(def_id.to_def_id());
 
     let span_str = || map.tcx.sess.source_map().span_to_snippet(map.span(id)).unwrap_or_default();
     let node_str = |prefix| format!("{} {}{}", prefix, span_str(), id_str);
@@ -1236,18 +1230,19 @@ fn hir_id_to_string(map: Map<'_>, id: HirId) -> String {
                 ItemKind::TraitAlias(..) => "trait alias",
                 ItemKind::Impl { .. } => "impl",
             };
-            format!("{} {}{}", item_str, path_str(), id_str)
+            format!("{} {}{}", item_str, path_str(item.owner_id.def_id), id_str)
+        }
+        Some(Node::ForeignItem(item)) => {
+            format!("foreign item {}{}", path_str(item.owner_id.def_id), id_str)
+        }
+        Some(Node::ImplItem(ii)) => {
+            let kind = match ii.kind {
+                ImplItemKind::Const(..) => "assoc const",
+                ImplItemKind::Fn(..) => "method",
+                ImplItemKind::Type(_) => "assoc type",
+            };
+            format!("{} {} in {}{}", kind, ii.ident, path_str(ii.owner_id.def_id), id_str)
         }
-        Some(Node::ForeignItem(_)) => format!("foreign item {}{}", path_str(), id_str),
-        Some(Node::ImplItem(ii)) => match ii.kind {
-            ImplItemKind::Const(..) => {
-                format!("assoc const {} in {}{}", ii.ident, path_str(), id_str)
-            }
-            ImplItemKind::Fn(..) => format!("method {} in {}{}", ii.ident, path_str(), id_str),
-            ImplItemKind::Type(_) => {
-                format!("assoc type {} in {}{}", ii.ident, path_str(), id_str)
-            }
-        },
         Some(Node::TraitItem(ti)) => {
             let kind = match ti.kind {
                 TraitItemKind::Const(..) => "assoc constant",
@@ -1255,13 +1250,13 @@ fn hir_id_to_string(map: Map<'_>, id: HirId) -> String {
                 TraitItemKind::Type(..) => "assoc type",
             };
 
-            format!("{} {} in {}{}", kind, ti.ident, path_str(), id_str)
+            format!("{} {} in {}{}", kind, ti.ident, path_str(ti.owner_id.def_id), id_str)
         }
         Some(Node::Variant(ref variant)) => {
-            format!("variant {} in {}{}", variant.ident, path_str(), id_str)
+            format!("variant {} in {}{}", variant.ident, path_str(variant.def_id), id_str)
         }
         Some(Node::Field(ref field)) => {
-            format!("field {} in {}{}", field.ident, path_str(), id_str)
+            format!("field {} in {}{}", field.ident, path_str(field.def_id), id_str)
         }
         Some(Node::AnonConst(_)) => node_str("const"),
         Some(Node::Expr(_)) => node_str("expr"),
@@ -1278,9 +1273,15 @@ fn hir_id_to_string(map: Map<'_>, id: HirId) -> String {
         Some(Node::Block(_)) => node_str("block"),
         Some(Node::Infer(_)) => node_str("infer"),
         Some(Node::Local(_)) => node_str("local"),
-        Some(Node::Ctor(..)) => format!("ctor {}{}", path_str(), id_str),
+        Some(Node::Ctor(ctor)) => format!(
+            "ctor {}{}",
+            ctor.ctor_def_id().map_or("<missing path>".into(), |def_id| path_str(def_id)),
+            id_str
+        ),
         Some(Node::Lifetime(_)) => node_str("lifetime"),
-        Some(Node::GenericParam(ref param)) => format!("generic_param {:?}{}", param, id_str),
+        Some(Node::GenericParam(ref param)) => {
+            format!("generic_param {}{}", path_str(param.def_id), id_str)
+        }
         Some(Node::Crate(..)) => String::from("root_crate"),
         None => format!("unknown node{}", id_str),
     }
@@ -1400,13 +1401,13 @@ impl<'hir> Visitor<'hir> for ItemCollector<'hir> {
     }
 
     fn visit_anon_const(&mut self, c: &'hir AnonConst) {
-        self.body_owners.push(self.tcx.hir().local_def_id(c.hir_id));
+        self.body_owners.push(c.def_id);
         intravisit::walk_anon_const(self, c)
     }
 
     fn visit_expr(&mut self, ex: &'hir Expr<'hir>) {
-        if matches!(ex.kind, ExprKind::Closure { .. }) {
-            self.body_owners.push(self.tcx.hir().local_def_id(ex.hir_id));
+        if let ExprKind::Closure(closure) = ex.kind {
+            self.body_owners.push(closure.def_id);
         }
         intravisit::walk_expr(self, ex)
     }
diff --git a/compiler/rustc_middle/src/hir/mod.rs b/compiler/rustc_middle/src/hir/mod.rs
index 1c6264ad0..02fd03c02 100644
--- a/compiler/rustc_middle/src/hir/mod.rs
+++ b/compiler/rustc_middle/src/hir/mod.rs
@@ -133,13 +133,8 @@ pub fn provide(providers: &mut Providers) {
         // Accessing the local_parent is ok since its value is hashed as part of `id`'s DefPathHash.
         tcx.opt_local_parent(id.def_id).map_or(CRATE_HIR_ID, |parent| {
             let mut parent_hir_id = tcx.hir().local_def_id_to_hir_id(parent);
-            if let Some(local_id) = tcx.hir_crate(()).owners[parent_hir_id.owner.def_id]
-                .unwrap()
-                .parenting
-                .get(&id.def_id)
-            {
-                parent_hir_id.local_id = *local_id;
-            }
+            parent_hir_id.local_id =
+                tcx.hir_crate(()).owners[parent_hir_id.owner.def_id].unwrap().parenting[&id.def_id];
             parent_hir_id
         })
     };
diff --git a/compiler/rustc_middle/src/infer/canonical.rs b/compiler/rustc_middle/src/infer/canonical.rs
index d3cf519b6..0331d764b 100644
--- a/compiler/rustc_middle/src/infer/canonical.rs
+++ b/compiler/rustc_middle/src/infer/canonical.rs
@@ -336,15 +336,17 @@ impl<'tcx> CanonicalVarValues<'tcx> {
                         tcx.mk_ty(ty::Bound(ty::INNERMOST, ty::BoundVar::from_u32(i).into())).into()
                     }
                     GenericArgKind::Lifetime(..) => {
-                        let br =
-                            ty::BoundRegion { var: ty::BoundVar::from_u32(i), kind: ty::BrAnon(i) };
+                        let br = ty::BoundRegion {
+                            var: ty::BoundVar::from_u32(i),
+                            kind: ty::BrAnon(i, None),
+                        };
                         tcx.mk_region(ty::ReLateBound(ty::INNERMOST, br)).into()
                     }
                     GenericArgKind::Const(ct) => tcx
-                        .mk_const(ty::ConstS {
-                            ty: ct.ty(),
-                            kind: ty::ConstKind::Bound(ty::INNERMOST, ty::BoundVar::from_u32(i)),
-                        })
+                        .mk_const(
+                            ty::ConstKind::Bound(ty::INNERMOST, ty::BoundVar::from_u32(i)),
+                            ct.ty(),
+                        )
                         .into(),
                 })
                 .collect(),
diff --git a/compiler/rustc_middle/src/lib.rs b/compiler/rustc_middle/src/lib.rs
index a58cbc376..7e4063c2f 100644
--- a/compiler/rustc_middle/src/lib.rs
+++ b/compiler/rustc_middle/src/lib.rs
@@ -30,8 +30,10 @@
 #![feature(core_intrinsics)]
 #![feature(discriminant_kind)]
 #![feature(exhaustive_patterns)]
+#![feature(generators)]
 #![feature(get_mut_unchecked)]
 #![feature(if_let_guard)]
+#![feature(iter_from_generator)]
 #![feature(negative_impls)]
 #![feature(never_type)]
 #![feature(extern_types)]
@@ -43,7 +45,6 @@
 #![feature(type_alias_impl_trait)]
 #![feature(associated_type_bounds)]
 #![feature(rustc_attrs)]
-#![cfg_attr(bootstrap, feature(half_open_range_patterns))]
 #![feature(control_flow_enum)]
 #![feature(associated_type_defaults)]
 #![feature(trusted_step)]
diff --git a/compiler/rustc_middle/src/lint.rs b/compiler/rustc_middle/src/lint.rs
index 79522bd0b..51df42f6d 100644
--- a/compiler/rustc_middle/src/lint.rs
+++ b/compiler/rustc_middle/src/lint.rs
@@ -276,7 +276,7 @@ pub fn explain_lint_level_source(
 
 /// The innermost function for emitting lints.
 ///
-/// If you are loocking to implement a lint, look for higher level functions,
+/// If you are looking to implement a lint, look for higher level functions,
 /// for example:
 /// - [`TyCtxt::emit_spanned_lint`]
 /// - [`TyCtxt::struct_span_lint_hir`]
diff --git a/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs b/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs
index 45d33a165..bea884c85 100644
--- a/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs
+++ b/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs
@@ -26,8 +26,10 @@ pub struct CodegenFnAttrs {
     /// The `#[target_feature(enable = "...")]` attribute and the enabled
     /// features (only enabled features are supported right now).
     pub target_features: Vec<Symbol>,
-    /// The `#[linkage = "..."]` attribute and the value we found.
+    /// The `#[linkage = "..."]` attribute on Rust-defined items and the value we found.
     pub linkage: Option<Linkage>,
+    /// The `#[linkage = "..."]` attribute on foreign items and the value we found.
+    pub import_linkage: Option<Linkage>,
     /// The `#[link_section = "..."]` attribute, or what executable section this
     /// should be placed in.
     pub link_section: Option<Symbol>,
@@ -113,6 +115,7 @@ impl CodegenFnAttrs {
             link_ordinal: None,
             target_features: vec![],
             linkage: None,
+            import_linkage: None,
             link_section: None,
             no_sanitize: SanitizerSet::empty(),
             instruction_set: None,
diff --git a/compiler/rustc_middle/src/middle/lang_items.rs b/compiler/rustc_middle/src/middle/lang_items.rs
index 31c20fa14..343ea1f00 100644
--- a/compiler/rustc_middle/src/middle/lang_items.rs
+++ b/compiler/rustc_middle/src/middle/lang_items.rs
@@ -27,7 +27,10 @@ impl<'tcx> TyCtxt<'tcx> {
         })
     }
 
-    pub fn fn_trait_kind_from_lang_item(self, id: DefId) -> Option<ty::ClosureKind> {
+    /// Given a [`DefId`] of a [`Fn`], [`FnMut`] or [`FnOnce`] traits,
+    /// returns a corresponding [`ty::ClosureKind`].
+    /// For any other [`DefId`] return `None`.
+    pub fn fn_trait_kind_from_def_id(self, id: DefId) -> Option<ty::ClosureKind> {
         let items = self.lang_items();
         match Some(id) {
             x if x == items.fn_trait() => Some(ty::ClosureKind::Fn),
@@ -37,8 +40,9 @@ impl<'tcx> TyCtxt<'tcx> {
         }
     }
 
-    pub fn is_weak_lang_item(self, item_def_id: DefId) -> bool {
-        self.lang_items().is_weak_lang_item(item_def_id)
+    /// Returns `true` if `id` is a `DefId` of [`Fn`], [`FnMut`] or [`FnOnce`] traits.
+    pub fn is_fn_trait(self, id: DefId) -> bool {
+        self.fn_trait_kind_from_def_id(id).is_some()
     }
 }
 
diff --git a/compiler/rustc_middle/src/middle/privacy.rs b/compiler/rustc_middle/src/middle/privacy.rs
index 9c68c7504..fc08d58cc 100644
--- a/compiler/rustc_middle/src/middle/privacy.rs
+++ b/compiler/rustc_middle/src/middle/privacy.rs
@@ -1,12 +1,12 @@
 //! A pass that checks to make sure private fields and methods aren't used
 //! outside their scopes. This pass will also generate a set of exported items
 //! which are available for use externally when compiled as a library.
-use crate::ty::{DefIdTree, Visibility};
+use crate::ty::{DefIdTree, TyCtxt, Visibility};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_macros::HashStable;
 use rustc_query_system::ich::StableHashingContext;
-use rustc_span::def_id::{DefId, LocalDefId};
+use rustc_span::def_id::LocalDefId;
 use std::hash::Hash;
 
 /// Represents the levels of effective visibility an item can have.
@@ -75,33 +75,33 @@ impl EffectiveVisibility {
 }
 
 /// Holds a map of effective visibilities for reachable HIR nodes.
-#[derive(Debug, Clone)]
+#[derive(Clone, Debug)]
 pub struct EffectiveVisibilities<Id = LocalDefId> {
     map: FxHashMap<Id, EffectiveVisibility>,
 }
 
-impl<Id: Hash + Eq + Copy> EffectiveVisibilities<Id> {
-    pub fn is_public_at_level(&self, id: Id, level: Level) -> bool {
+impl EffectiveVisibilities {
+    pub fn is_public_at_level(&self, id: LocalDefId, level: Level) -> bool {
         self.effective_vis(id)
             .map_or(false, |effective_vis| effective_vis.is_public_at_level(level))
     }
 
     /// See `Level::Reachable`.
-    pub fn is_reachable(&self, id: Id) -> bool {
+    pub fn is_reachable(&self, id: LocalDefId) -> bool {
         self.is_public_at_level(id, Level::Reachable)
     }
 
     /// See `Level::Reexported`.
-    pub fn is_exported(&self, id: Id) -> bool {
+    pub fn is_exported(&self, id: LocalDefId) -> bool {
         self.is_public_at_level(id, Level::Reexported)
     }
 
     /// See `Level::Direct`.
-    pub fn is_directly_public(&self, id: Id) -> bool {
+    pub fn is_directly_public(&self, id: LocalDefId) -> bool {
         self.is_public_at_level(id, Level::Direct)
     }
 
-    pub fn public_at_level(&self, id: Id) -> Option<Level> {
+    pub fn public_at_level(&self, id: LocalDefId) -> Option<Level> {
         self.effective_vis(id).and_then(|effective_vis| {
             for level in Level::all_levels() {
                 if effective_vis.is_public_at_level(level) {
@@ -112,31 +112,42 @@ impl<Id: Hash + Eq + Copy> EffectiveVisibilities<Id> {
         })
     }
 
-    pub fn effective_vis(&self, id: Id) -> Option<&EffectiveVisibility> {
-        self.map.get(&id)
-    }
-
-    pub fn iter(&self) -> impl Iterator<Item = (&Id, &EffectiveVisibility)> {
-        self.map.iter()
-    }
-
-    pub fn map_id<OutId: Hash + Eq + Copy>(
-        &self,
-        f: impl Fn(Id) -> OutId,
-    ) -> EffectiveVisibilities<OutId> {
-        EffectiveVisibilities { map: self.map.iter().map(|(k, v)| (f(*k), *v)).collect() }
+    // FIXME: Share code with `fn update`.
+    pub fn update_eff_vis(
+        &mut self,
+        def_id: LocalDefId,
+        eff_vis: &EffectiveVisibility,
+        tree: impl DefIdTree,
+    ) {
+        use std::collections::hash_map::Entry;
+        match self.map.entry(def_id) {
+            Entry::Occupied(mut occupied) => {
+                let old_eff_vis = occupied.get_mut();
+                for l in Level::all_levels() {
+                    let vis_at_level = eff_vis.at_level(l);
+                    let old_vis_at_level = old_eff_vis.at_level_mut(l);
+                    if vis_at_level != old_vis_at_level
+                        && vis_at_level.is_at_least(*old_vis_at_level, tree)
+                    {
+                        *old_vis_at_level = *vis_at_level
+                    }
+                }
+                old_eff_vis
+            }
+            Entry::Vacant(vacant) => vacant.insert(*eff_vis),
+        };
     }
 
     pub fn set_public_at_level(
         &mut self,
-        id: Id,
-        default_vis: impl FnOnce() -> Visibility,
+        id: LocalDefId,
+        lazy_private_vis: impl FnOnce() -> Visibility,
         level: Level,
     ) {
         let mut effective_vis = self
             .effective_vis(id)
             .copied()
-            .unwrap_or_else(|| EffectiveVisibility::from_vis(default_vis()));
+            .unwrap_or_else(|| EffectiveVisibility::from_vis(lazy_private_vis()));
         for l in Level::all_levels() {
             if l <= level {
                 *effective_vis.at_level_mut(l) = Visibility::Public;
@@ -144,61 +155,123 @@ impl<Id: Hash + Eq + Copy> EffectiveVisibilities<Id> {
         }
         self.map.insert(id, effective_vis);
     }
+
+    pub fn check_invariants(&self, tcx: TyCtxt<'_>, early: bool) {
+        if !cfg!(debug_assertions) {
+            return;
+        }
+        for (&def_id, ev) in &self.map {
+            // More direct visibility levels can never go farther than less direct ones,
+            // neither of effective visibilities can go farther than nominal visibility,
+            // and all effective visibilities are larger or equal than private visibility.
+            let private_vis = Visibility::Restricted(tcx.parent_module_from_def_id(def_id));
+            let span = tcx.def_span(def_id.to_def_id());
+            if !ev.direct.is_at_least(private_vis, tcx) {
+                span_bug!(span, "private {:?} > direct {:?}", private_vis, ev.direct);
+            }
+            if !ev.reexported.is_at_least(ev.direct, tcx) {
+                span_bug!(span, "direct {:?} > reexported {:?}", ev.direct, ev.reexported);
+            }
+            if !ev.reachable.is_at_least(ev.reexported, tcx) {
+                span_bug!(span, "reexported {:?} > reachable {:?}", ev.reexported, ev.reachable);
+            }
+            if !ev.reachable_through_impl_trait.is_at_least(ev.reachable, tcx) {
+                span_bug!(
+                    span,
+                    "reachable {:?} > reachable_through_impl_trait {:?}",
+                    ev.reachable,
+                    ev.reachable_through_impl_trait
+                );
+            }
+            let nominal_vis = tcx.visibility(def_id);
+            // FIXME: `rustc_privacy` is not yet updated for the new logic and can set
+            // effective visibilities that are larger than the nominal one.
+            if !nominal_vis.is_at_least(ev.reachable_through_impl_trait, tcx) && early {
+                span_bug!(
+                    span,
+                    "{:?}: reachable_through_impl_trait {:?} > nominal {:?}",
+                    def_id,
+                    ev.reachable_through_impl_trait,
+                    nominal_vis
+                );
+            }
+        }
+    }
+}
+
+pub trait IntoDefIdTree {
+    type Tree: DefIdTree;
+    fn tree(self) -> Self::Tree;
 }
 
-impl<Id: Hash + Eq + Copy + Into<DefId>> EffectiveVisibilities<Id> {
-    // `parent_id` is not necessarily a parent in source code tree,
-    // it is the node from which the maximum effective visibility is inherited.
-    pub fn update(
+impl<Id: Eq + Hash> EffectiveVisibilities<Id> {
+    pub fn iter(&self) -> impl Iterator<Item = (&Id, &EffectiveVisibility)> {
+        self.map.iter()
+    }
+
+    pub fn effective_vis(&self, id: Id) -> Option<&EffectiveVisibility> {
+        self.map.get(&id)
+    }
+
+    // FIXME: Share code with `fn update`.
+    pub fn effective_vis_or_private(
+        &mut self,
+        id: Id,
+        lazy_private_vis: impl FnOnce() -> Visibility,
+    ) -> &EffectiveVisibility {
+        self.map.entry(id).or_insert_with(|| EffectiveVisibility::from_vis(lazy_private_vis()))
+    }
+
+    pub fn update<T: IntoDefIdTree>(
         &mut self,
         id: Id,
         nominal_vis: Visibility,
-        default_vis: impl FnOnce() -> Visibility,
-        parent_id: Id,
+        lazy_private_vis: impl FnOnce(T) -> (Visibility, T),
+        inherited_effective_vis: EffectiveVisibility,
         level: Level,
-        tree: impl DefIdTree,
+        mut into_tree: T,
     ) -> bool {
         let mut changed = false;
-        let mut current_effective_vis = self.effective_vis(id).copied().unwrap_or_else(|| {
-            if id.into().is_crate_root() {
-                EffectiveVisibility::from_vis(Visibility::Public)
-            } else {
-                EffectiveVisibility::from_vis(default_vis())
+        let mut current_effective_vis = match self.map.get(&id).copied() {
+            Some(eff_vis) => eff_vis,
+            None => {
+                let private_vis;
+                (private_vis, into_tree) = lazy_private_vis(into_tree);
+                EffectiveVisibility::from_vis(private_vis)
             }
-        });
-        if let Some(inherited_effective_vis) = self.effective_vis(parent_id) {
-            let mut inherited_effective_vis_at_prev_level =
-                *inherited_effective_vis.at_level(level);
-            let mut calculated_effective_vis = inherited_effective_vis_at_prev_level;
-            for l in Level::all_levels() {
-                if level >= l {
-                    let inherited_effective_vis_at_level = *inherited_effective_vis.at_level(l);
-                    let current_effective_vis_at_level = current_effective_vis.at_level_mut(l);
-                    // effective visibility for id shouldn't be recalculated if
-                    // inherited from parent_id effective visibility isn't changed at next level
-                    if !(inherited_effective_vis_at_prev_level == inherited_effective_vis_at_level
-                        && level != l)
-                    {
-                        calculated_effective_vis =
-                            if nominal_vis.is_at_least(inherited_effective_vis_at_level, tree) {
-                                inherited_effective_vis_at_level
-                            } else {
-                                nominal_vis
-                            };
-                    }
-                    // effective visibility can't be decreased at next update call for the
-                    // same id
-                    if *current_effective_vis_at_level != calculated_effective_vis
-                        && calculated_effective_vis
-                            .is_at_least(*current_effective_vis_at_level, tree)
-                    {
-                        changed = true;
-                        *current_effective_vis_at_level = calculated_effective_vis;
-                    }
-                    inherited_effective_vis_at_prev_level = inherited_effective_vis_at_level;
+        };
+        let tree = into_tree.tree();
+
+        let mut inherited_effective_vis_at_prev_level = *inherited_effective_vis.at_level(level);
+        let mut calculated_effective_vis = inherited_effective_vis_at_prev_level;
+        for l in Level::all_levels() {
+            if level >= l {
+                let inherited_effective_vis_at_level = *inherited_effective_vis.at_level(l);
+                let current_effective_vis_at_level = current_effective_vis.at_level_mut(l);
+                // effective visibility for id shouldn't be recalculated if
+                // inherited from parent_id effective visibility isn't changed at next level
+                if !(inherited_effective_vis_at_prev_level == inherited_effective_vis_at_level
+                    && level != l)
+                {
+                    calculated_effective_vis =
+                        if nominal_vis.is_at_least(inherited_effective_vis_at_level, tree) {
+                            inherited_effective_vis_at_level
+                        } else {
+                            nominal_vis
+                        };
+                }
+                // effective visibility can't be decreased at next update call for the
+                // same id
+                if *current_effective_vis_at_level != calculated_effective_vis
+                    && calculated_effective_vis.is_at_least(*current_effective_vis_at_level, tree)
+                {
+                    changed = true;
+                    *current_effective_vis_at_level = calculated_effective_vis;
                 }
+                inherited_effective_vis_at_prev_level = inherited_effective_vis_at_level;
             }
         }
+
         self.map.insert(id, current_effective_vis);
         changed
     }
diff --git a/compiler/rustc_middle/src/mir/generic_graphviz.rs b/compiler/rustc_middle/src/mir/generic_graphviz.rs
index 11ac45943..ccae7e159 100644
--- a/compiler/rustc_middle/src/mir/generic_graphviz.rs
+++ b/compiler/rustc_middle/src/mir/generic_graphviz.rs
@@ -126,7 +126,7 @@ impl<
             write!(
                 w,
                 r#"<tr><td align="left" balign="left">{}</td></tr>"#,
-                dot::escape_html(&section).replace('\n', "<br/>")
+                dot::escape_html(&section)
             )?;
         }
 
@@ -147,7 +147,7 @@ impl<
             let src = self.node(source);
             let trg = self.node(target);
             let escaped_edge_label = if let Some(edge_label) = edge_labels.get(index) {
-                dot::escape_html(edge_label).replace('\n', r#"<br align="left"/>"#)
+                dot::escape_html(edge_label)
             } else {
                 "".to_owned()
             };
@@ -162,8 +162,7 @@ impl<
     where
         W: Write,
     {
-        let lines = label.split('\n').map(|s| dot::escape_html(s)).collect::<Vec<_>>();
-        let escaped_label = lines.join(r#"<br align="left"/>"#);
+        let escaped_label = dot::escape_html(label);
         writeln!(w, r#"    label=<<br/><br/>{}<br align="left"/><br/><br/><br/>>;"#, escaped_label)
     }
 
diff --git a/compiler/rustc_middle/src/mir/interpret/allocation.rs b/compiler/rustc_middle/src/mir/interpret/allocation.rs
index 37ec04b07..221105ac4 100644
--- a/compiler/rustc_middle/src/mir/interpret/allocation.rs
+++ b/compiler/rustc_middle/src/mir/interpret/allocation.rs
@@ -1,16 +1,20 @@
 //! The virtual memory representation of the MIR interpreter.
 
+mod init_mask;
+mod provenance_map;
+#[cfg(test)]
+mod tests;
+
 use std::borrow::Cow;
-use std::convert::{TryFrom, TryInto};
 use std::fmt;
 use std::hash;
-use std::iter;
-use std::ops::{Deref, Range};
+use std::ops::Range;
 use std::ptr;
 
+use either::{Left, Right};
+
 use rustc_ast::Mutability;
 use rustc_data_structures::intern::Interned;
-use rustc_data_structures::sorted_map::SortedMap;
 use rustc_span::DUMMY_SP;
 use rustc_target::abi::{Align, HasDataLayout, Size};
 
@@ -20,6 +24,10 @@ use super::{
     UnsupportedOpInfo,
 };
 use crate::ty;
+use init_mask::*;
+use provenance_map::*;
+
+pub use init_mask::{InitChunk, InitChunkIter};
 
 /// This type represents an Allocation in the Miri/CTFE core engine.
 ///
@@ -28,9 +36,9 @@ use crate::ty;
 /// module provides higher-level access.
 // Note: for performance reasons when interning, some of the `Allocation` fields can be partially
 // hashed. (see the `Hash` impl below for more details), so the impl is not derived.
-#[derive(Clone, Debug, Eq, PartialEq, PartialOrd, Ord, TyEncodable, TyDecodable)]
+#[derive(Clone, Eq, PartialEq, TyEncodable, TyDecodable)]
 #[derive(HashStable)]
-pub struct Allocation<Prov = AllocId, Extra = ()> {
+pub struct Allocation<Prov: Provenance = AllocId, Extra = ()> {
     /// The actual bytes of the allocation.
     /// Note that the bytes of a pointer represent the offset of the pointer.
     bytes: Box<[u8]>,
@@ -95,27 +103,25 @@ impl hash::Hash for Allocation {
 /// Interned types generally have an `Outer` type and an `Inner` type, where
 /// `Outer` is a newtype around `Interned<Inner>`, and all the operations are
 /// done on `Outer`, because all occurrences are interned. E.g. `Ty` is an
-/// outer type and `TyS` is its inner type.
+/// outer type and `TyKind` is its inner type.
 ///
 /// Here things are different because only const allocations are interned. This
 /// means that both the inner type (`Allocation`) and the outer type
 /// (`ConstAllocation`) are used quite a bit.
-#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, HashStable)]
+#[derive(Copy, Clone, PartialEq, Eq, Hash, HashStable)]
 #[rustc_pass_by_value]
-pub struct ConstAllocation<'tcx, Prov = AllocId, Extra = ()>(
-    pub Interned<'tcx, Allocation<Prov, Extra>>,
-);
+pub struct ConstAllocation<'tcx>(pub Interned<'tcx, Allocation>);
 
 impl<'tcx> fmt::Debug for ConstAllocation<'tcx> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        // This matches how `Allocation` is printed. We print it like this to
-        // avoid having to update expected output in a lot of tests.
-        write!(f, "{:?}", self.inner())
+        // The debug representation of this is very verbose and basically useless,
+        // so don't print it.
+        write!(f, "ConstAllocation {{ .. }}")
     }
 }
 
-impl<'tcx, Prov, Extra> ConstAllocation<'tcx, Prov, Extra> {
-    pub fn inner(self) -> &'tcx Allocation<Prov, Extra> {
+impl<'tcx> ConstAllocation<'tcx> {
+    pub fn inner(self) -> &'tcx Allocation {
         self.0.0
     }
 }
@@ -183,12 +189,21 @@ pub fn alloc_range(start: Size, size: Size) -> AllocRange {
     AllocRange { start, size }
 }
 
-impl AllocRange {
+impl From<Range<Size>> for AllocRange {
     #[inline]
-    pub fn from(r: Range<Size>) -> Self {
+    fn from(r: Range<Size>) -> Self {
         alloc_range(r.start, r.end - r.start) // `Size` subtraction (overflow-checked)
     }
+}
 
+impl From<Range<usize>> for AllocRange {
+    #[inline]
+    fn from(r: Range<usize>) -> Self {
+        AllocRange::from(Size::from_bytes(r.start)..Size::from_bytes(r.end))
+    }
+}
+
+impl AllocRange {
     #[inline(always)]
     pub fn end(self) -> Size {
         self.start + self.size // This does overflow checking.
@@ -205,7 +220,7 @@ impl AllocRange {
 }
 
 // The constructors are all without extra; the extra gets added by a machine hook later.
-impl<Prov> Allocation<Prov> {
+impl<Prov: Provenance> Allocation<Prov> {
     /// Creates an allocation initialized by the given bytes
     pub fn from_bytes<'a>(
         slice: impl Into<Cow<'a, [u8]>>,
@@ -263,7 +278,7 @@ impl<Prov> Allocation<Prov> {
 impl Allocation {
     /// Adjust allocation from the ones in tcx to a custom Machine instance
     /// with a different Provenance and Extra type.
-    pub fn adjust_from_tcx<Prov, Extra, Err>(
+    pub fn adjust_from_tcx<Prov: Provenance, Extra, Err>(
         self,
         cx: &impl HasDataLayout,
         extra: Extra,
@@ -271,10 +286,10 @@ impl Allocation {
     ) -> Result<Allocation<Prov, Extra>, Err> {
         // Compute new pointer provenance, which also adjusts the bytes.
         let mut bytes = self.bytes;
-        let mut new_provenance = Vec::with_capacity(self.provenance.0.len());
+        let mut new_provenance = Vec::with_capacity(self.provenance.ptrs().len());
         let ptr_size = cx.data_layout().pointer_size.bytes_usize();
         let endian = cx.data_layout().endian;
-        for &(offset, alloc_id) in self.provenance.iter() {
+        for &(offset, alloc_id) in self.provenance.ptrs().iter() {
             let idx = offset.bytes_usize();
             let ptr_bytes = &mut bytes[idx..idx + ptr_size];
             let bits = read_target_uint(endian, ptr_bytes).unwrap();
@@ -286,7 +301,7 @@ impl Allocation {
         // Create allocation.
         Ok(Allocation {
             bytes,
-            provenance: ProvenanceMap::from_presorted(new_provenance),
+            provenance: ProvenanceMap::from_presorted_ptrs(new_provenance),
             init_mask: self.init_mask,
             align: self.align,
             mutability: self.mutability,
@@ -296,7 +311,7 @@ impl Allocation {
 }
 
 /// Raw accessors. Provide access to otherwise private bytes.
-impl<Prov, Extra> Allocation<Prov, Extra> {
+impl<Prov: Provenance, Extra> Allocation<Prov, Extra> {
     pub fn len(&self) -> usize {
         self.bytes.len()
     }
@@ -349,9 +364,14 @@ impl<Prov: Provenance, Extra> Allocation<Prov, Extra> {
         cx: &impl HasDataLayout,
         range: AllocRange,
     ) -> AllocResult<&[u8]> {
-        self.check_init(range)?;
+        self.init_mask.is_range_initialized(range).map_err(|uninit_range| {
+            AllocError::InvalidUninitBytes(Some(UninitBytesAccess {
+                access: range,
+                uninit: uninit_range,
+            }))
+        })?;
         if !Prov::OFFSET_IS_ADDR {
-            if self.range_has_provenance(cx, range) {
+            if !self.provenance.range_empty(range, cx) {
                 return Err(AllocError::ReadPointerAsBytes);
             }
         }
@@ -370,7 +390,7 @@ impl<Prov: Provenance, Extra> Allocation<Prov, Extra> {
         range: AllocRange,
     ) -> AllocResult<&mut [u8]> {
         self.mark_init(range, true);
-        self.clear_provenance(cx, range)?;
+        self.provenance.clear(range, cx)?;
 
         Ok(&mut self.bytes[range.start.bytes_usize()..range.end().bytes_usize()])
     }
@@ -382,7 +402,7 @@ impl<Prov: Provenance, Extra> Allocation<Prov, Extra> {
         range: AllocRange,
     ) -> AllocResult<*mut [u8]> {
         self.mark_init(range, true);
-        self.clear_provenance(cx, range)?;
+        self.provenance.clear(range, cx)?;
 
         assert!(range.end().bytes_usize() <= self.bytes.len()); // need to do our own bounds-check
         let begin_ptr = self.bytes.as_mut_ptr().wrapping_add(range.start.bytes_usize());
@@ -393,6 +413,15 @@ impl<Prov: Provenance, Extra> Allocation<Prov, Extra> {
 
 /// Reading and writing.
 impl<Prov: Provenance, Extra> Allocation<Prov, Extra> {
+    /// Sets the init bit for the given range.
+    fn mark_init(&mut self, range: AllocRange, is_init: bool) {
+        if range.size.bytes() == 0 {
+            return;
+        }
+        assert!(self.mutability == Mutability::Mut);
+        self.init_mask.set_range(range, is_init);
+    }
+
     /// Reads a *non-ZST* scalar.
     ///
     /// If `read_provenance` is `true`, this will also read provenance; otherwise (if the machine
@@ -410,7 +439,7 @@ impl<Prov: Provenance, Extra> Allocation<Prov, Extra> {
         read_provenance: bool,
     ) -> AllocResult<Scalar<Prov>> {
         // First and foremost, if anything is uninit, bail.
-        if self.is_init(range).is_err() {
+        if self.init_mask.is_range_initialized(range).is_err() {
             return Err(AllocError::InvalidUninitBytes(None));
         }
 
@@ -423,7 +452,7 @@ impl<Prov: Provenance, Extra> Allocation<Prov, Extra> {
 
             // When reading data with provenance, the easy case is finding provenance exactly where we
             // are reading, then we can put data and provenance back together and return that.
-            if let Some(&prov) = self.provenance.get(&range.start) {
+            if let Some(prov) = self.provenance.get_ptr(range.start) {
                 // Now we can return the bits, with their appropriate provenance.
                 let ptr = Pointer::new(prov, Size::from_bytes(bits));
                 return Ok(Scalar::from_pointer(ptr, cx));
@@ -431,10 +460,9 @@ impl<Prov: Provenance, Extra> Allocation<Prov, Extra> {
 
             // If we can work on pointers byte-wise, join the byte-wise provenances.
             if Prov::OFFSET_IS_ADDR {
-                let mut prov = self.offset_get_provenance(cx, range.start);
-                for offset in 1..range.size.bytes() {
-                    let this_prov =
-                        self.offset_get_provenance(cx, range.start + Size::from_bytes(offset));
+                let mut prov = self.provenance.get(range.start, cx);
+                for offset in Size::from_bytes(1)..range.size {
+                    let this_prov = self.provenance.get(range.start + offset, cx);
                     prov = Prov::join(prov, this_prov);
                 }
                 // Now use this provenance.
@@ -452,7 +480,7 @@ impl<Prov: Provenance, Extra> Allocation<Prov, Extra> {
 
         // Fallback path for when we cannot treat provenance bytewise or ignore it.
         assert!(!Prov::OFFSET_IS_ADDR);
-        if self.range_has_provenance(cx, range) {
+        if !self.provenance.range_empty(range, cx) {
             return Err(AllocError::ReadPointerAsBytes);
         }
         // There is no provenance, we can just return the bits.
@@ -466,7 +494,6 @@ impl<Prov: Provenance, Extra> Allocation<Prov, Extra> {
     ///
     /// It is the caller's responsibility to check bounds and alignment beforehand.
     /// Most likely, you want to call `InterpCx::write_scalar` instead of this method.
-    #[instrument(skip(self, cx), level = "debug")]
     pub fn write_scalar(
         &mut self,
         cx: &impl HasDataLayout,
@@ -478,11 +505,11 @@ impl<Prov: Provenance, Extra> Allocation<Prov, Extra> {
         // `to_bits_or_ptr_internal` is the right method because we just want to store this data
         // as-is into memory.
         let (bytes, provenance) = match val.to_bits_or_ptr_internal(range.size)? {
-            Err(val) => {
-                let (provenance, offset) = val.into_parts();
+            Right(ptr) => {
+                let (provenance, offset) = ptr.into_parts();
                 (u128::from(offset.bytes()), Some(provenance))
             }
-            Ok(data) => (data, None),
+            Left(data) => (data, None),
         };
 
         let endian = cx.data_layout().endian;
@@ -491,7 +518,8 @@ impl<Prov: Provenance, Extra> Allocation<Prov, Extra> {
 
         // See if we have to also store some provenance.
         if let Some(provenance) = provenance {
-            self.provenance.0.insert(range.start, provenance);
+            assert_eq!(range.size, cx.data_layout().pointer_size);
+            self.provenance.insert_ptr(range.start, provenance, cx);
         }
 
         Ok(())
@@ -500,750 +528,25 @@ impl<Prov: Provenance, Extra> Allocation<Prov, Extra> {
     /// Write "uninit" to the given memory range.
     pub fn write_uninit(&mut self, cx: &impl HasDataLayout, range: AllocRange) -> AllocResult {
         self.mark_init(range, false);
-        self.clear_provenance(cx, range)?;
+        self.provenance.clear(range, cx)?;
         return Ok(());
     }
-}
-
-/// Provenance.
-impl<Prov: Copy, Extra> Allocation<Prov, Extra> {
-    /// Returns all provenance overlapping with the given pointer-offset pair.
-    fn range_get_provenance(&self, cx: &impl HasDataLayout, range: AllocRange) -> &[(Size, Prov)] {
-        // We have to go back `pointer_size - 1` bytes, as that one would still overlap with
-        // the beginning of this range.
-        let start = range.start.bytes().saturating_sub(cx.data_layout().pointer_size.bytes() - 1);
-        self.provenance.range(Size::from_bytes(start)..range.end())
-    }
-
-    /// Get the provenance of a single byte.
-    fn offset_get_provenance(&self, cx: &impl HasDataLayout, offset: Size) -> Option<Prov> {
-        let prov = self.range_get_provenance(cx, alloc_range(offset, Size::from_bytes(1)));
-        assert!(prov.len() <= 1);
-        prov.first().map(|(_offset, prov)| *prov)
-    }
-
-    /// Returns whether this allocation has progrnance overlapping with the given range.
-    ///
-    /// Note: this function exists to allow `range_get_provenance` to be private, in order to somewhat
-    /// limit access to provenance outside of the `Allocation` abstraction.
-    ///
-    pub fn range_has_provenance(&self, cx: &impl HasDataLayout, range: AllocRange) -> bool {
-        !self.range_get_provenance(cx, range).is_empty()
-    }
-
-    /// Removes all provenance inside the given range.
-    /// If there is provenance overlapping with the edges, it
-    /// are removed as well *and* the bytes they cover are marked as
-    /// uninitialized. This is a somewhat odd "spooky action at a distance",
-    /// but it allows strictly more code to run than if we would just error
-    /// immediately in that case.
-    fn clear_provenance(&mut self, cx: &impl HasDataLayout, range: AllocRange) -> AllocResult
-    where
-        Prov: Provenance,
-    {
-        // Find the start and end of the given range and its outermost provenance.
-        let (first, last) = {
-            // Find all provenance overlapping the given range.
-            let provenance = self.range_get_provenance(cx, range);
-            if provenance.is_empty() {
-                return Ok(());
-            }
-
-            (
-                provenance.first().unwrap().0,
-                provenance.last().unwrap().0 + cx.data_layout().pointer_size,
-            )
-        };
-        let start = range.start;
-        let end = range.end();
-
-        // We need to handle clearing the provenance from parts of a pointer.
-        // FIXME: Miri should preserve partial provenance; see
-        // https://github.com/rust-lang/miri/issues/2181.
-        if first < start {
-            if Prov::ERR_ON_PARTIAL_PTR_OVERWRITE {
-                return Err(AllocError::PartialPointerOverwrite(first));
-            }
-            warn!(
-                "Partial pointer overwrite! De-initializing memory at offsets {first:?}..{start:?}."
-            );
-            self.init_mask.set_range(first, start, false);
-        }
-        if last > end {
-            if Prov::ERR_ON_PARTIAL_PTR_OVERWRITE {
-                return Err(AllocError::PartialPointerOverwrite(
-                    last - cx.data_layout().pointer_size,
-                ));
-            }
-            warn!(
-                "Partial pointer overwrite! De-initializing memory at offsets {end:?}..{last:?}."
-            );
-            self.init_mask.set_range(end, last, false);
-        }
-
-        // Forget all the provenance.
-        // Since provenance do not overlap, we know that removing until `last` (exclusive) is fine,
-        // i.e., this will not remove any other provenance just after the ones we care about.
-        self.provenance.0.remove_range(first..last);
-
-        Ok(())
-    }
-}
-
-/// Stores the provenance information of pointers stored in memory.
-#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, TyEncodable, TyDecodable)]
-pub struct ProvenanceMap<Prov = AllocId>(SortedMap<Size, Prov>);
-
-impl<Prov> ProvenanceMap<Prov> {
-    pub fn new() -> Self {
-        ProvenanceMap(SortedMap::new())
-    }
-
-    // The caller must guarantee that the given provenance list is already sorted
-    // by address and contain no duplicates.
-    pub fn from_presorted(r: Vec<(Size, Prov)>) -> Self {
-        ProvenanceMap(SortedMap::from_presorted_elements(r))
-    }
-}
-
-impl<Prov> Deref for ProvenanceMap<Prov> {
-    type Target = SortedMap<Size, Prov>;
-
-    fn deref(&self) -> &Self::Target {
-        &self.0
-    }
-}
-
-/// A partial, owned list of provenance to transfer into another allocation.
-///
-/// Offsets are already adjusted to the destination allocation.
-pub struct AllocationProvenance<Prov> {
-    dest_provenance: Vec<(Size, Prov)>,
-}
-
-impl<Prov: Copy, Extra> Allocation<Prov, Extra> {
-    pub fn prepare_provenance_copy(
-        &self,
-        cx: &impl HasDataLayout,
-        src: AllocRange,
-        dest: Size,
-        count: u64,
-    ) -> AllocationProvenance<Prov> {
-        let provenance = self.range_get_provenance(cx, src);
-        if provenance.is_empty() {
-            return AllocationProvenance { dest_provenance: Vec::new() };
-        }
-
-        let size = src.size;
-        let mut new_provenance = Vec::with_capacity(provenance.len() * (count as usize));
-
-        // If `count` is large, this is rather wasteful -- we are allocating a big array here, which
-        // is mostly filled with redundant information since it's just N copies of the same `Prov`s
-        // at slightly adjusted offsets. The reason we do this is so that in `mark_provenance_range`
-        // we can use `insert_presorted`. That wouldn't work with an `Iterator` that just produces
-        // the right sequence of provenance for all N copies.
-        for i in 0..count {
-            new_provenance.extend(provenance.iter().map(|&(offset, reloc)| {
-                // compute offset for current repetition
-                let dest_offset = dest + size * i; // `Size` operations
-                (
-                    // shift offsets from source allocation to destination allocation
-                    (offset + dest_offset) - src.start, // `Size` operations
-                    reloc,
-                )
-            }));
-        }
-
-        AllocationProvenance { dest_provenance: new_provenance }
-    }
 
-    /// Applies a provenance copy.
-    /// The affected range, as defined in the parameters to `prepare_provenance_copy` is expected
+    /// Applies a previously prepared provenance copy.
+    /// The affected range, as defined in the parameters to `provenance().prepare_copy` is expected
     /// to be clear of provenance.
     ///
     /// This is dangerous to use as it can violate internal `Allocation` invariants!
     /// It only exists to support an efficient implementation of `mem_copy_repeatedly`.
-    pub fn mark_provenance_range(&mut self, provenance: AllocationProvenance<Prov>) {
-        self.provenance.0.insert_presorted(provenance.dest_provenance);
-    }
-}
-
-////////////////////////////////////////////////////////////////////////////////
-// Uninitialized byte tracking
-////////////////////////////////////////////////////////////////////////////////
-
-type Block = u64;
-
-/// A bitmask where each bit refers to the byte with the same index. If the bit is `true`, the byte
-/// is initialized. If it is `false` the byte is uninitialized.
-// Note: for performance reasons when interning, some of the `InitMask` fields can be partially
-// hashed. (see the `Hash` impl below for more details), so the impl is not derived.
-#[derive(Clone, Debug, Eq, PartialEq, PartialOrd, Ord, TyEncodable, TyDecodable)]
-#[derive(HashStable)]
-pub struct InitMask {
-    blocks: Vec<Block>,
-    len: Size,
-}
-
-// Const allocations are only hashed for interning. However, they can be large, making the hashing
-// expensive especially since it uses `FxHash`: it's better suited to short keys, not potentially
-// big buffers like the allocation's init mask. We can partially hash some fields when they're
-// large.
-impl hash::Hash for InitMask {
-    fn hash<H: hash::Hasher>(&self, state: &mut H) {
-        const MAX_BLOCKS_TO_HASH: usize = MAX_BYTES_TO_HASH / std::mem::size_of::<Block>();
-        const MAX_BLOCKS_LEN: usize = MAX_HASHED_BUFFER_LEN / std::mem::size_of::<Block>();
-
-        // Partially hash the `blocks` buffer when it is large. To limit collisions with common
-        // prefixes and suffixes, we hash the length and some slices of the buffer.
-        let block_count = self.blocks.len();
-        if block_count > MAX_BLOCKS_LEN {
-            // Hash the buffer's length.
-            block_count.hash(state);
-
-            // And its head and tail.
-            self.blocks[..MAX_BLOCKS_TO_HASH].hash(state);
-            self.blocks[block_count - MAX_BLOCKS_TO_HASH..].hash(state);
-        } else {
-            self.blocks.hash(state);
-        }
-
-        // Hash the other fields as usual.
-        self.len.hash(state);
-    }
-}
-
-impl InitMask {
-    pub const BLOCK_SIZE: u64 = 64;
-
-    #[inline]
-    fn bit_index(bits: Size) -> (usize, usize) {
-        // BLOCK_SIZE is the number of bits that can fit in a `Block`.
-        // Each bit in a `Block` represents the initialization state of one byte of an allocation,
-        // so we use `.bytes()` here.
-        let bits = bits.bytes();
-        let a = bits / InitMask::BLOCK_SIZE;
-        let b = bits % InitMask::BLOCK_SIZE;
-        (usize::try_from(a).unwrap(), usize::try_from(b).unwrap())
-    }
-
-    #[inline]
-    fn size_from_bit_index(block: impl TryInto<u64>, bit: impl TryInto<u64>) -> Size {
-        let block = block.try_into().ok().unwrap();
-        let bit = bit.try_into().ok().unwrap();
-        Size::from_bytes(block * InitMask::BLOCK_SIZE + bit)
-    }
-
-    pub fn new(size: Size, state: bool) -> Self {
-        let mut m = InitMask { blocks: vec![], len: Size::ZERO };
-        m.grow(size, state);
-        m
-    }
-
-    pub fn set_range(&mut self, start: Size, end: Size, new_state: bool) {
-        let len = self.len;
-        if end > len {
-            self.grow(end - len, new_state);
-        }
-        self.set_range_inbounds(start, end, new_state);
-    }
-
-    pub fn set_range_inbounds(&mut self, start: Size, end: Size, new_state: bool) {
-        let (blocka, bita) = Self::bit_index(start);
-        let (blockb, bitb) = Self::bit_index(end);
-        if blocka == blockb {
-            // First set all bits except the first `bita`,
-            // then unset the last `64 - bitb` bits.
-            let range = if bitb == 0 {
-                u64::MAX << bita
-            } else {
-                (u64::MAX << bita) & (u64::MAX >> (64 - bitb))
-            };
-            if new_state {
-                self.blocks[blocka] |= range;
-            } else {
-                self.blocks[blocka] &= !range;
-            }
-            return;
-        }
-        // across block boundaries
-        if new_state {
-            // Set `bita..64` to `1`.
-            self.blocks[blocka] |= u64::MAX << bita;
-            // Set `0..bitb` to `1`.
-            if bitb != 0 {
-                self.blocks[blockb] |= u64::MAX >> (64 - bitb);
-            }
-            // Fill in all the other blocks (much faster than one bit at a time).
-            for block in (blocka + 1)..blockb {
-                self.blocks[block] = u64::MAX;
-            }
-        } else {
-            // Set `bita..64` to `0`.
-            self.blocks[blocka] &= !(u64::MAX << bita);
-            // Set `0..bitb` to `0`.
-            if bitb != 0 {
-                self.blocks[blockb] &= !(u64::MAX >> (64 - bitb));
-            }
-            // Fill in all the other blocks (much faster than one bit at a time).
-            for block in (blocka + 1)..blockb {
-                self.blocks[block] = 0;
-            }
-        }
-    }
-
-    #[inline]
-    pub fn get(&self, i: Size) -> bool {
-        let (block, bit) = Self::bit_index(i);
-        (self.blocks[block] & (1 << bit)) != 0
+    pub fn provenance_apply_copy(&mut self, copy: ProvenanceCopy<Prov>) {
+        self.provenance.apply_copy(copy)
     }
 
-    #[inline]
-    pub fn set(&mut self, i: Size, new_state: bool) {
-        let (block, bit) = Self::bit_index(i);
-        self.set_bit(block, bit, new_state);
-    }
-
-    #[inline]
-    fn set_bit(&mut self, block: usize, bit: usize, new_state: bool) {
-        if new_state {
-            self.blocks[block] |= 1 << bit;
-        } else {
-            self.blocks[block] &= !(1 << bit);
-        }
-    }
-
-    pub fn grow(&mut self, amount: Size, new_state: bool) {
-        if amount.bytes() == 0 {
-            return;
-        }
-        let unused_trailing_bits =
-            u64::try_from(self.blocks.len()).unwrap() * Self::BLOCK_SIZE - self.len.bytes();
-        if amount.bytes() > unused_trailing_bits {
-            let additional_blocks = amount.bytes() / Self::BLOCK_SIZE + 1;
-            self.blocks.extend(
-                // FIXME(oli-obk): optimize this by repeating `new_state as Block`.
-                iter::repeat(0).take(usize::try_from(additional_blocks).unwrap()),
-            );
-        }
-        let start = self.len;
-        self.len += amount;
-        self.set_range_inbounds(start, start + amount, new_state); // `Size` operation
-    }
-
-    /// Returns the index of the first bit in `start..end` (end-exclusive) that is equal to is_init.
-    fn find_bit(&self, start: Size, end: Size, is_init: bool) -> Option<Size> {
-        /// A fast implementation of `find_bit`,
-        /// which skips over an entire block at a time if it's all 0s (resp. 1s),
-        /// and finds the first 1 (resp. 0) bit inside a block using `trailing_zeros` instead of a loop.
-        ///
-        /// Note that all examples below are written with 8 (instead of 64) bit blocks for simplicity,
-        /// and with the least significant bit (and lowest block) first:
-        /// ```text
-        ///        00000000|00000000
-        ///        ^      ^ ^      ^
-        /// index: 0      7 8      15
-        /// ```
-        /// Also, if not stated, assume that `is_init = true`, that is, we are searching for the first 1 bit.
-        fn find_bit_fast(
-            init_mask: &InitMask,
-            start: Size,
-            end: Size,
-            is_init: bool,
-        ) -> Option<Size> {
-            /// Search one block, returning the index of the first bit equal to `is_init`.
-            fn search_block(
-                bits: Block,
-                block: usize,
-                start_bit: usize,
-                is_init: bool,
-            ) -> Option<Size> {
-                // For the following examples, assume this function was called with:
-                //   bits = 0b00111011
-                //   start_bit = 3
-                //   is_init = false
-                // Note that, for the examples in this function, the most significant bit is written first,
-                // which is backwards compared to the comments in `find_bit`/`find_bit_fast`.
-
-                // Invert bits so we're always looking for the first set bit.
-                //        ! 0b00111011
-                //   bits = 0b11000100
-                let bits = if is_init { bits } else { !bits };
-                // Mask off unused start bits.
-                //          0b11000100
-                //        & 0b11111000
-                //   bits = 0b11000000
-                let bits = bits & (!0 << start_bit);
-                // Find set bit, if any.
-                //   bit = trailing_zeros(0b11000000)
-                //   bit = 6
-                if bits == 0 {
-                    None
-                } else {
-                    let bit = bits.trailing_zeros();
-                    Some(InitMask::size_from_bit_index(block, bit))
-                }
-            }
-
-            if start >= end {
-                return None;
-            }
-
-            // Convert `start` and `end` to block indexes and bit indexes within each block.
-            // We must convert `end` to an inclusive bound to handle block boundaries correctly.
-            //
-            // For example:
-            //
-            //   (a) 00000000|00000000    (b) 00000000|
-            //       ^~~~~~~~~~~^             ^~~~~~~~~^
-            //     start       end          start     end
-            //
-            // In both cases, the block index of `end` is 1.
-            // But we do want to search block 1 in (a), and we don't in (b).
-            //
-            // We subtract 1 from both end positions to make them inclusive:
-            //
-            //   (a) 00000000|00000000    (b) 00000000|
-            //       ^~~~~~~~~~^              ^~~~~~~^
-            //     start    end_inclusive   start end_inclusive
-            //
-            // For (a), the block index of `end_inclusive` is 1, and for (b), it's 0.
-            // This provides the desired behavior of searching blocks 0 and 1 for (a),
-            // and searching only block 0 for (b).
-            // There is no concern of overflows since we checked for `start >= end` above.
-            let (start_block, start_bit) = InitMask::bit_index(start);
-            let end_inclusive = Size::from_bytes(end.bytes() - 1);
-            let (end_block_inclusive, _) = InitMask::bit_index(end_inclusive);
-
-            // Handle first block: need to skip `start_bit` bits.
-            //
-            // We need to handle the first block separately,
-            // because there may be bits earlier in the block that should be ignored,
-            // such as the bit marked (1) in this example:
-            //
-            //       (1)
-            //       -|------
-            //   (c) 01000000|00000000|00000001
-            //          ^~~~~~~~~~~~~~~~~~^
-            //        start              end
-            if let Some(i) =
-                search_block(init_mask.blocks[start_block], start_block, start_bit, is_init)
-            {
-                // If the range is less than a block, we may find a matching bit after `end`.
-                //
-                // For example, we shouldn't successfully find bit (2), because it's after `end`:
-                //
-                //             (2)
-                //       -------|
-                //   (d) 00000001|00000000|00000001
-                //        ^~~~~^
-                //      start end
-                //
-                // An alternative would be to mask off end bits in the same way as we do for start bits,
-                // but performing this check afterwards is faster and simpler to implement.
-                if i < end {
-                    return Some(i);
-                } else {
-                    return None;
-                }
-            }
-
-            // Handle remaining blocks.
-            //
-            // We can skip over an entire block at once if it's all 0s (resp. 1s).
-            // The block marked (3) in this example is the first block that will be handled by this loop,
-            // and it will be skipped for that reason:
-            //
-            //                   (3)
-            //                --------
-            //   (e) 01000000|00000000|00000001
-            //          ^~~~~~~~~~~~~~~~~~^
-            //        start              end
-            if start_block < end_block_inclusive {
-                // This loop is written in a specific way for performance.
-                // Notably: `..end_block_inclusive + 1` is used for an inclusive range instead of `..=end_block_inclusive`,
-                // and `.zip(start_block + 1..)` is used to track the index instead of `.enumerate().skip().take()`,
-                // because both alternatives result in significantly worse codegen.
-                // `end_block_inclusive + 1` is guaranteed not to wrap, because `end_block_inclusive <= end / BLOCK_SIZE`,
-                // and `BLOCK_SIZE` (the number of bits per block) will always be at least 8 (1 byte).
-                for (&bits, block) in init_mask.blocks[start_block + 1..end_block_inclusive + 1]
-                    .iter()
-                    .zip(start_block + 1..)
-                {
-                    if let Some(i) = search_block(bits, block, 0, is_init) {
-                        // If this is the last block, we may find a matching bit after `end`.
-                        //
-                        // For example, we shouldn't successfully find bit (4), because it's after `end`:
-                        //
-                        //                               (4)
-                        //                         -------|
-                        //   (f) 00000001|00000000|00000001
-                        //          ^~~~~~~~~~~~~~~~~~^
-                        //        start              end
-                        //
-                        // As above with example (d), we could handle the end block separately and mask off end bits,
-                        // but unconditionally searching an entire block at once and performing this check afterwards
-                        // is faster and much simpler to implement.
-                        if i < end {
-                            return Some(i);
-                        } else {
-                            return None;
-                        }
-                    }
-                }
-            }
-
-            None
-        }
-
-        #[cfg_attr(not(debug_assertions), allow(dead_code))]
-        fn find_bit_slow(
-            init_mask: &InitMask,
-            start: Size,
-            end: Size,
-            is_init: bool,
-        ) -> Option<Size> {
-            (start..end).find(|&i| init_mask.get(i) == is_init)
-        }
-
-        let result = find_bit_fast(self, start, end, is_init);
-
-        debug_assert_eq!(
-            result,
-            find_bit_slow(self, start, end, is_init),
-            "optimized implementation of find_bit is wrong for start={:?} end={:?} is_init={} init_mask={:#?}",
-            start,
-            end,
-            is_init,
-            self
-        );
-
-        result
-    }
-}
-
-/// A contiguous chunk of initialized or uninitialized memory.
-pub enum InitChunk {
-    Init(Range<Size>),
-    Uninit(Range<Size>),
-}
-
-impl InitChunk {
-    #[inline]
-    pub fn is_init(&self) -> bool {
-        match self {
-            Self::Init(_) => true,
-            Self::Uninit(_) => false,
-        }
-    }
-
-    #[inline]
-    pub fn range(&self) -> Range<Size> {
-        match self {
-            Self::Init(r) => r.clone(),
-            Self::Uninit(r) => r.clone(),
-        }
-    }
-}
-
-impl InitMask {
-    /// Checks whether the range `start..end` (end-exclusive) is entirely initialized.
-    ///
-    /// Returns `Ok(())` if it's initialized. Otherwise returns a range of byte
-    /// indexes for the first contiguous span of the uninitialized access.
-    #[inline]
-    pub fn is_range_initialized(&self, start: Size, end: Size) -> Result<(), AllocRange> {
-        if end > self.len {
-            return Err(AllocRange::from(self.len..end));
-        }
-
-        let uninit_start = self.find_bit(start, end, false);
-
-        match uninit_start {
-            Some(uninit_start) => {
-                let uninit_end = self.find_bit(uninit_start, end, true).unwrap_or(end);
-                Err(AllocRange::from(uninit_start..uninit_end))
-            }
-            None => Ok(()),
-        }
-    }
-
-    /// Returns an iterator, yielding a range of byte indexes for each contiguous region
-    /// of initialized or uninitialized bytes inside the range `start..end` (end-exclusive).
-    ///
-    /// The iterator guarantees the following:
-    /// - Chunks are nonempty.
-    /// - Chunks are adjacent (each range's start is equal to the previous range's end).
-    /// - Chunks span exactly `start..end` (the first starts at `start`, the last ends at `end`).
-    /// - Chunks alternate between [`InitChunk::Init`] and [`InitChunk::Uninit`].
-    #[inline]
-    pub fn range_as_init_chunks(&self, start: Size, end: Size) -> InitChunkIter<'_> {
-        assert!(end <= self.len);
-
-        let is_init = if start < end {
-            self.get(start)
-        } else {
-            // `start..end` is empty: there are no chunks, so use some arbitrary value
-            false
-        };
-
-        InitChunkIter { init_mask: self, is_init, start, end }
-    }
-}
-
-/// Yields [`InitChunk`]s. See [`InitMask::range_as_init_chunks`].
-#[derive(Clone)]
-pub struct InitChunkIter<'a> {
-    init_mask: &'a InitMask,
-    /// Whether the next chunk we will return is initialized.
-    /// If there are no more chunks, contains some arbitrary value.
-    is_init: bool,
-    /// The current byte index into `init_mask`.
-    start: Size,
-    /// The end byte index into `init_mask`.
-    end: Size,
-}
-
-impl<'a> Iterator for InitChunkIter<'a> {
-    type Item = InitChunk;
-
-    #[inline]
-    fn next(&mut self) -> Option<Self::Item> {
-        if self.start >= self.end {
-            return None;
-        }
-
-        let end_of_chunk =
-            self.init_mask.find_bit(self.start, self.end, !self.is_init).unwrap_or(self.end);
-        let range = self.start..end_of_chunk;
-
-        let ret =
-            Some(if self.is_init { InitChunk::Init(range) } else { InitChunk::Uninit(range) });
-
-        self.is_init = !self.is_init;
-        self.start = end_of_chunk;
-
-        ret
-    }
-}
-
-/// Uninitialized bytes.
-impl<Prov: Copy, Extra> Allocation<Prov, Extra> {
-    /// Checks whether the given range  is entirely initialized.
-    ///
-    /// Returns `Ok(())` if it's initialized. Otherwise returns the range of byte
-    /// indexes of the first contiguous uninitialized access.
-    fn is_init(&self, range: AllocRange) -> Result<(), AllocRange> {
-        self.init_mask.is_range_initialized(range.start, range.end()) // `Size` addition
-    }
-
-    /// Checks that a range of bytes is initialized. If not, returns the `InvalidUninitBytes`
-    /// error which will report the first range of bytes which is uninitialized.
-    fn check_init(&self, range: AllocRange) -> AllocResult {
-        self.is_init(range).map_err(|uninit_range| {
-            AllocError::InvalidUninitBytes(Some(UninitBytesAccess {
-                access: range,
-                uninit: uninit_range,
-            }))
-        })
-    }
-
-    fn mark_init(&mut self, range: AllocRange, is_init: bool) {
-        if range.size.bytes() == 0 {
-            return;
-        }
-        assert!(self.mutability == Mutability::Mut);
-        self.init_mask.set_range(range.start, range.end(), is_init);
-    }
-}
-
-/// Run-length encoding of the uninit mask.
-/// Used to copy parts of a mask multiple times to another allocation.
-pub struct InitMaskCompressed {
-    /// Whether the first range is initialized.
-    initial: bool,
-    /// The lengths of ranges that are run-length encoded.
-    /// The initialization state of the ranges alternate starting with `initial`.
-    ranges: smallvec::SmallVec<[u64; 1]>,
-}
-
-impl InitMaskCompressed {
-    pub fn no_bytes_init(&self) -> bool {
-        // The `ranges` are run-length encoded and of alternating initialization state.
-        // So if `ranges.len() > 1` then the second block is an initialized range.
-        !self.initial && self.ranges.len() == 1
-    }
-}
-
-/// Transferring the initialization mask to other allocations.
-impl<Prov, Extra> Allocation<Prov, Extra> {
-    /// Creates a run-length encoding of the initialization mask; panics if range is empty.
-    ///
-    /// This is essentially a more space-efficient version of
-    /// `InitMask::range_as_init_chunks(...).collect::<Vec<_>>()`.
-    pub fn compress_uninit_range(&self, range: AllocRange) -> InitMaskCompressed {
-        // Since we are copying `size` bytes from `src` to `dest + i * size` (`for i in 0..repeat`),
-        // a naive initialization mask copying algorithm would repeatedly have to read the initialization mask from
-        // the source and write it to the destination. Even if we optimized the memory accesses,
-        // we'd be doing all of this `repeat` times.
-        // Therefore we precompute a compressed version of the initialization mask of the source value and
-        // then write it back `repeat` times without computing any more information from the source.
-
-        // A precomputed cache for ranges of initialized / uninitialized bits
-        // 0000010010001110 will become
-        // `[5, 1, 2, 1, 3, 3, 1]`,
-        // where each element toggles the state.
-
-        let mut ranges = smallvec::SmallVec::<[u64; 1]>::new();
-
-        let mut chunks = self.init_mask.range_as_init_chunks(range.start, range.end()).peekable();
-
-        let initial = chunks.peek().expect("range should be nonempty").is_init();
-
-        // Here we rely on `range_as_init_chunks` to yield alternating init/uninit chunks.
-        for chunk in chunks {
-            let len = chunk.range().end.bytes() - chunk.range().start.bytes();
-            ranges.push(len);
-        }
-
-        InitMaskCompressed { ranges, initial }
-    }
-
-    /// Applies multiple instances of the run-length encoding to the initialization mask.
+    /// Applies a previously prepared copy of the init mask.
     ///
     /// This is dangerous to use as it can violate internal `Allocation` invariants!
     /// It only exists to support an efficient implementation of `mem_copy_repeatedly`.
-    pub fn mark_compressed_init_range(
-        &mut self,
-        defined: &InitMaskCompressed,
-        range: AllocRange,
-        repeat: u64,
-    ) {
-        // An optimization where we can just overwrite an entire range of initialization
-        // bits if they are going to be uniformly `1` or `0`.
-        if defined.ranges.len() <= 1 {
-            self.init_mask.set_range_inbounds(
-                range.start,
-                range.start + range.size * repeat, // `Size` operations
-                defined.initial,
-            );
-            return;
-        }
-
-        for mut j in 0..repeat {
-            j *= range.size.bytes();
-            j += range.start.bytes();
-            let mut cur = defined.initial;
-            for range in &defined.ranges {
-                let old_j = j;
-                j += range;
-                self.init_mask.set_range_inbounds(
-                    Size::from_bytes(old_j),
-                    Size::from_bytes(j),
-                    cur,
-                );
-                cur = !cur;
-            }
-        }
+    pub fn init_mask_apply_copy(&mut self, copy: InitCopy, range: AllocRange, repeat: u64) {
+        self.init_mask.apply_copy(copy, range, repeat)
     }
 }
diff --git a/compiler/rustc_middle/src/mir/interpret/allocation/init_mask.rs b/compiler/rustc_middle/src/mir/interpret/allocation/init_mask.rs
new file mode 100644
index 000000000..82e9a961a
--- /dev/null
+++ b/compiler/rustc_middle/src/mir/interpret/allocation/init_mask.rs
@@ -0,0 +1,530 @@
+use std::hash;
+use std::iter;
+use std::ops::Range;
+
+use rustc_target::abi::Size;
+
+use super::AllocRange;
+
+type Block = u64;
+
+/// A bitmask where each bit refers to the byte with the same index. If the bit is `true`, the byte
+/// is initialized. If it is `false` the byte is uninitialized.
+// Note: for performance reasons when interning, some of the `InitMask` fields can be partially
+// hashed. (see the `Hash` impl below for more details), so the impl is not derived.
+#[derive(Clone, Debug, Eq, PartialEq, TyEncodable, TyDecodable)]
+#[derive(HashStable)]
+pub struct InitMask {
+    blocks: Vec<Block>,
+    len: Size,
+}
+
+// Const allocations are only hashed for interning. However, they can be large, making the hashing
+// expensive especially since it uses `FxHash`: it's better suited to short keys, not potentially
+// big buffers like the allocation's init mask. We can partially hash some fields when they're
+// large.
+impl hash::Hash for InitMask {
+    fn hash<H: hash::Hasher>(&self, state: &mut H) {
+        const MAX_BLOCKS_TO_HASH: usize = super::MAX_BYTES_TO_HASH / std::mem::size_of::<Block>();
+        const MAX_BLOCKS_LEN: usize = super::MAX_HASHED_BUFFER_LEN / std::mem::size_of::<Block>();
+
+        // Partially hash the `blocks` buffer when it is large. To limit collisions with common
+        // prefixes and suffixes, we hash the length and some slices of the buffer.
+        let block_count = self.blocks.len();
+        if block_count > MAX_BLOCKS_LEN {
+            // Hash the buffer's length.
+            block_count.hash(state);
+
+            // And its head and tail.
+            self.blocks[..MAX_BLOCKS_TO_HASH].hash(state);
+            self.blocks[block_count - MAX_BLOCKS_TO_HASH..].hash(state);
+        } else {
+            self.blocks.hash(state);
+        }
+
+        // Hash the other fields as usual.
+        self.len.hash(state);
+    }
+}
+
+impl InitMask {
+    pub const BLOCK_SIZE: u64 = 64;
+
+    pub fn new(size: Size, state: bool) -> Self {
+        let mut m = InitMask { blocks: vec![], len: Size::ZERO };
+        m.grow(size, state);
+        m
+    }
+
+    #[inline]
+    fn bit_index(bits: Size) -> (usize, usize) {
+        // BLOCK_SIZE is the number of bits that can fit in a `Block`.
+        // Each bit in a `Block` represents the initialization state of one byte of an allocation,
+        // so we use `.bytes()` here.
+        let bits = bits.bytes();
+        let a = bits / InitMask::BLOCK_SIZE;
+        let b = bits % InitMask::BLOCK_SIZE;
+        (usize::try_from(a).unwrap(), usize::try_from(b).unwrap())
+    }
+
+    #[inline]
+    fn size_from_bit_index(block: impl TryInto<u64>, bit: impl TryInto<u64>) -> Size {
+        let block = block.try_into().ok().unwrap();
+        let bit = bit.try_into().ok().unwrap();
+        Size::from_bytes(block * InitMask::BLOCK_SIZE + bit)
+    }
+
+    /// Checks whether the `range` is entirely initialized.
+    ///
+    /// Returns `Ok(())` if it's initialized. Otherwise returns a range of byte
+    /// indexes for the first contiguous span of the uninitialized access.
+    #[inline]
+    pub fn is_range_initialized(&self, range: AllocRange) -> Result<(), AllocRange> {
+        let end = range.end();
+        if end > self.len {
+            return Err(AllocRange::from(self.len..end));
+        }
+
+        let uninit_start = self.find_bit(range.start, end, false);
+
+        match uninit_start {
+            Some(uninit_start) => {
+                let uninit_end = self.find_bit(uninit_start, end, true).unwrap_or(end);
+                Err(AllocRange::from(uninit_start..uninit_end))
+            }
+            None => Ok(()),
+        }
+    }
+
+    pub fn set_range(&mut self, range: AllocRange, new_state: bool) {
+        let end = range.end();
+        let len = self.len;
+        if end > len {
+            self.grow(end - len, new_state);
+        }
+        self.set_range_inbounds(range.start, end, new_state);
+    }
+
+    fn set_range_inbounds(&mut self, start: Size, end: Size, new_state: bool) {
+        let (blocka, bita) = Self::bit_index(start);
+        let (blockb, bitb) = Self::bit_index(end);
+        if blocka == blockb {
+            // First set all bits except the first `bita`,
+            // then unset the last `64 - bitb` bits.
+            let range = if bitb == 0 {
+                u64::MAX << bita
+            } else {
+                (u64::MAX << bita) & (u64::MAX >> (64 - bitb))
+            };
+            if new_state {
+                self.blocks[blocka] |= range;
+            } else {
+                self.blocks[blocka] &= !range;
+            }
+            return;
+        }
+        // across block boundaries
+        if new_state {
+            // Set `bita..64` to `1`.
+            self.blocks[blocka] |= u64::MAX << bita;
+            // Set `0..bitb` to `1`.
+            if bitb != 0 {
+                self.blocks[blockb] |= u64::MAX >> (64 - bitb);
+            }
+            // Fill in all the other blocks (much faster than one bit at a time).
+            for block in (blocka + 1)..blockb {
+                self.blocks[block] = u64::MAX;
+            }
+        } else {
+            // Set `bita..64` to `0`.
+            self.blocks[blocka] &= !(u64::MAX << bita);
+            // Set `0..bitb` to `0`.
+            if bitb != 0 {
+                self.blocks[blockb] &= !(u64::MAX >> (64 - bitb));
+            }
+            // Fill in all the other blocks (much faster than one bit at a time).
+            for block in (blocka + 1)..blockb {
+                self.blocks[block] = 0;
+            }
+        }
+    }
+
+    #[inline]
+    pub fn get(&self, i: Size) -> bool {
+        let (block, bit) = Self::bit_index(i);
+        (self.blocks[block] & (1 << bit)) != 0
+    }
+
+    fn grow(&mut self, amount: Size, new_state: bool) {
+        if amount.bytes() == 0 {
+            return;
+        }
+        let unused_trailing_bits =
+            u64::try_from(self.blocks.len()).unwrap() * Self::BLOCK_SIZE - self.len.bytes();
+        if amount.bytes() > unused_trailing_bits {
+            let additional_blocks = amount.bytes() / Self::BLOCK_SIZE + 1;
+            self.blocks.extend(
+                // FIXME(oli-obk): optimize this by repeating `new_state as Block`.
+                iter::repeat(0).take(usize::try_from(additional_blocks).unwrap()),
+            );
+        }
+        let start = self.len;
+        self.len += amount;
+        self.set_range_inbounds(start, start + amount, new_state); // `Size` operation
+    }
+
+    /// Returns the index of the first bit in `start..end` (end-exclusive) that is equal to is_init.
+    fn find_bit(&self, start: Size, end: Size, is_init: bool) -> Option<Size> {
+        /// A fast implementation of `find_bit`,
+        /// which skips over an entire block at a time if it's all 0s (resp. 1s),
+        /// and finds the first 1 (resp. 0) bit inside a block using `trailing_zeros` instead of a loop.
+        ///
+        /// Note that all examples below are written with 8 (instead of 64) bit blocks for simplicity,
+        /// and with the least significant bit (and lowest block) first:
+        /// ```text
+        ///        00000000|00000000
+        ///        ^      ^ ^      ^
+        /// index: 0      7 8      15
+        /// ```
+        /// Also, if not stated, assume that `is_init = true`, that is, we are searching for the first 1 bit.
+        fn find_bit_fast(
+            init_mask: &InitMask,
+            start: Size,
+            end: Size,
+            is_init: bool,
+        ) -> Option<Size> {
+            /// Search one block, returning the index of the first bit equal to `is_init`.
+            fn search_block(
+                bits: Block,
+                block: usize,
+                start_bit: usize,
+                is_init: bool,
+            ) -> Option<Size> {
+                // For the following examples, assume this function was called with:
+                //   bits = 0b00111011
+                //   start_bit = 3
+                //   is_init = false
+                // Note that, for the examples in this function, the most significant bit is written first,
+                // which is backwards compared to the comments in `find_bit`/`find_bit_fast`.
+
+                // Invert bits so we're always looking for the first set bit.
+                //        ! 0b00111011
+                //   bits = 0b11000100
+                let bits = if is_init { bits } else { !bits };
+                // Mask off unused start bits.
+                //          0b11000100
+                //        & 0b11111000
+                //   bits = 0b11000000
+                let bits = bits & (!0 << start_bit);
+                // Find set bit, if any.
+                //   bit = trailing_zeros(0b11000000)
+                //   bit = 6
+                if bits == 0 {
+                    None
+                } else {
+                    let bit = bits.trailing_zeros();
+                    Some(InitMask::size_from_bit_index(block, bit))
+                }
+            }
+
+            if start >= end {
+                return None;
+            }
+
+            // Convert `start` and `end` to block indexes and bit indexes within each block.
+            // We must convert `end` to an inclusive bound to handle block boundaries correctly.
+            //
+            // For example:
+            //
+            //   (a) 00000000|00000000    (b) 00000000|
+            //       ^~~~~~~~~~~^             ^~~~~~~~~^
+            //     start       end          start     end
+            //
+            // In both cases, the block index of `end` is 1.
+            // But we do want to search block 1 in (a), and we don't in (b).
+            //
+            // We subtract 1 from both end positions to make them inclusive:
+            //
+            //   (a) 00000000|00000000    (b) 00000000|
+            //       ^~~~~~~~~~^              ^~~~~~~^
+            //     start    end_inclusive   start end_inclusive
+            //
+            // For (a), the block index of `end_inclusive` is 1, and for (b), it's 0.
+            // This provides the desired behavior of searching blocks 0 and 1 for (a),
+            // and searching only block 0 for (b).
+            // There is no concern of overflows since we checked for `start >= end` above.
+            let (start_block, start_bit) = InitMask::bit_index(start);
+            let end_inclusive = Size::from_bytes(end.bytes() - 1);
+            let (end_block_inclusive, _) = InitMask::bit_index(end_inclusive);
+
+            // Handle first block: need to skip `start_bit` bits.
+            //
+            // We need to handle the first block separately,
+            // because there may be bits earlier in the block that should be ignored,
+            // such as the bit marked (1) in this example:
+            //
+            //       (1)
+            //       -|------
+            //   (c) 01000000|00000000|00000001
+            //          ^~~~~~~~~~~~~~~~~~^
+            //        start              end
+            if let Some(i) =
+                search_block(init_mask.blocks[start_block], start_block, start_bit, is_init)
+            {
+                // If the range is less than a block, we may find a matching bit after `end`.
+                //
+                // For example, we shouldn't successfully find bit (2), because it's after `end`:
+                //
+                //             (2)
+                //       -------|
+                //   (d) 00000001|00000000|00000001
+                //        ^~~~~^
+                //      start end
+                //
+                // An alternative would be to mask off end bits in the same way as we do for start bits,
+                // but performing this check afterwards is faster and simpler to implement.
+                if i < end {
+                    return Some(i);
+                } else {
+                    return None;
+                }
+            }
+
+            // Handle remaining blocks.
+            //
+            // We can skip over an entire block at once if it's all 0s (resp. 1s).
+            // The block marked (3) in this example is the first block that will be handled by this loop,
+            // and it will be skipped for that reason:
+            //
+            //                   (3)
+            //                --------
+            //   (e) 01000000|00000000|00000001
+            //          ^~~~~~~~~~~~~~~~~~^
+            //        start              end
+            if start_block < end_block_inclusive {
+                // This loop is written in a specific way for performance.
+                // Notably: `..end_block_inclusive + 1` is used for an inclusive range instead of `..=end_block_inclusive`,
+                // and `.zip(start_block + 1..)` is used to track the index instead of `.enumerate().skip().take()`,
+                // because both alternatives result in significantly worse codegen.
+                // `end_block_inclusive + 1` is guaranteed not to wrap, because `end_block_inclusive <= end / BLOCK_SIZE`,
+                // and `BLOCK_SIZE` (the number of bits per block) will always be at least 8 (1 byte).
+                for (&bits, block) in init_mask.blocks[start_block + 1..end_block_inclusive + 1]
+                    .iter()
+                    .zip(start_block + 1..)
+                {
+                    if let Some(i) = search_block(bits, block, 0, is_init) {
+                        // If this is the last block, we may find a matching bit after `end`.
+                        //
+                        // For example, we shouldn't successfully find bit (4), because it's after `end`:
+                        //
+                        //                               (4)
+                        //                         -------|
+                        //   (f) 00000001|00000000|00000001
+                        //          ^~~~~~~~~~~~~~~~~~^
+                        //        start              end
+                        //
+                        // As above with example (d), we could handle the end block separately and mask off end bits,
+                        // but unconditionally searching an entire block at once and performing this check afterwards
+                        // is faster and much simpler to implement.
+                        if i < end {
+                            return Some(i);
+                        } else {
+                            return None;
+                        }
+                    }
+                }
+            }
+
+            None
+        }
+
+        #[cfg_attr(not(debug_assertions), allow(dead_code))]
+        fn find_bit_slow(
+            init_mask: &InitMask,
+            start: Size,
+            end: Size,
+            is_init: bool,
+        ) -> Option<Size> {
+            (start..end).find(|&i| init_mask.get(i) == is_init)
+        }
+
+        let result = find_bit_fast(self, start, end, is_init);
+
+        debug_assert_eq!(
+            result,
+            find_bit_slow(self, start, end, is_init),
+            "optimized implementation of find_bit is wrong for start={:?} end={:?} is_init={} init_mask={:#?}",
+            start,
+            end,
+            is_init,
+            self
+        );
+
+        result
+    }
+}
+
+/// A contiguous chunk of initialized or uninitialized memory.
+pub enum InitChunk {
+    Init(Range<Size>),
+    Uninit(Range<Size>),
+}
+
+impl InitChunk {
+    #[inline]
+    pub fn is_init(&self) -> bool {
+        match self {
+            Self::Init(_) => true,
+            Self::Uninit(_) => false,
+        }
+    }
+
+    #[inline]
+    pub fn range(&self) -> Range<Size> {
+        match self {
+            Self::Init(r) => r.clone(),
+            Self::Uninit(r) => r.clone(),
+        }
+    }
+}
+
+impl InitMask {
+    /// Returns an iterator, yielding a range of byte indexes for each contiguous region
+    /// of initialized or uninitialized bytes inside the range `start..end` (end-exclusive).
+    ///
+    /// The iterator guarantees the following:
+    /// - Chunks are nonempty.
+    /// - Chunks are adjacent (each range's start is equal to the previous range's end).
+    /// - Chunks span exactly `start..end` (the first starts at `start`, the last ends at `end`).
+    /// - Chunks alternate between [`InitChunk::Init`] and [`InitChunk::Uninit`].
+    #[inline]
+    pub fn range_as_init_chunks(&self, range: AllocRange) -> InitChunkIter<'_> {
+        let start = range.start;
+        let end = range.end();
+        assert!(end <= self.len);
+
+        let is_init = if start < end {
+            self.get(start)
+        } else {
+            // `start..end` is empty: there are no chunks, so use some arbitrary value
+            false
+        };
+
+        InitChunkIter { init_mask: self, is_init, start, end }
+    }
+}
+
+/// Yields [`InitChunk`]s. See [`InitMask::range_as_init_chunks`].
+#[derive(Clone)]
+pub struct InitChunkIter<'a> {
+    init_mask: &'a InitMask,
+    /// Whether the next chunk we will return is initialized.
+    /// If there are no more chunks, contains some arbitrary value.
+    is_init: bool,
+    /// The current byte index into `init_mask`.
+    start: Size,
+    /// The end byte index into `init_mask`.
+    end: Size,
+}
+
+impl<'a> Iterator for InitChunkIter<'a> {
+    type Item = InitChunk;
+
+    #[inline]
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.start >= self.end {
+            return None;
+        }
+
+        let end_of_chunk =
+            self.init_mask.find_bit(self.start, self.end, !self.is_init).unwrap_or(self.end);
+        let range = self.start..end_of_chunk;
+
+        let ret =
+            Some(if self.is_init { InitChunk::Init(range) } else { InitChunk::Uninit(range) });
+
+        self.is_init = !self.is_init;
+        self.start = end_of_chunk;
+
+        ret
+    }
+}
+
+/// Run-length encoding of the uninit mask.
+/// Used to copy parts of a mask multiple times to another allocation.
+pub struct InitCopy {
+    /// Whether the first range is initialized.
+    initial: bool,
+    /// The lengths of ranges that are run-length encoded.
+    /// The initialization state of the ranges alternate starting with `initial`.
+    ranges: smallvec::SmallVec<[u64; 1]>,
+}
+
+impl InitCopy {
+    pub fn no_bytes_init(&self) -> bool {
+        // The `ranges` are run-length encoded and of alternating initialization state.
+        // So if `ranges.len() > 1` then the second block is an initialized range.
+        !self.initial && self.ranges.len() == 1
+    }
+}
+
+/// Transferring the initialization mask to other allocations.
+impl InitMask {
+    /// Creates a run-length encoding of the initialization mask; panics if range is empty.
+    ///
+    /// This is essentially a more space-efficient version of
+    /// `InitMask::range_as_init_chunks(...).collect::<Vec<_>>()`.
+    pub fn prepare_copy(&self, range: AllocRange) -> InitCopy {
+        // Since we are copying `size` bytes from `src` to `dest + i * size` (`for i in 0..repeat`),
+        // a naive initialization mask copying algorithm would repeatedly have to read the initialization mask from
+        // the source and write it to the destination. Even if we optimized the memory accesses,
+        // we'd be doing all of this `repeat` times.
+        // Therefore we precompute a compressed version of the initialization mask of the source value and
+        // then write it back `repeat` times without computing any more information from the source.
+
+        // A precomputed cache for ranges of initialized / uninitialized bits
+        // 0000010010001110 will become
+        // `[5, 1, 2, 1, 3, 3, 1]`,
+        // where each element toggles the state.
+
+        let mut ranges = smallvec::SmallVec::<[u64; 1]>::new();
+
+        let mut chunks = self.range_as_init_chunks(range).peekable();
+
+        let initial = chunks.peek().expect("range should be nonempty").is_init();
+
+        // Here we rely on `range_as_init_chunks` to yield alternating init/uninit chunks.
+        for chunk in chunks {
+            let len = chunk.range().end.bytes() - chunk.range().start.bytes();
+            ranges.push(len);
+        }
+
+        InitCopy { ranges, initial }
+    }
+
+    /// Applies multiple instances of the run-length encoding to the initialization mask.
+    pub fn apply_copy(&mut self, defined: InitCopy, range: AllocRange, repeat: u64) {
+        // An optimization where we can just overwrite an entire range of initialization
+        // bits if they are going to be uniformly `1` or `0`.
+        if defined.ranges.len() <= 1 {
+            self.set_range_inbounds(
+                range.start,
+                range.start + range.size * repeat, // `Size` operations
+                defined.initial,
+            );
+            return;
+        }
+
+        for mut j in 0..repeat {
+            j *= range.size.bytes();
+            j += range.start.bytes();
+            let mut cur = defined.initial;
+            for range in &defined.ranges {
+                let old_j = j;
+                j += range;
+                self.set_range_inbounds(Size::from_bytes(old_j), Size::from_bytes(j), cur);
+                cur = !cur;
+            }
+        }
+    }
+}
diff --git a/compiler/rustc_middle/src/mir/interpret/allocation/provenance_map.rs b/compiler/rustc_middle/src/mir/interpret/allocation/provenance_map.rs
new file mode 100644
index 000000000..ddd3f3943
--- /dev/null
+++ b/compiler/rustc_middle/src/mir/interpret/allocation/provenance_map.rs
@@ -0,0 +1,321 @@
+//! Store the provenance for each byte in the range, with a more efficient
+//! representation for the common case where PTR_SIZE consecutive bytes have the same provenance.
+
+use std::cmp;
+
+use rustc_data_structures::sorted_map::SortedMap;
+use rustc_target::abi::{HasDataLayout, Size};
+
+use super::{alloc_range, AllocError, AllocId, AllocRange, AllocResult, Provenance};
+use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
+
+/// Stores the provenance information of pointers stored in memory.
+#[derive(Clone, PartialEq, Eq, Hash, Debug)]
+#[derive(HashStable)]
+pub struct ProvenanceMap<Prov = AllocId> {
+    /// Provenance in this map applies from the given offset for an entire pointer-size worth of
+    /// bytes. Two entires in this map are always at least a pointer size apart.
+    ptrs: SortedMap<Size, Prov>,
+    /// Provenance in this map only applies to the given single byte.
+    /// This map is disjoint from the previous. It will always be empty when
+    /// `Prov::OFFSET_IS_ADDR` is false.
+    bytes: Option<Box<SortedMap<Size, Prov>>>,
+}
+
+impl<D: Decoder, Prov: Provenance + Decodable<D>> Decodable<D> for ProvenanceMap<Prov> {
+    fn decode(d: &mut D) -> Self {
+        assert!(!Prov::OFFSET_IS_ADDR); // only `AllocId` is ever serialized
+        Self { ptrs: Decodable::decode(d), bytes: None }
+    }
+}
+
+impl<S: Encoder, Prov: Provenance + Encodable<S>> Encodable<S> for ProvenanceMap<Prov> {
+    fn encode(&self, s: &mut S) {
+        let Self { ptrs, bytes } = self;
+        assert!(!Prov::OFFSET_IS_ADDR); // only `AllocId` is ever serialized
+        debug_assert!(bytes.is_none());
+        ptrs.encode(s)
+    }
+}
+
+impl<Prov> ProvenanceMap<Prov> {
+    pub fn new() -> Self {
+        ProvenanceMap { ptrs: SortedMap::new(), bytes: None }
+    }
+
+    /// The caller must guarantee that the given provenance list is already sorted
+    /// by address and contain no duplicates.
+    pub fn from_presorted_ptrs(r: Vec<(Size, Prov)>) -> Self {
+        ProvenanceMap { ptrs: SortedMap::from_presorted_elements(r), bytes: None }
+    }
+}
+
+impl ProvenanceMap {
+    /// Give access to the ptr-sized provenances (which can also be thought of as relocations, and
+    /// indeed that is how codegen treats them).
+    ///
+    /// Only exposed with `AllocId` provenance, since it panics if there is bytewise provenance.
+    #[inline]
+    pub fn ptrs(&self) -> &SortedMap<Size, AllocId> {
+        debug_assert!(self.bytes.is_none()); // `AllocId::OFFSET_IS_ADDR` is false so this cannot fail
+        &self.ptrs
+    }
+}
+
+impl<Prov: Provenance> ProvenanceMap<Prov> {
+    /// Returns all ptr-sized provenance in the given range.
+    /// If the range has length 0, returns provenance that crosses the edge between `start-1` and
+    /// `start`.
+    fn range_get_ptrs(&self, range: AllocRange, cx: &impl HasDataLayout) -> &[(Size, Prov)] {
+        // We have to go back `pointer_size - 1` bytes, as that one would still overlap with
+        // the beginning of this range.
+        let adjusted_start = Size::from_bytes(
+            range.start.bytes().saturating_sub(cx.data_layout().pointer_size.bytes() - 1),
+        );
+        self.ptrs.range(adjusted_start..range.end())
+    }
+
+    /// Returns all byte-wise provenance in the given range.
+    fn range_get_bytes(&self, range: AllocRange) -> &[(Size, Prov)] {
+        if let Some(bytes) = self.bytes.as_ref() {
+            bytes.range(range.start..range.end())
+        } else {
+            &[]
+        }
+    }
+
+    /// Get the provenance of a single byte.
+    pub fn get(&self, offset: Size, cx: &impl HasDataLayout) -> Option<Prov> {
+        let prov = self.range_get_ptrs(alloc_range(offset, Size::from_bytes(1)), cx);
+        debug_assert!(prov.len() <= 1);
+        if let Some(entry) = prov.first() {
+            // If it overlaps with this byte, it is on this byte.
+            debug_assert!(self.bytes.as_ref().map_or(true, |b| b.get(&offset).is_none()));
+            Some(entry.1)
+        } else {
+            // Look up per-byte provenance.
+            self.bytes.as_ref().and_then(|b| b.get(&offset).copied())
+        }
+    }
+
+    /// Check if here is ptr-sized provenance at the given index.
+    /// Does not mean anything for bytewise provenance! But can be useful as an optimization.
+    pub fn get_ptr(&self, offset: Size) -> Option<Prov> {
+        self.ptrs.get(&offset).copied()
+    }
+
+    /// Returns whether this allocation has provenance overlapping with the given range.
+    ///
+    /// Note: this function exists to allow `range_get_provenance` to be private, in order to somewhat
+    /// limit access to provenance outside of the `Allocation` abstraction.
+    ///
+    pub fn range_empty(&self, range: AllocRange, cx: &impl HasDataLayout) -> bool {
+        self.range_get_ptrs(range, cx).is_empty() && self.range_get_bytes(range).is_empty()
+    }
+
+    /// Yields all the provenances stored in this map.
+    pub fn provenances(&self) -> impl Iterator<Item = Prov> + '_ {
+        let bytes = self.bytes.iter().flat_map(|b| b.values());
+        self.ptrs.values().chain(bytes).copied()
+    }
+
+    pub fn insert_ptr(&mut self, offset: Size, prov: Prov, cx: &impl HasDataLayout) {
+        debug_assert!(self.range_empty(alloc_range(offset, cx.data_layout().pointer_size), cx));
+        self.ptrs.insert(offset, prov);
+    }
+
+    /// Removes all provenance inside the given range.
+    /// If there is provenance overlapping with the edges, might result in an error.
+    pub fn clear(&mut self, range: AllocRange, cx: &impl HasDataLayout) -> AllocResult {
+        let start = range.start;
+        let end = range.end();
+        // Clear the bytewise part -- this is easy.
+        if Prov::OFFSET_IS_ADDR {
+            if let Some(bytes) = self.bytes.as_mut() {
+                bytes.remove_range(start..end);
+            }
+        } else {
+            debug_assert!(self.bytes.is_none());
+        }
+
+        // For the ptr-sized part, find the first (inclusive) and last (exclusive) byte of
+        // provenance that overlaps with the given range.
+        let (first, last) = {
+            // Find all provenance overlapping the given range.
+            let provenance = self.range_get_ptrs(range, cx);
+            if provenance.is_empty() {
+                // No provenance in this range, we are done.
+                return Ok(());
+            }
+
+            (
+                provenance.first().unwrap().0,
+                provenance.last().unwrap().0 + cx.data_layout().pointer_size,
+            )
+        };
+
+        // We need to handle clearing the provenance from parts of a pointer.
+        if first < start {
+            if !Prov::OFFSET_IS_ADDR {
+                // We can't split up the provenance into less than a pointer.
+                return Err(AllocError::PartialPointerOverwrite(first));
+            }
+            // Insert the remaining part in the bytewise provenance.
+            let prov = self.ptrs[&first];
+            let bytes = self.bytes.get_or_insert_with(Box::default);
+            for offset in first..start {
+                bytes.insert(offset, prov);
+            }
+        }
+        if last > end {
+            let begin_of_last = last - cx.data_layout().pointer_size;
+            if !Prov::OFFSET_IS_ADDR {
+                // We can't split up the provenance into less than a pointer.
+                return Err(AllocError::PartialPointerOverwrite(begin_of_last));
+            }
+            // Insert the remaining part in the bytewise provenance.
+            let prov = self.ptrs[&begin_of_last];
+            let bytes = self.bytes.get_or_insert_with(Box::default);
+            for offset in end..last {
+                bytes.insert(offset, prov);
+            }
+        }
+
+        // Forget all the provenance.
+        // Since provenance do not overlap, we know that removing until `last` (exclusive) is fine,
+        // i.e., this will not remove any other provenance just after the ones we care about.
+        self.ptrs.remove_range(first..last);
+
+        Ok(())
+    }
+}
+
+/// A partial, owned list of provenance to transfer into another allocation.
+///
+/// Offsets are already adjusted to the destination allocation.
+pub struct ProvenanceCopy<Prov> {
+    dest_ptrs: Option<Box<[(Size, Prov)]>>,
+    dest_bytes: Option<Box<[(Size, Prov)]>>,
+}
+
+impl<Prov: Provenance> ProvenanceMap<Prov> {
+    pub fn prepare_copy(
+        &self,
+        src: AllocRange,
+        dest: Size,
+        count: u64,
+        cx: &impl HasDataLayout,
+    ) -> AllocResult<ProvenanceCopy<Prov>> {
+        let shift_offset = move |idx, offset| {
+            // compute offset for current repetition
+            let dest_offset = dest + src.size * idx; // `Size` operations
+            // shift offsets from source allocation to destination allocation
+            (offset - src.start) + dest_offset // `Size` operations
+        };
+        let ptr_size = cx.data_layout().pointer_size;
+
+        // # Pointer-sized provenances
+        // Get the provenances that are entirely within this range.
+        // (Different from `range_get_ptrs` which asks if they overlap the range.)
+        // Only makes sense if we are copying at least one pointer worth of bytes.
+        let mut dest_ptrs_box = None;
+        if src.size >= ptr_size {
+            let adjusted_end = Size::from_bytes(src.end().bytes() - (ptr_size.bytes() - 1));
+            let ptrs = self.ptrs.range(src.start..adjusted_end);
+            // If `count` is large, this is rather wasteful -- we are allocating a big array here, which
+            // is mostly filled with redundant information since it's just N copies of the same `Prov`s
+            // at slightly adjusted offsets. The reason we do this is so that in `mark_provenance_range`
+            // we can use `insert_presorted`. That wouldn't work with an `Iterator` that just produces
+            // the right sequence of provenance for all N copies.
+            // Basically, this large array would have to be created anyway in the target allocation.
+            let mut dest_ptrs = Vec::with_capacity(ptrs.len() * (count as usize));
+            for i in 0..count {
+                dest_ptrs
+                    .extend(ptrs.iter().map(|&(offset, reloc)| (shift_offset(i, offset), reloc)));
+            }
+            debug_assert_eq!(dest_ptrs.len(), dest_ptrs.capacity());
+            dest_ptrs_box = Some(dest_ptrs.into_boxed_slice());
+        };
+
+        // # Byte-sized provenances
+        // This includes the existing bytewise provenance in the range, and ptr provenance
+        // that overlaps with the begin/end of the range.
+        let mut dest_bytes_box = None;
+        let begin_overlap = self.range_get_ptrs(alloc_range(src.start, Size::ZERO), cx).first();
+        let end_overlap = self.range_get_ptrs(alloc_range(src.end(), Size::ZERO), cx).first();
+        if !Prov::OFFSET_IS_ADDR {
+            // There can't be any bytewise provenance, and we cannot split up the begin/end overlap.
+            if let Some(entry) = begin_overlap {
+                return Err(AllocError::PartialPointerCopy(entry.0));
+            }
+            if let Some(entry) = end_overlap {
+                return Err(AllocError::PartialPointerCopy(entry.0));
+            }
+            debug_assert!(self.bytes.is_none());
+        } else {
+            let mut bytes = Vec::new();
+            // First, if there is a part of a pointer at the start, add that.
+            if let Some(entry) = begin_overlap {
+                trace!("start overlapping entry: {entry:?}");
+                // For really small copies, make sure we don't run off the end of the `src` range.
+                let entry_end = cmp::min(entry.0 + ptr_size, src.end());
+                for offset in src.start..entry_end {
+                    bytes.push((offset, entry.1));
+                }
+            } else {
+                trace!("no start overlapping entry");
+            }
+            // Then the main part, bytewise provenance from `self.bytes`.
+            if let Some(all_bytes) = self.bytes.as_ref() {
+                bytes.extend(all_bytes.range(src.start..src.end()));
+            }
+            // And finally possibly parts of a pointer at the end.
+            if let Some(entry) = end_overlap {
+                trace!("end overlapping entry: {entry:?}");
+                // For really small copies, make sure we don't start before `src` does.
+                let entry_start = cmp::max(entry.0, src.start);
+                for offset in entry_start..src.end() {
+                    if bytes.last().map_or(true, |bytes_entry| bytes_entry.0 < offset) {
+                        // The last entry, if it exists, has a lower offset than us.
+                        bytes.push((offset, entry.1));
+                    } else {
+                        // There already is an entry for this offset in there! This can happen when the
+                        // start and end range checks actually end up hitting the same pointer, so we
+                        // already added this in the "pointer at the start" part above.
+                        assert!(entry.0 <= src.start);
+                    }
+                }
+            } else {
+                trace!("no end overlapping entry");
+            }
+            trace!("byte provenances: {bytes:?}");
+
+            // And again a buffer for the new list on the target side.
+            let mut dest_bytes = Vec::with_capacity(bytes.len() * (count as usize));
+            for i in 0..count {
+                dest_bytes
+                    .extend(bytes.iter().map(|&(offset, reloc)| (shift_offset(i, offset), reloc)));
+            }
+            debug_assert_eq!(dest_bytes.len(), dest_bytes.capacity());
+            dest_bytes_box = Some(dest_bytes.into_boxed_slice());
+        }
+
+        Ok(ProvenanceCopy { dest_ptrs: dest_ptrs_box, dest_bytes: dest_bytes_box })
+    }
+
+    /// Applies a provenance copy.
+    /// The affected range, as defined in the parameters to `prepare_copy` is expected
+    /// to be clear of provenance.
+    pub fn apply_copy(&mut self, copy: ProvenanceCopy<Prov>) {
+        if let Some(dest_ptrs) = copy.dest_ptrs {
+            self.ptrs.insert_presorted(dest_ptrs.into());
+        }
+        if Prov::OFFSET_IS_ADDR {
+            if let Some(dest_bytes) = copy.dest_bytes && !dest_bytes.is_empty() {
+                self.bytes.get_or_insert_with(Box::default).insert_presorted(dest_bytes.into());
+            }
+        } else {
+            debug_assert!(copy.dest_bytes.is_none());
+        }
+    }
+}
diff --git a/compiler/rustc_middle/src/mir/interpret/allocation/tests.rs b/compiler/rustc_middle/src/mir/interpret/allocation/tests.rs
new file mode 100644
index 000000000..c9c3c50c5
--- /dev/null
+++ b/compiler/rustc_middle/src/mir/interpret/allocation/tests.rs
@@ -0,0 +1,19 @@
+use super::*;
+
+#[test]
+fn uninit_mask() {
+    let mut mask = InitMask::new(Size::from_bytes(500), false);
+    assert!(!mask.get(Size::from_bytes(499)));
+    mask.set_range(alloc_range(Size::from_bytes(499), Size::from_bytes(1)), true);
+    assert!(mask.get(Size::from_bytes(499)));
+    mask.set_range((100..256).into(), true);
+    for i in 0..100 {
+        assert!(!mask.get(Size::from_bytes(i)), "{i} should not be set");
+    }
+    for i in 100..256 {
+        assert!(mask.get(Size::from_bytes(i)), "{i} should be set");
+    }
+    for i in 256..499 {
+        assert!(!mask.get(Size::from_bytes(i)), "{i} should not be set");
+    }
+}
diff --git a/compiler/rustc_middle/src/mir/interpret/error.rs b/compiler/rustc_middle/src/mir/interpret/error.rs
index b5a50cc15..bd9cd53e1 100644
--- a/compiler/rustc_middle/src/mir/interpret/error.rs
+++ b/compiler/rustc_middle/src/mir/interpret/error.rs
@@ -16,8 +16,6 @@ pub enum ErrorHandled {
     /// Already reported an error for this evaluation, and the compilation is
     /// *guaranteed* to fail. Warnings/lints *must not* produce `Reported`.
     Reported(ErrorGuaranteed),
-    /// Already emitted a lint for this evaluation.
-    Linted,
     /// Don't emit an error, the evaluation failed because the MIR was generic
     /// and the substs didn't fully monomorphize it.
     TooGeneric,
@@ -89,18 +87,6 @@ fn print_backtrace(backtrace: &Backtrace) {
     eprintln!("\n\nAn error occurred in miri:\n{}", backtrace);
 }
 
-impl From<ErrorHandled> for InterpErrorInfo<'_> {
-    fn from(err: ErrorHandled) -> Self {
-        match err {
-            ErrorHandled::Reported(ErrorGuaranteed { .. }) | ErrorHandled::Linted => {
-                err_inval!(ReferencedConstant)
-            }
-            ErrorHandled::TooGeneric => err_inval!(TooGeneric),
-        }
-        .into()
-    }
-}
-
 impl From<ErrorGuaranteed> for InterpErrorInfo<'_> {
     fn from(err: ErrorGuaranteed) -> Self {
         InterpError::InvalidProgram(InvalidProgramInfo::AlreadyReported(err)).into()
@@ -138,9 +124,6 @@ impl<'tcx> From<InterpError<'tcx>> for InterpErrorInfo<'tcx> {
 pub enum InvalidProgramInfo<'tcx> {
     /// Resolution can fail if we are in a too generic context.
     TooGeneric,
-    /// Cannot compute this constant because it depends on another one
-    /// which already produced an error.
-    ReferencedConstant,
     /// Abort in case errors are already reported.
     AlreadyReported(ErrorGuaranteed),
     /// An error occurred during layout computation.
@@ -158,9 +141,11 @@ impl fmt::Display for InvalidProgramInfo<'_> {
         use InvalidProgramInfo::*;
         match self {
             TooGeneric => write!(f, "encountered overly generic constant"),
-            ReferencedConstant => write!(f, "referenced constant has errors"),
             AlreadyReported(ErrorGuaranteed { .. }) => {
-                write!(f, "encountered constants with type errors, stopping evaluation")
+                write!(
+                    f,
+                    "an error has already been reported elsewhere (this should not usually be printed)"
+                )
             }
             Layout(ref err) => write!(f, "{err}"),
             FnAbiAdjustForForeignAbi(ref err) => write!(f, "{err}"),
@@ -401,16 +386,15 @@ impl fmt::Display for UndefinedBehaviorInfo {
 pub enum UnsupportedOpInfo {
     /// Free-form case. Only for errors that are never caught!
     Unsupported(String),
-    /// Overwriting parts of a pointer; the resulting state cannot be represented in our
-    /// `Allocation` data structure. See <https://github.com/rust-lang/miri/issues/2181>.
-    PartialPointerOverwrite(Pointer<AllocId>),
-    /// Attempting to `copy` parts of a pointer to somewhere else; the resulting state cannot be
-    /// represented in our `Allocation` data structure. See
-    /// <https://github.com/rust-lang/miri/issues/2181>.
-    PartialPointerCopy(Pointer<AllocId>),
     //
     // The variants below are only reachable from CTFE/const prop, miri will never emit them.
     //
+    /// Overwriting parts of a pointer; without knowing absolute addresses, the resulting state
+    /// cannot be represented by the CTFE interpreter.
+    PartialPointerOverwrite(Pointer<AllocId>),
+    /// Attempting to `copy` parts of a pointer to somewhere else; without knowing absolute
+    /// addresses, the resulting state cannot be represented by the CTFE interpreter.
+    PartialPointerCopy(Pointer<AllocId>),
     /// Encountered a pointer where we needed raw bytes.
     ReadPointerAsBytes,
     /// Accessing thread local statics
diff --git a/compiler/rustc_middle/src/mir/interpret/mod.rs b/compiler/rustc_middle/src/mir/interpret/mod.rs
index 5e3dfcbcc..d79cd8b7a 100644
--- a/compiler/rustc_middle/src/mir/interpret/mod.rs
+++ b/compiler/rustc_middle/src/mir/interpret/mod.rs
@@ -106,6 +106,7 @@ use rustc_ast::LitKind;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sync::{HashMapExt, Lock};
 use rustc_data_structures::tiny_list::TinyList;
+use rustc_errors::ErrorGuaranteed;
 use rustc_hir::def_id::DefId;
 use rustc_macros::HashStable;
 use rustc_middle::ty::print::with_no_trimmed_paths;
@@ -127,8 +128,8 @@ pub use self::error::{
 pub use self::value::{get_slice_bytes, ConstAlloc, ConstValue, Scalar};
 
 pub use self::allocation::{
-    alloc_range, AllocRange, Allocation, ConstAllocation, InitChunk, InitChunkIter, InitMask,
-    ProvenanceMap,
+    alloc_range, AllocError, AllocRange, AllocResult, Allocation, ConstAllocation, InitChunk,
+    InitChunkIter,
 };
 
 pub use self::pointer::{Pointer, PointerArithmetic, Provenance};
@@ -176,7 +177,7 @@ pub enum LitToConstError {
     /// This is used for graceful error handling (`delay_span_bug`) in
     /// type checking (`Const::from_anon_const`).
     TypeError,
-    Reported,
+    Reported(ErrorGuaranteed),
 }
 
 #[derive(Copy, Clone, Eq, Hash, Ord, PartialEq, PartialOrd)]
diff --git a/compiler/rustc_middle/src/mir/interpret/pointer.rs b/compiler/rustc_middle/src/mir/interpret/pointer.rs
index 23c2ce647..9c270ba1e 100644
--- a/compiler/rustc_middle/src/mir/interpret/pointer.rs
+++ b/compiler/rustc_middle/src/mir/interpret/pointer.rs
@@ -103,8 +103,7 @@ impl<T: HasDataLayout> PointerArithmetic for T {}
 /// This trait abstracts over the kind of provenance that is associated with a `Pointer`. It is
 /// mostly opaque; the `Machine` trait extends it with some more operations that also have access to
 /// some global state.
-/// We don't actually care about this `Debug` bound (we use `Provenance::fmt` to format the entire
-/// pointer), but `derive` adds some unnecessary bounds.
+/// The `Debug` rendering is used to distplay bare provenance, and for the default impl of `fmt`.
 pub trait Provenance: Copy + fmt::Debug {
     /// Says whether the `offset` field of `Pointer`s with this provenance is the actual physical address.
     /// - If `false`, the offset *must* be relative. This means the bytes representing a pointer are
@@ -115,14 +114,23 @@ pub trait Provenance: Copy + fmt::Debug {
     ///   pointer, and implement ptr-to-int transmutation by stripping provenance.
     const OFFSET_IS_ADDR: bool;
 
-    /// We also use this trait to control whether to abort execution when a pointer is being partially overwritten
-    /// (this avoids a separate trait in `allocation.rs` just for this purpose).
-    const ERR_ON_PARTIAL_PTR_OVERWRITE: bool;
-
     /// Determines how a pointer should be printed.
+    ///
+    /// Default impl is only good for when `OFFSET_IS_ADDR == true`.
     fn fmt(ptr: &Pointer<Self>, f: &mut fmt::Formatter<'_>) -> fmt::Result
     where
-        Self: Sized;
+        Self: Sized,
+    {
+        assert!(Self::OFFSET_IS_ADDR);
+        let (prov, addr) = ptr.into_parts(); // address is absolute
+        write!(f, "{:#x}", addr.bytes())?;
+        if f.alternate() {
+            write!(f, "{prov:#?}")?;
+        } else {
+            write!(f, "{prov:?}")?;
+        }
+        Ok(())
+    }
 
     /// If `OFFSET_IS_ADDR == false`, provenance must always be able to
     /// identify the allocation this ptr points to (i.e., this must return `Some`).
@@ -139,9 +147,6 @@ impl Provenance for AllocId {
     // so ptr-to-int casts are not possible (since we do not know the global physical offset).
     const OFFSET_IS_ADDR: bool = false;
 
-    // For now, do not allow this, so that we keep our options open.
-    const ERR_ON_PARTIAL_PTR_OVERWRITE: bool = true;
-
     fn fmt(ptr: &Pointer<Self>, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         // Forward `alternate` flag to `alloc_id` printing.
         if f.alternate() {
@@ -168,7 +173,7 @@ impl Provenance for AllocId {
 /// Represents a pointer in the Miri engine.
 ///
 /// Pointers are "tagged" with provenance information; typically the `AllocId` they belong to.
-#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, TyEncodable, TyDecodable, Hash)]
+#[derive(Copy, Clone, Eq, PartialEq, TyEncodable, TyDecodable, Hash)]
 #[derive(HashStable)]
 pub struct Pointer<Prov = AllocId> {
     pub(super) offset: Size, // kept private to avoid accidental misinterpretation (meaning depends on `Prov` type)
diff --git a/compiler/rustc_middle/src/mir/interpret/queries.rs b/compiler/rustc_middle/src/mir/interpret/queries.rs
index 473894ac1..b6c6e9d55 100644
--- a/compiler/rustc_middle/src/mir/interpret/queries.rs
+++ b/compiler/rustc_middle/src/mir/interpret/queries.rs
@@ -175,6 +175,8 @@ impl<'tcx> TyCtxt<'tcx> {
 
 impl<'tcx> TyCtxtAt<'tcx> {
     /// Evaluate a static's initializer, returning the allocation of the initializer's memory.
+    ///
+    /// The span is entirely ignored here, but still helpful for better query cycle errors.
     pub fn eval_static_initializer(
         self,
         def_id: DefId,
@@ -187,6 +189,8 @@ impl<'tcx> TyCtxtAt<'tcx> {
     }
 
     /// Evaluate anything constant-like, returning the allocation of the final memory.
+    ///
+    /// The span is entirely ignored here, but still helpful for better query cycle errors.
     fn eval_to_allocation(
         self,
         gid: GlobalId<'tcx>,
diff --git a/compiler/rustc_middle/src/mir/interpret/value.rs b/compiler/rustc_middle/src/mir/interpret/value.rs
index ac5fddb7a..e6636e50e 100644
--- a/compiler/rustc_middle/src/mir/interpret/value.rs
+++ b/compiler/rustc_middle/src/mir/interpret/value.rs
@@ -1,6 +1,8 @@
 use std::convert::{TryFrom, TryInto};
 use std::fmt;
 
+use either::{Either, Left, Right};
+
 use rustc_apfloat::{
     ieee::{Double, Single},
     Float,
@@ -18,15 +20,15 @@ use super::{
 /// Represents the result of const evaluation via the `eval_to_allocation` query.
 #[derive(Copy, Clone, HashStable, TyEncodable, TyDecodable, Debug, Hash, Eq, PartialEq)]
 pub struct ConstAlloc<'tcx> {
-    // the value lives here, at offset 0, and that allocation definitely is an `AllocKind::Memory`
-    // (so you can use `AllocMap::unwrap_memory`).
+    /// The value lives here, at offset 0, and that allocation definitely is an `AllocKind::Memory`
+    /// (so you can use `AllocMap::unwrap_memory`).
     pub alloc_id: AllocId,
     pub ty: Ty<'tcx>,
 }
 
 /// Represents a constant value in Rust. `Scalar` and `Slice` are optimizations for
 /// array length computations, enum discriminants and the pattern matching logic.
-#[derive(Copy, Clone, Debug, Eq, PartialEq, PartialOrd, Ord, TyEncodable, TyDecodable, Hash)]
+#[derive(Copy, Clone, Debug, Eq, PartialEq, TyEncodable, TyDecodable, Hash)]
 #[derive(HashStable, Lift)]
 pub enum ConstValue<'tcx> {
     /// Used only for types with `layout::abi::Scalar` ABI.
@@ -108,7 +110,7 @@ impl<'tcx> ConstValue<'tcx> {
 ///
 /// These variants would be private if there was a convenient way to achieve that in Rust.
 /// Do *not* match on a `Scalar`! Use the various `to_*` methods instead.
-#[derive(Clone, Copy, Eq, PartialEq, Ord, PartialOrd, TyEncodable, TyDecodable, Hash)]
+#[derive(Clone, Copy, Eq, PartialEq, TyEncodable, TyDecodable, Hash)]
 #[derive(HashStable)]
 pub enum Scalar<Prov = AllocId> {
     /// The raw bytes of a simple value.
@@ -293,10 +295,10 @@ impl<Prov> Scalar<Prov> {
     pub fn to_bits_or_ptr_internal(
         self,
         target_size: Size,
-    ) -> Result<Result<u128, Pointer<Prov>>, ScalarSizeMismatch> {
+    ) -> Result<Either<u128, Pointer<Prov>>, ScalarSizeMismatch> {
         assert_ne!(target_size.bytes(), 0, "you should never look at the bits of a ZST");
         Ok(match self {
-            Scalar::Int(int) => Ok(int.to_bits(target_size).map_err(|size| {
+            Scalar::Int(int) => Left(int.to_bits(target_size).map_err(|size| {
                 ScalarSizeMismatch { target_size: target_size.bytes(), data_size: size.bytes() }
             })?),
             Scalar::Ptr(ptr, sz) => {
@@ -306,7 +308,7 @@ impl<Prov> Scalar<Prov> {
                         data_size: sz.into(),
                     });
                 }
-                Err(ptr)
+                Right(ptr)
             }
         })
     }
@@ -318,8 +320,8 @@ impl<'tcx, Prov: Provenance> Scalar<Prov> {
             .to_bits_or_ptr_internal(cx.pointer_size())
             .map_err(|s| err_ub!(ScalarSizeMismatch(s)))?
         {
-            Err(ptr) => Ok(ptr.into()),
-            Ok(bits) => {
+            Right(ptr) => Ok(ptr.into()),
+            Left(bits) => {
                 let addr = u64::try_from(bits).unwrap();
                 Ok(Pointer::from_addr(addr))
             }
diff --git a/compiler/rustc_middle/src/mir/mod.rs b/compiler/rustc_middle/src/mir/mod.rs
index 79db35a76..a513444e1 100644
--- a/compiler/rustc_middle/src/mir/mod.rs
+++ b/compiler/rustc_middle/src/mir/mod.rs
@@ -10,7 +10,7 @@ use crate::ty::codec::{TyDecoder, TyEncoder};
 use crate::ty::fold::{FallibleTypeFolder, TypeFoldable};
 use crate::ty::print::{FmtPrinter, Printer};
 use crate::ty::visit::{TypeVisitable, TypeVisitor};
-use crate::ty::{self, List, Ty, TyCtxt};
+use crate::ty::{self, DefIdTree, List, Ty, TyCtxt};
 use crate::ty::{AdtDef, InstanceDef, ScalarInt, UserTypeAnnotationIndex};
 use crate::ty::{GenericArg, InternalSubsts, SubstsRef};
 
@@ -100,13 +100,9 @@ impl<'tcx> HasLocalDecls<'tcx> for Body<'tcx> {
 /// pass will be named after the type, and it will consist of a main
 /// loop that goes over each available MIR and applies `run_pass`.
 pub trait MirPass<'tcx> {
-    fn name(&self) -> Cow<'_, str> {
+    fn name(&self) -> &str {
         let name = std::any::type_name::<Self>();
-        if let Some(tail) = name.rfind(':') {
-            Cow::from(&name[tail + 1..])
-        } else {
-            Cow::from(name)
-        }
+        if let Some((_, tail)) = name.rsplit_once(':') { tail } else { name }
     }
 
     /// Returns `true` if this pass is enabled with the current combination of compiler flags.
@@ -138,33 +134,46 @@ impl MirPhase {
             }
         }
     }
-}
 
-impl Display for MirPhase {
-    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
-        match self {
-            MirPhase::Built => write!(f, "built"),
-            MirPhase::Analysis(p) => write!(f, "analysis-{}", p),
-            MirPhase::Runtime(p) => write!(f, "runtime-{}", p),
+    /// Parses an `MirPhase` from a pair of strings. Panics if this isn't possible for any reason.
+    pub fn parse(dialect: String, phase: Option<String>) -> Self {
+        match &*dialect.to_ascii_lowercase() {
+            "built" => {
+                assert!(phase.is_none(), "Cannot specify a phase for `Built` MIR");
+                MirPhase::Built
+            }
+            "analysis" => Self::Analysis(AnalysisPhase::parse(phase)),
+            "runtime" => Self::Runtime(RuntimePhase::parse(phase)),
+            _ => panic!("Unknown MIR dialect {}", dialect),
         }
     }
 }
 
-impl Display for AnalysisPhase {
-    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
-        match self {
-            AnalysisPhase::Initial => write!(f, "initial"),
-            AnalysisPhase::PostCleanup => write!(f, "post_cleanup"),
+impl AnalysisPhase {
+    pub fn parse(phase: Option<String>) -> Self {
+        let Some(phase) = phase else {
+            return Self::Initial;
+        };
+
+        match &*phase.to_ascii_lowercase() {
+            "initial" => Self::Initial,
+            "post_cleanup" | "post-cleanup" | "postcleanup" => Self::PostCleanup,
+            _ => panic!("Unknown analysis phase {}", phase),
         }
     }
 }
 
-impl Display for RuntimePhase {
-    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
-        match self {
-            RuntimePhase::Initial => write!(f, "initial"),
-            RuntimePhase::PostCleanup => write!(f, "post_cleanup"),
-            RuntimePhase::Optimized => write!(f, "optimized"),
+impl RuntimePhase {
+    pub fn parse(phase: Option<String>) -> Self {
+        let Some(phase) = phase else {
+            return Self::Initial;
+        };
+
+        match &*phase.to_ascii_lowercase() {
+            "initial" => Self::Initial,
+            "post_cleanup" | "post-cleanup" | "postcleanup" => Self::PostCleanup,
+            "optimized" => Self::Optimized,
+            _ => panic!("Unknown runtime phase {}", phase),
         }
     }
 }
@@ -293,6 +302,13 @@ pub struct Body<'tcx> {
     /// potentially allow things like `[u8; std::mem::size_of::<T>() * 0]` due to this.
     pub is_polymorphic: bool,
 
+    /// The phase at which this MIR should be "injected" into the compilation process.
+    ///
+    /// Everything that comes before this `MirPhase` should be skipped.
+    ///
+    /// This is only `Some` if the function that this body comes from was annotated with `rustc_custom_mir`.
+    pub injection_phase: Option<MirPhase>,
+
     pub tainted_by_errors: Option<ErrorGuaranteed>,
 }
 
@@ -319,7 +335,7 @@ impl<'tcx> Body<'tcx> {
 
         let mut body = Body {
             phase: MirPhase::Built,
-            pass_count: 1,
+            pass_count: 0,
             source,
             basic_blocks: BasicBlocks::new(basic_blocks),
             source_scopes,
@@ -339,6 +355,7 @@ impl<'tcx> Body<'tcx> {
             span,
             required_consts: Vec::new(),
             is_polymorphic: false,
+            injection_phase: None,
             tainted_by_errors,
         };
         body.is_polymorphic = body.has_non_region_param();
@@ -353,7 +370,7 @@ impl<'tcx> Body<'tcx> {
     pub fn new_cfg_only(basic_blocks: IndexVec<BasicBlock, BasicBlockData<'tcx>>) -> Self {
         let mut body = Body {
             phase: MirPhase::Built,
-            pass_count: 1,
+            pass_count: 0,
             source: MirSource::item(CRATE_DEF_ID.to_def_id()),
             basic_blocks: BasicBlocks::new(basic_blocks),
             source_scopes: IndexVec::new(),
@@ -366,6 +383,7 @@ impl<'tcx> Body<'tcx> {
             required_consts: Vec::new(),
             var_debug_info: Vec::new(),
             is_polymorphic: false,
+            injection_phase: None,
             tainted_by_errors: None,
         };
         body.is_polymorphic = body.has_non_region_param();
@@ -508,6 +526,14 @@ impl<'tcx> Body<'tcx> {
     pub fn generator_kind(&self) -> Option<GeneratorKind> {
         self.generator.as_ref().map(|generator| generator.generator_kind)
     }
+
+    #[inline]
+    pub fn should_skip(&self) -> bool {
+        let Some(injection_phase) = self.injection_phase else {
+            return false;
+        };
+        injection_phase > self.phase
+    }
 }
 
 #[derive(Copy, Clone, PartialEq, Eq, Debug, TyEncodable, TyDecodable, HashStable)]
@@ -1012,6 +1038,18 @@ pub enum VarDebugInfoContents<'tcx> {
     /// based on a `Local`, not a `Static`, and contains no indexing.
     Place(Place<'tcx>),
     Const(Constant<'tcx>),
+    /// The user variable's data is split across several fragments,
+    /// each described by a `VarDebugInfoFragment`.
+    /// See DWARF 5's "2.6.1.2 Composite Location Descriptions"
+    /// and LLVM's `DW_OP_LLVM_fragment` for more details on
+    /// the underlying debuginfo feature this relies on.
+    Composite {
+        /// Type of the original user variable.
+        ty: Ty<'tcx>,
+        /// All the parts of the original user variable, which ended
+        /// up in disjoint places, due to optimizations.
+        fragments: Vec<VarDebugInfoFragment<'tcx>>,
+    },
 }
 
 impl<'tcx> Debug for VarDebugInfoContents<'tcx> {
@@ -1019,7 +1057,48 @@ impl<'tcx> Debug for VarDebugInfoContents<'tcx> {
         match self {
             VarDebugInfoContents::Const(c) => write!(fmt, "{}", c),
             VarDebugInfoContents::Place(p) => write!(fmt, "{:?}", p),
+            VarDebugInfoContents::Composite { ty, fragments } => {
+                write!(fmt, "{:?}{{ ", ty)?;
+                for f in fragments.iter() {
+                    write!(fmt, "{:?}, ", f)?;
+                }
+                write!(fmt, "}}")
+            }
+        }
+    }
+}
+
+#[derive(Clone, TyEncodable, TyDecodable, HashStable, TypeFoldable, TypeVisitable)]
+pub struct VarDebugInfoFragment<'tcx> {
+    /// Where in the composite user variable this fragment is,
+    /// represented as a "projection" into the composite variable.
+    /// At lower levels, this corresponds to a byte/bit range.
+    // NOTE(eddyb) there's an unenforced invariant that this contains
+    // only `Field`s, and not into `enum` variants or `union`s.
+    // FIXME(eddyb) support this for `enum`s by either using DWARF's
+    // more advanced control-flow features (unsupported by LLVM?)
+    // to match on the discriminant, or by using custom type debuginfo
+    // with non-overlapping variants for the composite variable.
+    pub projection: Vec<PlaceElem<'tcx>>,
+
+    /// Where the data for this fragment can be found.
+    // NOTE(eddyb) There's an unenforced invariant that this `Place` is
+    // contains no indexing (with a non-constant index).
+    pub contents: Place<'tcx>,
+}
+
+impl Debug for VarDebugInfoFragment<'_> {
+    fn fmt(&self, fmt: &mut Formatter<'_>) -> fmt::Result {
+        for elem in self.projection.iter() {
+            match elem {
+                ProjectionElem::Field(field, _) => {
+                    write!(fmt, ".{:?}", field.index())?;
+                }
+                _ => bug!("unsupported fragment projection `{:?}`", elem),
+            }
         }
+
+        write!(fmt, " => {:?}", self.contents)
     }
 }
 
@@ -1186,6 +1265,11 @@ impl<'tcx> BasicBlockData<'tcx> {
     pub fn visitable(&self, index: usize) -> &dyn MirVisitable<'tcx> {
         if index < self.statements.len() { &self.statements[index] } else { &self.terminator }
     }
+
+    /// Does the block have no statements and an unreachable terminator?
+    pub fn is_empty_unreachable(&self) -> bool {
+        self.statements.is_empty() && matches!(self.terminator().kind, TerminatorKind::Unreachable)
+    }
 }
 
 impl<O> AssertKind<O> {
@@ -1477,7 +1561,7 @@ impl<'tcx> Place<'tcx> {
     /// If MirPhase >= Derefered and if projection contains Deref,
     /// It's guaranteed to be in the first place
     pub fn has_deref(&self) -> bool {
-        // To make sure this is not accidently used in wrong mir phase
+        // To make sure this is not accidentally used in wrong mir phase
         debug_assert!(
             self.projection.is_empty() || !self.projection[1..].contains(&PlaceElem::Deref)
         );
@@ -1767,7 +1851,7 @@ impl<'tcx> Operand<'tcx> {
         substs: SubstsRef<'tcx>,
         span: Span,
     ) -> Self {
-        let ty = tcx.bound_type_of(def_id).subst(tcx, substs);
+        let ty = tcx.mk_fn_def(def_id, substs);
         Operand::Constant(Box::new(Constant {
             span,
             user_ty: None,
@@ -1893,6 +1977,7 @@ impl BorrowKind {
         }
     }
 
+    // FIXME: won't be used after diagnostic migration
     pub fn describe_mutability(&self) -> &str {
         match *self {
             BorrowKind::Shared | BorrowKind::Shallow | BorrowKind::Unique => "immutable",
@@ -1997,10 +2082,10 @@ impl<'tcx> Debug for Rvalue<'tcx> {
                                 .print_def_path(variant_def.def_id, substs)?
                                 .into_buffer();
 
-                            match variant_def.ctor_kind {
-                                CtorKind::Const => fmt.write_str(&name),
-                                CtorKind::Fn => fmt_tuple(fmt, &name),
-                                CtorKind::Fictive => {
+                            match variant_def.ctor_kind() {
+                                Some(CtorKind::Const) => fmt.write_str(&name),
+                                Some(CtorKind::Fn) => fmt_tuple(fmt, &name),
+                                None => {
                                     let mut struct_fmt = fmt.debug_struct(&name);
                                     for (field, place) in iter::zip(&variant_def.fields, places) {
                                         struct_fmt.field(field.name.as_str(), place);
@@ -2186,8 +2271,10 @@ impl<'tcx> ConstantKind<'tcx> {
                 // FIXME: We might want to have a `try_eval`-like function on `Unevaluated`
                 match tcx.const_eval_resolve(param_env, uneval, None) {
                     Ok(val) => Self::Val(val, ty),
-                    Err(ErrorHandled::TooGeneric | ErrorHandled::Linted) => self,
-                    Err(_) => Self::Ty(tcx.const_error(ty)),
+                    Err(ErrorHandled::TooGeneric) => self,
+                    Err(ErrorHandled::Reported(guar)) => {
+                        Self::Ty(tcx.const_error_with_guaranteed(ty, guar))
+                    }
                 }
             }
         }
@@ -2403,16 +2490,11 @@ impl<'tcx> ConstantKind<'tcx> {
             ExprKind::Path(QPath::Resolved(_, &Path { res: Res::Def(ConstParam, def_id), .. })) => {
                 // Find the name and index of the const parameter by indexing the generics of
                 // the parent item and construct a `ParamConst`.
-                let hir_id = tcx.hir().local_def_id_to_hir_id(def_id.expect_local());
-                let item_id = tcx.hir().get_parent_node(hir_id);
-                let item_def_id = tcx.hir().local_def_id(item_id);
-                let generics = tcx.generics_of(item_def_id.to_def_id());
+                let item_def_id = tcx.parent(def_id);
+                let generics = tcx.generics_of(item_def_id);
                 let index = generics.param_def_id_to_index[&def_id];
-                let name = tcx.hir().name(hir_id);
-                let ty_const = tcx.mk_const(ty::ConstS {
-                    kind: ty::ConstKind::Param(ty::ParamConst::new(index, name)),
-                    ty,
-                });
+                let name = tcx.item_name(def_id);
+                let ty_const = tcx.mk_const(ty::ParamConst::new(index, name), ty);
                 debug!(?ty_const);
 
                 return Self::Ty(ty_const);
@@ -2839,14 +2921,14 @@ fn pretty_print_const_value<'tcx>(
                             let cx = cx.print_value_path(variant_def.def_id, substs)?;
                             fmt.write_str(&cx.into_buffer())?;
 
-                            match variant_def.ctor_kind {
-                                CtorKind::Const => {}
-                                CtorKind::Fn => {
+                            match variant_def.ctor_kind() {
+                                Some(CtorKind::Const) => {}
+                                Some(CtorKind::Fn) => {
                                     fmt.write_str("(")?;
                                     comma_sep(fmt, fields)?;
                                     fmt.write_str(")")?;
                                 }
-                                CtorKind::Fictive => {
+                                None => {
                                     fmt.write_str(" {{ ")?;
                                     let mut first = true;
                                     for (field_def, field) in iter::zip(&variant_def.fields, fields)
diff --git a/compiler/rustc_middle/src/mir/pretty.rs b/compiler/rustc_middle/src/mir/pretty.rs
index 05dcfba77..2a4ff4b88 100644
--- a/compiler/rustc_middle/src/mir/pretty.rs
+++ b/compiler/rustc_middle/src/mir/pretty.rs
@@ -12,11 +12,10 @@ use rustc_data_structures::fx::FxHashMap;
 use rustc_hir::def_id::DefId;
 use rustc_index::vec::Idx;
 use rustc_middle::mir::interpret::{
-    read_target_uint, AllocId, Allocation, ConstAllocation, ConstValue, GlobalAlloc, Pointer,
-    Provenance,
+    alloc_range, read_target_uint, AllocId, Allocation, ConstAllocation, ConstValue, GlobalAlloc,
+    Pointer, Provenance,
 };
 use rustc_middle::mir::visit::Visitor;
-use rustc_middle::mir::MirSource;
 use rustc_middle::mir::*;
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_target::abi::Size;
@@ -74,7 +73,7 @@ pub enum PassWhere {
 #[inline]
 pub fn dump_mir<'tcx, F>(
     tcx: TyCtxt<'tcx>,
-    pass_num: Option<&dyn Display>,
+    pass_num: bool,
     pass_name: &str,
     disambiguator: &dyn Display,
     body: &Body<'tcx>,
@@ -111,7 +110,7 @@ pub fn dump_enabled<'tcx>(tcx: TyCtxt<'tcx>, pass_name: &str, def_id: DefId) ->
 
 fn dump_matched_mir_node<'tcx, F>(
     tcx: TyCtxt<'tcx>,
-    pass_num: Option<&dyn Display>,
+    pass_num: bool,
     pass_name: &str,
     disambiguator: &dyn Display,
     body: &Body<'tcx>,
@@ -120,8 +119,7 @@ fn dump_matched_mir_node<'tcx, F>(
     F: FnMut(PassWhere, &mut dyn Write) -> io::Result<()>,
 {
     let _: io::Result<()> = try {
-        let mut file =
-            create_dump_file(tcx, "mir", pass_num, pass_name, disambiguator, body.source)?;
+        let mut file = create_dump_file(tcx, "mir", pass_num, pass_name, disambiguator, body)?;
         // see notes on #41697 above
         let def_path =
             ty::print::with_forced_impl_filename_line!(tcx.def_path_str(body.source.def_id()));
@@ -143,16 +141,14 @@ fn dump_matched_mir_node<'tcx, F>(
 
     if tcx.sess.opts.unstable_opts.dump_mir_graphviz {
         let _: io::Result<()> = try {
-            let mut file =
-                create_dump_file(tcx, "dot", pass_num, pass_name, disambiguator, body.source)?;
+            let mut file = create_dump_file(tcx, "dot", pass_num, pass_name, disambiguator, body)?;
             write_mir_fn_graphviz(tcx, body, false, &mut file)?;
         };
     }
 
     if let Some(spanview) = tcx.sess.opts.unstable_opts.dump_mir_spanview {
         let _: io::Result<()> = try {
-            let file_basename =
-                dump_file_basename(tcx, pass_num, pass_name, disambiguator, body.source);
+            let file_basename = dump_file_basename(tcx, pass_num, pass_name, disambiguator, body);
             let mut file = create_dump_file_with_basename(tcx, &file_basename, "html")?;
             if body.source.def_id().is_local() {
                 write_mir_fn_spanview(tcx, body, spanview, &file_basename, &mut file)?;
@@ -165,11 +161,12 @@ fn dump_matched_mir_node<'tcx, F>(
 /// where we should dump a MIR representation output files.
 fn dump_file_basename<'tcx>(
     tcx: TyCtxt<'tcx>,
-    pass_num: Option<&dyn Display>,
+    pass_num: bool,
     pass_name: &str,
     disambiguator: &dyn Display,
-    source: MirSource<'tcx>,
+    body: &Body<'tcx>,
 ) -> String {
+    let source = body.source;
     let promotion_id = match source.promoted {
         Some(id) => format!("-{:?}", id),
         None => String::new(),
@@ -178,9 +175,10 @@ fn dump_file_basename<'tcx>(
     let pass_num = if tcx.sess.opts.unstable_opts.dump_mir_exclude_pass_number {
         String::new()
     } else {
-        match pass_num {
-            None => ".-------".to_string(),
-            Some(pass_num) => format!(".{}", pass_num),
+        if pass_num {
+            format!(".{:03}-{:03}", body.phase.phase_index(), body.pass_count)
+        } else {
+            ".-------".to_string()
         }
     };
 
@@ -250,14 +248,14 @@ fn create_dump_file_with_basename(
 pub fn create_dump_file<'tcx>(
     tcx: TyCtxt<'tcx>,
     extension: &str,
-    pass_num: Option<&dyn Display>,
+    pass_num: bool,
     pass_name: &str,
     disambiguator: &dyn Display,
-    source: MirSource<'tcx>,
+    body: &Body<'tcx>,
 ) -> io::Result<io::BufWriter<fs::File>> {
     create_dump_file_with_basename(
         tcx,
-        &dump_file_basename(tcx, pass_num, pass_name, disambiguator, source),
+        &dump_file_basename(tcx, pass_num, pass_name, disambiguator, body),
         extension,
     )
 }
@@ -476,6 +474,7 @@ impl<'tcx> Visitor<'tcx> for ExtraComments<'tcx> {
                     // These variants shouldn't exist in the MIR.
                     ty::ConstKind::Placeholder(_)
                     | ty::ConstKind::Infer(_)
+                    | ty::ConstKind::Expr(_)
                     | ty::ConstKind::Bound(..) => bug!("unexpected MIR constant: {:?}", literal),
                 },
                 ConstantKind::Unevaluated(uv, _) => {
@@ -685,7 +684,7 @@ pub fn write_allocations<'tcx>(
     fn alloc_ids_from_alloc(
         alloc: ConstAllocation<'_>,
     ) -> impl DoubleEndedIterator<Item = AllocId> + '_ {
-        alloc.inner().provenance().values().map(|id| *id)
+        alloc.inner().provenance().ptrs().values().map(|id| *id)
     }
 
     fn alloc_ids_from_const_val(val: ConstValue<'_>) -> impl Iterator<Item = AllocId> + '_ {
@@ -788,7 +787,7 @@ pub fn write_allocations<'tcx>(
 /// After the hex dump, an ascii dump follows, replacing all unprintable characters (control
 /// characters or characters whose value is larger than 127) with a `.`
 /// This also prints provenance adequately.
-pub fn display_allocation<'a, 'tcx, Prov, Extra>(
+pub fn display_allocation<'a, 'tcx, Prov: Provenance, Extra>(
     tcx: TyCtxt<'tcx>,
     alloc: &'a Allocation<Prov, Extra>,
 ) -> RenderAllocation<'a, 'tcx, Prov, Extra> {
@@ -796,7 +795,7 @@ pub fn display_allocation<'a, 'tcx, Prov, Extra>(
 }
 
 #[doc(hidden)]
-pub struct RenderAllocation<'a, 'tcx, Prov, Extra> {
+pub struct RenderAllocation<'a, 'tcx, Prov: Provenance, Extra> {
     tcx: TyCtxt<'tcx>,
     alloc: &'a Allocation<Prov, Extra>,
 }
@@ -882,9 +881,9 @@ fn write_allocation_bytes<'tcx, Prov: Provenance, Extra>(
         if i != line_start {
             write!(w, " ")?;
         }
-        if let Some(&prov) = alloc.provenance().get(&i) {
+        if let Some(prov) = alloc.provenance().get_ptr(i) {
             // Memory with provenance must be defined
-            assert!(alloc.init_mask().is_range_initialized(i, i + ptr_size).is_ok());
+            assert!(alloc.init_mask().is_range_initialized(alloc_range(i, ptr_size)).is_ok());
             let j = i.bytes_usize();
             let offset = alloc
                 .inspect_with_uninit_and_ptr_outside_interpreter(j..j + ptr_size.bytes_usize());
@@ -904,9 +903,9 @@ fn write_allocation_bytes<'tcx, Prov: Provenance, Extra>(
                 let overflow = ptr_size - remainder;
                 let remainder_width = provenance_width(remainder.bytes_usize()) - 2;
                 let overflow_width = provenance_width(overflow.bytes_usize() - 1) + 1;
-                ascii.push('╾');
-                for _ in 0..remainder.bytes() - 1 {
-                    ascii.push('─');
+                ascii.push('╾'); // HEAVY LEFT AND LIGHT RIGHT
+                for _ in 1..remainder.bytes() {
+                    ascii.push('─'); // LIGHT HORIZONTAL
                 }
                 if overflow_width > remainder_width && overflow_width >= target.len() {
                     // The case where the provenance fits into the part in the next line
@@ -926,7 +925,7 @@ fn write_allocation_bytes<'tcx, Prov: Provenance, Extra>(
                 for _ in 0..overflow.bytes() - 1 {
                     ascii.push('─');
                 }
-                ascii.push('╼');
+                ascii.push('╼'); // LIGHT LEFT AND HEAVY RIGHT
                 i += ptr_size;
                 continue;
             } else {
@@ -941,7 +940,23 @@ fn write_allocation_bytes<'tcx, Prov: Provenance, Extra>(
                 ascii.push('╼');
                 i += ptr_size;
             }
-        } else if alloc.init_mask().is_range_initialized(i, i + Size::from_bytes(1)).is_ok() {
+        } else if let Some(prov) = alloc.provenance().get(i, &tcx) {
+            // Memory with provenance must be defined
+            assert!(
+                alloc.init_mask().is_range_initialized(alloc_range(i, Size::from_bytes(1))).is_ok()
+            );
+            ascii.push('━'); // HEAVY HORIZONTAL
+            // We have two characters to display this, which is obviously not enough.
+            // Format is similar to "oversized" above.
+            let j = i.bytes_usize();
+            let c = alloc.inspect_with_uninit_and_ptr_outside_interpreter(j..j + 1)[0];
+            write!(w, "╾{:02x}{:#?} (1 ptr byte)╼", c, prov)?;
+            i += Size::from_bytes(1);
+        } else if alloc
+            .init_mask()
+            .is_range_initialized(alloc_range(i, Size::from_bytes(1)))
+            .is_ok()
+        {
             let j = i.bytes_usize();
 
             // Checked definedness (and thus range) and provenance. This access also doesn't
diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs
index 85ef51f12..5ba053820 100644
--- a/compiler/rustc_middle/src/mir/syntax.rs
+++ b/compiler/rustc_middle/src/mir/syntax.rs
@@ -6,6 +6,7 @@
 use super::{BasicBlock, Constant, Field, Local, SwitchTargets, UserTypeProjection};
 
 use crate::mir::coverage::{CodeRegion, CoverageKind};
+use crate::traits::Reveal;
 use crate::ty::adjustment::PointerCast;
 use crate::ty::subst::SubstsRef;
 use crate::ty::{self, List, Ty};
@@ -85,10 +86,30 @@ pub enum MirPhase {
     ///
     /// Also note that the lint pass which reports eg `200_u8 + 200_u8` as an error is run as a part
     /// of analysis to runtime MIR lowering. To ensure lints are reported reliably, this means that
-    /// transformations which may supress such errors should not run on analysis MIR.
+    /// transformations which may suppress such errors should not run on analysis MIR.
     Runtime(RuntimePhase),
 }
 
+impl MirPhase {
+    pub fn name(&self) -> &'static str {
+        match *self {
+            MirPhase::Built => "built",
+            MirPhase::Analysis(AnalysisPhase::Initial) => "analysis",
+            MirPhase::Analysis(AnalysisPhase::PostCleanup) => "analysis-post-cleanup",
+            MirPhase::Runtime(RuntimePhase::Initial) => "runtime",
+            MirPhase::Runtime(RuntimePhase::PostCleanup) => "runtime-post-cleanup",
+            MirPhase::Runtime(RuntimePhase::Optimized) => "runtime-optimized",
+        }
+    }
+
+    pub fn reveal(&self) -> Reveal {
+        match *self {
+            MirPhase::Built | MirPhase::Analysis(_) => Reveal::UserFacing,
+            MirPhase::Runtime(_) => Reveal::All,
+        }
+    }
+}
+
 /// See [`MirPhase::Analysis`].
 #[derive(Copy, Clone, TyEncodable, TyDecodable, Debug, PartialEq, Eq, PartialOrd, Ord)]
 #[derive(HashStable)]
@@ -387,7 +408,7 @@ impl std::fmt::Display for NonDivergingIntrinsic<'_> {
 #[derive(Copy, Clone, TyEncodable, TyDecodable, Debug, PartialEq, Eq, Hash, HashStable)]
 #[rustc_pass_by_value]
 pub enum RetagKind {
-    /// The initial retag when entering a function.
+    /// The initial retag of arguments when entering a function.
     FnEntry,
     /// Retag preparing for a two-phase borrow.
     TwoPhase,
@@ -1185,7 +1206,8 @@ pub enum NullOp {
     AlignOf,
 }
 
-#[derive(Copy, Clone, Debug, PartialEq, Eq, TyEncodable, TyDecodable, Hash, HashStable)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[derive(HashStable, TyEncodable, TyDecodable, TypeFoldable, TypeVisitable)]
 pub enum UnOp {
     /// The `!` operator for logical inversion
     Not,
@@ -1193,7 +1215,8 @@ pub enum UnOp {
     Neg,
 }
 
-#[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Eq, TyEncodable, TyDecodable, Hash, HashStable)]
+#[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Ord, Eq, Hash)]
+#[derive(TyEncodable, TyDecodable, HashStable, TypeFoldable, TypeVisitable)]
 pub enum BinOp {
     /// The `+` operator (addition)
     Add,
diff --git a/compiler/rustc_middle/src/mir/type_foldable.rs b/compiler/rustc_middle/src/mir/type_foldable.rs
index 4c0974f86..0705b4cff 100644
--- a/compiler/rustc_middle/src/mir/type_foldable.rs
+++ b/compiler/rustc_middle/src/mir/type_foldable.rs
@@ -16,9 +16,7 @@ TrivialTypeTraversalAndLiftImpls! {
     UserTypeAnnotationIndex,
     BorrowKind,
     CastKind,
-    BinOp,
     NullOp,
-    UnOp,
     hir::Movability,
     BasicBlock,
     SwitchTargets,
diff --git a/compiler/rustc_middle/src/mir/visit.rs b/compiler/rustc_middle/src/mir/visit.rs
index ddcf3711b..b21f50ae5 100644
--- a/compiler/rustc_middle/src/mir/visit.rs
+++ b/compiler/rustc_middle/src/mir/visit.rs
@@ -847,6 +847,17 @@ macro_rules! make_mir_visitor {
                             PlaceContext::NonUse(NonUseContext::VarDebugInfo),
                             location
                         ),
+                    VarDebugInfoContents::Composite { ty, fragments } => {
+                        // FIXME(eddyb) use a better `TyContext` here.
+                        self.visit_ty($(& $mutability)? *ty, TyContext::Location(location));
+                        for VarDebugInfoFragment { projection: _, contents } in fragments {
+                            self.visit_place(
+                                contents,
+                                PlaceContext::NonUse(NonUseContext::VarDebugInfo),
+                                location,
+                            );
+                        }
+                    }
                 }
             }
 
@@ -1320,6 +1331,15 @@ impl PlaceContext {
         )
     }
 
+    /// Returns `true` if this place context represents an address-of.
+    pub fn is_address_of(&self) -> bool {
+        matches!(
+            self,
+            PlaceContext::NonMutatingUse(NonMutatingUseContext::AddressOf)
+                | PlaceContext::MutatingUse(MutatingUseContext::AddressOf)
+        )
+    }
+
     /// Returns `true` if this place context represents a storage live or storage dead marker.
     #[inline]
     pub fn is_storage_marker(&self) -> bool {
diff --git a/compiler/rustc_middle/src/query/keys.rs b/compiler/rustc_middle/src/query/keys.rs
new file mode 100644
index 000000000..880632561
--- /dev/null
+++ b/compiler/rustc_middle/src/query/keys.rs
@@ -0,0 +1,594 @@
+//! Defines the set of legal keys that can be used in queries.
+
+use crate::infer::canonical::Canonical;
+use crate::mir;
+use crate::traits;
+use crate::ty::fast_reject::SimplifiedType;
+use crate::ty::subst::{GenericArg, SubstsRef};
+use crate::ty::{self, layout::TyAndLayout, Ty, TyCtxt};
+use rustc_hir::def_id::{CrateNum, DefId, LocalDefId, LOCAL_CRATE};
+use rustc_hir::hir_id::{HirId, OwnerId};
+use rustc_query_system::query::{DefaultCacheSelector, VecCacheSelector};
+use rustc_span::symbol::{Ident, Symbol};
+use rustc_span::{Span, DUMMY_SP};
+
+/// The `Key` trait controls what types can legally be used as the key
+/// for a query.
+pub trait Key: Sized {
+    type CacheSelector = DefaultCacheSelector<Self>;
+
+    /// Given an instance of this key, what crate is it referring to?
+    /// This is used to find the provider.
+    fn query_crate_is_local(&self) -> bool;
+
+    /// In the event that a cycle occurs, if no explicit span has been
+    /// given for a query with key `self`, what span should we use?
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span;
+
+    /// If the key is a [`DefId`] or `DefId`--equivalent, return that `DefId`.
+    /// Otherwise, return `None`.
+    fn key_as_def_id(&self) -> Option<DefId> {
+        None
+    }
+
+    fn ty_adt_id(&self) -> Option<DefId> {
+        None
+    }
+}
+
+impl Key for () {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl<'tcx> Key for ty::InstanceDef<'tcx> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        tcx.def_span(self.def_id())
+    }
+}
+
+impl<'tcx> Key for ty::Instance<'tcx> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        tcx.def_span(self.def_id())
+    }
+}
+
+impl<'tcx> Key for mir::interpret::GlobalId<'tcx> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        self.instance.default_span(tcx)
+    }
+}
+
+impl<'tcx> Key for (Ty<'tcx>, Option<ty::PolyExistentialTraitRef<'tcx>>) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl<'tcx> Key for mir::interpret::LitToConstInput<'tcx> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+
+    fn default_span(&self, _tcx: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl Key for CrateNum {
+    type CacheSelector = VecCacheSelector<Self>;
+
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        *self == LOCAL_CRATE
+    }
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl Key for OwnerId {
+    type CacheSelector = VecCacheSelector<Self>;
+
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        self.to_def_id().default_span(tcx)
+    }
+    fn key_as_def_id(&self) -> Option<DefId> {
+        Some(self.to_def_id())
+    }
+}
+
+impl Key for LocalDefId {
+    type CacheSelector = VecCacheSelector<Self>;
+
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        self.to_def_id().default_span(tcx)
+    }
+    fn key_as_def_id(&self) -> Option<DefId> {
+        Some(self.to_def_id())
+    }
+}
+
+impl Key for DefId {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        self.krate == LOCAL_CRATE
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        tcx.def_span(*self)
+    }
+    #[inline(always)]
+    fn key_as_def_id(&self) -> Option<DefId> {
+        Some(*self)
+    }
+}
+
+impl Key for ty::WithOptConstParam<LocalDefId> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        self.did.default_span(tcx)
+    }
+}
+
+impl Key for SimplifiedType {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl Key for (DefId, DefId) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        self.0.krate == LOCAL_CRATE
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        self.1.default_span(tcx)
+    }
+}
+
+impl<'tcx> Key for (ty::Instance<'tcx>, LocalDefId) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        self.0.default_span(tcx)
+    }
+}
+
+impl Key for (DefId, LocalDefId) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        self.0.krate == LOCAL_CRATE
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        self.1.default_span(tcx)
+    }
+}
+
+impl Key for (LocalDefId, DefId) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        self.0.default_span(tcx)
+    }
+}
+
+impl Key for (LocalDefId, LocalDefId) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        self.0.default_span(tcx)
+    }
+}
+
+impl Key for (DefId, Option<Ident>) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        self.0.krate == LOCAL_CRATE
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        tcx.def_span(self.0)
+    }
+    #[inline(always)]
+    fn key_as_def_id(&self) -> Option<DefId> {
+        Some(self.0)
+    }
+}
+
+impl Key for (DefId, LocalDefId, Ident) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        self.0.krate == LOCAL_CRATE
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        self.1.default_span(tcx)
+    }
+}
+
+impl Key for (CrateNum, DefId) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        self.0 == LOCAL_CRATE
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        self.1.default_span(tcx)
+    }
+}
+
+impl Key for (CrateNum, SimplifiedType) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        self.0 == LOCAL_CRATE
+    }
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl Key for (DefId, SimplifiedType) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        self.0.krate == LOCAL_CRATE
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        self.0.default_span(tcx)
+    }
+}
+
+impl<'tcx> Key for SubstsRef<'tcx> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl<'tcx> Key for (DefId, SubstsRef<'tcx>) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        self.0.krate == LOCAL_CRATE
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        self.0.default_span(tcx)
+    }
+}
+
+impl<'tcx> Key for (ty::UnevaluatedConst<'tcx>, ty::UnevaluatedConst<'tcx>) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        (self.0).def.did.krate == LOCAL_CRATE
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        (self.0).def.did.default_span(tcx)
+    }
+}
+
+impl<'tcx> Key for (LocalDefId, DefId, SubstsRef<'tcx>) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        self.0.default_span(tcx)
+    }
+}
+
+impl<'tcx> Key for (ty::ParamEnv<'tcx>, ty::PolyTraitRef<'tcx>) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        self.1.def_id().krate == LOCAL_CRATE
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        tcx.def_span(self.1.def_id())
+    }
+}
+
+impl<'tcx> Key for (ty::Const<'tcx>, mir::Field) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl<'tcx> Key for mir::interpret::ConstAlloc<'tcx> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl<'tcx> Key for ty::PolyTraitRef<'tcx> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        self.def_id().krate == LOCAL_CRATE
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        tcx.def_span(self.def_id())
+    }
+}
+
+impl<'tcx> Key for ty::PolyExistentialTraitRef<'tcx> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        self.def_id().krate == LOCAL_CRATE
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        tcx.def_span(self.def_id())
+    }
+}
+
+impl<'tcx> Key for (ty::PolyTraitRef<'tcx>, ty::PolyTraitRef<'tcx>) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        self.0.def_id().krate == LOCAL_CRATE
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        tcx.def_span(self.0.def_id())
+    }
+}
+
+impl<'tcx> Key for GenericArg<'tcx> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl<'tcx> Key for mir::ConstantKind<'tcx> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl<'tcx> Key for ty::Const<'tcx> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl<'tcx> Key for Ty<'tcx> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+    fn ty_adt_id(&self) -> Option<DefId> {
+        match self.kind() {
+            ty::Adt(adt, _) => Some(adt.did()),
+            _ => None,
+        }
+    }
+}
+
+impl<'tcx> Key for TyAndLayout<'tcx> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl<'tcx> Key for (Ty<'tcx>, Ty<'tcx>) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl<'tcx> Key for &'tcx ty::List<ty::Predicate<'tcx>> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl<'tcx> Key for ty::ParamEnv<'tcx> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl<'tcx, T: Key> Key for ty::ParamEnvAnd<'tcx, T> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        self.value.query_crate_is_local()
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        self.value.default_span(tcx)
+    }
+}
+
+impl Key for Symbol {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, _tcx: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl Key for Option<Symbol> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, _tcx: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+/// Canonical query goals correspond to abstract trait operations that
+/// are not tied to any crate in particular.
+impl<'tcx, T> Key for Canonical<'tcx, T> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+
+    fn default_span(&self, _tcx: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl Key for (Symbol, u32, u32) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+
+    fn default_span(&self, _tcx: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl<'tcx> Key for (DefId, Ty<'tcx>, SubstsRef<'tcx>, ty::ParamEnv<'tcx>) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+
+    fn default_span(&self, _tcx: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl<'tcx> Key for (ty::Predicate<'tcx>, traits::WellFormedLoc) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+
+    fn default_span(&self, _tcx: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl<'tcx> Key for (ty::PolyFnSig<'tcx>, &'tcx ty::List<Ty<'tcx>>) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl<'tcx> Key for (ty::Instance<'tcx>, &'tcx ty::List<Ty<'tcx>>) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        self.0.default_span(tcx)
+    }
+}
+
+impl<'tcx> Key for (Ty<'tcx>, ty::ValTree<'tcx>) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl Key for HirId {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        tcx.hir().span(*self)
+    }
+
+    #[inline(always)]
+    fn key_as_def_id(&self) -> Option<DefId> {
+        None
+    }
+}
diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs
index 3d720f09b..34415e2a1 100644
--- a/compiler/rustc_middle/src/query/mod.rs
+++ b/compiler/rustc_middle/src/query/mod.rs
@@ -7,6 +7,9 @@
 use crate::ty::{self, print::describe_as_module, TyCtxt};
 use rustc_span::def_id::LOCAL_CRATE;
 
+mod keys;
+pub use keys::Key;
+
 // Each of these queries corresponds to a function pointer field in the
 // `Providers` struct for requesting a value of that type, and a method
 // on `tcx: TyCtxt` (and `tcx.at(span)`) for doing that request in a way
@@ -30,7 +33,7 @@ rustc_queries! {
     }
 
     query resolver_for_lowering(_: ()) -> &'tcx Steal<ty::ResolverAstLowering> {
-        eval_always
+        feedable
         no_hash
         desc { "getting the resolver for lowering" }
     }
@@ -172,6 +175,12 @@ rustc_queries! {
         separate_provide_extern
     }
 
+    query is_type_alias_impl_trait(key: DefId) -> bool
+    {
+        desc { "determine whether the opaque is a type-alias impl trait" }
+        separate_provide_extern
+    }
+
     query analysis(key: ()) -> Result<(), ErrorGuaranteed> {
         eval_always
         desc { "running analysis passes on this crate" }
@@ -271,6 +280,10 @@ rustc_queries! {
         desc { |tcx| "elaborating item bounds for `{}`", tcx.def_path_str(key) }
     }
 
+    /// Look up all native libraries this crate depends on.
+    /// These are assembled from the following places:
+    /// - `extern` blocks (depending on their `link` attributes)
+    /// - the `libs` (`-l`) option
     query native_libraries(_: CrateNum) -> Vec<NativeLib> {
         arena_cache
         desc { "looking up the native libraries of a linked crate" }
@@ -393,7 +406,7 @@ rustc_queries! {
     /// Try to build an abstract representation of the given constant.
     query thir_abstract_const(
         key: DefId
-    ) -> Result<Option<&'tcx [ty::abstract_const::Node<'tcx>]>, ErrorGuaranteed> {
+    ) -> Result<Option<ty::Const<'tcx>>, ErrorGuaranteed> {
         desc {
             |tcx| "building an abstract representation for `{}`", tcx.def_path_str(key),
         }
@@ -402,7 +415,7 @@ rustc_queries! {
     /// Try to build an abstract representation of the given constant.
     query thir_abstract_const_of_const_arg(
         key: (LocalDefId, DefId)
-    ) -> Result<Option<&'tcx [ty::abstract_const::Node<'tcx>]>, ErrorGuaranteed> {
+    ) -> Result<Option<ty::Const<'tcx>>, ErrorGuaranteed> {
         desc {
             |tcx|
             "building an abstract representation for the const argument `{}`",
@@ -410,15 +423,6 @@ rustc_queries! {
         }
     }
 
-    query try_unify_abstract_consts(key:
-        ty::ParamEnvAnd<'tcx, (ty::UnevaluatedConst<'tcx>, ty::UnevaluatedConst<'tcx>
-    )>) -> bool {
-        desc {
-            |tcx| "trying to unify the generic constants `{}` and `{}`",
-            tcx.def_path_str(key.value.0.def.did), tcx.def_path_str(key.value.1.def.did)
-        }
-    }
-
     query mir_drops_elaborated_and_const_checked(
         key: ty::WithOptConstParam<LocalDefId>
     ) -> &'tcx Steal<mir::Body<'tcx>> {
@@ -564,7 +568,7 @@ rustc_queries! {
 
     /// Returns the inferred outlives predicates (e.g., for `struct
     /// Foo<'a, T> { x: &'a T }`, this would return `T: 'a`).
-    query inferred_outlives_of(key: DefId) -> &'tcx [(ty::Predicate<'tcx>, Span)] {
+    query inferred_outlives_of(key: DefId) -> &'tcx [(ty::Clause<'tcx>, Span)] {
         desc { |tcx| "computing inferred outlives predicates of `{}`", tcx.def_path_str(key) }
         cache_on_disk_if { key.is_local() }
         separate_provide_extern
@@ -1111,6 +1115,7 @@ rustc_queries! {
         desc { |tcx| "looking up span for `{}`", tcx.def_path_str(def_id) }
         cache_on_disk_if { def_id.is_local() }
         separate_provide_extern
+        feedable
     }
 
     /// Gets the span for the identifier of the definition.
@@ -1391,6 +1396,13 @@ rustc_queries! {
         desc { "checking if the crate has_global_allocator" }
         separate_provide_extern
     }
+    query has_alloc_error_handler(_: CrateNum) -> bool {
+        // This query depends on untracked global state in CStore
+        eval_always
+        fatal_cycle
+        desc { "checking if the crate has_alloc_error_handler" }
+        separate_provide_extern
+    }
     query has_panic_handler(_: CrateNum) -> bool {
         fatal_cycle
         desc { "checking if the crate has_panic_handler" }
@@ -1532,6 +1544,7 @@ rustc_queries! {
         desc { "available upstream drop-glue for `{:?}`", substs }
     }
 
+    /// Returns a list of all `extern` blocks of a crate.
     query foreign_modules(_: CrateNum) -> FxHashMap<DefId, ForeignModule> {
         arena_cache
         desc { "looking up the foreign modules of a linked crate" }
@@ -1543,9 +1556,12 @@ rustc_queries! {
     query entry_fn(_: ()) -> Option<(DefId, EntryFnType)> {
         desc { "looking up the entry function of a crate" }
     }
+
+    /// Finds the `rustc_proc_macro_decls` item of a crate.
     query proc_macro_decls_static(_: ()) -> Option<LocalDefId> {
-        desc { "looking up the derive registrar for a crate" }
+        desc { "looking up the proc macro declarations for a crate" }
     }
+
     // The macro which defines `rustc_metadata::provide_extern` depends on this query's name.
     // Changing the name should cause a compiler error, but in case that changes, be aware.
     query crate_hash(_: CrateNum) -> Svh {
@@ -1553,17 +1569,24 @@ rustc_queries! {
         desc { "looking up the hash a crate" }
         separate_provide_extern
     }
+
+    /// Gets the hash for the host proc macro. Used to support -Z dual-proc-macro.
     query crate_host_hash(_: CrateNum) -> Option<Svh> {
         eval_always
         desc { "looking up the hash of a host version of a crate" }
         separate_provide_extern
     }
+
+    /// Gets the extra data to put in each output filename for a crate.
+    /// For example, compiling the `foo` crate with `extra-filename=-a` creates a `libfoo-b.rlib` file.
     query extra_filename(_: CrateNum) -> String {
         arena_cache
         eval_always
         desc { "looking up the extra filename for a crate" }
         separate_provide_extern
     }
+
+    /// Gets the paths where the crate came from in the file system.
     query crate_extern_paths(_: CrateNum) -> Vec<PathBuf> {
         arena_cache
         eval_always
@@ -1587,23 +1610,15 @@ rustc_queries! {
         separate_provide_extern
     }
 
+    /// Get the corresponding native library from the `native_libraries` query
     query native_library(def_id: DefId) -> Option<&'tcx NativeLib> {
         desc { |tcx| "getting the native library for `{}`", tcx.def_path_str(def_id) }
     }
 
-    /// Does lifetime resolution, but does not descend into trait items. This
-    /// should only be used for resolving lifetimes of on trait definitions,
-    /// and is used to avoid cycles. Importantly, `resolve_lifetimes` still visits
-    /// the same lifetimes and is responsible for diagnostics.
-    /// See `rustc_resolve::late::lifetimes for details.
-    query resolve_lifetimes_trait_definition(_: LocalDefId) -> ResolveLifetimes {
-        arena_cache
-        desc { "resolving lifetimes for a trait definition" }
-    }
     /// Does lifetime resolution on items. Importantly, we can't resolve
     /// lifetimes directly on things like trait methods, because of trait params.
     /// See `rustc_resolve::late::lifetimes for details.
-    query resolve_lifetimes(_: LocalDefId) -> ResolveLifetimes {
+    query resolve_lifetimes(_: hir::OwnerId) -> ResolveLifetimes {
         arena_cache
         desc { "resolving lifetimes" }
     }
@@ -1634,6 +1649,8 @@ rustc_queries! {
     /// a generic type parameter will panic if you call this method on it:
     ///
     /// ```
+    /// use std::fmt::Debug;
+    ///
     /// pub trait Foo<T: Debug> {}
     /// ```
     ///
@@ -1705,7 +1722,7 @@ rustc_queries! {
     }
 
     /// Returns the lang items defined in another crate by loading it from metadata.
-    query defined_lang_items(_: CrateNum) -> &'tcx [(DefId, usize)] {
+    query defined_lang_items(_: CrateNum) -> &'tcx [(DefId, LangItem)] {
         desc { "calculating the lang items defined in a crate" }
         separate_provide_extern
     }
@@ -1761,6 +1778,10 @@ rustc_queries! {
         eval_always
         desc { "getting the allocator kind for the current crate" }
     }
+    query alloc_error_handler_kind(_: ()) -> Option<AllocatorKind> {
+        eval_always
+        desc { "alloc error handler kind for the current crate" }
+    }
 
     query upvars_mentioned(def_id: DefId) -> Option<&'tcx FxIndexMap<hir::HirId, hir::Upvar>> {
         desc { |tcx| "collecting upvars mentioned in `{}`", tcx.def_path_str(def_id) }
@@ -2054,19 +2075,8 @@ rustc_queries! {
         remap_env_constness
     }
 
-    query normalize_opaque_types(key: &'tcx ty::List<ty::Predicate<'tcx>>) -> &'tcx ty::List<ty::Predicate<'tcx>> {
-        desc { "normalizing opaque types in `{:?}`", key }
-    }
-
-    /// Checks whether a type is definitely uninhabited. This is
-    /// conservative: for some types that are uninhabited we return `false`,
-    /// but we only return `true` for types that are definitely uninhabited.
-    /// `ty.conservative_is_privately_uninhabited` implies that any value of type `ty`
-    /// will be `Abi::Uninhabited`. (Note that uninhabited types may have nonzero
-    /// size, to account for partial initialisation. See #49298 for details.)
-    query conservative_is_privately_uninhabited(key: ty::ParamEnvAnd<'tcx, Ty<'tcx>>) -> bool {
-        desc { "conservatively checking if `{}` is privately uninhabited", key.value }
-        remap_env_constness
+    query reveal_opaque_types_in_bounds(key: &'tcx ty::List<ty::Predicate<'tcx>>) -> &'tcx ty::List<ty::Predicate<'tcx>> {
+        desc { "revealing opaque types in `{:?}`", key }
     }
 
     query limits(key: ()) -> Limits {
diff --git a/compiler/rustc_middle/src/thir.rs b/compiler/rustc_middle/src/thir.rs
index ea7a507d7..8bef9dfe0 100644
--- a/compiler/rustc_middle/src/thir.rs
+++ b/compiler/rustc_middle/src/thir.rs
@@ -10,7 +10,6 @@
 
 use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
 use rustc_hir as hir;
-use rustc_hir::def::CtorKind;
 use rustc_hir::def_id::DefId;
 use rustc_hir::RangeEnd;
 use rustc_index::newtype_index;
@@ -751,7 +750,7 @@ impl<'tcx> fmt::Display for Pat<'tcx> {
 
                     // Only for Adt we can have `S {...}`,
                     // which we handle separately here.
-                    if variant.ctor_kind == CtorKind::Fictive {
+                    if variant.ctor.is_none() {
                         write!(f, " {{ ")?;
 
                         let mut printed = 0;
diff --git a/compiler/rustc_middle/src/traits/mod.rs b/compiler/rustc_middle/src/traits/mod.rs
index e73d44bbb..143435cb2 100644
--- a/compiler/rustc_middle/src/traits/mod.rs
+++ b/compiler/rustc_middle/src/traits/mod.rs
@@ -203,13 +203,20 @@ pub struct UnifyReceiverContext<'tcx> {
     pub substs: SubstsRef<'tcx>,
 }
 
-#[derive(Clone, Debug, PartialEq, Eq, Hash, Lift, Default)]
+#[derive(Clone, PartialEq, Eq, Hash, Lift, Default)]
 pub struct InternedObligationCauseCode<'tcx> {
     /// `None` for `ObligationCauseCode::MiscObligation` (a common case, occurs ~60% of
     /// the time). `Some` otherwise.
     code: Option<Lrc<ObligationCauseCode<'tcx>>>,
 }
 
+impl<'tcx> std::fmt::Debug for InternedObligationCauseCode<'tcx> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let cause: &ObligationCauseCode<'_> = self;
+        cause.fmt(f)
+    }
+}
+
 impl<'tcx> ObligationCauseCode<'tcx> {
     #[inline(always)]
     fn into(self) -> InternedObligationCauseCode<'tcx> {
@@ -431,6 +438,8 @@ pub enum ObligationCauseCode<'tcx> {
     },
 
     AscribeUserTypeProvePredicate(Span),
+
+    RustCall,
 }
 
 /// The 'location' at which we try to perform HIR-based wf checking.
@@ -462,7 +471,7 @@ pub struct ImplDerivedObligationCause<'tcx> {
 }
 
 impl<'tcx> ObligationCauseCode<'tcx> {
-    // Return the base obligation, ignoring derived obligations.
+    /// Returns the base obligation, ignoring derived obligations.
     pub fn peel_derives(&self) -> &Self {
         let mut base_cause = self;
         while let Some((parent_code, _)) = base_cause.parent() {
@@ -567,9 +576,6 @@ pub enum SelectionError<'tcx> {
     /// Signaling that an error has already been emitted, to avoid
     /// multiple errors being shown.
     ErrorReporting,
-    /// Multiple applicable `impl`s where found. The `DefId`s correspond to
-    /// all the `impl`s' Items.
-    Ambiguous(Vec<DefId>),
 }
 
 /// When performing resolution, it is typically the case that there
@@ -645,15 +651,12 @@ pub enum ImplSource<'tcx, N> {
     /// Same as above, but for a function pointer type with the given signature.
     FnPointer(ImplSourceFnPointerData<'tcx, N>),
 
-    /// ImplSource for a builtin `DeterminantKind` trait implementation.
-    DiscriminantKind(ImplSourceDiscriminantKindData),
-
-    /// ImplSource for a builtin `Pointee` trait implementation.
-    Pointee(ImplSourcePointeeData),
-
     /// ImplSource automatically generated for a generator.
     Generator(ImplSourceGeneratorData<'tcx, N>),
 
+    /// ImplSource automatically generated for a generator backing an async future.
+    Future(ImplSourceFutureData<'tcx, N>),
+
     /// ImplSource for a trait alias.
     TraitAlias(ImplSourceTraitAliasData<'tcx, N>),
 
@@ -670,10 +673,9 @@ impl<'tcx, N> ImplSource<'tcx, N> {
             ImplSource::AutoImpl(d) => d.nested,
             ImplSource::Closure(c) => c.nested,
             ImplSource::Generator(c) => c.nested,
+            ImplSource::Future(c) => c.nested,
             ImplSource::Object(d) => d.nested,
             ImplSource::FnPointer(d) => d.nested,
-            ImplSource::DiscriminantKind(ImplSourceDiscriminantKindData)
-            | ImplSource::Pointee(ImplSourcePointeeData) => vec![],
             ImplSource::TraitAlias(d) => d.nested,
             ImplSource::TraitUpcasting(d) => d.nested,
             ImplSource::ConstDestruct(i) => i.nested,
@@ -688,10 +690,9 @@ impl<'tcx, N> ImplSource<'tcx, N> {
             ImplSource::AutoImpl(d) => &d.nested,
             ImplSource::Closure(c) => &c.nested,
             ImplSource::Generator(c) => &c.nested,
+            ImplSource::Future(c) => &c.nested,
             ImplSource::Object(d) => &d.nested,
             ImplSource::FnPointer(d) => &d.nested,
-            ImplSource::DiscriminantKind(ImplSourceDiscriminantKindData)
-            | ImplSource::Pointee(ImplSourcePointeeData) => &[],
             ImplSource::TraitAlias(d) => &d.nested,
             ImplSource::TraitUpcasting(d) => &d.nested,
             ImplSource::ConstDestruct(i) => &i.nested,
@@ -731,16 +732,15 @@ impl<'tcx, N> ImplSource<'tcx, N> {
                 substs: c.substs,
                 nested: c.nested.into_iter().map(f).collect(),
             }),
+            ImplSource::Future(c) => ImplSource::Future(ImplSourceFutureData {
+                generator_def_id: c.generator_def_id,
+                substs: c.substs,
+                nested: c.nested.into_iter().map(f).collect(),
+            }),
             ImplSource::FnPointer(p) => ImplSource::FnPointer(ImplSourceFnPointerData {
                 fn_ty: p.fn_ty,
                 nested: p.nested.into_iter().map(f).collect(),
             }),
-            ImplSource::DiscriminantKind(ImplSourceDiscriminantKindData) => {
-                ImplSource::DiscriminantKind(ImplSourceDiscriminantKindData)
-            }
-            ImplSource::Pointee(ImplSourcePointeeData) => {
-                ImplSource::Pointee(ImplSourcePointeeData)
-            }
             ImplSource::TraitAlias(d) => ImplSource::TraitAlias(ImplSourceTraitAliasData {
                 alias_def_id: d.alias_def_id,
                 substs: d.substs,
@@ -792,6 +792,16 @@ pub struct ImplSourceGeneratorData<'tcx, N> {
 
 #[derive(Clone, PartialEq, Eq, TyEncodable, TyDecodable, HashStable, Lift)]
 #[derive(TypeFoldable, TypeVisitable)]
+pub struct ImplSourceFutureData<'tcx, N> {
+    pub generator_def_id: DefId,
+    pub substs: SubstsRef<'tcx>,
+    /// Nested obligations. This can be non-empty if the generator
+    /// signature contains associated types.
+    pub nested: Vec<N>,
+}
+
+#[derive(Clone, PartialEq, Eq, TyEncodable, TyDecodable, HashStable, Lift)]
+#[derive(TypeFoldable, TypeVisitable)]
 pub struct ImplSourceClosureData<'tcx, N> {
     pub closure_def_id: DefId,
     pub substs: SubstsRef<'tcx>,
@@ -850,13 +860,6 @@ pub struct ImplSourceFnPointerData<'tcx, N> {
     pub nested: Vec<N>,
 }
 
-// FIXME(@lcnr): This should be  refactored and merged with other builtin vtables.
-#[derive(Clone, Debug, PartialEq, Eq, TyEncodable, TyDecodable, HashStable)]
-pub struct ImplSourceDiscriminantKindData;
-
-#[derive(Clone, Debug, PartialEq, Eq, TyEncodable, TyDecodable, HashStable)]
-pub struct ImplSourcePointeeData;
-
 #[derive(Clone, PartialEq, Eq, TyEncodable, TyDecodable, HashStable, Lift)]
 #[derive(TypeFoldable, TypeVisitable)]
 pub struct ImplSourceConstDestructData<N> {
@@ -918,10 +921,13 @@ impl ObjectSafetyViolation {
             }
             ObjectSafetyViolation::Method(
                 name,
-                MethodViolationCode::ReferencesImplTraitInTrait,
+                MethodViolationCode::ReferencesImplTraitInTrait(_),
                 _,
             ) => format!("method `{}` references an `impl Trait` type in its return type", name)
                 .into(),
+            ObjectSafetyViolation::Method(name, MethodViolationCode::AsyncFn, _) => {
+                format!("method `{}` is `async`", name).into()
+            }
             ObjectSafetyViolation::Method(
                 name,
                 MethodViolationCode::WhereClauseReferencesSelf,
@@ -1029,7 +1035,10 @@ pub enum MethodViolationCode {
     ReferencesSelfOutput,
 
     /// e.g., `fn foo(&self) -> impl Sized`
-    ReferencesImplTraitInTrait,
+    ReferencesImplTraitInTrait(Span),
+
+    /// e.g., `async fn foo(&self)`
+    AsyncFn,
 
     /// e.g., `fn foo(&self) where Self: Clone`
     WhereClauseReferencesSelf,
diff --git a/compiler/rustc_middle/src/traits/select.rs b/compiler/rustc_middle/src/traits/select.rs
index 85ead3171..ec69864c9 100644
--- a/compiler/rustc_middle/src/traits/select.rs
+++ b/compiler/rustc_middle/src/traits/select.rs
@@ -105,6 +105,12 @@ pub type EvaluationCache<'tcx> = Cache<
 /// parameter environment.
 #[derive(PartialEq, Eq, Debug, Clone, TypeFoldable, TypeVisitable)]
 pub enum SelectionCandidate<'tcx> {
+    /// A builtin implementation for some specific traits, used in cases
+    /// where we cannot rely an ordinary library implementations.
+    ///
+    /// The most notable examples are `sized`, `Copy` and `Clone`. This is also
+    /// used for the `DiscriminantKind` and `Pointee` trait, both of which have
+    /// an associated type.
     BuiltinCandidate {
         /// `false` if there are no *further* obligations.
         has_nested: bool,
@@ -131,18 +137,16 @@ pub enum SelectionCandidate<'tcx> {
     /// generated for a generator.
     GeneratorCandidate,
 
+    /// Implementation of a `Future` trait by one of the generator types
+    /// generated for an async construct.
+    FutureCandidate,
+
     /// Implementation of a `Fn`-family trait by one of the anonymous
     /// types generated for a fn pointer type (e.g., `fn(int) -> int`)
     FnPointerCandidate {
         is_const: bool,
     },
 
-    /// Builtin implementation of `DiscriminantKind`.
-    DiscriminantKindCandidate,
-
-    /// Builtin implementation of `Pointee`.
-    PointeeCandidate,
-
     TraitAliasCandidate,
 
     /// Matching `dyn Trait` with a supertrait of `Trait`. The index is the
diff --git a/compiler/rustc_middle/src/traits/specialization_graph.rs b/compiler/rustc_middle/src/traits/specialization_graph.rs
index 0a2819fee..cccedc9ec 100644
--- a/compiler/rustc_middle/src/traits/specialization_graph.rs
+++ b/compiler/rustc_middle/src/traits/specialization_graph.rs
@@ -1,3 +1,4 @@
+use crate::error::StrictCoherenceNeedsNegativeCoherence;
 use crate::ty::fast_reject::SimplifiedType;
 use crate::ty::visit::TypeVisitable;
 use crate::ty::{self, TyCtxt};
@@ -65,9 +66,21 @@ impl OverlapMode {
 
         if with_negative_coherence {
             if strict_coherence { OverlapMode::Strict } else { OverlapMode::WithNegative }
-        } else if strict_coherence {
-            bug!("To use strict_coherence you need to set with_negative_coherence feature flag");
         } else {
+            if strict_coherence {
+                let attr_span = trait_id
+                    .as_local()
+                    .into_iter()
+                    .flat_map(|local_def_id| {
+                        tcx.hir().attrs(tcx.hir().local_def_id_to_hir_id(local_def_id))
+                    })
+                    .find(|attr| attr.has_name(sym::rustc_strict_coherence))
+                    .map(|attr| attr.span);
+                tcx.sess.emit_err(StrictCoherenceNeedsNegativeCoherence {
+                    span: tcx.def_span(trait_id),
+                    attr_span,
+                });
+            }
             OverlapMode::Stable
         }
     }
@@ -249,7 +262,7 @@ pub fn ancestors<'tcx>(
 
     if let Some(reported) = specialization_graph.has_errored {
         Err(reported)
-    } else if let Some(reported) = tcx.type_of(start_from_impl).error_reported() {
+    } else if let Err(reported) = tcx.type_of(start_from_impl).error_reported() {
         Err(reported)
     } else {
         Ok(Ancestors {
diff --git a/compiler/rustc_middle/src/traits/structural_impls.rs b/compiler/rustc_middle/src/traits/structural_impls.rs
index 7fbd57ac7..6acb7745d 100644
--- a/compiler/rustc_middle/src/traits/structural_impls.rs
+++ b/compiler/rustc_middle/src/traits/structural_impls.rs
@@ -15,11 +15,9 @@ impl<'tcx, N: fmt::Debug> fmt::Debug for traits::ImplSource<'tcx, N> {
 
             super::ImplSource::Generator(ref d) => write!(f, "{:?}", d),
 
-            super::ImplSource::FnPointer(ref d) => write!(f, "({:?})", d),
-
-            super::ImplSource::DiscriminantKind(ref d) => write!(f, "{:?}", d),
+            super::ImplSource::Future(ref d) => write!(f, "{:?}", d),
 
-            super::ImplSource::Pointee(ref d) => write!(f, "{:?}", d),
+            super::ImplSource::FnPointer(ref d) => write!(f, "({:?})", d),
 
             super::ImplSource::Object(ref d) => write!(f, "{:?}", d),
 
@@ -58,6 +56,16 @@ impl<'tcx, N: fmt::Debug> fmt::Debug for traits::ImplSourceGeneratorData<'tcx, N
     }
 }
 
+impl<'tcx, N: fmt::Debug> fmt::Debug for traits::ImplSourceFutureData<'tcx, N> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "ImplSourceFutureData(generator_def_id={:?}, substs={:?}, nested={:?})",
+            self.generator_def_id, self.substs, self.nested
+        )
+    }
+}
+
 impl<'tcx, N: fmt::Debug> fmt::Debug for traits::ImplSourceClosureData<'tcx, N> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(
@@ -125,11 +133,3 @@ impl<N: fmt::Debug> fmt::Debug for traits::ImplSourceConstDestructData<N> {
         write!(f, "ImplSourceConstDestructData(nested={:?})", self.nested)
     }
 }
-
-///////////////////////////////////////////////////////////////////////////
-// Lift implementations
-
-TrivialTypeTraversalAndLiftImpls! {
-    super::ImplSourceDiscriminantKindData,
-    super::ImplSourcePointeeData,
-}
diff --git a/compiler/rustc_middle/src/ty/_match.rs b/compiler/rustc_middle/src/ty/_match.rs
index e6aab30a1..cd147d7e5 100644
--- a/compiler/rustc_middle/src/ty/_match.rs
+++ b/compiler/rustc_middle/src/ty/_match.rs
@@ -36,6 +36,11 @@ impl<'tcx> TypeRelation<'tcx> for Match<'tcx> {
     fn tcx(&self) -> TyCtxt<'tcx> {
         self.tcx
     }
+
+    fn intercrate(&self) -> bool {
+        false
+    }
+
     fn param_env(&self) -> ty::ParamEnv<'tcx> {
         self.param_env
     }
@@ -43,6 +48,10 @@ impl<'tcx> TypeRelation<'tcx> for Match<'tcx> {
         true
     } // irrelevant
 
+    fn mark_ambiguous(&mut self) {
+        bug!()
+    }
+
     fn relate_with_variance<T: Relate<'tcx>>(
         &mut self,
         _: ty::Variance,
diff --git a/compiler/rustc_middle/src/ty/abstract_const.rs b/compiler/rustc_middle/src/ty/abstract_const.rs
index 1aa4df778..5de758ad9 100644
--- a/compiler/rustc_middle/src/ty/abstract_const.rs
+++ b/compiler/rustc_middle/src/ty/abstract_const.rs
@@ -1,98 +1,13 @@
 //! A subset of a mir body used for const evaluatability checking.
-use crate::mir;
-use crate::ty::visit::TypeVisitable;
-use crate::ty::{self, DelaySpanBugEmitted, EarlyBinder, SubstsRef, Ty, TyCtxt};
+use crate::ty::{
+    self, Const, EarlyBinder, Ty, TyCtxt, TypeFoldable, TypeFolder, TypeSuperFoldable,
+    TypeVisitable,
+};
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir::def_id::DefId;
-use std::cmp;
-use std::ops::ControlFlow;
 
-rustc_index::newtype_index! {
-    /// An index into an `AbstractConst`.
-    pub struct NodeId {
-        derive [HashStable]
-        DEBUG_FORMAT = "n{}",
-    }
-}
-
-/// A tree representing an anonymous constant.
-///
-/// This is only able to represent a subset of `MIR`,
-/// and should not leak any information about desugarings.
-#[derive(Debug, Clone, Copy)]
-pub struct AbstractConst<'tcx> {
-    // FIXME: Consider adding something like `IndexSlice`
-    // and use this here.
-    inner: &'tcx [Node<'tcx>],
-    substs: SubstsRef<'tcx>,
-}
-
-impl<'tcx> AbstractConst<'tcx> {
-    pub fn new(
-        tcx: TyCtxt<'tcx>,
-        uv: ty::UnevaluatedConst<'tcx>,
-    ) -> Result<Option<AbstractConst<'tcx>>, ErrorGuaranteed> {
-        let inner = tcx.thir_abstract_const_opt_const_arg(uv.def)?;
-        debug!("AbstractConst::new({:?}) = {:?}", uv, inner);
-        Ok(inner.map(|inner| AbstractConst { inner, substs: tcx.erase_regions(uv.substs) }))
-    }
-
-    pub fn from_const(
-        tcx: TyCtxt<'tcx>,
-        ct: ty::Const<'tcx>,
-    ) -> Result<Option<AbstractConst<'tcx>>, ErrorGuaranteed> {
-        match ct.kind() {
-            ty::ConstKind::Unevaluated(uv) => AbstractConst::new(tcx, uv),
-            ty::ConstKind::Error(DelaySpanBugEmitted { reported, .. }) => Err(reported),
-            _ => Ok(None),
-        }
-    }
-
-    #[inline]
-    pub fn subtree(self, node: NodeId) -> AbstractConst<'tcx> {
-        AbstractConst { inner: &self.inner[..=node.index()], substs: self.substs }
-    }
-
-    #[inline]
-    pub fn root(self, tcx: TyCtxt<'tcx>) -> Node<'tcx> {
-        let node = self.inner.last().copied().unwrap();
-        match node {
-            Node::Leaf(leaf) => Node::Leaf(EarlyBinder(leaf).subst(tcx, self.substs)),
-            Node::Cast(kind, operand, ty) => {
-                Node::Cast(kind, operand, EarlyBinder(ty).subst(tcx, self.substs))
-            }
-            // Don't perform substitution on the following as they can't directly contain generic params
-            Node::Binop(_, _, _) | Node::UnaryOp(_, _) | Node::FunctionCall(_, _) => node,
-        }
-    }
-
-    pub fn unify_failure_kind(self, tcx: TyCtxt<'tcx>) -> FailureKind {
-        let mut failure_kind = FailureKind::Concrete;
-        walk_abstract_const::<!, _>(tcx, self, |node| {
-            match node.root(tcx) {
-                Node::Leaf(leaf) => {
-                    if leaf.has_non_region_infer() {
-                        failure_kind = FailureKind::MentionsInfer;
-                    } else if leaf.has_non_region_param() {
-                        failure_kind = cmp::min(failure_kind, FailureKind::MentionsParam);
-                    }
-                }
-                Node::Cast(_, _, ty) => {
-                    if ty.has_non_region_infer() {
-                        failure_kind = FailureKind::MentionsInfer;
-                    } else if ty.has_non_region_param() {
-                        failure_kind = cmp::min(failure_kind, FailureKind::MentionsParam);
-                    }
-                }
-                Node::Binop(_, _, _) | Node::UnaryOp(_, _) | Node::FunctionCall(_, _) => {}
-            }
-            ControlFlow::CONTINUE
-        });
-        failure_kind
-    }
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, HashStable, TyEncodable, TyDecodable)]
+#[derive(Hash, Debug, Clone, Copy, Ord, PartialOrd, PartialEq, Eq)]
+#[derive(TyDecodable, TyEncodable, HashStable, TypeVisitable, TypeFoldable)]
 pub enum CastKind {
     /// thir::ExprKind::As
     As,
@@ -100,16 +15,6 @@ pub enum CastKind {
     Use,
 }
 
-/// A node of an `AbstractConst`.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, HashStable, TyEncodable, TyDecodable)]
-pub enum Node<'tcx> {
-    Leaf(ty::Const<'tcx>),
-    Binop(mir::BinOp, NodeId, NodeId),
-    UnaryOp(mir::UnOp, NodeId),
-    FunctionCall(NodeId, &'tcx [NodeId]),
-    Cast(CastKind, NodeId, Ty<'tcx>),
-}
-
 #[derive(Debug, Copy, Clone, PartialEq, Eq, HashStable, TyEncodable, TyDecodable)]
 pub enum NotConstEvaluatable {
     Error(ErrorGuaranteed),
@@ -127,68 +32,53 @@ TrivialTypeTraversalAndLiftImpls! {
     NotConstEvaluatable,
 }
 
+pub type BoundAbstractConst<'tcx> = Result<Option<EarlyBinder<ty::Const<'tcx>>>, ErrorGuaranteed>;
+
 impl<'tcx> TyCtxt<'tcx> {
-    #[inline]
-    pub fn thir_abstract_const_opt_const_arg(
+    /// Returns a const without substs applied
+    pub fn bound_abstract_const(
         self,
-        def: ty::WithOptConstParam<DefId>,
-    ) -> Result<Option<&'tcx [Node<'tcx>]>, ErrorGuaranteed> {
-        if let Some((did, param_did)) = def.as_const_arg() {
+        uv: ty::WithOptConstParam<DefId>,
+    ) -> BoundAbstractConst<'tcx> {
+        let ac = if let Some((did, param_did)) = uv.as_const_arg() {
             self.thir_abstract_const_of_const_arg((did, param_did))
         } else {
-            self.thir_abstract_const(def.did)
-        }
+            self.thir_abstract_const(uv.did)
+        };
+        Ok(ac?.map(|ac| EarlyBinder(ac)))
     }
-}
 
-#[instrument(skip(tcx, f), level = "debug")]
-pub fn walk_abstract_const<'tcx, R, F>(
-    tcx: TyCtxt<'tcx>,
-    ct: AbstractConst<'tcx>,
-    mut f: F,
-) -> ControlFlow<R>
-where
-    F: FnMut(AbstractConst<'tcx>) -> ControlFlow<R>,
-{
-    #[instrument(skip(tcx, f), level = "debug")]
-    fn recurse<'tcx, R>(
-        tcx: TyCtxt<'tcx>,
-        ct: AbstractConst<'tcx>,
-        f: &mut dyn FnMut(AbstractConst<'tcx>) -> ControlFlow<R>,
-    ) -> ControlFlow<R> {
-        f(ct)?;
-        let root = ct.root(tcx);
-        debug!(?root);
-        match root {
-            Node::Leaf(_) => ControlFlow::CONTINUE,
-            Node::Binop(_, l, r) => {
-                recurse(tcx, ct.subtree(l), f)?;
-                recurse(tcx, ct.subtree(r), f)
+    pub fn expand_abstract_consts<T: TypeFoldable<'tcx>>(self, ac: T) -> T {
+        struct Expander<'tcx> {
+            tcx: TyCtxt<'tcx>,
+        }
+
+        impl<'tcx> TypeFolder<'tcx> for Expander<'tcx> {
+            fn tcx(&self) -> TyCtxt<'tcx> {
+                self.tcx
             }
-            Node::UnaryOp(_, v) => recurse(tcx, ct.subtree(v), f),
-            Node::FunctionCall(func, args) => {
-                recurse(tcx, ct.subtree(func), f)?;
-                args.iter().try_for_each(|&arg| recurse(tcx, ct.subtree(arg), f))
+            fn fold_ty(&mut self, ty: Ty<'tcx>) -> Ty<'tcx> {
+                if ty.has_type_flags(ty::TypeFlags::HAS_CT_PROJECTION) {
+                    ty.super_fold_with(self)
+                } else {
+                    ty
+                }
+            }
+            fn fold_const(&mut self, c: Const<'tcx>) -> Const<'tcx> {
+                let ct = match c.kind() {
+                    ty::ConstKind::Unevaluated(uv) => match self.tcx.bound_abstract_const(uv.def) {
+                        Err(e) => self.tcx.const_error_with_guaranteed(c.ty(), e),
+                        Ok(Some(bac)) => {
+                            let substs = self.tcx.erase_regions(uv.substs);
+                            bac.subst(self.tcx, substs)
+                        }
+                        Ok(None) => c,
+                    },
+                    _ => c,
+                };
+                ct.super_fold_with(self)
             }
-            Node::Cast(_, operand, _) => recurse(tcx, ct.subtree(operand), f),
         }
+        ac.fold_with(&mut Expander { tcx: self })
     }
-
-    recurse(tcx, ct, &mut f)
-}
-
-// We were unable to unify the abstract constant with
-// a constant found in the caller bounds, there are
-// now three possible cases here.
-#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)]
-pub enum FailureKind {
-    /// The abstract const still references an inference
-    /// variable, in this case we return `TooGeneric`.
-    MentionsInfer,
-    /// The abstract const references a generic parameter,
-    /// this means that we emit an error here.
-    MentionsParam,
-    /// The substs are concrete enough that we can simply
-    /// try and evaluate the given constant.
-    Concrete,
 }
diff --git a/compiler/rustc_middle/src/ty/adjustment.rs b/compiler/rustc_middle/src/ty/adjustment.rs
index 4682ac96b..7036c4a7b 100644
--- a/compiler/rustc_middle/src/ty/adjustment.rs
+++ b/compiler/rustc_middle/src/ty/adjustment.rs
@@ -1,7 +1,5 @@
-use crate::ty::subst::SubstsRef;
 use crate::ty::{self, Ty, TyCtxt};
 use rustc_hir as hir;
-use rustc_hir::def_id::DefId;
 use rustc_hir::lang_items::LangItem;
 use rustc_macros::HashStable;
 use rustc_span::Span;
@@ -121,7 +119,8 @@ pub struct OverloadedDeref<'tcx> {
 }
 
 impl<'tcx> OverloadedDeref<'tcx> {
-    pub fn method_call(&self, tcx: TyCtxt<'tcx>, source: Ty<'tcx>) -> (DefId, SubstsRef<'tcx>) {
+    /// Get the zst function item type for this method call.
+    pub fn method_call(&self, tcx: TyCtxt<'tcx>, source: Ty<'tcx>) -> Ty<'tcx> {
         let trait_def_id = match self.mutbl {
             hir::Mutability::Not => tcx.require_lang_item(LangItem::Deref, None),
             hir::Mutability::Mut => tcx.require_lang_item(LangItem::DerefMut, None),
@@ -132,7 +131,7 @@ impl<'tcx> OverloadedDeref<'tcx> {
             .find(|m| m.kind == ty::AssocKind::Fn)
             .unwrap()
             .def_id;
-        (method_def_id, tcx.mk_substs_trait(source, &[]))
+        tcx.mk_fn_def(method_def_id, tcx.mk_substs_trait(source, []))
     }
 }
 
@@ -160,6 +159,18 @@ pub enum AutoBorrowMutability {
     Not,
 }
 
+impl AutoBorrowMutability {
+    /// Creates an `AutoBorrowMutability` from a mutability and allowance of two phase borrows.
+    ///
+    /// Note that when `mutbl.is_not()`, `allow_two_phase_borrow` is ignored
+    pub fn new(mutbl: hir::Mutability, allow_two_phase_borrow: AllowTwoPhase) -> Self {
+        match mutbl {
+            hir::Mutability::Not => Self::Not,
+            hir::Mutability::Mut => Self::Mut { allow_two_phase_borrow },
+        }
+    }
+}
+
 impl From<AutoBorrowMutability> for hir::Mutability {
     fn from(m: AutoBorrowMutability) -> Self {
         match m {
diff --git a/compiler/rustc_middle/src/ty/adt.rs b/compiler/rustc_middle/src/ty/adt.rs
index b0a2412ab..d3d667f68 100644
--- a/compiler/rustc_middle/src/ty/adt.rs
+++ b/compiler/rustc_middle/src/ty/adt.rs
@@ -14,7 +14,7 @@ use rustc_index::vec::{Idx, IndexVec};
 use rustc_query_system::ich::StableHashingContext;
 use rustc_session::DataTypeKind;
 use rustc_span::symbol::sym;
-use rustc_target::abi::VariantIdx;
+use rustc_target::abi::{ReprOptions, VariantIdx};
 
 use std::cell::RefCell;
 use std::cmp::Ordering;
@@ -22,9 +22,7 @@ use std::hash::{Hash, Hasher};
 use std::ops::Range;
 use std::str;
 
-use super::{
-    Destructor, FieldDef, GenericPredicates, ReprOptions, Ty, TyCtxt, VariantDef, VariantDiscr,
-};
+use super::{Destructor, FieldDef, GenericPredicates, Ty, TyCtxt, VariantDef, VariantDiscr};
 
 bitflags! {
     #[derive(HashStable, TyEncodable, TyDecodable)]
@@ -230,7 +228,7 @@ impl AdtDefData {
             AdtKind::Struct => AdtFlags::IS_STRUCT,
         };
 
-        if kind == AdtKind::Struct && variants[VariantIdx::new(0)].ctor_def_id.is_some() {
+        if kind == AdtKind::Struct && variants[VariantIdx::new(0)].ctor.is_some() {
             flags |= AdtFlags::HAS_CTOR;
         }
 
@@ -386,11 +384,9 @@ impl<'tcx> AdtDef<'tcx> {
         //    Baz = 3,
         // }
         // ```
-        if self
-            .variants()
-            .iter()
-            .any(|v| matches!(v.discr, VariantDiscr::Explicit(_)) && v.ctor_kind != CtorKind::Const)
-        {
+        if self.variants().iter().any(|v| {
+            matches!(v.discr, VariantDiscr::Explicit(_)) && v.ctor_kind() != Some(CtorKind::Const)
+        }) {
             return false;
         }
         self.variants().iter().all(|v| v.fields.is_empty())
@@ -405,7 +401,7 @@ impl<'tcx> AdtDef<'tcx> {
     pub fn variant_with_ctor_id(self, cid: DefId) -> &'tcx VariantDef {
         self.variants()
             .iter()
-            .find(|v| v.ctor_def_id == Some(cid))
+            .find(|v| v.ctor_def_id() == Some(cid))
             .expect("variant_with_ctor_id: unknown variant")
     }
 
@@ -422,7 +418,7 @@ impl<'tcx> AdtDef<'tcx> {
     pub fn variant_index_with_ctor_id(self, cid: DefId) -> VariantIdx {
         self.variants()
             .iter_enumerated()
-            .find(|(_, v)| v.ctor_def_id == Some(cid))
+            .find(|(_, v)| v.ctor_def_id() == Some(cid))
             .expect("variant_index_with_ctor_id: unknown variant")
             .0
     }
@@ -463,9 +459,7 @@ impl<'tcx> AdtDef<'tcx> {
             }
             Err(err) => {
                 let msg = match err {
-                    ErrorHandled::Reported(_) | ErrorHandled::Linted => {
-                        "enum discriminant evaluation failed"
-                    }
+                    ErrorHandled::Reported(_) => "enum discriminant evaluation failed",
                     ErrorHandled::TooGeneric => "enum discriminant depends on generics",
                 };
                 tcx.sess.delay_span_bug(tcx.def_span(expr_did), msg);
diff --git a/compiler/rustc_middle/src/ty/closure.rs b/compiler/rustc_middle/src/ty/closure.rs
index 0d6c26a58..d00553cba 100644
--- a/compiler/rustc_middle/src/ty/closure.rs
+++ b/compiler/rustc_middle/src/ty/closure.rs
@@ -5,6 +5,7 @@ use crate::{mir, ty};
 
 use std::fmt::Write;
 
+use hir::LangItem;
 use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
 use rustc_hir as hir;
 use rustc_hir::def_id::{DefId, LocalDefId};
@@ -14,8 +15,8 @@ use super::{Ty, TyCtxt};
 
 use self::BorrowKind::*;
 
-// Captures are represented using fields inside a structure.
-// This represents accessing self in the closure structure
+/// Captures are represented using fields inside a structure.
+/// This represents accessing self in the closure structure
 pub const CAPTURE_STRUCT_LOCAL: mir::Local = mir::Local::from_u32(1);
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, TyEncodable, TyDecodable, HashStable)]
@@ -90,20 +91,26 @@ pub enum ClosureKind {
 }
 
 impl<'tcx> ClosureKind {
-    // This is the initial value used when doing upvar inference.
+    /// This is the initial value used when doing upvar inference.
     pub const LATTICE_BOTTOM: ClosureKind = ClosureKind::Fn;
 
     /// Returns `true` if a type that impls this closure kind
     /// must also implement `other`.
     pub fn extends(self, other: ty::ClosureKind) -> bool {
-        matches!(
-            (self, other),
-            (ClosureKind::Fn, ClosureKind::Fn)
-                | (ClosureKind::Fn, ClosureKind::FnMut)
-                | (ClosureKind::Fn, ClosureKind::FnOnce)
-                | (ClosureKind::FnMut, ClosureKind::FnMut)
-                | (ClosureKind::FnMut, ClosureKind::FnOnce)
-                | (ClosureKind::FnOnce, ClosureKind::FnOnce)
+        self <= other
+    }
+
+    /// Converts `self` to a [`DefId`] of the corresponding trait.
+    ///
+    /// Note: the inverse of this function is [`TyCtxt::fn_trait_kind_from_def_id`].
+    pub fn to_def_id(&self, tcx: TyCtxt<'_>) -> DefId {
+        tcx.require_lang_item(
+            match self {
+                ClosureKind::Fn => LangItem::Fn,
+                ClosureKind::FnMut => LangItem::FnMut,
+                ClosureKind::FnOnce => LangItem::FnOnce,
+            },
+            None,
         )
     }
 
@@ -116,26 +123,6 @@ impl<'tcx> ClosureKind {
             ClosureKind::FnOnce => tcx.types.i32,
         }
     }
-
-    pub fn from_def_id(tcx: TyCtxt<'_>, def_id: DefId) -> Option<ClosureKind> {
-        if Some(def_id) == tcx.lang_items().fn_once_trait() {
-            Some(ClosureKind::FnOnce)
-        } else if Some(def_id) == tcx.lang_items().fn_mut_trait() {
-            Some(ClosureKind::FnMut)
-        } else if Some(def_id) == tcx.lang_items().fn_trait() {
-            Some(ClosureKind::Fn)
-        } else {
-            None
-        }
-    }
-
-    pub fn to_def_id(&self, tcx: TyCtxt<'_>) -> DefId {
-        match self {
-            ClosureKind::Fn => tcx.lang_items().fn_once_trait().unwrap(),
-            ClosureKind::FnMut => tcx.lang_items().fn_mut_trait().unwrap(),
-            ClosureKind::FnOnce => tcx.lang_items().fn_trait().unwrap(),
-        }
-    }
 }
 
 /// A composite describing a `Place` that is captured by a closure.
diff --git a/compiler/rustc_middle/src/ty/codec.rs b/compiler/rustc_middle/src/ty/codec.rs
index 14ec88b7e..75f2d45ea 100644
--- a/compiler/rustc_middle/src/ty/codec.rs
+++ b/compiler/rustc_middle/src/ty/codec.rs
@@ -298,7 +298,7 @@ impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D> for ty::List<Ty
 }
 
 impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D>
-    for ty::List<ty::Binder<'tcx, ty::ExistentialPredicate<'tcx>>>
+    for ty::List<ty::PolyExistentialPredicate<'tcx>>
 {
     fn decode(decoder: &mut D) -> &'tcx Self {
         let len = decoder.read_usize();
@@ -310,7 +310,8 @@ impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D>
 
 impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> Decodable<D> for ty::Const<'tcx> {
     fn decode(decoder: &mut D) -> Self {
-        decoder.interner().mk_const(Decodable::decode(decoder))
+        let consts: ty::ConstS<'tcx> = Decodable::decode(decoder);
+        decoder.interner().mk_const(consts.kind, consts.ty)
     }
 }
 
@@ -344,9 +345,7 @@ impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D>
     }
 }
 
-impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D>
-    for [ty::abstract_const::Node<'tcx>]
-{
+impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D> for [(ty::Clause<'tcx>, Span)] {
     fn decode(decoder: &mut D) -> &'tcx Self {
         decoder.interner().arena.alloc_from_iter(
             (0..decoder.read_usize()).map(|_| Decodable::decode(decoder)).collect::<Vec<_>>(),
@@ -355,30 +354,29 @@ impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D>
 }
 
 impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D>
-    for [ty::abstract_const::NodeId]
+    for ty::List<ty::BoundVariableKind>
 {
     fn decode(decoder: &mut D) -> &'tcx Self {
-        decoder.interner().arena.alloc_from_iter(
-            (0..decoder.read_usize()).map(|_| Decodable::decode(decoder)).collect::<Vec<_>>(),
+        let len = decoder.read_usize();
+        decoder.interner().mk_bound_variable_kinds(
+            (0..len).map::<ty::BoundVariableKind, _>(|_| Decodable::decode(decoder)),
         )
     }
 }
 
-impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D>
-    for ty::List<ty::BoundVariableKind>
-{
+impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D> for ty::List<ty::Const<'tcx>> {
     fn decode(decoder: &mut D) -> &'tcx Self {
         let len = decoder.read_usize();
-        decoder.interner().mk_bound_variable_kinds(
-            (0..len).map::<ty::BoundVariableKind, _>(|_| Decodable::decode(decoder)),
-        )
+        decoder
+            .interner()
+            .mk_const_list((0..len).map::<ty::Const<'tcx>, _>(|_| Decodable::decode(decoder)))
     }
 }
 
 impl_decodable_via_ref! {
     &'tcx ty::TypeckResults<'tcx>,
     &'tcx ty::List<Ty<'tcx>>,
-    &'tcx ty::List<ty::Binder<'tcx, ty::ExistentialPredicate<'tcx>>>,
+    &'tcx ty::List<ty::PolyExistentialPredicate<'tcx>>,
     &'tcx traits::ImplSource<'tcx, ()>,
     &'tcx mir::Body<'tcx>,
     &'tcx mir::UnsafetyCheckResult,
diff --git a/compiler/rustc_middle/src/ty/consts.rs b/compiler/rustc_middle/src/ty/consts.rs
index f998e6083..c2be08e49 100644
--- a/compiler/rustc_middle/src/ty/consts.rs
+++ b/compiler/rustc_middle/src/ty/consts.rs
@@ -1,8 +1,6 @@
 use crate::mir::interpret::LitToConstInput;
-use crate::mir::ConstantKind;
-use crate::ty::{self, InternalSubsts, ParamEnv, ParamEnvAnd, Ty, TyCtxt};
+use crate::ty::{self, DefIdTree, InternalSubsts, ParamEnv, ParamEnvAnd, Ty, TyCtxt};
 use rustc_data_structures::intern::Interned;
-use rustc_errors::ErrorGuaranteed;
 use rustc_hir as hir;
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_macros::HashStable;
@@ -77,13 +75,13 @@ impl<'tcx> Const<'tcx> {
 
         match Self::try_eval_lit_or_param(tcx, ty, expr) {
             Some(v) => v,
-            None => tcx.mk_const(ty::ConstS {
-                kind: ty::ConstKind::Unevaluated(ty::UnevaluatedConst {
+            None => tcx.mk_const(
+                ty::UnevaluatedConst {
                     def: def.to_global(),
                     substs: InternalSubsts::identity_for_item(tcx, def.did.to_def_id()),
-                }),
+                },
                 ty,
-            }),
+            ),
         }
     }
 
@@ -132,27 +130,16 @@ impl<'tcx> Const<'tcx> {
             ExprKind::Path(QPath::Resolved(_, &Path { res: Res::Def(ConstParam, def_id), .. })) => {
                 // Find the name and index of the const parameter by indexing the generics of
                 // the parent item and construct a `ParamConst`.
-                let hir_id = tcx.hir().local_def_id_to_hir_id(def_id.expect_local());
-                let item_id = tcx.hir().get_parent_node(hir_id);
-                let item_def_id = tcx.hir().local_def_id(item_id);
-                let generics = tcx.generics_of(item_def_id.to_def_id());
+                let item_def_id = tcx.parent(def_id);
+                let generics = tcx.generics_of(item_def_id);
                 let index = generics.param_def_id_to_index[&def_id];
-                let name = tcx.hir().name(hir_id);
-                Some(tcx.mk_const(ty::ConstS {
-                    kind: ty::ConstKind::Param(ty::ParamConst::new(index, name)),
-                    ty,
-                }))
+                let name = tcx.item_name(def_id);
+                Some(tcx.mk_const(ty::ParamConst::new(index, name), ty))
             }
             _ => None,
         }
     }
 
-    /// Interns the given value as a constant.
-    #[inline]
-    pub fn from_value(tcx: TyCtxt<'tcx>, val: ty::ValTree<'tcx>, ty: Ty<'tcx>) -> Self {
-        tcx.mk_const(ConstS { kind: ConstKind::Value(val), ty })
-    }
-
     /// Panics if self.kind != ty::ConstKind::Value
     pub fn to_valtree(self) -> ty::ValTree<'tcx> {
         match self.kind() {
@@ -161,11 +148,6 @@ impl<'tcx> Const<'tcx> {
         }
     }
 
-    pub fn from_scalar_int(tcx: TyCtxt<'tcx>, i: ScalarInt, ty: Ty<'tcx>) -> Self {
-        let valtree = ty::ValTree::from_scalar_int(i);
-        Self::from_value(tcx, valtree, ty)
-    }
-
     #[inline]
     /// Creates a constant with the given integer value and interns it.
     pub fn from_bits(tcx: TyCtxt<'tcx>, bits: u128, ty: ParamEnvAnd<'tcx, Ty<'tcx>>) -> Self {
@@ -173,14 +155,16 @@ impl<'tcx> Const<'tcx> {
             .layout_of(ty)
             .unwrap_or_else(|e| panic!("could not compute layout for {:?}: {:?}", ty, e))
             .size;
-        Self::from_scalar_int(tcx, ScalarInt::try_from_uint(bits, size).unwrap(), ty.value)
+        tcx.mk_const(
+            ty::ValTree::from_scalar_int(ScalarInt::try_from_uint(bits, size).unwrap()),
+            ty.value,
+        )
     }
 
     #[inline]
     /// Creates an interned zst constant.
     pub fn zero_sized(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Self {
-        let valtree = ty::ValTree::zst();
-        Self::from_value(tcx, valtree, ty)
+        tcx.mk_const(ty::ValTree::zst(), ty)
     }
 
     #[inline]
@@ -227,8 +211,8 @@ impl<'tcx> Const<'tcx> {
     pub fn eval(self, tcx: TyCtxt<'tcx>, param_env: ParamEnv<'tcx>) -> Const<'tcx> {
         if let Some(val) = self.kind().try_eval_for_typeck(tcx, param_env) {
             match val {
-                Ok(val) => Const::from_value(tcx, val, self.ty()),
-                Err(ErrorGuaranteed { .. }) => tcx.const_error(self.ty()),
+                Ok(val) => tcx.mk_const(val, self.ty()),
+                Err(guar) => tcx.const_error_with_guaranteed(self.ty(), guar),
             }
         } else {
             // Either the constant isn't evaluatable or ValTree creation failed.
@@ -237,20 +221,6 @@ impl<'tcx> Const<'tcx> {
     }
 
     #[inline]
-    /// Tries to evaluate the constant if it is `Unevaluated` and creates a ConstValue if the
-    /// evaluation succeeds. If it doesn't succeed, returns the unevaluated constant.
-    pub fn eval_for_mir(self, tcx: TyCtxt<'tcx>, param_env: ParamEnv<'tcx>) -> ConstantKind<'tcx> {
-        if let Some(val) = self.kind().try_eval_for_mir(tcx, param_env) {
-            match val {
-                Ok(const_val) => ConstantKind::from_value(const_val, self.ty()),
-                Err(ErrorGuaranteed { .. }) => ConstantKind::Ty(tcx.const_error(self.ty())),
-            }
-        } else {
-            ConstantKind::Ty(self)
-        }
-    }
-
-    #[inline]
     /// Panics if the value cannot be evaluated or doesn't contain a valid integer of the given type.
     pub fn eval_bits(self, tcx: TyCtxt<'tcx>, param_env: ParamEnv<'tcx>, ty: Ty<'tcx>) -> u128 {
         self.try_eval_bits(tcx, param_env, ty)
@@ -272,9 +242,9 @@ impl<'tcx> Const<'tcx> {
 pub fn const_param_default<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> Const<'tcx> {
     let default_def_id = match tcx.hir().get_by_def_id(def_id.expect_local()) {
         hir::Node::GenericParam(hir::GenericParam {
-            kind: hir::GenericParamKind::Const { ty: _, default: Some(ac) },
+            kind: hir::GenericParamKind::Const { default: Some(ac), .. },
             ..
-        }) => tcx.hir().local_def_id(ac.hir_id),
+        }) => ac.def_id,
         _ => span_bug!(
             tcx.def_span(def_id),
             "`const_param_default` expected a generic parameter with a constant"
diff --git a/compiler/rustc_middle/src/ty/consts/int.rs b/compiler/rustc_middle/src/ty/consts/int.rs
index 7436f0f6f..f3186e1c3 100644
--- a/compiler/rustc_middle/src/ty/consts/int.rs
+++ b/compiler/rustc_middle/src/ty/consts/int.rs
@@ -245,6 +245,18 @@ impl ScalarInt {
         self.to_bits(size)
     }
 
+    // Tries to convert the `ScalarInt` to `bool`. Fails if the `size` of the `ScalarInt`
+    // in not equal to `Size { raw: 1 }` or if the value is not 0 or 1 and returns the `size`
+    // value of the `ScalarInt` in that case.
+    #[inline]
+    pub fn try_to_bool(self) -> Result<bool, Size> {
+        match self.try_to_u8()? {
+            0 => Ok(false),
+            1 => Ok(true),
+            _ => Err(self.size()),
+        }
+    }
+
     // Tries to convert the `ScalarInt` to `u8`. Fails if the `size` of the `ScalarInt`
     // in not equal to `Size { raw: 1 }` and returns the `size` value of the `ScalarInt` in
     // that case.
diff --git a/compiler/rustc_middle/src/ty/consts/kind.rs b/compiler/rustc_middle/src/ty/consts/kind.rs
index 4ab761e07..becc2b805 100644
--- a/compiler/rustc_middle/src/ty/consts/kind.rs
+++ b/compiler/rustc_middle/src/ty/consts/kind.rs
@@ -1,10 +1,12 @@
 use std::convert::TryInto;
 
+use super::Const;
 use crate::mir;
 use crate::mir::interpret::{AllocId, ConstValue, Scalar};
+use crate::ty::abstract_const::CastKind;
 use crate::ty::subst::{InternalSubsts, SubstsRef};
 use crate::ty::ParamEnv;
-use crate::ty::{self, TyCtxt, TypeVisitable};
+use crate::ty::{self, List, Ty, TyCtxt, TypeVisitable};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir::def_id::DefId;
@@ -47,6 +49,7 @@ impl<'tcx> UnevaluatedConst<'tcx> {
 /// Represents a constant in Rust.
 #[derive(Copy, Clone, Debug, Eq, PartialEq, PartialOrd, Ord, TyEncodable, TyDecodable)]
 #[derive(Hash, HashStable, TypeFoldable, TypeVisitable)]
+#[derive(derive_more::From)]
 pub enum ConstKind<'tcx> {
     /// A const generic parameter.
     Param(ty::ParamConst),
@@ -69,9 +72,31 @@ pub enum ConstKind<'tcx> {
 
     /// A placeholder for a const which could not be computed; this is
     /// propagated to avoid useless error messages.
-    Error(ty::DelaySpanBugEmitted),
+    #[from(ignore)]
+    Error(ErrorGuaranteed),
+
+    /// Expr which contains an expression which has partially evaluated items.
+    Expr(Expr<'tcx>),
+}
+
+impl<'tcx> From<ty::ConstVid<'tcx>> for ConstKind<'tcx> {
+    fn from(const_vid: ty::ConstVid<'tcx>) -> Self {
+        InferConst::Var(const_vid).into()
+    }
 }
 
+#[derive(Copy, Clone, Debug, Eq, PartialEq, PartialOrd, Ord, Hash)]
+#[derive(HashStable, TyEncodable, TyDecodable, TypeVisitable, TypeFoldable)]
+pub enum Expr<'tcx> {
+    Binop(mir::BinOp, Const<'tcx>, Const<'tcx>),
+    UnOp(mir::UnOp, Const<'tcx>),
+    FunctionCall(Const<'tcx>, &'tcx List<Const<'tcx>>),
+    Cast(CastKind, Const<'tcx>, Ty<'tcx>),
+}
+
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
+static_assert_size!(Expr<'_>, 24);
+
 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 static_assert_size!(ConstKind<'_>, 32);
 
@@ -226,7 +251,7 @@ impl<'tcx> ConstKind<'tcx> {
                         // (which may be identity substs, see above),
                         // can leak through `val` into the const we return.
                         Ok(val) => Some(Ok(EvalResult::ValTree(val?))),
-                        Err(ErrorHandled::TooGeneric | ErrorHandled::Linted) => None,
+                        Err(ErrorHandled::TooGeneric) => None,
                         Err(ErrorHandled::Reported(e)) => Some(Err(e)),
                     }
                 }
@@ -237,7 +262,7 @@ impl<'tcx> ConstKind<'tcx> {
                         // (which may be identity substs, see above),
                         // can leak through `val` into the const we return.
                         Ok(val) => Some(Ok(EvalResult::ConstVal(val))),
-                        Err(ErrorHandled::TooGeneric | ErrorHandled::Linted) => None,
+                        Err(ErrorHandled::TooGeneric) => None,
                         Err(ErrorHandled::Reported(e)) => Some(Err(e)),
                     }
                 }
diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index 3d7e2a083..b44bc14ec 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -1,5 +1,7 @@
 //! Type context book-keeping.
 
+#![allow(rustc::usage_of_ty_tykind)]
+
 use crate::arena::Arena;
 use crate::dep_graph::{DepGraph, DepKindStruct};
 use crate::hir::place::Place as HirPlace;
@@ -17,17 +19,17 @@ use crate::traits;
 use crate::ty::query::{self, TyCtxtAt};
 use crate::ty::{
     self, AdtDef, AdtDefData, AdtKind, Binder, BindingMode, BoundVar, CanonicalPolyFnSig,
-    ClosureSizeProfileData, Const, ConstS, ConstVid, DefIdTree, ExistentialPredicate, FloatTy,
-    FloatVar, FloatVid, GenericParamDefKind, InferConst, InferTy, IntTy, IntVar, IntVid, List,
-    ParamConst, ParamTy, PolyFnSig, Predicate, PredicateKind, PredicateS, ProjectionTy, Region,
-    RegionKind, ReprOptions, TraitObjectVisitor, Ty, TyKind, TyS, TyVar, TyVid, TypeAndMut, UintTy,
+    ClosureSizeProfileData, Const, ConstS, DefIdTree, FloatTy, FloatVar, FloatVid,
+    GenericParamDefKind, InferTy, IntTy, IntVar, IntVid, List, ParamConst, ParamTy,
+    PolyExistentialPredicate, PolyFnSig, Predicate, PredicateKind, ProjectionTy, Region,
+    RegionKind, ReprOptions, TraitObjectVisitor, Ty, TyKind, TyVar, TyVid, TypeAndMut, UintTy,
     Visibility,
 };
 use crate::ty::{GenericArg, GenericArgKind, InternalSubsts, SubstsRef, UserSubsts};
 use rustc_ast as ast;
 use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
-use rustc_data_structures::intern::{Interned, WithStableHash};
+use rustc_data_structures::intern::Interned;
 use rustc_data_structures::memmap::Mmap;
 use rustc_data_structures::profiling::SelfProfilerRef;
 use rustc_data_structures::sharded::{IntoPointer, ShardedHashMap};
@@ -41,7 +43,7 @@ use rustc_errors::{
 };
 use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
-use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LocalDefId, LOCAL_CRATE};
+use rustc_hir::def_id::{CrateNum, DefId, LocalDefId, LocalDefIdMap, LOCAL_CRATE};
 use rustc_hir::definitions::Definitions;
 use rustc_hir::hir_id::OwnerId;
 use rustc_hir::intravisit::Visitor;
@@ -53,6 +55,7 @@ use rustc_hir::{
 use rustc_index::vec::{Idx, IndexVec};
 use rustc_macros::HashStable;
 use rustc_middle::mir::FakeReadCause;
+use rustc_query_system::dep_graph::DepNodeIndex;
 use rustc_query_system::ich::StableHashingContext;
 use rustc_serialize::opaque::{FileEncodeResult, FileEncoder};
 use rustc_session::config::{CrateType, OutputFilenames};
@@ -67,6 +70,7 @@ use rustc_span::{Span, DUMMY_SP};
 use rustc_target::abi::{Layout, LayoutS, TargetDataLayout, VariantIdx};
 use rustc_target::spec::abi;
 use rustc_type_ir::sty::TyKind::*;
+use rustc_type_ir::WithCachedTypeInfo;
 use rustc_type_ir::{DynKind, InternAs, InternIteratorElement, Interner, TypeFlags};
 
 use std::any::Any;
@@ -80,7 +84,7 @@ use std::mem;
 use std::ops::{Bound, Deref};
 use std::sync::Arc;
 
-use super::{ImplPolarity, ResolverOutputs, RvalueScopes};
+use super::{ImplPolarity, RvalueScopes};
 
 pub trait OnDiskCache<'tcx>: rustc_data_structures::sync::Sync {
     /// Creates a new `OnDiskCache` instance from the serialized data in `data`.
@@ -109,7 +113,7 @@ impl<'tcx> Interner for TyCtxt<'tcx> {
     type Mutability = hir::Mutability;
     type Movability = hir::Movability;
     type PolyFnSig = PolyFnSig<'tcx>;
-    type ListBinderExistentialPredicate = &'tcx List<Binder<'tcx, ExistentialPredicate<'tcx>>>;
+    type ListBinderExistentialPredicate = &'tcx List<PolyExistentialPredicate<'tcx>>;
     type BinderListTy = Binder<'tcx, &'tcx List<Ty<'tcx>>>;
     type ListTy = &'tcx List<Ty<'tcx>>;
     type ProjectionTy = ty::ProjectionTy<'tcx>;
@@ -117,7 +121,7 @@ impl<'tcx> Interner for TyCtxt<'tcx> {
     type BoundTy = ty::BoundTy;
     type PlaceholderType = ty::PlaceholderType;
     type InferTy = InferTy;
-    type DelaySpanBugEmitted = DelaySpanBugEmitted;
+    type ErrorGuaranteed = ErrorGuaranteed;
     type PredicateKind = ty::PredicateKind<'tcx>;
     type AllocId = crate::mir::interpret::AllocId;
 
@@ -128,15 +132,6 @@ impl<'tcx> Interner for TyCtxt<'tcx> {
     type PlaceholderRegion = ty::PlaceholderRegion;
 }
 
-/// A type that is not publicly constructable. This prevents people from making [`TyKind::Error`]s
-/// except through the error-reporting functions on a [`tcx`][TyCtxt].
-#[derive(Copy, Clone, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)]
-#[derive(TyEncodable, TyDecodable, HashStable)]
-pub struct DelaySpanBugEmitted {
-    pub reported: ErrorGuaranteed,
-    _priv: (),
-}
-
 type InternedSet<'tcx, T> = ShardedHashMap<InternedInSet<'tcx, T>, ()>;
 
 pub struct CtxtInterners<'tcx> {
@@ -145,20 +140,20 @@ pub struct CtxtInterners<'tcx> {
 
     // Specifically use a speedy hash algorithm for these hash sets, since
     // they're accessed quite often.
-    type_: InternedSet<'tcx, WithStableHash<TyS<'tcx>>>,
+    type_: InternedSet<'tcx, WithCachedTypeInfo<TyKind<'tcx>>>,
+    const_lists: InternedSet<'tcx, List<ty::Const<'tcx>>>,
     substs: InternedSet<'tcx, InternalSubsts<'tcx>>,
     canonical_var_infos: InternedSet<'tcx, List<CanonicalVarInfo<'tcx>>>,
     region: InternedSet<'tcx, RegionKind<'tcx>>,
-    poly_existential_predicates:
-        InternedSet<'tcx, List<ty::Binder<'tcx, ExistentialPredicate<'tcx>>>>,
-    predicate: InternedSet<'tcx, PredicateS<'tcx>>,
+    poly_existential_predicates: InternedSet<'tcx, List<PolyExistentialPredicate<'tcx>>>,
+    predicate: InternedSet<'tcx, WithCachedTypeInfo<ty::Binder<'tcx, PredicateKind<'tcx>>>>,
     predicates: InternedSet<'tcx, List<Predicate<'tcx>>>,
     projs: InternedSet<'tcx, List<ProjectionKind>>,
     place_elems: InternedSet<'tcx, List<PlaceElem<'tcx>>>,
     const_: InternedSet<'tcx, ConstS<'tcx>>,
     const_allocation: InternedSet<'tcx, Allocation>,
     bound_variable_kinds: InternedSet<'tcx, List<ty::BoundVariableKind>>,
-    layout: InternedSet<'tcx, LayoutS<'tcx>>,
+    layout: InternedSet<'tcx, LayoutS<VariantIdx>>,
     adt_def: InternedSet<'tcx, AdtDefData>,
 }
 
@@ -167,6 +162,7 @@ impl<'tcx> CtxtInterners<'tcx> {
         CtxtInterners {
             arena,
             type_: Default::default(),
+            const_lists: Default::default(),
             substs: Default::default(),
             region: Default::default(),
             poly_existential_predicates: Default::default(),
@@ -198,53 +194,64 @@ impl<'tcx> CtxtInterners<'tcx> {
             self.type_
                 .intern(kind, |kind| {
                     let flags = super::flags::FlagComputation::for_kind(&kind);
+                    let stable_hash =
+                        self.stable_hash(&flags, sess, definitions, cstore, source_span, &kind);
 
-                    // It's impossible to hash inference variables (and will ICE), so we don't need to try to cache them.
-                    // Without incremental, we rarely stable-hash types, so let's not do it proactively.
-                    let stable_hash = if flags.flags.intersects(TypeFlags::NEEDS_INFER)
-                        || sess.opts.incremental.is_none()
-                    {
-                        Fingerprint::ZERO
-                    } else {
-                        let mut hasher = StableHasher::new();
-                        let mut hcx = StableHashingContext::ignore_spans(
-                            sess,
-                            definitions,
-                            cstore,
-                            source_span,
-                        );
-                        kind.hash_stable(&mut hcx, &mut hasher);
-                        hasher.finish()
-                    };
-
-                    let ty_struct = TyS {
-                        kind,
+                    InternedInSet(self.arena.alloc(WithCachedTypeInfo {
+                        internee: kind,
+                        stable_hash,
                         flags: flags.flags,
                         outer_exclusive_binder: flags.outer_exclusive_binder,
-                    };
-
-                    InternedInSet(
-                        self.arena.alloc(WithStableHash { internee: ty_struct, stable_hash }),
-                    )
+                    }))
                 })
                 .0,
         ))
     }
 
+    fn stable_hash<'a, T: HashStable<StableHashingContext<'a>>>(
+        &self,
+        flags: &ty::flags::FlagComputation,
+        sess: &'a Session,
+        definitions: &'a rustc_hir::definitions::Definitions,
+        cstore: &'a CrateStoreDyn,
+        source_span: &'a IndexVec<LocalDefId, Span>,
+        val: &T,
+    ) -> Fingerprint {
+        // It's impossible to hash inference variables (and will ICE), so we don't need to try to cache them.
+        // Without incremental, we rarely stable-hash types, so let's not do it proactively.
+        if flags.flags.intersects(TypeFlags::NEEDS_INFER) || sess.opts.incremental.is_none() {
+            Fingerprint::ZERO
+        } else {
+            let mut hasher = StableHasher::new();
+            let mut hcx = StableHashingContext::new(sess, definitions, cstore, source_span);
+            val.hash_stable(&mut hcx, &mut hasher);
+            hasher.finish()
+        }
+    }
+
     #[inline(never)]
-    fn intern_predicate(&self, kind: Binder<'tcx, PredicateKind<'tcx>>) -> Predicate<'tcx> {
+    fn intern_predicate(
+        &self,
+        kind: Binder<'tcx, PredicateKind<'tcx>>,
+        sess: &Session,
+        definitions: &rustc_hir::definitions::Definitions,
+        cstore: &CrateStoreDyn,
+        source_span: &IndexVec<LocalDefId, Span>,
+    ) -> Predicate<'tcx> {
         Predicate(Interned::new_unchecked(
             self.predicate
                 .intern(kind, |kind| {
                     let flags = super::flags::FlagComputation::for_predicate(kind);
 
-                    let predicate_struct = PredicateS {
-                        kind,
+                    let stable_hash =
+                        self.stable_hash(&flags, sess, definitions, cstore, source_span, &kind);
+
+                    InternedInSet(self.arena.alloc(WithCachedTypeInfo {
+                        internee: kind,
+                        stable_hash,
                         flags: flags.flags,
                         outer_exclusive_binder: flags.outer_exclusive_binder,
-                    };
-
-                    InternedInSet(self.arena.alloc(predicate_struct))
+                    }))
                 })
                 .0,
         ))
@@ -456,7 +463,7 @@ pub struct TypeckResults<'tcx> {
 
     /// Stores the canonicalized types provided by the user. See also
     /// `AscribeUserType` statement in MIR.
-    pub user_provided_sigs: DefIdMap<CanonicalPolyFnSig<'tcx>>,
+    pub user_provided_sigs: LocalDefIdMap<CanonicalPolyFnSig<'tcx>>,
 
     adjustments: ItemLocalMap<Vec<ty::adjustment::Adjustment<'tcx>>>,
 
@@ -660,6 +667,14 @@ impl<'tcx> TypeckResults<'tcx> {
         LocalTableInContextMut { hir_owner: self.hir_owner, data: &mut self.field_indices }
     }
 
+    pub fn field_index(&self, id: hir::HirId) -> usize {
+        self.field_indices().get(id).cloned().expect("no index for a field")
+    }
+
+    pub fn opt_field_index(&self, id: hir::HirId) -> Option<usize> {
+        self.field_indices().get(id).cloned()
+    }
+
     pub fn user_provided_types(&self) -> LocalTableInContext<'_, CanonicalUserType<'tcx>> {
         LocalTableInContext { hir_owner: self.hir_owner, data: &self.user_provided_types }
     }
@@ -725,22 +740,24 @@ impl<'tcx> TypeckResults<'tcx> {
         self.node_substs.get(&id.local_id).cloned()
     }
 
-    // Returns the type of a pattern as a monotype. Like @expr_ty, this function
-    // doesn't provide type parameter substitutions.
+    /// Returns the type of a pattern as a monotype. Like [`expr_ty`], this function
+    /// doesn't provide type parameter substitutions.
+    ///
+    /// [`expr_ty`]: TypeckResults::expr_ty
     pub fn pat_ty(&self, pat: &hir::Pat<'_>) -> Ty<'tcx> {
         self.node_type(pat.hir_id)
     }
 
-    // Returns the type of an expression as a monotype.
-    //
-    // NB (1): This is the PRE-ADJUSTMENT TYPE for the expression.  That is, in
-    // some cases, we insert `Adjustment` annotations such as auto-deref or
-    // auto-ref.  The type returned by this function does not consider such
-    // adjustments.  See `expr_ty_adjusted()` instead.
-    //
-    // NB (2): This type doesn't provide type parameter substitutions; e.g., if you
-    // ask for the type of "id" in "id(3)", it will return "fn(&isize) -> isize"
-    // instead of "fn(ty) -> T with T = isize".
+    /// Returns the type of an expression as a monotype.
+    ///
+    /// NB (1): This is the PRE-ADJUSTMENT TYPE for the expression.  That is, in
+    /// some cases, we insert `Adjustment` annotations such as auto-deref or
+    /// auto-ref.  The type returned by this function does not consider such
+    /// adjustments.  See `expr_ty_adjusted()` instead.
+    ///
+    /// NB (2): This type doesn't provide type parameter substitutions; e.g., if you
+    /// ask for the type of `id` in `id(3)`, it will return `fn(&isize) -> isize`
+    /// instead of `fn(ty) -> T with T = isize`.
     pub fn expr_ty(&self, expr: &hir::Expr<'_>) -> Ty<'tcx> {
         self.node_type(expr.hir_id)
     }
@@ -1007,18 +1024,46 @@ impl<'tcx> CommonConsts<'tcx> {
     }
 }
 
-// This struct contains information regarding the `ReFree(FreeRegion)` corresponding to a lifetime
-// conflict.
+/// This struct contains information regarding the `ReFree(FreeRegion)` corresponding to a lifetime
+/// conflict.
 #[derive(Debug)]
 pub struct FreeRegionInfo {
-    // `LocalDefId` corresponding to FreeRegion
+    /// `LocalDefId` corresponding to FreeRegion
     pub def_id: LocalDefId,
-    // the bound region corresponding to FreeRegion
+    /// the bound region corresponding to FreeRegion
     pub boundregion: ty::BoundRegionKind,
-    // checks if bound region is in Impl Item
+    /// checks if bound region is in Impl Item
     pub is_impl_item: bool,
 }
 
+/// This struct should only be created by `create_def`.
+#[derive(Copy, Clone)]
+pub struct TyCtxtFeed<'tcx, KEY: Copy> {
+    pub tcx: TyCtxt<'tcx>,
+    // Do not allow direct access, as downstream code must not mutate this field.
+    key: KEY,
+}
+
+impl<'tcx> TyCtxt<'tcx> {
+    pub fn feed_unit_query(self) -> TyCtxtFeed<'tcx, ()> {
+        TyCtxtFeed { tcx: self, key: () }
+    }
+}
+
+impl<'tcx, KEY: Copy> TyCtxtFeed<'tcx, KEY> {
+    #[inline(always)]
+    pub fn key(&self) -> KEY {
+        self.key
+    }
+}
+
+impl<'tcx> TyCtxtFeed<'tcx, LocalDefId> {
+    #[inline(always)]
+    pub fn def_id(&self) -> LocalDefId {
+        self.key
+    }
+}
+
 /// The central data structure of the compiler. It stores references
 /// to the various **arenas** and also houses the results of the
 /// various **compiler queries** that have been performed. See the
@@ -1071,7 +1116,6 @@ pub struct GlobalCtxt<'tcx> {
 
     /// Output of the resolver.
     pub(crate) untracked_resolutions: ty::ResolverGlobalCtxt,
-    untracked_resolver_for_lowering: Steal<ty::ResolverAstLowering>,
     /// The entire crate as AST. This field serves as the input for the hir_crate query,
     /// which lowers it from AST to HIR. It must not be read or used by anything else.
     pub untracked_crate: Steal<Lrc<ast::Crate>>,
@@ -1201,8 +1245,9 @@ impl<'tcx> TyCtxt<'tcx> {
             debug!("layout_scalar_valid_range: attr={:?}", attr);
             if let Some(
                 &[
-                    ast::NestedMetaItem::Literal(ast::Lit {
-                        kind: ast::LitKind::Int(a, _), ..
+                    ast::NestedMetaItem::Lit(ast::MetaItemLit {
+                        kind: ast::LitKind::Int(a, _),
+                        ..
                     }),
                 ],
             ) = attr.meta_item_list().as_deref()
@@ -1233,21 +1278,17 @@ impl<'tcx> TyCtxt<'tcx> {
         lint_store: Lrc<dyn Any + sync::Send + sync::Sync>,
         arena: &'tcx WorkerLocal<Arena<'tcx>>,
         hir_arena: &'tcx WorkerLocal<hir::Arena<'tcx>>,
-        resolver_outputs: ResolverOutputs,
+        definitions: Definitions,
+        untracked_resolutions: ty::ResolverGlobalCtxt,
         krate: Lrc<ast::Crate>,
         dep_graph: DepGraph,
         on_disk_cache: Option<&'tcx dyn OnDiskCache<'tcx>>,
         queries: &'tcx dyn query::QueryEngine<'tcx>,
         query_kinds: &'tcx [DepKindStruct<'tcx>],
-        crate_name: &str,
+        crate_name: Symbol,
         output_filenames: OutputFilenames,
     ) -> GlobalCtxt<'tcx> {
-        let ResolverOutputs {
-            definitions,
-            global_ctxt: untracked_resolutions,
-            ast_lowering: untracked_resolver_for_lowering,
-        } = resolver_outputs;
-        let data_layout = TargetDataLayout::parse(&s.target).unwrap_or_else(|err| {
+        let data_layout = s.target.parse_data_layout().unwrap_or_else(|err| {
             s.emit_fatal(err);
         });
         let interners = CtxtInterners::new(arena);
@@ -1275,7 +1316,6 @@ impl<'tcx> TyCtxt<'tcx> {
             lifetimes: common_lifetimes,
             consts: common_consts,
             untracked_resolutions,
-            untracked_resolver_for_lowering: Steal::new(untracked_resolver_for_lowering),
             untracked_crate: Steal::new(krate),
             on_disk_cache,
             queries,
@@ -1285,13 +1325,19 @@ impl<'tcx> TyCtxt<'tcx> {
             pred_rcache: Default::default(),
             selection_cache: Default::default(),
             evaluation_cache: Default::default(),
-            crate_name: Symbol::intern(crate_name),
+            crate_name,
             data_layout,
             alloc_map: Lock::new(interpret::AllocMap::new()),
             output_filenames: Arc::new(output_filenames),
         }
     }
 
+    /// Constructs a `TyKind::Error` type with current `ErrorGuaranteed`
+    #[track_caller]
+    pub fn ty_error_with_guaranteed(self, reported: ErrorGuaranteed) -> Ty<'tcx> {
+        self.mk_ty(Error(reported))
+    }
+
     /// Constructs a `TyKind::Error` type and registers a `delay_span_bug` to ensure it gets used.
     #[track_caller]
     pub fn ty_error(self) -> Ty<'tcx> {
@@ -1303,7 +1349,17 @@ impl<'tcx> TyCtxt<'tcx> {
     #[track_caller]
     pub fn ty_error_with_message<S: Into<MultiSpan>>(self, span: S, msg: &str) -> Ty<'tcx> {
         let reported = self.sess.delay_span_bug(span, msg);
-        self.mk_ty(Error(DelaySpanBugEmitted { reported, _priv: () }))
+        self.mk_ty(Error(reported))
+    }
+
+    /// Like [TyCtxt::ty_error] but for constants, with current `ErrorGuaranteed`
+    #[track_caller]
+    pub fn const_error_with_guaranteed(
+        self,
+        ty: Ty<'tcx>,
+        reported: ErrorGuaranteed,
+    ) -> Const<'tcx> {
+        self.mk_const(ty::ConstKind::Error(reported), ty)
     }
 
     /// Like [TyCtxt::ty_error] but for constants.
@@ -1325,10 +1381,7 @@ impl<'tcx> TyCtxt<'tcx> {
         msg: &str,
     ) -> Const<'tcx> {
         let reported = self.sess.delay_span_bug(span, msg);
-        self.mk_const(ty::ConstS {
-            kind: ty::ConstKind::Error(DelaySpanBugEmitted { reported, _priv: () }),
-            ty,
-        })
+        self.mk_const(ty::ConstKind::Error(reported), ty)
     }
 
     pub fn consider_optimizing<T: Fn() -> String>(self, msg: T) -> bool {
@@ -1357,6 +1410,11 @@ impl<'tcx> TyCtxt<'tcx> {
         self.diagnostic_items(did.krate).name_to_id.get(&name) == Some(&did)
     }
 
+    /// Returns `true` if the node pointed to by `def_id` is a generator for an async construct.
+    pub fn generator_is_async(self, def_id: DefId) -> bool {
+        matches!(self.generator_kind(def_id), Some(hir::GeneratorKind::Async(_)))
+    }
+
     pub fn stability(self) -> &'tcx stability::Index {
         self.stability_index(())
     }
@@ -1460,14 +1518,19 @@ impl<'tcx> TyCtxt<'tcx> {
             self.def_path(def_id).to_string_no_crate_verbose()
         )
     }
+}
 
+impl<'tcx> TyCtxtAt<'tcx> {
     /// Create a new definition within the incr. comp. engine.
-    pub fn create_def(self, parent: LocalDefId, data: hir::definitions::DefPathData) -> LocalDefId {
+    pub fn create_def(
+        self,
+        parent: LocalDefId,
+        data: hir::definitions::DefPathData,
+    ) -> TyCtxtFeed<'tcx, LocalDefId> {
         // This function modifies `self.definitions` using a side-effect.
         // We need to ensure that these side effects are re-run by the incr. comp. engine.
         // Depending on the forever-red node will tell the graph that the calling query
         // needs to be re-evaluated.
-        use rustc_query_system::dep_graph::DepNodeIndex;
         self.dep_graph.read_index(DepNodeIndex::FOREVER_RED_NODE);
 
         // The following call has the side effect of modifying the tables inside `definitions`.
@@ -1484,23 +1547,42 @@ impl<'tcx> TyCtxt<'tcx> {
         // This is fine because:
         // - those queries are `eval_always` so we won't miss their result changing;
         // - this write will have happened before these queries are called.
-        self.definitions.write().create_def(parent, data)
+        let key = self.definitions.write().create_def(parent, data);
+
+        let feed = TyCtxtFeed { tcx: self.tcx, key };
+        feed.def_span(self.span);
+        feed
     }
+}
 
+impl<'tcx> TyCtxt<'tcx> {
     pub fn iter_local_def_id(self) -> impl Iterator<Item = LocalDefId> + 'tcx {
-        // Create a dependency to the crate to be sure we re-execute this when the amount of
+        // Create a dependency to the red node to be sure we re-execute this when the amount of
         // definitions change.
-        self.ensure().hir_crate(());
-        // Leak a read lock once we start iterating on definitions, to prevent adding new ones
-        // while iterating.  If some query needs to add definitions, it should be `ensure`d above.
-        let definitions = self.definitions.leak();
-        definitions.iter_local_def_id()
+        self.dep_graph.read_index(DepNodeIndex::FOREVER_RED_NODE);
+
+        let definitions = &self.definitions;
+        std::iter::from_generator(|| {
+            let mut i = 0;
+
+            // Recompute the number of definitions each time, because our caller may be creating
+            // new ones.
+            while i < { definitions.read().num_definitions() } {
+                let local_def_index = rustc_span::def_id::DefIndex::from_usize(i);
+                yield LocalDefId { local_def_index };
+                i += 1;
+            }
+
+            // Leak a read lock once we finish iterating on definitions, to prevent adding new ones.
+            definitions.leak();
+        })
     }
 
     pub fn def_path_table(self) -> &'tcx rustc_hir::definitions::DefPathTable {
         // Create a dependency to the crate to be sure we re-execute this when the amount of
         // definitions change.
-        self.ensure().hir_crate(());
+        self.dep_graph.read_index(DepNodeIndex::FOREVER_RED_NODE);
+
         // Leak a read lock once we start iterating on definitions, to prevent adding new ones
         // while iterating.  If some query needs to add definitions, it should be `ensure`d above.
         let definitions = self.definitions.leak();
@@ -1659,7 +1741,7 @@ impl<'tcx> TyCtxt<'tcx> {
         }
     }
 
-    // Checks if the bound region is in Impl Item.
+    /// Checks if the bound region is in Impl Item.
     pub fn is_bound_region_in_impl_item(self, suitable_region_binding_scope: LocalDefId) -> bool {
         let container_id = self.parent(suitable_region_binding_scope.to_def_id());
         if self.impl_trait_ref(container_id).is_some() {
@@ -1810,7 +1892,7 @@ nop_lift! {const_; Const<'a> => Const<'tcx>}
 nop_lift! {const_allocation; ConstAllocation<'a> => ConstAllocation<'tcx>}
 nop_lift! {predicate; Predicate<'a> => Predicate<'tcx>}
 
-nop_list_lift! {poly_existential_predicates; ty::Binder<'a, ExistentialPredicate<'a>> => ty::Binder<'tcx, ExistentialPredicate<'tcx>>}
+nop_list_lift! {poly_existential_predicates; PolyExistentialPredicate<'a> => PolyExistentialPredicate<'tcx>}
 nop_list_lift! {predicates; Predicate<'a> => Predicate<'tcx>}
 nop_list_lift! {canonical_var_infos; CanonicalVarInfo<'a> => CanonicalVarInfo<'tcx>}
 nop_list_lift! {projs; ProjectionKind => ProjectionKind}
@@ -2026,7 +2108,7 @@ macro_rules! sty_debug_print {
                 let shards = tcx.interners.type_.lock_shards();
                 let types = shards.iter().flat_map(|shard| shard.keys());
                 for &InternedInSet(t) in types {
-                    let variant = match t.kind {
+                    let variant = match t.internee {
                         ty::Bool | ty::Char | ty::Int(..) | ty::Uint(..) |
                             ty::Float(..) | ty::Str | ty::Never => continue,
                         ty::Error(_) => /* unimportant */ continue,
@@ -2136,49 +2218,26 @@ impl<'tcx, T: 'tcx + ?Sized> IntoPointer for InternedInSet<'tcx, T> {
 }
 
 #[allow(rustc::usage_of_ty_tykind)]
-impl<'tcx> Borrow<TyKind<'tcx>> for InternedInSet<'tcx, WithStableHash<TyS<'tcx>>> {
-    fn borrow<'a>(&'a self) -> &'a TyKind<'tcx> {
-        &self.0.kind
-    }
-}
-
-impl<'tcx> PartialEq for InternedInSet<'tcx, WithStableHash<TyS<'tcx>>> {
-    fn eq(&self, other: &InternedInSet<'tcx, WithStableHash<TyS<'tcx>>>) -> bool {
-        // The `Borrow` trait requires that `x.borrow() == y.borrow()` equals
-        // `x == y`.
-        self.0.kind == other.0.kind
-    }
-}
-
-impl<'tcx> Eq for InternedInSet<'tcx, WithStableHash<TyS<'tcx>>> {}
-
-impl<'tcx> Hash for InternedInSet<'tcx, WithStableHash<TyS<'tcx>>> {
-    fn hash<H: Hasher>(&self, s: &mut H) {
-        // The `Borrow` trait requires that `x.borrow().hash(s) == x.hash(s)`.
-        self.0.kind.hash(s)
-    }
-}
-
-impl<'tcx> Borrow<Binder<'tcx, PredicateKind<'tcx>>> for InternedInSet<'tcx, PredicateS<'tcx>> {
-    fn borrow<'a>(&'a self) -> &'a Binder<'tcx, PredicateKind<'tcx>> {
-        &self.0.kind
+impl<'tcx, T> Borrow<T> for InternedInSet<'tcx, WithCachedTypeInfo<T>> {
+    fn borrow<'a>(&'a self) -> &'a T {
+        &self.0.internee
     }
 }
 
-impl<'tcx> PartialEq for InternedInSet<'tcx, PredicateS<'tcx>> {
-    fn eq(&self, other: &InternedInSet<'tcx, PredicateS<'tcx>>) -> bool {
+impl<'tcx, T: PartialEq> PartialEq for InternedInSet<'tcx, WithCachedTypeInfo<T>> {
+    fn eq(&self, other: &InternedInSet<'tcx, WithCachedTypeInfo<T>>) -> bool {
         // The `Borrow` trait requires that `x.borrow() == y.borrow()` equals
         // `x == y`.
-        self.0.kind == other.0.kind
+        self.0.internee == other.0.internee
     }
 }
 
-impl<'tcx> Eq for InternedInSet<'tcx, PredicateS<'tcx>> {}
+impl<'tcx, T: Eq> Eq for InternedInSet<'tcx, WithCachedTypeInfo<T>> {}
 
-impl<'tcx> Hash for InternedInSet<'tcx, PredicateS<'tcx>> {
+impl<'tcx, T: Hash> Hash for InternedInSet<'tcx, WithCachedTypeInfo<T>> {
     fn hash<H: Hasher>(&self, s: &mut H) {
         // The `Borrow` trait requires that `x.borrow().hash(s) == x.hash(s)`.
-        self.0.kind.hash(s)
+        self.0.internee.hash(s)
     }
 }
 
@@ -2243,9 +2302,9 @@ macro_rules! direct_interners {
 
 direct_interners! {
     region: mk_region(RegionKind<'tcx>): Region -> Region<'tcx>,
-    const_: mk_const(ConstS<'tcx>): Const -> Const<'tcx>,
+    const_: mk_const_internal(ConstS<'tcx>): Const -> Const<'tcx>,
     const_allocation: intern_const_alloc(Allocation): ConstAllocation -> ConstAllocation<'tcx>,
-    layout: intern_layout(LayoutS<'tcx>): Layout -> Layout<'tcx>,
+    layout: intern_layout(LayoutS<VariantIdx>): Layout -> Layout<'tcx>,
     adt_def: intern_adt_def(AdtDefData): AdtDef -> AdtDef<'tcx>,
 }
 
@@ -2262,10 +2321,11 @@ macro_rules! slice_interners {
 }
 
 slice_interners!(
+    const_lists: _intern_const_list(Const<'tcx>),
     substs: _intern_substs(GenericArg<'tcx>),
     canonical_var_infos: _intern_canonical_var_infos(CanonicalVarInfo<'tcx>),
     poly_existential_predicates:
-        _intern_poly_existential_predicates(ty::Binder<'tcx, ExistentialPredicate<'tcx>>),
+        _intern_poly_existential_predicates(PolyExistentialPredicate<'tcx>),
     predicates: _intern_predicates(Predicate<'tcx>),
     projs: _intern_projs(ProjectionKind),
     place_elems: _intern_place_elems(PlaceElem<'tcx>),
@@ -2294,10 +2354,10 @@ impl<'tcx> TyCtxt<'tcx> {
     /// Given a `ty`, return whether it's an `impl Future<...>`.
     pub fn ty_is_opaque_future(self, ty: Ty<'_>) -> bool {
         let ty::Opaque(def_id, _) = ty.kind() else { return false };
-        let future_trait = self.lang_items().future_trait().unwrap();
+        let future_trait = self.require_lang_item(LangItem::Future, None);
 
         self.explicit_item_bounds(def_id).iter().any(|(predicate, _)| {
-            let ty::PredicateKind::Trait(trait_predicate) = predicate.kind().skip_binder() else {
+            let ty::PredicateKind::Clause(ty::Clause::Trait(trait_predicate)) = predicate.kind().skip_binder() else {
                 return false;
             };
             trait_predicate.trait_ref.def_id == future_trait
@@ -2320,7 +2380,9 @@ impl<'tcx> TyCtxt<'tcx> {
             let generic_predicates = self.super_predicates_of(trait_did);
 
             for (predicate, _) in generic_predicates.predicates {
-                if let ty::PredicateKind::Trait(data) = predicate.kind().skip_binder() {
+                if let ty::PredicateKind::Clause(ty::Clause::Trait(data)) =
+                    predicate.kind().skip_binder()
+                {
                     if set.insert(data.def_id()) {
                         stack.push(data.def_id());
                     }
@@ -2374,7 +2436,14 @@ impl<'tcx> TyCtxt<'tcx> {
 
     #[inline]
     pub fn mk_predicate(self, binder: Binder<'tcx, PredicateKind<'tcx>>) -> Predicate<'tcx> {
-        self.interners.intern_predicate(binder)
+        self.interners.intern_predicate(
+            binder,
+            self.sess,
+            &self.definitions.read(),
+            &*self.untracked_resolutions.cstore,
+            // This is only used to create a stable hashing context.
+            &self.untracked_resolutions.source_span,
+        )
     }
 
     #[inline]
@@ -2456,7 +2525,7 @@ impl<'tcx> TyCtxt<'tcx> {
 
     #[inline]
     pub fn mk_lang_item(self, ty: Ty<'tcx>, item: LangItem) -> Option<Ty<'tcx>> {
-        let def_id = self.lang_items().require(item).ok()?;
+        let def_id = self.lang_items().get(item)?;
         Some(self.mk_generic_adt(def_id, ty))
     }
 
@@ -2517,7 +2586,7 @@ impl<'tcx> TyCtxt<'tcx> {
         self.mk_ty(Tuple(self.intern_type_list(&ts)))
     }
 
-    pub fn mk_tup<I: InternAs<[Ty<'tcx>], Ty<'tcx>>>(self, iter: I) -> I::Output {
+    pub fn mk_tup<I: InternAs<Ty<'tcx>, Ty<'tcx>>>(self, iter: I) -> I::Output {
         iter.intern_with(|ts| self.mk_ty(Tuple(self.intern_type_list(&ts))))
     }
 
@@ -2533,6 +2602,11 @@ impl<'tcx> TyCtxt<'tcx> {
 
     #[inline]
     pub fn mk_fn_def(self, def_id: DefId, substs: SubstsRef<'tcx>) -> Ty<'tcx> {
+        debug_assert_eq!(
+            self.generics_of(def_id).count(),
+            substs.len(),
+            "wrong number of generic parameters for {def_id:?}: {substs:?}",
+        );
         self.mk_ty(FnDef(def_id, substs))
     }
 
@@ -2544,7 +2618,7 @@ impl<'tcx> TyCtxt<'tcx> {
     #[inline]
     pub fn mk_dynamic(
         self,
-        obj: &'tcx List<ty::Binder<'tcx, ExistentialPredicate<'tcx>>>,
+        obj: &'tcx List<PolyExistentialPredicate<'tcx>>,
         reg: ty::Region<'tcx>,
         repr: DynKind,
     ) -> Ty<'tcx> {
@@ -2553,6 +2627,11 @@ impl<'tcx> TyCtxt<'tcx> {
 
     #[inline]
     pub fn mk_projection(self, item_def_id: DefId, substs: SubstsRef<'tcx>) -> Ty<'tcx> {
+        debug_assert_eq!(
+            self.generics_of(item_def_id).count(),
+            substs.len(),
+            "wrong number of generic parameters for {item_def_id:?}: {substs:?}",
+        );
         self.mk_ty(Projection(ProjectionTy { item_def_id, substs }))
     }
 
@@ -2582,8 +2661,8 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     #[inline]
-    pub fn mk_const_var(self, v: ConstVid<'tcx>, ty: Ty<'tcx>) -> Const<'tcx> {
-        self.mk_const(ty::ConstS { kind: ty::ConstKind::Infer(InferConst::Var(v)), ty })
+    pub fn mk_const(self, kind: impl Into<ty::ConstKind<'tcx>>, ty: Ty<'tcx>) -> Const<'tcx> {
+        self.mk_const_internal(ty::ConstS { kind: kind.into(), ty })
     }
 
     #[inline]
@@ -2602,29 +2681,22 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     #[inline]
-    pub fn mk_const_infer(self, ic: InferConst<'tcx>, ty: Ty<'tcx>) -> ty::Const<'tcx> {
-        self.mk_const(ty::ConstS { kind: ty::ConstKind::Infer(ic), ty })
-    }
-
-    #[inline]
     pub fn mk_ty_param(self, index: u32, name: Symbol) -> Ty<'tcx> {
         self.mk_ty(Param(ParamTy { index, name }))
     }
 
-    #[inline]
-    pub fn mk_const_param(self, index: u32, name: Symbol, ty: Ty<'tcx>) -> Const<'tcx> {
-        self.mk_const(ty::ConstS { kind: ty::ConstKind::Param(ParamConst { index, name }), ty })
-    }
-
     pub fn mk_param_from_def(self, param: &ty::GenericParamDef) -> GenericArg<'tcx> {
         match param.kind {
             GenericParamDefKind::Lifetime => {
                 self.mk_region(ty::ReEarlyBound(param.to_early_bound_region_data())).into()
             }
             GenericParamDefKind::Type { .. } => self.mk_ty_param(param.index, param.name).into(),
-            GenericParamDefKind::Const { .. } => {
-                self.mk_const_param(param.index, param.name, self.type_of(param.def_id)).into()
-            }
+            GenericParamDefKind::Const { .. } => self
+                .mk_const(
+                    ParamConst { index: param.index, name: param.name },
+                    self.type_of(param.def_id),
+                )
+                .into(),
         }
     }
 
@@ -2677,8 +2749,8 @@ impl<'tcx> TyCtxt<'tcx> {
 
     pub fn intern_poly_existential_predicates(
         self,
-        eps: &[ty::Binder<'tcx, ExistentialPredicate<'tcx>>],
-    ) -> &'tcx List<ty::Binder<'tcx, ExistentialPredicate<'tcx>>> {
+        eps: &[PolyExistentialPredicate<'tcx>],
+    ) -> &'tcx List<PolyExistentialPredicate<'tcx>> {
         assert!(!eps.is_empty());
         assert!(
             eps.array_windows()
@@ -2700,6 +2772,17 @@ impl<'tcx> TyCtxt<'tcx> {
         }
     }
 
+    pub fn mk_const_list<I: InternAs<ty::Const<'tcx>, &'tcx List<ty::Const<'tcx>>>>(
+        self,
+        iter: I,
+    ) -> I::Output {
+        iter.intern_with(|xs| self.intern_const_list(xs))
+    }
+
+    pub fn intern_const_list(self, cs: &[ty::Const<'tcx>]) -> &'tcx List<ty::Const<'tcx>> {
+        if cs.is_empty() { List::empty() } else { self._intern_const_list(cs) }
+    }
+
     pub fn intern_type_list(self, ts: &[Ty<'tcx>]) -> &'tcx List<Ty<'tcx>> {
         if ts.is_empty() {
             List::empty()
@@ -2762,10 +2845,7 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     pub fn mk_poly_existential_predicates<
-        I: InternAs<
-            [ty::Binder<'tcx, ExistentialPredicate<'tcx>>],
-            &'tcx List<ty::Binder<'tcx, ExistentialPredicate<'tcx>>>,
-        >,
+        I: InternAs<PolyExistentialPredicate<'tcx>, &'tcx List<PolyExistentialPredicate<'tcx>>>,
     >(
         self,
         iter: I,
@@ -2773,37 +2853,58 @@ impl<'tcx> TyCtxt<'tcx> {
         iter.intern_with(|xs| self.intern_poly_existential_predicates(xs))
     }
 
-    pub fn mk_predicates<I: InternAs<[Predicate<'tcx>], &'tcx List<Predicate<'tcx>>>>(
+    pub fn mk_predicates<I: InternAs<Predicate<'tcx>, &'tcx List<Predicate<'tcx>>>>(
         self,
         iter: I,
     ) -> I::Output {
         iter.intern_with(|xs| self.intern_predicates(xs))
     }
 
-    pub fn mk_type_list<I: InternAs<[Ty<'tcx>], &'tcx List<Ty<'tcx>>>>(self, iter: I) -> I::Output {
+    pub fn mk_type_list<I: InternAs<Ty<'tcx>, &'tcx List<Ty<'tcx>>>>(self, iter: I) -> I::Output {
         iter.intern_with(|xs| self.intern_type_list(xs))
     }
 
-    pub fn mk_substs<I: InternAs<[GenericArg<'tcx>], &'tcx List<GenericArg<'tcx>>>>(
+    pub fn mk_substs<I: InternAs<GenericArg<'tcx>, &'tcx List<GenericArg<'tcx>>>>(
         self,
         iter: I,
     ) -> I::Output {
         iter.intern_with(|xs| self.intern_substs(xs))
     }
 
-    pub fn mk_place_elems<I: InternAs<[PlaceElem<'tcx>], &'tcx List<PlaceElem<'tcx>>>>(
+    pub fn mk_place_elems<I: InternAs<PlaceElem<'tcx>, &'tcx List<PlaceElem<'tcx>>>>(
         self,
         iter: I,
     ) -> I::Output {
         iter.intern_with(|xs| self.intern_place_elems(xs))
     }
 
-    pub fn mk_substs_trait(self, self_ty: Ty<'tcx>, rest: &[GenericArg<'tcx>]) -> SubstsRef<'tcx> {
-        self.mk_substs(iter::once(self_ty.into()).chain(rest.iter().cloned()))
+    pub fn mk_substs_trait(
+        self,
+        self_ty: Ty<'tcx>,
+        rest: impl IntoIterator<Item = GenericArg<'tcx>>,
+    ) -> SubstsRef<'tcx> {
+        self.mk_substs(iter::once(self_ty.into()).chain(rest))
+    }
+
+    pub fn mk_trait_ref(
+        self,
+        trait_def_id: DefId,
+        substs: impl IntoIterator<Item = impl Into<GenericArg<'tcx>>>,
+    ) -> ty::TraitRef<'tcx> {
+        let substs = substs.into_iter().map(Into::into);
+        let n = self.generics_of(trait_def_id).count();
+        debug_assert_eq!(
+            (n, Some(n)),
+            substs.size_hint(),
+            "wrong number of generic parameters for {trait_def_id:?}: {:?} \nDid you accidentally include the self-type in the params list?",
+            substs.collect::<Vec<_>>(),
+        );
+        let substs = self.mk_substs(substs);
+        ty::TraitRef::new(trait_def_id, substs)
     }
 
     pub fn mk_bound_variable_kinds<
-        I: InternAs<[ty::BoundVariableKind], &'tcx List<ty::BoundVariableKind>>,
+        I: InternAs<ty::BoundVariableKind, &'tcx List<ty::BoundVariableKind>>,
     >(
         self,
         iter: I,
@@ -2820,7 +2921,9 @@ impl<'tcx> TyCtxt<'tcx> {
         span: impl Into<MultiSpan>,
         decorator: impl for<'a> DecorateLint<'a, ()>,
     ) {
-        self.struct_span_lint_hir(lint, hir_id, span, decorator.msg(), |diag| {
+        let msg = decorator.msg();
+        let (level, src) = self.lint_level_at_node(lint, hir_id);
+        struct_lint_level(self.sess, lint, level, src, Some(span.into()), msg, |diag| {
             decorator.decorate_lint(diag)
         })
     }
@@ -2830,6 +2933,7 @@ impl<'tcx> TyCtxt<'tcx> {
     /// Return value of the `decorate` closure is ignored, see [`struct_lint_level`] for a detailed explanation.
     ///
     /// [`struct_lint_level`]: rustc_middle::lint::struct_lint_level#decorate-signature
+    #[rustc_lint_diagnostics]
     pub fn struct_span_lint_hir(
         self,
         lint: &'static Lint,
@@ -2860,6 +2964,7 @@ impl<'tcx> TyCtxt<'tcx> {
     /// Return value of the `decorate` closure is ignored, see [`struct_lint_level`] for a detailed explanation.
     ///
     /// [`struct_lint_level`]: rustc_middle::lint::struct_lint_level#decorate-signature
+    #[rustc_lint_diagnostics]
     pub fn struct_lint_node(
         self,
         lint: &'static Lint,
@@ -2954,6 +3059,15 @@ impl<'tcx> TyCtxtAt<'tcx> {
     pub fn ty_error_with_message(self, msg: &str) -> Ty<'tcx> {
         self.tcx.ty_error_with_message(self.span, msg)
     }
+
+    pub fn mk_trait_ref(
+        self,
+        trait_lang_item: LangItem,
+        substs: impl IntoIterator<Item = impl Into<ty::GenericArg<'tcx>>>,
+    ) -> ty::TraitRef<'tcx> {
+        let trait_def_id = self.require_lang_item(trait_lang_item, Some(self.span));
+        self.tcx.mk_trait_ref(trait_def_id, substs)
+    }
 }
 
 /// Parameter attributes that can only be determined by examining the body of a function instead
@@ -2979,7 +3093,6 @@ fn ptr_eq<T, U>(t: *const T, u: *const U) -> bool {
 
 pub fn provide(providers: &mut ty::query::Providers) {
     providers.resolutions = |tcx, ()| &tcx.untracked_resolutions;
-    providers.resolver_for_lowering = |tcx, ()| &tcx.untracked_resolver_for_lowering;
     providers.module_reexports =
         |tcx, id| tcx.resolutions(()).reexport_map.get(&id).map(|v| &v[..]);
     providers.crate_name = |tcx, id| {
diff --git a/compiler/rustc_middle/src/ty/diagnostics.rs b/compiler/rustc_middle/src/ty/diagnostics.rs
index b8fd01e6a..511d51cd6 100644
--- a/compiler/rustc_middle/src/ty/diagnostics.rs
+++ b/compiler/rustc_middle/src/ty/diagnostics.rs
@@ -151,7 +151,6 @@ enum SuggestChangingConstraintsMessage<'a> {
 }
 
 fn suggest_removing_unsized_bound(
-    tcx: TyCtxt<'_>,
     generics: &hir::Generics<'_>,
     suggestions: &mut Vec<(Span, String, SuggestChangingConstraintsMessage<'_>)>,
     param: &hir::GenericParam<'_>,
@@ -160,17 +159,16 @@ fn suggest_removing_unsized_bound(
     // See if there's a `?Sized` bound that can be removed to suggest that.
     // First look at the `where` clause because we can have `where T: ?Sized`,
     // then look at params.
-    let param_def_id = tcx.hir().local_def_id(param.hir_id);
     for (where_pos, predicate) in generics.predicates.iter().enumerate() {
         let WherePredicate::BoundPredicate(predicate) = predicate else {
             continue;
         };
-        if !predicate.is_param_bound(param_def_id.to_def_id()) {
+        if !predicate.is_param_bound(param.def_id.to_def_id()) {
             continue;
         };
 
         for (pos, bound) in predicate.bounds.iter().enumerate() {
-            let    hir::GenericBound::Trait(poly, hir::TraitBoundModifier::Maybe) = bound else {
+            let hir::GenericBound::Trait(poly, hir::TraitBoundModifier::Maybe) = bound else {
                 continue;
             };
             if poly.trait_ref.trait_def_id() != def_id {
@@ -232,7 +230,7 @@ pub fn suggest_constraining_type_params<'a>(
                     param.span,
                     &format!("this type parameter needs to be `{}`", constraint),
                 );
-                suggest_removing_unsized_bound(tcx, generics, &mut suggestions, param, def_id);
+                suggest_removing_unsized_bound(generics, &mut suggestions, param, def_id);
             }
         }
 
@@ -283,8 +281,7 @@ pub fn suggest_constraining_type_params<'a>(
         //          --
         //          |
         //          replace with: `T: Bar +`
-        let param_def_id = tcx.hir().local_def_id(param.hir_id);
-        if let Some(span) = generics.bounds_span_for_suggestions(param_def_id) {
+        if let Some(span) = generics.bounds_span_for_suggestions(param.def_id) {
             suggest_restrict(span, true);
             continue;
         }
@@ -358,6 +355,12 @@ pub fn suggest_constraining_type_params<'a>(
         ));
     }
 
+    // FIXME: remove the suggestions that are from derive, as the span is not correct
+    suggestions = suggestions
+        .into_iter()
+        .filter(|(span, _, _)| !span.in_derive_expansion())
+        .collect::<Vec<_>>();
+
     if suggestions.len() == 1 {
         let (span, suggestion, msg) = suggestions.pop().unwrap();
 
@@ -400,7 +403,7 @@ impl<'v> hir::intravisit::Visitor<'v> for TraitObjectVisitor<'v> {
             hir::TyKind::TraitObject(
                 _,
                 hir::Lifetime {
-                    name:
+                    res:
                         hir::LifetimeName::ImplicitObjectLifetimeDefault | hir::LifetimeName::Static,
                     ..
                 },
@@ -424,10 +427,9 @@ pub struct StaticLifetimeVisitor<'tcx>(pub Vec<Span>, pub crate::hir::map::Map<'
 
 impl<'v> hir::intravisit::Visitor<'v> for StaticLifetimeVisitor<'v> {
     fn visit_lifetime(&mut self, lt: &'v hir::Lifetime) {
-        if let hir::LifetimeName::ImplicitObjectLifetimeDefault | hir::LifetimeName::Static =
-            lt.name
+        if let hir::LifetimeName::ImplicitObjectLifetimeDefault | hir::LifetimeName::Static = lt.res
         {
-            self.0.push(lt.span);
+            self.0.push(lt.ident.span);
         }
     }
 }
@@ -511,11 +513,3 @@ impl<'tcx> TypeVisitor<'tcx> for IsSuggestableVisitor<'tcx> {
         c.super_visit_with(self)
     }
 }
-
-#[derive(Diagnostic)]
-#[diag(borrowck_const_not_used_in_type_alias)]
-pub(super) struct ConstNotUsedTraitAlias {
-    pub ct: String,
-    #[primary_span]
-    pub span: Span,
-}
diff --git a/compiler/rustc_middle/src/ty/error.rs b/compiler/rustc_middle/src/ty/error.rs
index 4e6cdb786..aa61c39b8 100644
--- a/compiler/rustc_middle/src/ty/error.rs
+++ b/compiler/rustc_middle/src/ty/error.rs
@@ -12,7 +12,12 @@ use rustc_span::{BytePos, Span};
 use rustc_target::spec::abi;
 
 use std::borrow::Cow;
+use std::collections::hash_map::DefaultHasher;
 use std::fmt;
+use std::hash::{Hash, Hasher};
+use std::path::PathBuf;
+
+use super::print::PrettyPrinter;
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq, TypeFoldable, TypeVisitable, Lift)]
 pub struct ExpectedFound<T> {
@@ -64,10 +69,7 @@ pub enum TypeError<'tcx> {
     CyclicTy(Ty<'tcx>),
     CyclicConst(ty::Const<'tcx>),
     ProjectionMismatched(ExpectedFound<DefId>),
-    ExistentialMismatch(
-        ExpectedFound<&'tcx ty::List<ty::Binder<'tcx, ty::ExistentialPredicate<'tcx>>>>,
-    ),
-    ObjectUnsafeCoercion(DefId),
+    ExistentialMismatch(ExpectedFound<&'tcx ty::List<ty::PolyExistentialPredicate<'tcx>>>),
     ConstMismatch(ExpectedFound<ty::Const<'tcx>>),
 
     IntrinsicCast,
@@ -219,7 +221,6 @@ impl<'tcx> fmt::Display for TypeError<'tcx> {
                 f,
                 "cannot coerce functions with `#[target_feature]` to safe function pointers"
             ),
-            ObjectUnsafeCoercion(_) => write!(f, "coercion to object-unsafe trait object"),
         }
     }
 }
@@ -246,8 +247,7 @@ impl<'tcx> TypeError<'tcx> {
             | ProjectionMismatched(_)
             | ExistentialMismatch(_)
             | ConstMismatch(_)
-            | IntrinsicCast
-            | ObjectUnsafeCoercion(_) => true,
+            | IntrinsicCast => true,
         }
     }
 }
@@ -430,7 +430,9 @@ impl<'tcx> TyCtxt<'tcx> {
                     (ty::Projection(_), ty::Projection(_)) => {
                         diag.note("an associated type was expected, but a different one was found");
                     }
-                    (ty::Param(p), ty::Projection(proj)) | (ty::Projection(proj), ty::Param(p)) => {
+                    (ty::Param(p), ty::Projection(proj)) | (ty::Projection(proj), ty::Param(p))
+                        if self.def_kind(proj.item_def_id) != DefKind::ImplTraitPlaceholder =>
+                    {
                         let generics = self.generics_of(body_owner_def_id);
                         let p_span = self.def_span(generics.type_param(p, self).def_id);
                         if !sp.contains(p_span) {
@@ -983,6 +985,47 @@ fn foo(&self) -> Self::T { String::new() }
         false
     }
 
+    pub fn short_ty_string(self, ty: Ty<'tcx>) -> (String, Option<PathBuf>) {
+        let width = self.sess.diagnostic_width();
+        let length_limit = width.saturating_sub(30);
+        let mut type_limit = 50;
+        let regular = FmtPrinter::new(self, hir::def::Namespace::TypeNS)
+            .pretty_print_type(ty)
+            .expect("could not write to `String`")
+            .into_buffer();
+        if regular.len() <= width {
+            return (regular, None);
+        }
+        let mut short;
+        loop {
+            // Look for the longest properly trimmed path that still fits in lenght_limit.
+            short = FmtPrinter::new_with_limit(
+                self,
+                hir::def::Namespace::TypeNS,
+                rustc_session::Limit(type_limit),
+            )
+            .pretty_print_type(ty)
+            .expect("could not write to `String`")
+            .into_buffer();
+            if short.len() <= length_limit || type_limit == 0 {
+                break;
+            }
+            type_limit -= 1;
+        }
+        if regular == short {
+            return (regular, None);
+        }
+        // Multiple types might be shortened in a single error, ensure we create a file for each.
+        let mut s = DefaultHasher::new();
+        ty.hash(&mut s);
+        let hash = s.finish();
+        let path = self.output_filenames(()).temp_path_ext(&format!("long-type-{hash}.txt"), None);
+        match std::fs::write(&path, &regular) {
+            Ok(_) => (short, Some(path)),
+            Err(_) => (regular, None),
+        }
+    }
+
     fn format_generic_args(self, args: &[ty::GenericArg<'tcx>]) -> String {
         FmtPrinter::new(self, hir::def::Namespace::TypeNS)
             .path_generic_args(Ok, args)
diff --git a/compiler/rustc_middle/src/ty/fast_reject.rs b/compiler/rustc_middle/src/ty/fast_reject.rs
index 3be0bc4de..c9c09c93a 100644
--- a/compiler/rustc_middle/src/ty/fast_reject.rs
+++ b/compiler/rustc_middle/src/ty/fast_reject.rs
@@ -42,7 +42,6 @@ where
     ClosureSimplifiedType(D),
     GeneratorSimplifiedType(D),
     GeneratorWitnessSimplifiedType(usize),
-    OpaqueSimplifiedType(D),
     FunctionSimplifiedType(usize),
     PlaceholderSimplifiedType,
 }
@@ -127,7 +126,7 @@ pub fn simplify_type<'tcx>(
             TreatParams::AsPlaceholder => Some(PlaceholderSimplifiedType),
             TreatParams::AsInfer => None,
         },
-        ty::Projection(_) => match treat_params {
+        ty::Opaque(..) | ty::Projection(_) => match treat_params {
             // When treating `ty::Param` as a placeholder, projections also
             // don't unify with anything else as long as they are fully normalized.
             //
@@ -138,7 +137,6 @@ pub fn simplify_type<'tcx>(
             }
             TreatParams::AsPlaceholder | TreatParams::AsInfer => None,
         },
-        ty::Opaque(def_id, _) => Some(OpaqueSimplifiedType(def_id)),
         ty::Foreign(def_id) => Some(ForeignSimplifiedType(def_id)),
         ty::Bound(..) | ty::Infer(_) | ty::Error(_) => None,
     }
@@ -151,8 +149,7 @@ impl<D: Copy + Debug + Eq> SimplifiedTypeGen<D> {
             | ForeignSimplifiedType(d)
             | TraitSimplifiedType(d)
             | ClosureSimplifiedType(d)
-            | GeneratorSimplifiedType(d)
-            | OpaqueSimplifiedType(d) => Some(d),
+            | GeneratorSimplifiedType(d) => Some(d),
             _ => None,
         }
     }
@@ -182,7 +179,6 @@ impl<D: Copy + Debug + Eq> SimplifiedTypeGen<D> {
             ClosureSimplifiedType(d) => ClosureSimplifiedType(map(d)),
             GeneratorSimplifiedType(d) => GeneratorSimplifiedType(map(d)),
             GeneratorWitnessSimplifiedType(n) => GeneratorWitnessSimplifiedType(n),
-            OpaqueSimplifiedType(d) => OpaqueSimplifiedType(map(d)),
             FunctionSimplifiedType(n) => FunctionSimplifiedType(n),
             PlaceholderSimplifiedType => PlaceholderSimplifiedType,
         }
@@ -229,7 +225,7 @@ impl DeepRejectCtxt {
         match impl_ty.kind() {
             // Start by checking whether the type in the impl may unify with
             // pretty much everything. Just return `true` in that case.
-            ty::Param(_) | ty::Projection(_) | ty::Error(_) => return true,
+            ty::Param(_) | ty::Projection(_) | ty::Error(_) | ty::Opaque(..) => return true,
             // These types only unify with inference variables or their own
             // variant.
             ty::Bool
@@ -247,8 +243,7 @@ impl DeepRejectCtxt {
             | ty::Never
             | ty::Tuple(..)
             | ty::FnPtr(..)
-            | ty::Foreign(..)
-            | ty::Opaque(..) => {}
+            | ty::Foreign(..) => {}
             ty::FnDef(..)
             | ty::Closure(..)
             | ty::Generator(..)
@@ -328,10 +323,7 @@ impl DeepRejectCtxt {
                 _ => false,
             },
 
-            // Opaque types in impls should be forbidden, but that doesn't
-            // stop compilation. So this match arm should never return true
-            // if compilation succeeds.
-            ty::Opaque(..) => matches!(k, ty::Opaque(..)),
+            ty::Opaque(..) => true,
 
             // Impls cannot contain these types as these cannot be named directly.
             ty::FnDef(..) | ty::Closure(..) | ty::Generator(..) => false,
@@ -364,7 +356,10 @@ impl DeepRejectCtxt {
 
     pub fn consts_may_unify(self, obligation_ct: ty::Const<'_>, impl_ct: ty::Const<'_>) -> bool {
         match impl_ct.kind() {
-            ty::ConstKind::Param(_) | ty::ConstKind::Unevaluated(_) | ty::ConstKind::Error(_) => {
+            ty::ConstKind::Expr(_)
+            | ty::ConstKind::Param(_)
+            | ty::ConstKind::Unevaluated(_)
+            | ty::ConstKind::Error(_) => {
                 return true;
             }
             ty::ConstKind::Value(_) => {}
@@ -382,7 +377,9 @@ impl DeepRejectCtxt {
 
             // As we don't necessarily eagerly evaluate constants,
             // they might unify with any value.
-            ty::ConstKind::Unevaluated(_) | ty::ConstKind::Error(_) => true,
+            ty::ConstKind::Expr(_) | ty::ConstKind::Unevaluated(_) | ty::ConstKind::Error(_) => {
+                true
+            }
             ty::ConstKind::Value(obl) => match k {
                 ty::ConstKind::Value(imp) => obl == imp,
                 _ => true,
diff --git a/compiler/rustc_middle/src/ty/flags.rs b/compiler/rustc_middle/src/ty/flags.rs
index 7201737be..046a2660a 100644
--- a/compiler/rustc_middle/src/ty/flags.rs
+++ b/compiler/rustc_middle/src/ty/flags.rs
@@ -6,7 +6,7 @@ use std::slice;
 pub struct FlagComputation {
     pub flags: TypeFlags,
 
-    // see `Ty::outer_exclusive_binder` for details
+    /// see `Ty::outer_exclusive_binder` for details
     pub outer_exclusive_binder: ty::DebruijnIndex,
 }
 
@@ -216,14 +216,17 @@ impl FlagComputation {
 
     fn add_predicate_atom(&mut self, atom: ty::PredicateKind<'_>) {
         match atom {
-            ty::PredicateKind::Trait(trait_pred) => {
+            ty::PredicateKind::Clause(ty::Clause::Trait(trait_pred)) => {
                 self.add_substs(trait_pred.trait_ref.substs);
             }
-            ty::PredicateKind::RegionOutlives(ty::OutlivesPredicate(a, b)) => {
+            ty::PredicateKind::Clause(ty::Clause::RegionOutlives(ty::OutlivesPredicate(a, b))) => {
                 self.add_region(a);
                 self.add_region(b);
             }
-            ty::PredicateKind::TypeOutlives(ty::OutlivesPredicate(ty, region)) => {
+            ty::PredicateKind::Clause(ty::Clause::TypeOutlives(ty::OutlivesPredicate(
+                ty,
+                region,
+            ))) => {
                 self.add_ty(ty);
                 self.add_region(region);
             }
@@ -235,7 +238,10 @@ impl FlagComputation {
                 self.add_ty(a);
                 self.add_ty(b);
             }
-            ty::PredicateKind::Projection(ty::ProjectionPredicate { projection_ty, term }) => {
+            ty::PredicateKind::Clause(ty::Clause::Projection(ty::ProjectionPredicate {
+                projection_ty,
+                term,
+            })) => {
                 self.add_projection_ty(projection_ty);
                 match term.unpack() {
                     ty::TermKind::Ty(ty) => self.add_ty(ty),
@@ -259,6 +265,7 @@ impl FlagComputation {
             ty::PredicateKind::TypeWellFormedFromEnv(ty) => {
                 self.add_ty(ty);
             }
+            ty::PredicateKind::Ambiguous => {}
         }
     }
 
@@ -306,6 +313,26 @@ impl FlagComputation {
                 self.add_flags(TypeFlags::STILL_FURTHER_SPECIALIZABLE);
             }
             ty::ConstKind::Value(_) => {}
+            ty::ConstKind::Expr(e) => {
+                use ty::Expr;
+                match e {
+                    Expr::Binop(_, l, r) => {
+                        self.add_const(l);
+                        self.add_const(r);
+                    }
+                    Expr::UnOp(_, v) => self.add_const(v),
+                    Expr::FunctionCall(f, args) => {
+                        self.add_const(f);
+                        for arg in args {
+                            self.add_const(arg);
+                        }
+                    }
+                    Expr::Cast(_, c, t) => {
+                        self.add_ty(t);
+                        self.add_const(c);
+                    }
+                }
+            }
             ty::ConstKind::Error(_) => self.add_flags(TypeFlags::HAS_ERROR),
         }
     }
diff --git a/compiler/rustc_middle/src/ty/fold.rs b/compiler/rustc_middle/src/ty/fold.rs
index 54f1499eb..d431d008d 100644
--- a/compiler/rustc_middle/src/ty/fold.rs
+++ b/compiler/rustc_middle/src/ty/fold.rs
@@ -407,6 +407,7 @@ where
         match *t.kind() {
             ty::Bound(debruijn, bound_ty) if debruijn == self.current_index => {
                 let ty = self.delegate.replace_ty(bound_ty);
+                debug_assert!(!ty.has_vars_bound_above(ty::INNERMOST));
                 ty::fold::shift_vars(self.tcx, ty, self.current_index.as_u32())
             }
             _ if t.has_vars_bound_at_or_above(self.current_index) => t.super_fold_with(self),
@@ -437,6 +438,7 @@ where
         match ct.kind() {
             ty::ConstKind::Bound(debruijn, bound_const) if debruijn == self.current_index => {
                 let ct = self.delegate.replace_const(bound_const, ct.ty());
+                debug_assert!(!ct.has_vars_bound_above(ty::INNERMOST));
                 ty::fold::shift_vars(self.tcx, ct, self.current_index.as_u32())
             }
             _ => ct.super_fold_with(self),
@@ -566,10 +568,7 @@ impl<'tcx> TyCtxt<'tcx> {
                     ))
                 },
                 consts: &mut |c, ty: Ty<'tcx>| {
-                    self.mk_const(ty::ConstS {
-                        kind: ty::ConstKind::Bound(ty::INNERMOST, shift_bv(c)),
-                        ty,
-                    })
+                    self.mk_const(ty::ConstKind::Bound(ty::INNERMOST, shift_bv(c)), ty)
                 },
             },
         )
@@ -601,7 +600,7 @@ impl<'tcx> TyCtxt<'tcx> {
             .replace_late_bound_regions(sig, |_| {
                 let br = ty::BoundRegion {
                     var: ty::BoundVar::from_u32(counter),
-                    kind: ty::BrAnon(counter),
+                    kind: ty::BrAnon(counter, None),
                 };
                 let r = self.mk_region(ty::ReLateBound(ty::INNERMOST, br));
                 counter += 1;
@@ -609,7 +608,7 @@ impl<'tcx> TyCtxt<'tcx> {
             })
             .0;
         let bound_vars = self.mk_bound_variable_kinds(
-            (0..counter).map(|i| ty::BoundVariableKind::Region(ty::BrAnon(i))),
+            (0..counter).map(|i| ty::BoundVariableKind::Region(ty::BrAnon(i, None))),
         );
         Binder::bind_with_vars(inner, bound_vars)
     }
@@ -629,7 +628,9 @@ impl<'tcx> TyCtxt<'tcx> {
                 let index = entry.index();
                 let var = ty::BoundVar::from_usize(index);
                 let kind = entry
-                    .or_insert_with(|| ty::BoundVariableKind::Region(ty::BrAnon(index as u32)))
+                    .or_insert_with(|| {
+                        ty::BoundVariableKind::Region(ty::BrAnon(index as u32, None))
+                    })
                     .expect_region();
                 let br = ty::BoundRegion { var, kind };
                 self.tcx.mk_region(ty::ReLateBound(ty::INNERMOST, br))
@@ -648,7 +649,7 @@ impl<'tcx> TyCtxt<'tcx> {
                 let index = entry.index();
                 let var = ty::BoundVar::from_usize(index);
                 let () = entry.or_insert_with(|| ty::BoundVariableKind::Const).expect_const();
-                self.tcx.mk_const(ty::ConstS { ty, kind: ty::ConstKind::Bound(ty::INNERMOST, var) })
+                self.tcx.mk_const(ty::ConstKind::Bound(ty::INNERMOST, var), ty)
             }
         }
 
@@ -698,14 +699,10 @@ impl<'tcx> TypeFolder<'tcx> for Shifter<'tcx> {
 
     fn fold_region(&mut self, r: ty::Region<'tcx>) -> ty::Region<'tcx> {
         match *r {
-            ty::ReLateBound(debruijn, br) => {
-                if self.amount == 0 || debruijn < self.current_index {
-                    r
-                } else {
-                    let debruijn = debruijn.shifted_in(self.amount);
-                    let shifted = ty::ReLateBound(debruijn, br);
-                    self.tcx.mk_region(shifted)
-                }
+            ty::ReLateBound(debruijn, br) if debruijn >= self.current_index => {
+                let debruijn = debruijn.shifted_in(self.amount);
+                let shifted = ty::ReLateBound(debruijn, br);
+                self.tcx.mk_region(shifted)
             }
             _ => r,
         }
@@ -713,34 +710,30 @@ impl<'tcx> TypeFolder<'tcx> for Shifter<'tcx> {
 
     fn fold_ty(&mut self, ty: Ty<'tcx>) -> Ty<'tcx> {
         match *ty.kind() {
-            ty::Bound(debruijn, bound_ty) => {
-                if self.amount == 0 || debruijn < self.current_index {
-                    ty
-                } else {
-                    let debruijn = debruijn.shifted_in(self.amount);
-                    self.tcx.mk_ty(ty::Bound(debruijn, bound_ty))
-                }
+            ty::Bound(debruijn, bound_ty) if debruijn >= self.current_index => {
+                let debruijn = debruijn.shifted_in(self.amount);
+                self.tcx.mk_ty(ty::Bound(debruijn, bound_ty))
             }
 
-            _ => ty.super_fold_with(self),
+            _ if ty.has_vars_bound_at_or_above(self.current_index) => ty.super_fold_with(self),
+            _ => ty,
         }
     }
 
     fn fold_const(&mut self, ct: ty::Const<'tcx>) -> ty::Const<'tcx> {
-        if let ty::ConstKind::Bound(debruijn, bound_ct) = ct.kind() {
-            if self.amount == 0 || debruijn < self.current_index {
-                ct
-            } else {
-                let debruijn = debruijn.shifted_in(self.amount);
-                self.tcx.mk_const(ty::ConstS {
-                    kind: ty::ConstKind::Bound(debruijn, bound_ct),
-                    ty: ct.ty(),
-                })
-            }
+        if let ty::ConstKind::Bound(debruijn, bound_ct) = ct.kind()
+            && debruijn >= self.current_index
+        {
+            let debruijn = debruijn.shifted_in(self.amount);
+            self.tcx.mk_const(ty::ConstKind::Bound(debruijn, bound_ct), ct.ty())
         } else {
             ct.super_fold_with(self)
         }
     }
+
+    fn fold_predicate(&mut self, p: ty::Predicate<'tcx>) -> ty::Predicate<'tcx> {
+        if p.has_vars_bound_at_or_above(self.current_index) { p.super_fold_with(self) } else { p }
+    }
 }
 
 pub fn shift_region<'tcx>(
@@ -762,5 +755,9 @@ where
 {
     debug!("shift_vars(value={:?}, amount={})", value, amount);
 
+    if amount == 0 || !value.has_escaping_bound_vars() {
+        return value;
+    }
+
     value.fold_with(&mut Shifter::new(tcx, amount))
 }
diff --git a/compiler/rustc_middle/src/ty/generics.rs b/compiler/rustc_middle/src/ty/generics.rs
index 19754d145..48329da3e 100644
--- a/compiler/rustc_middle/src/ty/generics.rs
+++ b/compiler/rustc_middle/src/ty/generics.rs
@@ -3,7 +3,7 @@ use crate::ty::{EarlyBinder, SubstsRef};
 use rustc_ast as ast;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_hir::def_id::DefId;
-use rustc_span::symbol::Symbol;
+use rustc_span::symbol::{kw, Symbol};
 use rustc_span::Span;
 
 use super::{EarlyBoundRegion, InstantiatedPredicates, ParamConst, ParamTy, Predicate, TyCtxt};
@@ -78,6 +78,15 @@ impl GenericParamDef {
         }
     }
 
+    pub fn is_anonymous_lifetime(&self) -> bool {
+        match self.kind {
+            GenericParamDefKind::Lifetime => {
+                self.name == kw::UnderscoreLifetime || self.name == kw::Empty
+            }
+            _ => false,
+        }
+    }
+
     pub fn default_value<'tcx>(
         &self,
         tcx: TyCtxt<'tcx>,
@@ -92,6 +101,20 @@ impl GenericParamDef {
             _ => None,
         }
     }
+
+    pub fn to_error<'tcx>(
+        &self,
+        tcx: TyCtxt<'tcx>,
+        preceding_substs: &[ty::GenericArg<'tcx>],
+    ) -> ty::GenericArg<'tcx> {
+        match &self.kind {
+            ty::GenericParamDefKind::Lifetime => tcx.lifetimes.re_static.into(),
+            ty::GenericParamDefKind::Type { .. } => tcx.ty_error().into(),
+            ty::GenericParamDefKind::Const { .. } => {
+                tcx.const_error(tcx.bound_type_of(self.def_id).subst(tcx, preceding_substs)).into()
+            }
+        }
+    }
 }
 
 #[derive(Default)]
diff --git a/compiler/rustc_middle/src/ty/impls_ty.rs b/compiler/rustc_middle/src/ty/impls_ty.rs
index d1c0d62ac..3e59c0b96 100644
--- a/compiler/rustc_middle/src/ty/impls_ty.rs
+++ b/compiler/rustc_middle/src/ty/impls_ty.rs
@@ -112,19 +112,6 @@ impl<'a> HashStable<StableHashingContext<'a>> for mir::interpret::AllocId {
     }
 }
 
-// `Relocations` with default type parameters is a sorted map.
-impl<'a, Prov> HashStable<StableHashingContext<'a>> for mir::interpret::ProvenanceMap<Prov>
-where
-    Prov: HashStable<StableHashingContext<'a>>,
-{
-    fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
-        self.len().hash_stable(hcx, hasher);
-        for reloc in self.iter() {
-            reloc.hash_stable(hcx, hasher);
-        }
-    }
-}
-
 impl<'a> ToStableHashKey<StableHashingContext<'a>> for region::Scope {
     type KeyType = region::Scope;
 
diff --git a/compiler/rustc_middle/src/ty/inhabitedness/inhabited_predicate.rs b/compiler/rustc_middle/src/ty/inhabitedness/inhabited_predicate.rs
index b7aa45572..33f727297 100644
--- a/compiler/rustc_middle/src/ty/inhabitedness/inhabited_predicate.rs
+++ b/compiler/rustc_middle/src/ty/inhabitedness/inhabited_predicate.rs
@@ -41,6 +41,13 @@ impl<'tcx> InhabitedPredicate<'tcx> {
         self.apply_inner(tcx, param_env, &|_| Err(())).ok()
     }
 
+    /// Same as `apply`, but `NotInModule(_)` predicates yield `false`. That is,
+    /// privately uninhabited types are considered always uninhabited.
+    pub fn apply_ignore_module(self, tcx: TyCtxt<'tcx>, param_env: ParamEnv<'tcx>) -> bool {
+        let Ok(result) = self.apply_inner::<!>(tcx, param_env, &|_| Ok(true));
+        result
+    }
+
     fn apply_inner<E>(
         self,
         tcx: TyCtxt<'tcx>,
diff --git a/compiler/rustc_middle/src/ty/inhabitedness/mod.rs b/compiler/rustc_middle/src/ty/inhabitedness/mod.rs
index 279a728ea..ace81bc4f 100644
--- a/compiler/rustc_middle/src/ty/inhabitedness/mod.rs
+++ b/compiler/rustc_middle/src/ty/inhabitedness/mod.rs
@@ -5,7 +5,7 @@
 //!
 //! # Example
 //! ```rust
-//! enum Void {}
+//! #![feature(never_type)]
 //! mod a {
 //!     pub mod b {
 //!         pub struct SecretlyUninhabited {
@@ -15,6 +15,7 @@
 //! }
 //!
 //! mod c {
+//!     enum Void {}
 //!     pub struct AlsoSecretlyUninhabited {
 //!         _priv: Void,
 //!     }
@@ -28,14 +29,14 @@
 //! }
 //! ```
 //! In this code, the type `Foo` will only be visibly uninhabited inside the
-//! modules `b`, `c` and `d`. Calling `uninhabited_predicate` on `Foo` will
+//! modules `b`, `c` and `d`. Calling `inhabited_predicate` on `Foo` will
 //! return `NotInModule(b) AND NotInModule(c)`.
 //!
 //! We need this information for pattern-matching on `Foo` or types that contain
 //! `Foo`.
 //!
 //! # Example
-//! ```rust
+//! ```ignore(illustrative)
 //! let foo_result: Result<T, Foo> = ... ;
 //! let Ok(t) = foo_result;
 //! ```
@@ -56,57 +57,6 @@ pub(crate) fn provide(providers: &mut ty::query::Providers) {
         ty::query::Providers { inhabited_predicate_adt, inhabited_predicate_type, ..*providers };
 }
 
-impl<'tcx> TyCtxt<'tcx> {
-    /// Checks whether a type is visibly uninhabited from a particular module.
-    ///
-    /// # Example
-    /// ```
-    /// #![feature(never_type)]
-    /// # fn main() {}
-    /// enum Void {}
-    /// mod a {
-    ///     pub mod b {
-    ///         pub struct SecretlyUninhabited {
-    ///             _priv: !,
-    ///         }
-    ///     }
-    /// }
-    ///
-    /// mod c {
-    ///     use super::Void;
-    ///     pub struct AlsoSecretlyUninhabited {
-    ///         _priv: Void,
-    ///     }
-    ///     mod d {
-    ///     }
-    /// }
-    ///
-    /// struct Foo {
-    ///     x: a::b::SecretlyUninhabited,
-    ///     y: c::AlsoSecretlyUninhabited,
-    /// }
-    /// ```
-    /// In this code, the type `Foo` will only be visibly uninhabited inside the
-    /// modules b, c and d. This effects pattern-matching on `Foo` or types that
-    /// contain `Foo`.
-    ///
-    /// # Example
-    /// ```ignore (illustrative)
-    /// let foo_result: Result<T, Foo> = ... ;
-    /// let Ok(t) = foo_result;
-    /// ```
-    /// This code should only compile in modules where the uninhabitedness of Foo is
-    /// visible.
-    pub fn is_ty_uninhabited_from(
-        self,
-        module: DefId,
-        ty: Ty<'tcx>,
-        param_env: ty::ParamEnv<'tcx>,
-    ) -> bool {
-        !ty.inhabited_predicate(self).apply(self, param_env, module)
-    }
-}
-
 /// Returns an `InhabitedPredicate` that is generic over type parameters and
 /// requires calling [`InhabitedPredicate::subst`]
 fn inhabited_predicate_adt(tcx: TyCtxt<'_>, def_id: DefId) -> InhabitedPredicate<'_> {
@@ -170,6 +120,64 @@ impl<'tcx> Ty<'tcx> {
             _ => InhabitedPredicate::True,
         }
     }
+
+    /// Checks whether a type is visibly uninhabited from a particular module.
+    ///
+    /// # Example
+    /// ```
+    /// #![feature(never_type)]
+    /// # fn main() {}
+    /// enum Void {}
+    /// mod a {
+    ///     pub mod b {
+    ///         pub struct SecretlyUninhabited {
+    ///             _priv: !,
+    ///         }
+    ///     }
+    /// }
+    ///
+    /// mod c {
+    ///     use super::Void;
+    ///     pub struct AlsoSecretlyUninhabited {
+    ///         _priv: Void,
+    ///     }
+    ///     mod d {
+    ///     }
+    /// }
+    ///
+    /// struct Foo {
+    ///     x: a::b::SecretlyUninhabited,
+    ///     y: c::AlsoSecretlyUninhabited,
+    /// }
+    /// ```
+    /// In this code, the type `Foo` will only be visibly uninhabited inside the
+    /// modules b, c and d. This effects pattern-matching on `Foo` or types that
+    /// contain `Foo`.
+    ///
+    /// # Example
+    /// ```ignore (illustrative)
+    /// let foo_result: Result<T, Foo> = ... ;
+    /// let Ok(t) = foo_result;
+    /// ```
+    /// This code should only compile in modules where the uninhabitedness of Foo is
+    /// visible.
+    pub fn is_inhabited_from(
+        self,
+        tcx: TyCtxt<'tcx>,
+        module: DefId,
+        param_env: ty::ParamEnv<'tcx>,
+    ) -> bool {
+        self.inhabited_predicate(tcx).apply(tcx, param_env, module)
+    }
+
+    /// Returns true if the type is uninhabited without regard to visibility
+    pub fn is_privately_uninhabited(
+        self,
+        tcx: TyCtxt<'tcx>,
+        param_env: ty::ParamEnv<'tcx>,
+    ) -> bool {
+        !self.inhabited_predicate(tcx).apply_ignore_module(tcx, param_env)
+    }
 }
 
 /// N.B. this query should only be called through `Ty::inhabited_predicate`
diff --git a/compiler/rustc_middle/src/ty/instance.rs b/compiler/rustc_middle/src/ty/instance.rs
index 6c1414f7b..586460986 100644
--- a/compiler/rustc_middle/src/ty/instance.rs
+++ b/compiler/rustc_middle/src/ty/instance.rs
@@ -276,28 +276,45 @@ impl<'tcx> InstanceDef<'tcx> {
     }
 }
 
-impl<'tcx> fmt::Display for Instance<'tcx> {
+fn fmt_instance(
+    f: &mut fmt::Formatter<'_>,
+    instance: &Instance<'_>,
+    type_length: rustc_session::Limit,
+) -> fmt::Result {
+    ty::tls::with(|tcx| {
+        let substs = tcx.lift(instance.substs).expect("could not lift for printing");
+
+        let s = FmtPrinter::new_with_limit(tcx, Namespace::ValueNS, type_length)
+            .print_def_path(instance.def_id(), substs)?
+            .into_buffer();
+        f.write_str(&s)
+    })?;
+
+    match instance.def {
+        InstanceDef::Item(_) => Ok(()),
+        InstanceDef::VTableShim(_) => write!(f, " - shim(vtable)"),
+        InstanceDef::ReifyShim(_) => write!(f, " - shim(reify)"),
+        InstanceDef::Intrinsic(_) => write!(f, " - intrinsic"),
+        InstanceDef::Virtual(_, num) => write!(f, " - virtual#{}", num),
+        InstanceDef::FnPtrShim(_, ty) => write!(f, " - shim({})", ty),
+        InstanceDef::ClosureOnceShim { .. } => write!(f, " - shim"),
+        InstanceDef::DropGlue(_, None) => write!(f, " - shim(None)"),
+        InstanceDef::DropGlue(_, Some(ty)) => write!(f, " - shim(Some({}))", ty),
+        InstanceDef::CloneShim(_, ty) => write!(f, " - shim({})", ty),
+    }
+}
+
+pub struct ShortInstance<'a, 'tcx>(pub &'a Instance<'tcx>, pub usize);
+
+impl<'a, 'tcx> fmt::Display for ShortInstance<'a, 'tcx> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        ty::tls::with(|tcx| {
-            let substs = tcx.lift(self.substs).expect("could not lift for printing");
-            let s = FmtPrinter::new(tcx, Namespace::ValueNS)
-                .print_def_path(self.def_id(), substs)?
-                .into_buffer();
-            f.write_str(&s)
-        })?;
+        fmt_instance(f, self.0, rustc_session::Limit(self.1))
+    }
+}
 
-        match self.def {
-            InstanceDef::Item(_) => Ok(()),
-            InstanceDef::VTableShim(_) => write!(f, " - shim(vtable)"),
-            InstanceDef::ReifyShim(_) => write!(f, " - shim(reify)"),
-            InstanceDef::Intrinsic(_) => write!(f, " - intrinsic"),
-            InstanceDef::Virtual(_, num) => write!(f, " - virtual#{}", num),
-            InstanceDef::FnPtrShim(_, ty) => write!(f, " - shim({})", ty),
-            InstanceDef::ClosureOnceShim { .. } => write!(f, " - shim"),
-            InstanceDef::DropGlue(_, None) => write!(f, " - shim(None)"),
-            InstanceDef::DropGlue(_, Some(ty)) => write!(f, " - shim(Some({}))", ty),
-            InstanceDef::CloneShim(_, ty) => write!(f, " - shim({})", ty),
-        }
+impl<'tcx> fmt::Display for Instance<'tcx> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        ty::tls::with(|tcx| fmt_instance(f, self, tcx.type_length_limit()))
     }
 }
 
@@ -511,12 +528,12 @@ impl<'tcx> Instance<'tcx> {
         Instance::resolve(tcx, ty::ParamEnv::reveal_all(), def_id, substs).unwrap().unwrap()
     }
 
+    #[instrument(level = "debug", skip(tcx), ret)]
     pub fn fn_once_adapter_instance(
         tcx: TyCtxt<'tcx>,
         closure_did: DefId,
         substs: ty::SubstsRef<'tcx>,
     ) -> Option<Instance<'tcx>> {
-        debug!("fn_once_adapter_shim({:?}, {:?})", closure_did, substs);
         let fn_once = tcx.require_lang_item(LangItem::FnOnce, None);
         let call_once = tcx
             .associated_items(fn_once)
@@ -534,9 +551,9 @@ impl<'tcx> Instance<'tcx> {
         let sig =
             tcx.try_normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), sig).ok()?;
         assert_eq!(sig.inputs().len(), 1);
-        let substs = tcx.mk_substs_trait(self_ty, &[sig.inputs()[0].into()]);
+        let substs = tcx.mk_substs_trait(self_ty, [sig.inputs()[0].into()]);
 
-        debug!("fn_once_adapter_shim: self_ty={:?} sig={:?}", self_ty, sig);
+        debug!(?self_ty, ?sig);
         Some(Instance { def, substs })
     }
 
diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs
index 3312f44c6..488fd5678 100644
--- a/compiler/rustc_middle/src/ty/layout.rs
+++ b/compiler/rustc_middle/src/ty/layout.rs
@@ -1,8 +1,6 @@
 use crate::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use crate::ty::normalize_erasing_regions::NormalizationError;
 use crate::ty::{self, ReprOptions, Ty, TyCtxt, TypeVisitable};
-use rustc_ast as ast;
-use rustc_attr as attr;
 use rustc_errors::{DiagnosticBuilder, Handler, IntoDiagnostic};
 use rustc_hir as hir;
 use rustc_hir::def_id::DefId;
@@ -20,7 +18,6 @@ use std::ops::Bound;
 
 pub trait IntegerExt {
     fn to_ty<'tcx>(&self, tcx: TyCtxt<'tcx>, signed: bool) -> Ty<'tcx>;
-    fn from_attr<C: HasDataLayout>(cx: &C, ity: attr::IntType) -> Integer;
     fn from_int_ty<C: HasDataLayout>(cx: &C, ity: ty::IntTy) -> Integer;
     fn from_uint_ty<C: HasDataLayout>(cx: &C, uty: ty::UintTy) -> Integer;
     fn repr_discr<'tcx>(
@@ -49,22 +46,6 @@ impl IntegerExt for Integer {
         }
     }
 
-    /// Gets the Integer type from an attr::IntType.
-    fn from_attr<C: HasDataLayout>(cx: &C, ity: attr::IntType) -> Integer {
-        let dl = cx.data_layout();
-
-        match ity {
-            attr::SignedInt(ast::IntTy::I8) | attr::UnsignedInt(ast::UintTy::U8) => I8,
-            attr::SignedInt(ast::IntTy::I16) | attr::UnsignedInt(ast::UintTy::U16) => I16,
-            attr::SignedInt(ast::IntTy::I32) | attr::UnsignedInt(ast::UintTy::U32) => I32,
-            attr::SignedInt(ast::IntTy::I64) | attr::UnsignedInt(ast::UintTy::U64) => I64,
-            attr::SignedInt(ast::IntTy::I128) | attr::UnsignedInt(ast::UintTy::U128) => I128,
-            attr::SignedInt(ast::IntTy::Isize) | attr::UnsignedInt(ast::UintTy::Usize) => {
-                dl.ptr_sized_integer()
-            }
-        }
-    }
-
     fn from_int_ty<C: HasDataLayout>(cx: &C, ity: ty::IntTy) -> Integer {
         match ity {
             ty::IntTy::I8 => I8,
@@ -189,8 +170,8 @@ pub enum LayoutError<'tcx> {
     NormalizationFailure(Ty<'tcx>, NormalizationError<'tcx>),
 }
 
-impl<'a> IntoDiagnostic<'a, !> for LayoutError<'a> {
-    fn into_diagnostic(self, handler: &'a Handler) -> DiagnosticBuilder<'a, !> {
+impl IntoDiagnostic<'_, !> for LayoutError<'_> {
+    fn into_diagnostic(self, handler: &Handler) -> DiagnosticBuilder<'_, !> {
         let mut diag = handler.struct_fatal("");
 
         match self {
@@ -237,6 +218,18 @@ pub struct LayoutCx<'tcx, C> {
     pub param_env: ty::ParamEnv<'tcx>,
 }
 
+impl<'tcx> LayoutCalculator for LayoutCx<'tcx, TyCtxt<'tcx>> {
+    type TargetDataLayoutRef = &'tcx TargetDataLayout;
+
+    fn delay_bug(&self, txt: &str) {
+        self.tcx.sess.delay_span_bug(DUMMY_SP, txt);
+    }
+
+    fn current_data_layout(&self) -> Self::TargetDataLayoutRef {
+        &self.tcx.data_layout
+    }
+}
+
 /// Type size "skeleton", i.e., the only information determining a type's size.
 /// While this is conservative, (aside from constant sizes, only pointers,
 /// newtypes thereof and null pointer optimized enums are allowed), it is
@@ -610,7 +603,7 @@ where
                 })
             }
 
-            Variants::Multiple { ref variants, .. } => variants[variant_index],
+            Variants::Multiple { ref variants, .. } => cx.tcx().intern_layout(variants[variant_index].clone()),
         };
 
         assert_eq!(*layout.variants(), Variants::Single { index: variant_index });
@@ -1126,8 +1119,8 @@ impl<'tcx> fmt::Display for FnAbiError<'tcx> {
     }
 }
 
-impl<'tcx> IntoDiagnostic<'tcx, !> for FnAbiError<'tcx> {
-    fn into_diagnostic(self, handler: &'tcx Handler) -> DiagnosticBuilder<'tcx, !> {
+impl IntoDiagnostic<'_, !> for FnAbiError<'_> {
+    fn into_diagnostic(self, handler: &Handler) -> DiagnosticBuilder<'_, !> {
         handler.struct_fatal(self.to_string())
     }
 }
diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs
index a42d05706..3f99efd25 100644
--- a/compiler/rustc_middle/src/ty/mod.rs
+++ b/compiler/rustc_middle/src/ty/mod.rs
@@ -9,6 +9,8 @@
 //!
 //! ["The `ty` module: representing types"]: https://rustc-dev-guide.rust-lang.org/ty.html
 
+#![allow(rustc::usage_of_ty_tykind)]
+
 pub use self::fold::{FallibleTypeFolder, TypeFoldable, TypeFolder, TypeSuperFoldable};
 pub use self::visit::{TypeSuperVisitable, TypeVisitable, TypeVisitor};
 pub use self::AssocItemContainer::*;
@@ -26,13 +28,12 @@ use crate::ty::util::Discr;
 pub use adt::*;
 pub use assoc::*;
 pub use generics::*;
-use hir::OpaqueTyOrigin;
 use rustc_ast as ast;
 use rustc_ast::node_id::NodeMap;
 use rustc_attr as attr;
 use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexMap, FxIndexSet};
-use rustc_data_structures::intern::{Interned, WithStableHash};
+use rustc_data_structures::intern::Interned;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::tagged_ptr::CopyTaggedPtr;
 use rustc_hir as hir;
@@ -48,7 +49,9 @@ use rustc_session::cstore::CrateStoreDyn;
 use rustc_span::hygiene::MacroKind;
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::{ExpnId, Span};
-use rustc_target::abi::{Align, VariantIdx};
+use rustc_target::abi::{Align, Integer, IntegerType, VariantIdx};
+pub use rustc_target::abi::{ReprFlags, ReprOptions};
+use rustc_type_ir::WithCachedTypeInfo;
 pub use subst::*;
 pub use vtable::*;
 
@@ -76,15 +79,15 @@ pub use self::closure::{
     CAPTURE_STRUCT_LOCAL,
 };
 pub use self::consts::{
-    Const, ConstInt, ConstKind, ConstS, InferConst, ScalarInt, UnevaluatedConst, ValTree,
+    Const, ConstInt, ConstKind, ConstS, Expr, InferConst, ScalarInt, UnevaluatedConst, ValTree,
 };
 pub use self::context::{
     tls, CanonicalUserType, CanonicalUserTypeAnnotation, CanonicalUserTypeAnnotations,
-    CtxtInterners, DeducedParamAttrs, DelaySpanBugEmitted, FreeRegionInfo, GeneratorDiagnosticData,
-    GeneratorInteriorTypeCause, GlobalCtxt, Lift, OnDiskCache, TyCtxt, TypeckResults, UserType,
-    UserTypeAnnotationIndex,
+    CtxtInterners, DeducedParamAttrs, FreeRegionInfo, GeneratorDiagnosticData,
+    GeneratorInteriorTypeCause, GlobalCtxt, Lift, OnDiskCache, TyCtxt, TyCtxtFeed, TypeckResults,
+    UserType, UserTypeAnnotationIndex,
 };
-pub use self::instance::{Instance, InstanceDef};
+pub use self::instance::{Instance, InstanceDef, ShortInstance};
 pub use self::list::List;
 pub use self::parameterized::ParameterizedOverTcx;
 pub use self::rvalue_scopes::RvalueScopes;
@@ -94,9 +97,10 @@ pub use self::sty::{
     BoundVariableKind, CanonicalPolyFnSig, ClosureSubsts, ClosureSubstsParts, ConstVid,
     EarlyBoundRegion, ExistentialPredicate, ExistentialProjection, ExistentialTraitRef, FnSig,
     FreeRegion, GenSig, GeneratorSubsts, GeneratorSubstsParts, InlineConstSubsts,
-    InlineConstSubstsParts, ParamConst, ParamTy, PolyExistentialProjection,
-    PolyExistentialTraitRef, PolyFnSig, PolyGenSig, PolyTraitRef, ProjectionTy, Region, RegionKind,
-    RegionVid, TraitRef, TyKind, TypeAndMut, UpvarSubsts, VarianceDiagInfo,
+    InlineConstSubstsParts, ParamConst, ParamTy, PolyExistentialPredicate,
+    PolyExistentialProjection, PolyExistentialTraitRef, PolyFnSig, PolyGenSig, PolyTraitRef,
+    ProjectionTy, Region, RegionKind, RegionVid, TraitRef, TyKind, TypeAndMut, UpvarSubsts,
+    VarianceDiagInfo,
 };
 pub use self::trait_def::TraitDef;
 
@@ -206,6 +210,8 @@ pub struct ResolverAstLowering {
     /// A small map keeping true kinds of built-in macros that appear to be fn-like on
     /// the surface (`macro` items in libcore), but are actually attributes or derives.
     pub builtin_macro_kinds: FxHashMap<LocalDefId, MacroKind>,
+    /// List functions and methods for which lifetime elision was successful.
+    pub lifetime_elision_allowed: FxHashSet<ast::NodeId>,
 }
 
 #[derive(Clone, Copy, Debug)]
@@ -410,7 +416,7 @@ impl Visibility<DefId> {
         self.map_id(|id| id.expect_local())
     }
 
-    // Returns `true` if this item is visible anywhere in the local crate.
+    /// Returns `true` if this item is visible anywhere in the local crate.
     pub fn is_visible_locally(self) -> bool {
         match self {
             Visibility::Public => true,
@@ -441,118 +447,44 @@ pub struct CReaderCacheKey {
     pub pos: usize,
 }
 
-/// Represents a type.
-///
-/// IMPORTANT:
-/// - This is a very "dumb" struct (with no derives and no `impls`).
-/// - Values of this type are always interned and thus unique, and are stored
-///   as an `Interned<TyS>`.
-/// - `Ty` (which contains a reference to a `Interned<TyS>`) or `Interned<TyS>`
-///   should be used everywhere instead of `TyS`. In particular, `Ty` has most
-///   of the relevant methods.
-#[derive(PartialEq, Eq, PartialOrd, Ord)]
-#[allow(rustc::usage_of_ty_tykind)]
-pub(crate) struct TyS<'tcx> {
-    /// This field shouldn't be used directly and may be removed in the future.
-    /// Use `Ty::kind()` instead.
-    kind: TyKind<'tcx>,
-
-    /// This field provides fast access to information that is also contained
-    /// in `kind`.
-    ///
-    /// This field shouldn't be used directly and may be removed in the future.
-    /// Use `Ty::flags()` instead.
-    flags: TypeFlags,
-
-    /// This field provides fast access to information that is also contained
-    /// in `kind`.
-    ///
-    /// This is a kind of confusing thing: it stores the smallest
-    /// binder such that
-    ///
-    /// (a) the binder itself captures nothing but
-    /// (b) all the late-bound things within the type are captured
-    ///     by some sub-binder.
-    ///
-    /// So, for a type without any late-bound things, like `u32`, this
-    /// will be *innermost*, because that is the innermost binder that
-    /// captures nothing. But for a type `&'D u32`, where `'D` is a
-    /// late-bound region with De Bruijn index `D`, this would be `D + 1`
-    /// -- the binder itself does not capture `D`, but `D` is captured
-    /// by an inner binder.
-    ///
-    /// We call this concept an "exclusive" binder `D` because all
-    /// De Bruijn indices within the type are contained within `0..D`
-    /// (exclusive).
-    outer_exclusive_binder: ty::DebruijnIndex,
-}
-
-/// Use this rather than `TyS`, whenever possible.
+/// Use this rather than `TyKind`, whenever possible.
 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, HashStable)]
 #[rustc_diagnostic_item = "Ty"]
 #[rustc_pass_by_value]
-pub struct Ty<'tcx>(Interned<'tcx, WithStableHash<TyS<'tcx>>>);
+pub struct Ty<'tcx>(Interned<'tcx, WithCachedTypeInfo<TyKind<'tcx>>>);
 
 impl<'tcx> TyCtxt<'tcx> {
     /// A "bool" type used in rustc_mir_transform unit tests when we
     /// have not spun up a TyCtxt.
-    pub const BOOL_TY_FOR_UNIT_TESTING: Ty<'tcx> = Ty(Interned::new_unchecked(&WithStableHash {
-        internee: TyS {
-            kind: ty::Bool,
+    pub const BOOL_TY_FOR_UNIT_TESTING: Ty<'tcx> =
+        Ty(Interned::new_unchecked(&WithCachedTypeInfo {
+            internee: ty::Bool,
+            stable_hash: Fingerprint::ZERO,
             flags: TypeFlags::empty(),
             outer_exclusive_binder: DebruijnIndex::from_usize(0),
-        },
-        stable_hash: Fingerprint::ZERO,
-    }));
-}
-
-impl<'a, 'tcx> HashStable<StableHashingContext<'a>> for TyS<'tcx> {
-    #[inline]
-    fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
-        let TyS {
-            kind,
-
-            // The other fields just provide fast access to information that is
-            // also contained in `kind`, so no need to hash them.
-            flags: _,
-
-            outer_exclusive_binder: _,
-        } = self;
-
-        kind.hash_stable(hcx, hasher)
-    }
+        }));
 }
 
 impl ty::EarlyBoundRegion {
     /// Does this early bound region have a name? Early bound regions normally
     /// always have names except when using anonymous lifetimes (`'_`).
     pub fn has_name(&self) -> bool {
-        self.name != kw::UnderscoreLifetime
+        self.name != kw::UnderscoreLifetime && self.name != kw::Empty
     }
 }
 
-/// Represents a predicate.
-///
-/// See comments on `TyS`, which apply here too (albeit for
-/// `PredicateS`/`Predicate` rather than `TyS`/`Ty`).
-#[derive(Debug)]
-pub(crate) struct PredicateS<'tcx> {
-    kind: Binder<'tcx, PredicateKind<'tcx>>,
-    flags: TypeFlags,
-    /// See the comment for the corresponding field of [TyS].
-    outer_exclusive_binder: ty::DebruijnIndex,
-}
-
-/// Use this rather than `PredicateS`, whenever possible.
-#[derive(Clone, Copy, PartialEq, Eq, Hash)]
+/// Use this rather than `PredicateKind`, whenever possible.
+#[derive(Clone, Copy, PartialEq, Eq, Hash, HashStable)]
 #[rustc_pass_by_value]
-pub struct Predicate<'tcx>(Interned<'tcx, PredicateS<'tcx>>);
+pub struct Predicate<'tcx>(
+    Interned<'tcx, WithCachedTypeInfo<ty::Binder<'tcx, PredicateKind<'tcx>>>>,
+);
 
 impl<'tcx> Predicate<'tcx> {
     /// Gets the inner `Binder<'tcx, PredicateKind<'tcx>>`.
     #[inline]
     pub fn kind(self) -> Binder<'tcx, PredicateKind<'tcx>> {
-        self.0.kind
+        self.0.internee
     }
 
     #[inline(always)]
@@ -572,13 +504,15 @@ impl<'tcx> Predicate<'tcx> {
         let kind = self
             .kind()
             .map_bound(|kind| match kind {
-                PredicateKind::Trait(TraitPredicate { trait_ref, constness, polarity }) => {
-                    Some(PredicateKind::Trait(TraitPredicate {
-                        trait_ref,
-                        constness,
-                        polarity: polarity.flip()?,
-                    }))
-                }
+                PredicateKind::Clause(Clause::Trait(TraitPredicate {
+                    trait_ref,
+                    constness,
+                    polarity,
+                })) => Some(PredicateKind::Clause(Clause::Trait(TraitPredicate {
+                    trait_ref,
+                    constness,
+                    polarity: polarity.flip()?,
+                }))),
 
                 _ => None,
             })
@@ -588,14 +522,14 @@ impl<'tcx> Predicate<'tcx> {
     }
 
     pub fn without_const(mut self, tcx: TyCtxt<'tcx>) -> Self {
-        if let PredicateKind::Trait(TraitPredicate { trait_ref, constness, polarity }) = self.kind().skip_binder()
+        if let PredicateKind::Clause(Clause::Trait(TraitPredicate { trait_ref, constness, polarity })) = self.kind().skip_binder()
             && constness != BoundConstness::NotConst
         {
-            self = tcx.mk_predicate(self.kind().rebind(PredicateKind::Trait(TraitPredicate {
+            self = tcx.mk_predicate(self.kind().rebind(PredicateKind::Clause(Clause::Trait(TraitPredicate {
                 trait_ref,
                 constness: BoundConstness::NotConst,
                 polarity,
-            })));
+            }))));
         }
         self
     }
@@ -609,36 +543,22 @@ impl<'tcx> Predicate<'tcx> {
     pub fn allow_normalization(self) -> bool {
         match self.kind().skip_binder() {
             PredicateKind::WellFormed(_) => false,
-            PredicateKind::Trait(_)
-            | PredicateKind::RegionOutlives(_)
-            | PredicateKind::TypeOutlives(_)
-            | PredicateKind::Projection(_)
+            PredicateKind::Clause(Clause::Trait(_))
+            | PredicateKind::Clause(Clause::RegionOutlives(_))
+            | PredicateKind::Clause(Clause::TypeOutlives(_))
+            | PredicateKind::Clause(Clause::Projection(_))
             | PredicateKind::ObjectSafe(_)
             | PredicateKind::ClosureKind(_, _, _)
             | PredicateKind::Subtype(_)
             | PredicateKind::Coerce(_)
             | PredicateKind::ConstEvaluatable(_)
             | PredicateKind::ConstEquate(_, _)
+            | PredicateKind::Ambiguous
             | PredicateKind::TypeWellFormedFromEnv(_) => true,
         }
     }
 }
 
-impl<'a, 'tcx> HashStable<StableHashingContext<'a>> for Predicate<'tcx> {
-    fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
-        let PredicateS {
-            ref kind,
-
-            // The other fields just provide fast access to information that is
-            // also contained in `kind`, so no need to hash them.
-            flags: _,
-            outer_exclusive_binder: _,
-        } = self.0.0;
-
-        kind.hash_stable(hcx, hasher);
-    }
-}
-
 impl rustc_errors::IntoDiagnosticArg for Predicate<'_> {
     fn into_diagnostic_arg(self) -> rustc_errors::DiagnosticArgValue<'static> {
         rustc_errors::DiagnosticArgValue::Str(std::borrow::Cow::Owned(self.to_string()))
@@ -647,7 +567,9 @@ impl rustc_errors::IntoDiagnosticArg for Predicate<'_> {
 
 #[derive(Clone, Copy, PartialEq, Eq, Hash, TyEncodable, TyDecodable)]
 #[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
-pub enum PredicateKind<'tcx> {
+/// A clause is something that can appear in where bounds or be inferred
+/// by implied bounds.
+pub enum Clause<'tcx> {
     /// Corresponds to `where Foo: Bar<A, B, C>`. `Foo` here would be
     /// the `Self` type of the trait reference and `A`, `B`, and `C`
     /// would be the type parameters.
@@ -662,6 +584,13 @@ pub enum PredicateKind<'tcx> {
     /// `where <T as TraitRef>::Name == X`, approximately.
     /// See the `ProjectionPredicate` struct for details.
     Projection(ProjectionPredicate<'tcx>),
+}
+
+#[derive(Clone, Copy, PartialEq, Eq, Hash, TyEncodable, TyDecodable)]
+#[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
+pub enum PredicateKind<'tcx> {
+    /// Prove a clause
+    Clause(Clause<'tcx>),
 
     /// No syntax: `T` well-formed.
     WellFormed(GenericArg<'tcx>),
@@ -701,6 +630,10 @@ pub enum PredicateKind<'tcx> {
     ///
     /// Only used for Chalk.
     TypeWellFormedFromEnv(Ty<'tcx>),
+
+    /// A marker predicate that is always ambiguous.
+    /// Used for coherence to mark opaque types as possibly equal to each other but ambiguous.
+    Ambiguous,
 }
 
 /// The crate outlives map is computed during typeck and contains the
@@ -714,7 +647,7 @@ pub struct CratePredicatesMap<'tcx> {
     /// For each struct with outlive bounds, maps to a vector of the
     /// predicate of its outlive bounds. If an item has no outlives
     /// bounds, it will have no entry.
-    pub predicates: FxHashMap<DefId, &'tcx [(Predicate<'tcx>, Span)]>,
+    pub predicates: FxHashMap<DefId, &'tcx [(Clause<'tcx>, Span)]>,
 }
 
 impl<'tcx> Predicate<'tcx> {
@@ -851,6 +784,10 @@ impl<'tcx> TraitPredicate<'tcx> {
         }
     }
 
+    pub fn with_self_type(self, tcx: TyCtxt<'tcx>, self_ty: Ty<'tcx>) -> Self {
+        Self { trait_ref: self.trait_ref.with_self_type(tcx, self_ty), ..self }
+    }
+
     pub fn def_id(self) -> DefId {
         self.trait_ref.def_id
     }
@@ -902,9 +839,10 @@ impl<'tcx> PolyTraitPredicate<'tcx> {
     }
 }
 
+/// `A: B`
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, TyEncodable, TyDecodable)]
 #[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
-pub struct OutlivesPredicate<A, B>(pub A, pub B); // `A: B`
+pub struct OutlivesPredicate<A, B>(pub A, pub B);
 pub type RegionOutlivesPredicate<'tcx> = OutlivesPredicate<ty::Region<'tcx>, ty::Region<'tcx>>;
 pub type TypeOutlivesPredicate<'tcx> = OutlivesPredicate<Ty<'tcx>, ty::Region<'tcx>>;
 pub type PolyRegionOutlivesPredicate<'tcx> = ty::Binder<'tcx, RegionOutlivesPredicate<'tcx>>;
@@ -1003,7 +941,7 @@ impl<'tcx> Term<'tcx> {
         unsafe {
             match ptr & TAG_MASK {
                 TYPE_TAG => TermKind::Ty(Ty(Interned::new_unchecked(
-                    &*((ptr & !TAG_MASK) as *const WithStableHash<ty::TyS<'tcx>>),
+                    &*((ptr & !TAG_MASK) as *const WithCachedTypeInfo<ty::TyKind<'tcx>>),
                 ))),
                 CONST_TAG => TermKind::Const(ty::Const(Interned::new_unchecked(
                     &*((ptr & !TAG_MASK) as *const ty::ConstS<'tcx>),
@@ -1047,7 +985,7 @@ impl<'tcx> TermKind<'tcx> {
             TermKind::Ty(ty) => {
                 // Ensure we can use the tag bits.
                 assert_eq!(mem::align_of_val(&*ty.0.0) & TAG_MASK, 0);
-                (TYPE_TAG, ty.0.0 as *const WithStableHash<ty::TyS<'tcx>> as usize)
+                (TYPE_TAG, ty.0.0 as *const WithCachedTypeInfo<ty::TyKind<'tcx>> as usize)
             }
             TermKind::Const(ct) => {
                 // Ensure we can use the tag bits.
@@ -1125,12 +1063,12 @@ impl<'tcx> ToPolyTraitRef<'tcx> for PolyTraitPredicate<'tcx> {
     }
 }
 
-pub trait ToPredicate<'tcx> {
-    fn to_predicate(self, tcx: TyCtxt<'tcx>) -> Predicate<'tcx>;
+pub trait ToPredicate<'tcx, P = Predicate<'tcx>> {
+    fn to_predicate(self, tcx: TyCtxt<'tcx>) -> P;
 }
 
-impl<'tcx> ToPredicate<'tcx> for Predicate<'tcx> {
-    fn to_predicate(self, _tcx: TyCtxt<'tcx>) -> Predicate<'tcx> {
+impl<'tcx, T> ToPredicate<'tcx, T> for T {
+    fn to_predicate(self, _tcx: TyCtxt<'tcx>) -> T {
         self
     }
 }
@@ -1142,27 +1080,53 @@ impl<'tcx> ToPredicate<'tcx> for Binder<'tcx, PredicateKind<'tcx>> {
     }
 }
 
+impl<'tcx> ToPredicate<'tcx> for Clause<'tcx> {
+    #[inline(always)]
+    fn to_predicate(self, tcx: TyCtxt<'tcx>) -> Predicate<'tcx> {
+        tcx.mk_predicate(ty::Binder::dummy(ty::PredicateKind::Clause(self)))
+    }
+}
+
+impl<'tcx> ToPredicate<'tcx> for Binder<'tcx, TraitRef<'tcx>> {
+    #[inline(always)]
+    fn to_predicate(self, tcx: TyCtxt<'tcx>) -> Predicate<'tcx> {
+        let pred: PolyTraitPredicate<'tcx> = self.to_predicate(tcx);
+        pred.to_predicate(tcx)
+    }
+}
+
+impl<'tcx> ToPredicate<'tcx, PolyTraitPredicate<'tcx>> for Binder<'tcx, TraitRef<'tcx>> {
+    #[inline(always)]
+    fn to_predicate(self, _: TyCtxt<'tcx>) -> PolyTraitPredicate<'tcx> {
+        self.map_bound(|trait_ref| TraitPredicate {
+            trait_ref,
+            constness: ty::BoundConstness::NotConst,
+            polarity: ty::ImplPolarity::Positive,
+        })
+    }
+}
+
 impl<'tcx> ToPredicate<'tcx> for PolyTraitPredicate<'tcx> {
     fn to_predicate(self, tcx: TyCtxt<'tcx>) -> Predicate<'tcx> {
-        self.map_bound(PredicateKind::Trait).to_predicate(tcx)
+        self.map_bound(|p| PredicateKind::Clause(Clause::Trait(p))).to_predicate(tcx)
     }
 }
 
 impl<'tcx> ToPredicate<'tcx> for PolyRegionOutlivesPredicate<'tcx> {
     fn to_predicate(self, tcx: TyCtxt<'tcx>) -> Predicate<'tcx> {
-        self.map_bound(PredicateKind::RegionOutlives).to_predicate(tcx)
+        self.map_bound(|p| PredicateKind::Clause(Clause::RegionOutlives(p))).to_predicate(tcx)
     }
 }
 
 impl<'tcx> ToPredicate<'tcx> for PolyTypeOutlivesPredicate<'tcx> {
     fn to_predicate(self, tcx: TyCtxt<'tcx>) -> Predicate<'tcx> {
-        self.map_bound(PredicateKind::TypeOutlives).to_predicate(tcx)
+        self.map_bound(|p| PredicateKind::Clause(Clause::TypeOutlives(p))).to_predicate(tcx)
     }
 }
 
 impl<'tcx> ToPredicate<'tcx> for PolyProjectionPredicate<'tcx> {
     fn to_predicate(self, tcx: TyCtxt<'tcx>) -> Predicate<'tcx> {
-        self.map_bound(PredicateKind::Projection).to_predicate(tcx)
+        self.map_bound(|p| PredicateKind::Clause(Clause::Projection(p))).to_predicate(tcx)
     }
 }
 
@@ -1170,17 +1134,18 @@ impl<'tcx> Predicate<'tcx> {
     pub fn to_opt_poly_trait_pred(self) -> Option<PolyTraitPredicate<'tcx>> {
         let predicate = self.kind();
         match predicate.skip_binder() {
-            PredicateKind::Trait(t) => Some(predicate.rebind(t)),
-            PredicateKind::Projection(..)
+            PredicateKind::Clause(Clause::Trait(t)) => Some(predicate.rebind(t)),
+            PredicateKind::Clause(Clause::Projection(..))
             | PredicateKind::Subtype(..)
             | PredicateKind::Coerce(..)
-            | PredicateKind::RegionOutlives(..)
+            | PredicateKind::Clause(Clause::RegionOutlives(..))
             | PredicateKind::WellFormed(..)
             | PredicateKind::ObjectSafe(..)
             | PredicateKind::ClosureKind(..)
-            | PredicateKind::TypeOutlives(..)
+            | PredicateKind::Clause(Clause::TypeOutlives(..))
             | PredicateKind::ConstEvaluatable(..)
             | PredicateKind::ConstEquate(..)
+            | PredicateKind::Ambiguous
             | PredicateKind::TypeWellFormedFromEnv(..) => None,
         }
     }
@@ -1188,17 +1153,18 @@ impl<'tcx> Predicate<'tcx> {
     pub fn to_opt_poly_projection_pred(self) -> Option<PolyProjectionPredicate<'tcx>> {
         let predicate = self.kind();
         match predicate.skip_binder() {
-            PredicateKind::Projection(t) => Some(predicate.rebind(t)),
-            PredicateKind::Trait(..)
+            PredicateKind::Clause(Clause::Projection(t)) => Some(predicate.rebind(t)),
+            PredicateKind::Clause(Clause::Trait(..))
             | PredicateKind::Subtype(..)
             | PredicateKind::Coerce(..)
-            | PredicateKind::RegionOutlives(..)
+            | PredicateKind::Clause(Clause::RegionOutlives(..))
             | PredicateKind::WellFormed(..)
             | PredicateKind::ObjectSafe(..)
             | PredicateKind::ClosureKind(..)
-            | PredicateKind::TypeOutlives(..)
+            | PredicateKind::Clause(Clause::TypeOutlives(..))
             | PredicateKind::ConstEvaluatable(..)
             | PredicateKind::ConstEquate(..)
+            | PredicateKind::Ambiguous
             | PredicateKind::TypeWellFormedFromEnv(..) => None,
         }
     }
@@ -1206,17 +1172,18 @@ impl<'tcx> Predicate<'tcx> {
     pub fn to_opt_type_outlives(self) -> Option<PolyTypeOutlivesPredicate<'tcx>> {
         let predicate = self.kind();
         match predicate.skip_binder() {
-            PredicateKind::TypeOutlives(data) => Some(predicate.rebind(data)),
-            PredicateKind::Trait(..)
-            | PredicateKind::Projection(..)
+            PredicateKind::Clause(Clause::TypeOutlives(data)) => Some(predicate.rebind(data)),
+            PredicateKind::Clause(Clause::Trait(..))
+            | PredicateKind::Clause(Clause::Projection(..))
             | PredicateKind::Subtype(..)
             | PredicateKind::Coerce(..)
-            | PredicateKind::RegionOutlives(..)
+            | PredicateKind::Clause(Clause::RegionOutlives(..))
             | PredicateKind::WellFormed(..)
             | PredicateKind::ObjectSafe(..)
             | PredicateKind::ClosureKind(..)
             | PredicateKind::ConstEvaluatable(..)
             | PredicateKind::ConstEquate(..)
+            | PredicateKind::Ambiguous
             | PredicateKind::TypeWellFormedFromEnv(..) => None,
         }
     }
@@ -1257,7 +1224,7 @@ impl<'tcx> InstantiatedPredicates<'tcx> {
     }
 }
 
-#[derive(Copy, Clone, Debug, PartialEq, Eq, HashStable, TyEncodable, TyDecodable, Lift)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, HashStable, TyEncodable, TyDecodable, Lift)]
 #[derive(TypeFoldable, TypeVisitable)]
 pub struct OpaqueTypeKey<'tcx> {
     pub def_id: LocalDefId,
@@ -1319,7 +1286,6 @@ impl<'tcx> OpaqueHiddenType<'tcx> {
         tcx: TyCtxt<'tcx>,
         // typeck errors have subpar spans for opaque types, so delay error reporting until borrowck.
         ignore_errors: bool,
-        origin: OpaqueTyOrigin,
     ) -> Self {
         let OpaqueTypeKey { def_id, substs } = opaque_type_key;
 
@@ -1332,78 +1298,10 @@ impl<'tcx> OpaqueHiddenType<'tcx> {
         let id_substs = InternalSubsts::identity_for_item(tcx, def_id.to_def_id());
         debug!(?id_substs);
 
-        let map = substs.iter().zip(id_substs);
-
-        let map: FxHashMap<GenericArg<'tcx>, GenericArg<'tcx>> = match origin {
-            // HACK: The HIR lowering for async fn does not generate
-            // any `+ Captures<'x>` bounds for the `impl Future<...>`, so all async fns with lifetimes
-            // would now fail to compile. We should probably just make hir lowering fill this in properly.
-            OpaqueTyOrigin::AsyncFn(_) => map.collect(),
-            OpaqueTyOrigin::FnReturn(_) | OpaqueTyOrigin::TyAlias => {
-                // Opaque types may only use regions that are bound. So for
-                // ```rust
-                // type Foo<'a, 'b, 'c> = impl Trait<'a> + 'b;
-                // ```
-                // we may not use `'c` in the hidden type.
-                struct OpaqueTypeLifetimeCollector<'tcx> {
-                    lifetimes: FxHashSet<ty::Region<'tcx>>,
-                }
-
-                impl<'tcx> ty::TypeVisitor<'tcx> for OpaqueTypeLifetimeCollector<'tcx> {
-                    fn visit_region(&mut self, r: ty::Region<'tcx>) -> ControlFlow<Self::BreakTy> {
-                        self.lifetimes.insert(r);
-                        r.super_visit_with(self)
-                    }
-                }
-
-                let mut collector = OpaqueTypeLifetimeCollector { lifetimes: Default::default() };
-
-                for pred in tcx.bound_explicit_item_bounds(def_id.to_def_id()).transpose_iter() {
-                    let pred = pred.map_bound(|(pred, _)| *pred).subst(tcx, id_substs);
-
-                    trace!(pred=?pred.kind());
-
-                    // We only ignore opaque type substs if the opaque type is the outermost type.
-                    // The opaque type may be nested within itself via recursion in e.g.
-                    // type Foo<'a> = impl PartialEq<Foo<'a>>;
-                    // which thus mentions `'a` and should thus accept hidden types that borrow 'a
-                    // instead of requiring an additional `+ 'a`.
-                    match pred.kind().skip_binder() {
-                        ty::PredicateKind::Trait(TraitPredicate {
-                            trait_ref: ty::TraitRef { def_id: _, substs },
-                            constness: _,
-                            polarity: _,
-                        }) => {
-                            trace!(?substs);
-                            for subst in &substs[1..] {
-                                subst.visit_with(&mut collector);
-                            }
-                        }
-                        ty::PredicateKind::Projection(ty::ProjectionPredicate {
-                            projection_ty: ty::ProjectionTy { substs, item_def_id: _ },
-                            term,
-                        }) => {
-                            for subst in &substs[1..] {
-                                subst.visit_with(&mut collector);
-                            }
-                            term.visit_with(&mut collector);
-                        }
-                        _ => {
-                            pred.visit_with(&mut collector);
-                        }
-                    }
-                }
-                let lifetimes = collector.lifetimes;
-                trace!(?lifetimes);
-                map.filter(|(_, v)| {
-                    let ty::GenericArgKind::Lifetime(lt) = v.unpack() else {
-                        return true;
-                    };
-                    lifetimes.contains(&lt)
-                })
-                .collect()
-            }
-        };
+        // This zip may have several times the same lifetime in `substs` paired with a different
+        // lifetime from `id_substs`.  Simply `collect`ing the iterator is the correct behaviour:
+        // it will pick the last one, which is the one we introduced in the impl-trait desugaring.
+        let map = substs.iter().zip(id_substs).collect();
         debug!("map = {:#?}", map);
 
         // Convert the type from the function into a type valid outside
@@ -1748,7 +1646,7 @@ impl<'tcx> ParamEnv<'tcx> {
         }
 
         ParamEnv::new(
-            tcx.normalize_opaque_types(self.caller_bounds()),
+            tcx.reveal_opaque_types_in_bounds(self.caller_bounds()),
             Reveal::All,
             self.constness(),
         )
@@ -1852,15 +1750,13 @@ pub struct VariantDef {
     pub def_id: DefId,
     /// `DefId` that identifies the variant's constructor.
     /// If this variant is a struct variant, then this is `None`.
-    pub ctor_def_id: Option<DefId>,
+    pub ctor: Option<(CtorKind, DefId)>,
     /// Variant or struct name.
     pub name: Symbol,
     /// Discriminant of this variant.
     pub discr: VariantDiscr,
     /// Fields of this variant.
     pub fields: Vec<FieldDef>,
-    /// Type of constructor of variant.
-    pub ctor_kind: CtorKind,
     /// Flags of the variant (e.g. is field list non-exhaustive)?
     flags: VariantFlags,
 }
@@ -1885,19 +1781,18 @@ impl VariantDef {
     pub fn new(
         name: Symbol,
         variant_did: Option<DefId>,
-        ctor_def_id: Option<DefId>,
+        ctor: Option<(CtorKind, DefId)>,
         discr: VariantDiscr,
         fields: Vec<FieldDef>,
-        ctor_kind: CtorKind,
         adt_kind: AdtKind,
         parent_did: DefId,
         recovered: bool,
         is_field_list_non_exhaustive: bool,
     ) -> Self {
         debug!(
-            "VariantDef::new(name = {:?}, variant_did = {:?}, ctor_def_id = {:?}, discr = {:?},
-             fields = {:?}, ctor_kind = {:?}, adt_kind = {:?}, parent_did = {:?})",
-            name, variant_did, ctor_def_id, discr, fields, ctor_kind, adt_kind, parent_did,
+            "VariantDef::new(name = {:?}, variant_did = {:?}, ctor = {:?}, discr = {:?},
+             fields = {:?}, adt_kind = {:?}, parent_did = {:?})",
+            name, variant_did, ctor, discr, fields, adt_kind, parent_did,
         );
 
         let mut flags = VariantFlags::NO_VARIANT_FLAGS;
@@ -1909,15 +1804,7 @@ impl VariantDef {
             flags |= VariantFlags::IS_RECOVERED;
         }
 
-        VariantDef {
-            def_id: variant_did.unwrap_or(parent_did),
-            ctor_def_id,
-            name,
-            discr,
-            fields,
-            ctor_kind,
-            flags,
-        }
+        VariantDef { def_id: variant_did.unwrap_or(parent_did), ctor, name, discr, fields, flags }
     }
 
     /// Is this field list non-exhaustive?
@@ -1936,6 +1823,16 @@ impl VariantDef {
     pub fn ident(&self, tcx: TyCtxt<'_>) -> Ident {
         Ident::new(self.name, tcx.def_ident_span(self.def_id).unwrap())
     }
+
+    #[inline]
+    pub fn ctor_kind(&self) -> Option<CtorKind> {
+        self.ctor.map(|(kind, _)| kind)
+    }
+
+    #[inline]
+    pub fn ctor_def_id(&self) -> Option<DefId> {
+        self.ctor.map(|(_, def_id)| def_id)
+    }
 }
 
 impl PartialEq for VariantDef {
@@ -1948,26 +1845,8 @@ impl PartialEq for VariantDef {
         // definition of `VariantDef` changes, a compile-error will be produced,
         // reminding us to revisit this assumption.
 
-        let Self {
-            def_id: lhs_def_id,
-            ctor_def_id: _,
-            name: _,
-            discr: _,
-            fields: _,
-            ctor_kind: _,
-            flags: _,
-        } = &self;
-
-        let Self {
-            def_id: rhs_def_id,
-            ctor_def_id: _,
-            name: _,
-            discr: _,
-            fields: _,
-            ctor_kind: _,
-            flags: _,
-        } = other;
-
+        let Self { def_id: lhs_def_id, ctor: _, name: _, discr: _, fields: _, flags: _ } = &self;
+        let Self { def_id: rhs_def_id, ctor: _, name: _, discr: _, fields: _, flags: _ } = other;
         lhs_def_id == rhs_def_id
     }
 }
@@ -1984,9 +1863,7 @@ impl Hash for VariantDef {
         // of `VariantDef` changes, a compile-error will be produced, reminding
         // us to revisit this assumption.
 
-        let Self { def_id, ctor_def_id: _, name: _, discr: _, fields: _, ctor_kind: _, flags: _ } =
-            &self;
-
+        let Self { def_id, ctor: _, name: _, discr: _, fields: _, flags: _ } = &self;
         def_id.hash(s)
     }
 }
@@ -2047,163 +1924,6 @@ impl Hash for FieldDef {
     }
 }
 
-bitflags! {
-    #[derive(TyEncodable, TyDecodable, Default, HashStable)]
-    pub struct ReprFlags: u8 {
-        const IS_C               = 1 << 0;
-        const IS_SIMD            = 1 << 1;
-        const IS_TRANSPARENT     = 1 << 2;
-        // Internal only for now. If true, don't reorder fields.
-        const IS_LINEAR          = 1 << 3;
-        // If true, the type's layout can be randomized using
-        // the seed stored in `ReprOptions.layout_seed`
-        const RANDOMIZE_LAYOUT   = 1 << 4;
-        // Any of these flags being set prevent field reordering optimisation.
-        const IS_UNOPTIMISABLE   = ReprFlags::IS_C.bits
-                                 | ReprFlags::IS_SIMD.bits
-                                 | ReprFlags::IS_LINEAR.bits;
-    }
-}
-
-/// Represents the repr options provided by the user,
-#[derive(Copy, Clone, Debug, Eq, PartialEq, TyEncodable, TyDecodable, Default, HashStable)]
-pub struct ReprOptions {
-    pub int: Option<attr::IntType>,
-    pub align: Option<Align>,
-    pub pack: Option<Align>,
-    pub flags: ReprFlags,
-    /// The seed to be used for randomizing a type's layout
-    ///
-    /// Note: This could technically be a `[u8; 16]` (a `u128`) which would
-    /// be the "most accurate" hash as it'd encompass the item and crate
-    /// hash without loss, but it does pay the price of being larger.
-    /// Everything's a tradeoff, a `u64` seed should be sufficient for our
-    /// purposes (primarily `-Z randomize-layout`)
-    pub field_shuffle_seed: u64,
-}
-
-impl ReprOptions {
-    pub fn new(tcx: TyCtxt<'_>, did: DefId) -> ReprOptions {
-        let mut flags = ReprFlags::empty();
-        let mut size = None;
-        let mut max_align: Option<Align> = None;
-        let mut min_pack: Option<Align> = None;
-
-        // Generate a deterministically-derived seed from the item's path hash
-        // to allow for cross-crate compilation to actually work
-        let mut field_shuffle_seed = tcx.def_path_hash(did).0.to_smaller_hash();
-
-        // If the user defined a custom seed for layout randomization, xor the item's
-        // path hash with the user defined seed, this will allowing determinism while
-        // still allowing users to further randomize layout generation for e.g. fuzzing
-        if let Some(user_seed) = tcx.sess.opts.unstable_opts.layout_seed {
-            field_shuffle_seed ^= user_seed;
-        }
-
-        for attr in tcx.get_attrs(did, sym::repr) {
-            for r in attr::parse_repr_attr(&tcx.sess, attr) {
-                flags.insert(match r {
-                    attr::ReprC => ReprFlags::IS_C,
-                    attr::ReprPacked(pack) => {
-                        let pack = Align::from_bytes(pack as u64).unwrap();
-                        min_pack = Some(if let Some(min_pack) = min_pack {
-                            min_pack.min(pack)
-                        } else {
-                            pack
-                        });
-                        ReprFlags::empty()
-                    }
-                    attr::ReprTransparent => ReprFlags::IS_TRANSPARENT,
-                    attr::ReprSimd => ReprFlags::IS_SIMD,
-                    attr::ReprInt(i) => {
-                        size = Some(i);
-                        ReprFlags::empty()
-                    }
-                    attr::ReprAlign(align) => {
-                        max_align = max_align.max(Some(Align::from_bytes(align as u64).unwrap()));
-                        ReprFlags::empty()
-                    }
-                });
-            }
-        }
-
-        // If `-Z randomize-layout` was enabled for the type definition then we can
-        // consider performing layout randomization
-        if tcx.sess.opts.unstable_opts.randomize_layout {
-            flags.insert(ReprFlags::RANDOMIZE_LAYOUT);
-        }
-
-        // This is here instead of layout because the choice must make it into metadata.
-        if !tcx.consider_optimizing(|| format!("Reorder fields of {:?}", tcx.def_path_str(did))) {
-            flags.insert(ReprFlags::IS_LINEAR);
-        }
-
-        Self { int: size, align: max_align, pack: min_pack, flags, field_shuffle_seed }
-    }
-
-    #[inline]
-    pub fn simd(&self) -> bool {
-        self.flags.contains(ReprFlags::IS_SIMD)
-    }
-
-    #[inline]
-    pub fn c(&self) -> bool {
-        self.flags.contains(ReprFlags::IS_C)
-    }
-
-    #[inline]
-    pub fn packed(&self) -> bool {
-        self.pack.is_some()
-    }
-
-    #[inline]
-    pub fn transparent(&self) -> bool {
-        self.flags.contains(ReprFlags::IS_TRANSPARENT)
-    }
-
-    #[inline]
-    pub fn linear(&self) -> bool {
-        self.flags.contains(ReprFlags::IS_LINEAR)
-    }
-
-    /// Returns the discriminant type, given these `repr` options.
-    /// This must only be called on enums!
-    pub fn discr_type(&self) -> attr::IntType {
-        self.int.unwrap_or(attr::SignedInt(ast::IntTy::Isize))
-    }
-
-    /// Returns `true` if this `#[repr()]` should inhabit "smart enum
-    /// layout" optimizations, such as representing `Foo<&T>` as a
-    /// single pointer.
-    pub fn inhibit_enum_layout_opt(&self) -> bool {
-        self.c() || self.int.is_some()
-    }
-
-    /// Returns `true` if this `#[repr()]` should inhibit struct field reordering
-    /// optimizations, such as with `repr(C)`, `repr(packed(1))`, or `repr(<int>)`.
-    pub fn inhibit_struct_field_reordering_opt(&self) -> bool {
-        if let Some(pack) = self.pack {
-            if pack.bytes() == 1 {
-                return true;
-            }
-        }
-
-        self.flags.intersects(ReprFlags::IS_UNOPTIMISABLE) || self.int.is_some()
-    }
-
-    /// Returns `true` if this type is valid for reordering and `-Z randomize-layout`
-    /// was enabled for its declaration crate
-    pub fn can_randomize_type_layout(&self) -> bool {
-        !self.inhibit_struct_field_reordering_opt()
-            && self.flags.contains(ReprFlags::RANDOMIZE_LAYOUT)
-    }
-
-    /// Returns `true` if this `#[repr()]` should inhibit union ABI optimisations.
-    pub fn inhibit_union_abi_opt(&self) -> bool {
-        self.c()
-    }
-}
-
 impl<'tcx> FieldDef {
     /// Returns the type of this field. The resulting type is not normalized. The `subst` is
     /// typically obtained via the second field of [`TyKind::Adt`].
@@ -2271,6 +1991,81 @@ impl<'tcx> TyCtxt<'tcx> {
             .filter(move |item| item.kind == AssocKind::Fn && item.defaultness(self).has_value())
     }
 
+    pub fn repr_options_of_def(self, did: DefId) -> ReprOptions {
+        let mut flags = ReprFlags::empty();
+        let mut size = None;
+        let mut max_align: Option<Align> = None;
+        let mut min_pack: Option<Align> = None;
+
+        // Generate a deterministically-derived seed from the item's path hash
+        // to allow for cross-crate compilation to actually work
+        let mut field_shuffle_seed = self.def_path_hash(did).0.to_smaller_hash();
+
+        // If the user defined a custom seed for layout randomization, xor the item's
+        // path hash with the user defined seed, this will allowing determinism while
+        // still allowing users to further randomize layout generation for e.g. fuzzing
+        if let Some(user_seed) = self.sess.opts.unstable_opts.layout_seed {
+            field_shuffle_seed ^= user_seed;
+        }
+
+        for attr in self.get_attrs(did, sym::repr) {
+            for r in attr::parse_repr_attr(&self.sess, attr) {
+                flags.insert(match r {
+                    attr::ReprC => ReprFlags::IS_C,
+                    attr::ReprPacked(pack) => {
+                        let pack = Align::from_bytes(pack as u64).unwrap();
+                        min_pack = Some(if let Some(min_pack) = min_pack {
+                            min_pack.min(pack)
+                        } else {
+                            pack
+                        });
+                        ReprFlags::empty()
+                    }
+                    attr::ReprTransparent => ReprFlags::IS_TRANSPARENT,
+                    attr::ReprSimd => ReprFlags::IS_SIMD,
+                    attr::ReprInt(i) => {
+                        size = Some(match i {
+                            attr::IntType::SignedInt(x) => match x {
+                                ast::IntTy::Isize => IntegerType::Pointer(true),
+                                ast::IntTy::I8 => IntegerType::Fixed(Integer::I8, true),
+                                ast::IntTy::I16 => IntegerType::Fixed(Integer::I16, true),
+                                ast::IntTy::I32 => IntegerType::Fixed(Integer::I32, true),
+                                ast::IntTy::I64 => IntegerType::Fixed(Integer::I64, true),
+                                ast::IntTy::I128 => IntegerType::Fixed(Integer::I128, true),
+                            },
+                            attr::IntType::UnsignedInt(x) => match x {
+                                ast::UintTy::Usize => IntegerType::Pointer(false),
+                                ast::UintTy::U8 => IntegerType::Fixed(Integer::I8, false),
+                                ast::UintTy::U16 => IntegerType::Fixed(Integer::I16, false),
+                                ast::UintTy::U32 => IntegerType::Fixed(Integer::I32, false),
+                                ast::UintTy::U64 => IntegerType::Fixed(Integer::I64, false),
+                                ast::UintTy::U128 => IntegerType::Fixed(Integer::I128, false),
+                            },
+                        });
+                        ReprFlags::empty()
+                    }
+                    attr::ReprAlign(align) => {
+                        max_align = max_align.max(Some(Align::from_bytes(align as u64).unwrap()));
+                        ReprFlags::empty()
+                    }
+                });
+            }
+        }
+
+        // If `-Z randomize-layout` was enabled for the type definition then we can
+        // consider performing layout randomization
+        if self.sess.opts.unstable_opts.randomize_layout {
+            flags.insert(ReprFlags::RANDOMIZE_LAYOUT);
+        }
+
+        // This is here instead of layout because the choice must make it into metadata.
+        if !self.consider_optimizing(|| format!("Reorder fields of {:?}", self.def_path_str(did))) {
+            flags.insert(ReprFlags::IS_LINEAR);
+        }
+
+        ReprOptions { int: size, align: max_align, pack: min_pack, flags, field_shuffle_seed }
+    }
+
     /// Look up the name of a definition across crates. This does not look at HIR.
     pub fn opt_item_name(self, def_id: DefId) -> Option<Symbol> {
         if let Some(cnum) = def_id.as_crate_root() {
@@ -2322,10 +2117,6 @@ impl<'tcx> TyCtxt<'tcx> {
         }
     }
 
-    pub fn field_index(self, hir_id: hir::HirId, typeck_results: &TypeckResults<'_>) -> usize {
-        typeck_results.field_indices().get(hir_id).cloned().expect("no index for a field")
-    }
-
     pub fn find_field_index(self, ident: Ident, variant: &VariantDef) -> Option<usize> {
         variant
             .fields
@@ -2506,6 +2297,10 @@ impl<'tcx> TyCtxt<'tcx> {
         self.trait_def(trait_def_id).has_auto_impl
     }
 
+    pub fn trait_is_coinductive(self, trait_def_id: DefId) -> bool {
+        self.trait_is_auto(trait_def_id) || self.lang_items().sized_trait() == Some(trait_def_id)
+    }
+
     /// Returns layout of a generator. Layout might be unavailable if the
     /// generator is tainted by errors.
     pub fn generator_layout(self, def_id: DefId) -> Option<&'tcx GeneratorLayout<'tcx>> {
@@ -2550,11 +2345,11 @@ impl<'tcx> TyCtxt<'tcx> {
 
     /// Looks up the span of `impl_did` if the impl is local; otherwise returns `Err`
     /// with the name of the crate containing the impl.
-    pub fn span_of_impl(self, impl_did: DefId) -> Result<Span, Symbol> {
-        if let Some(impl_did) = impl_did.as_local() {
-            Ok(self.def_span(impl_did))
+    pub fn span_of_impl(self, impl_def_id: DefId) -> Result<Span, Symbol> {
+        if let Some(impl_def_id) = impl_def_id.as_local() {
+            Ok(self.def_span(impl_def_id))
         } else {
-            Err(self.crate_name(impl_did.krate))
+            Err(self.crate_name(impl_def_id.krate))
         }
     }
 
@@ -2782,8 +2577,7 @@ mod size_asserts {
     use super::*;
     use rustc_data_structures::static_assert_size;
     // tidy-alphabetical-start
-    static_assert_size!(PredicateS<'_>, 48);
-    static_assert_size!(TyS<'_>, 40);
-    static_assert_size!(WithStableHash<TyS<'_>>, 56);
+    static_assert_size!(PredicateKind<'_>, 32);
+    static_assert_size!(WithCachedTypeInfo<TyKind<'_>>, 56);
     // tidy-alphabetical-end
 }
diff --git a/compiler/rustc_middle/src/ty/opaque_types.rs b/compiler/rustc_middle/src/ty/opaque_types.rs
index b05c63109..98cd92007 100644
--- a/compiler/rustc_middle/src/ty/opaque_types.rs
+++ b/compiler/rustc_middle/src/ty/opaque_types.rs
@@ -1,7 +1,8 @@
+use crate::error::ConstNotUsedTraitAlias;
+use crate::ty::fold::{TypeFolder, TypeSuperFoldable};
+use crate::ty::subst::{GenericArg, GenericArgKind};
+use crate::ty::{self, Ty, TyCtxt, TypeFoldable};
 use rustc_data_structures::fx::FxHashMap;
-use rustc_middle::ty::fold::{TypeFolder, TypeSuperFoldable};
-use rustc_middle::ty::subst::{GenericArg, GenericArgKind};
-use rustc_middle::ty::{self, Ty, TyCtxt, TypeFoldable};
 use rustc_span::Span;
 
 /// Converts generic params of a TypeFoldable from one
@@ -201,7 +202,7 @@ impl<'tcx> TypeFolder<'tcx> for ReverseMapper<'tcx> {
                     Some(u) => panic!("const mapped to unexpected kind: {:?}", u),
                     None => {
                         if !self.ignore_errors {
-                            self.tcx.sess.emit_err(ty::ConstNotUsedTraitAlias {
+                            self.tcx.sess.emit_err(ConstNotUsedTraitAlias {
                                 ct: ct.to_string(),
                                 span: self.span,
                             });
diff --git a/compiler/rustc_middle/src/ty/parameterized.rs b/compiler/rustc_middle/src/ty/parameterized.rs
index e1e705a92..9e69544d2 100644
--- a/compiler/rustc_middle/src/ty/parameterized.rs
+++ b/compiler/rustc_middle/src/ty/parameterized.rs
@@ -4,9 +4,8 @@ use rustc_index::vec::{Idx, IndexVec};
 
 use crate::middle::exported_symbols::ExportedSymbol;
 use crate::mir::Body;
-use crate::ty::abstract_const::Node;
 use crate::ty::{
-    self, Const, FnSig, GeneratorDiagnosticData, GenericPredicates, Predicate, TraitRef, Ty,
+    self, Clause, Const, FnSig, GeneratorDiagnosticData, GenericPredicates, Predicate, TraitRef, Ty,
 };
 
 pub trait ParameterizedOverTcx: 'static {
@@ -54,6 +53,7 @@ trivially_parameterized_over_tcx! {
     usize,
     (),
     u32,
+    bool,
     std::string::String,
     crate::metadata::ModChild,
     crate::middle::codegen_fn_attrs::CodegenFnAttrs,
@@ -70,7 +70,7 @@ trivially_parameterized_over_tcx! {
     ty::adjustment::CoerceUnsizedInfo,
     ty::fast_reject::SimplifiedTypeGen<DefId>,
     rustc_ast::Attribute,
-    rustc_ast::MacArgs,
+    rustc_ast::DelimArgs,
     rustc_attr::ConstStability,
     rustc_attr::DefaultBodyStability,
     rustc_attr::Deprecation,
@@ -122,8 +122,8 @@ parameterized_over_tcx! {
     TraitRef,
     Const,
     Predicate,
+    Clause,
     GeneratorDiagnosticData,
     Body,
-    Node,
     ExportedSymbol,
 }
diff --git a/compiler/rustc_middle/src/ty/print/mod.rs b/compiler/rustc_middle/src/ty/print/mod.rs
index 44b9548db..667298b9b 100644
--- a/compiler/rustc_middle/src/ty/print/mod.rs
+++ b/compiler/rustc_middle/src/ty/print/mod.rs
@@ -63,7 +63,7 @@ pub trait Printer<'tcx>: Sized {
 
     fn print_dyn_existential(
         self,
-        predicates: &'tcx ty::List<ty::Binder<'tcx, ty::ExistentialPredicate<'tcx>>>,
+        predicates: &'tcx ty::List<ty::PolyExistentialPredicate<'tcx>>,
     ) -> Result<Self::DynExistential, Self::Error>;
 
     fn print_const(self, ct: ty::Const<'tcx>) -> Result<Self::Const, Self::Error>;
@@ -308,9 +308,7 @@ impl<'tcx, P: Printer<'tcx>> Print<'tcx, P> for Ty<'tcx> {
     }
 }
 
-impl<'tcx, P: Printer<'tcx>> Print<'tcx, P>
-    for &'tcx ty::List<ty::Binder<'tcx, ty::ExistentialPredicate<'tcx>>>
-{
+impl<'tcx, P: Printer<'tcx>> Print<'tcx, P> for &'tcx ty::List<ty::PolyExistentialPredicate<'tcx>> {
     type Output = P::DynExistential;
     type Error = P::Error;
     fn print(&self, cx: P) -> Result<Self::Output, Self::Error> {
diff --git a/compiler/rustc_middle/src/ty/print/pretty.rs b/compiler/rustc_middle/src/ty/print/pretty.rs
index ef9aa236b..5303341ba 100644
--- a/compiler/rustc_middle/src/ty/print/pretty.rs
+++ b/compiler/rustc_middle/src/ty/print/pretty.rs
@@ -11,8 +11,10 @@ use rustc_hir as hir;
 use rustc_hir::def::{self, CtorKind, DefKind, Namespace};
 use rustc_hir::def_id::{DefId, DefIdSet, CRATE_DEF_ID, LOCAL_CRATE};
 use rustc_hir::definitions::{DefPathData, DefPathDataName, DisambiguatedDefPathData};
+use rustc_hir::LangItem;
 use rustc_session::config::TrimmedDefPaths;
 use rustc_session::cstore::{ExternCrate, ExternCrateSource};
+use rustc_session::Limit;
 use rustc_span::symbol::{kw, Ident, Symbol};
 use rustc_target::abi::Size;
 use rustc_target::spec::abi::Abi;
@@ -63,7 +65,6 @@ thread_local! {
     static NO_TRIMMED_PATH: Cell<bool> = const { Cell::new(false) };
     static NO_QUERIES: Cell<bool> = const { Cell::new(false) };
     static NO_VISIBLE_PATH: Cell<bool> = const { Cell::new(false) };
-    static NO_VERBOSE_CONSTANTS: Cell<bool> = const { Cell::new(false) };
 }
 
 macro_rules! define_helper {
@@ -118,9 +119,6 @@ define_helper!(
     /// Prevent selection of visible paths. `Display` impl of DefId will prefer
     /// visible (public) reexports of types as paths.
     fn with_no_visible_paths(NoVisibleGuard, NO_VISIBLE_PATH);
-    /// Prevent verbose printing of constants. Verbose printing of constants is
-    /// never desirable in some contexts like `std::any::type_name`.
-    fn with_no_verbose_constants(NoVerboseConstantsGuard, NO_VERBOSE_CONSTANTS);
 );
 
 /// The "region highlights" are used to control region printing during
@@ -600,7 +598,7 @@ pub trait PrettyPrinter<'tcx>:
             }
             ty::FnPtr(ref bare_fn) => p!(print(bare_fn)),
             ty::Infer(infer_ty) => {
-                let verbose = self.tcx().sess.verbose();
+                let verbose = self.should_print_verbose();
                 if let ty::TyVar(ty_vid) = infer_ty {
                     if let Some(name) = self.ty_infer_name(ty_vid) {
                         p!(write("{}", name))
@@ -642,7 +640,7 @@ pub trait PrettyPrinter<'tcx>:
                 p!(print_def_path(def_id, &[]));
             }
             ty::Projection(ref data) => {
-                if !(self.tcx().sess.verbose() || NO_QUERIES.with(|q| q.get()))
+                if !(self.should_print_verbose() || NO_QUERIES.with(|q| q.get()))
                     && self.tcx().def_kind(data.item_def_id) == DefKind::ImplTraitPlaceholder
                 {
                     return self.pretty_print_opaque_impl_type(data.item_def_id, data.substs);
@@ -658,7 +656,7 @@ pub trait PrettyPrinter<'tcx>:
                 // only affect certain debug messages (e.g. messages printed
                 // from `rustc_middle::ty` during the computation of `tcx.predicates_of`),
                 // and should have no effect on any compiler output.
-                if self.tcx().sess.verbose() || NO_QUERIES.with(|q| q.get()) {
+                if self.should_print_verbose() || NO_QUERIES.with(|q| q.get()) {
                     p!(write("Opaque({:?}, {:?})", def_id, substs));
                     return Ok(self);
                 }
@@ -684,13 +682,19 @@ pub trait PrettyPrinter<'tcx>:
             ty::Str => p!("str"),
             ty::Generator(did, substs, movability) => {
                 p!(write("["));
-                match movability {
-                    hir::Movability::Movable => {}
-                    hir::Movability::Static => p!("static "),
+                let generator_kind = self.tcx().generator_kind(did).unwrap();
+                let should_print_movability =
+                    self.should_print_verbose() || generator_kind == hir::GeneratorKind::Gen;
+
+                if should_print_movability {
+                    match movability {
+                        hir::Movability::Movable => {}
+                        hir::Movability::Static => p!("static "),
+                    }
                 }
 
-                if !self.tcx().sess.verbose() {
-                    p!("generator");
+                if !self.should_print_verbose() {
+                    p!(write("{}", generator_kind));
                     // FIXME(eddyb) should use `def_span`.
                     if let Some(did) = did.as_local() {
                         let span = self.tcx().def_span(did);
@@ -725,7 +729,7 @@ pub trait PrettyPrinter<'tcx>:
             }
             ty::Closure(did, substs) => {
                 p!(write("["));
-                if !self.tcx().sess.verbose() {
+                if !self.should_print_verbose() {
                     p!(write("closure"));
                     // FIXME(eddyb) should use `def_span`.
                     if let Some(did) = did.as_local() {
@@ -763,7 +767,7 @@ pub trait PrettyPrinter<'tcx>:
             }
             ty::Array(ty, sz) => {
                 p!("[", print(ty), "; ");
-                if !NO_VERBOSE_CONSTANTS.with(|flag| flag.get()) && self.tcx().sess.verbose() {
+                if self.should_print_verbose() {
                     p!(write("{:?}", sz));
                 } else if let ty::ConstKind::Unevaluated(..) = sz.kind() {
                     // Do not try to evaluate unevaluated constants. If we are const evaluating an
@@ -805,7 +809,7 @@ pub trait PrettyPrinter<'tcx>:
             let bound_predicate = predicate.kind();
 
             match bound_predicate.skip_binder() {
-                ty::PredicateKind::Trait(pred) => {
+                ty::PredicateKind::Clause(ty::Clause::Trait(pred)) => {
                     let trait_ref = bound_predicate.rebind(pred.trait_ref);
 
                     // Don't print + Sized, but rather + ?Sized if absent.
@@ -816,7 +820,7 @@ pub trait PrettyPrinter<'tcx>:
 
                     self.insert_trait_and_projection(trait_ref, None, &mut traits, &mut fn_traits);
                 }
-                ty::PredicateKind::Projection(pred) => {
+                ty::PredicateKind::Clause(ty::Clause::Projection(pred)) => {
                     let proj_ref = bound_predicate.rebind(pred);
                     let trait_ref = proj_ref.required_poly_trait_ref(tcx);
 
@@ -830,7 +834,7 @@ pub trait PrettyPrinter<'tcx>:
                         &mut fn_traits,
                     );
                 }
-                ty::PredicateKind::TypeOutlives(outlives) => {
+                ty::PredicateKind::Clause(ty::Clause::TypeOutlives(outlives)) => {
                     lifetimes.push(outlives.1);
                 }
                 _ => {}
@@ -892,7 +896,7 @@ pub trait PrettyPrinter<'tcx>:
                                 // Group the return ty with its def id, if we had one.
                                 entry
                                     .return_ty
-                                    .map(|ty| (tcx.lang_items().fn_once_output().unwrap(), ty)),
+                                    .map(|ty| (tcx.require_lang_item(LangItem::FnOnce, None), ty)),
                             );
                         }
                         if let Some(trait_ref) = entry.fn_mut_trait_ref {
@@ -1063,7 +1067,7 @@ pub trait PrettyPrinter<'tcx>:
 
     fn pretty_print_dyn_existential(
         mut self,
-        predicates: &'tcx ty::List<ty::Binder<'tcx, ty::ExistentialPredicate<'tcx>>>,
+        predicates: &'tcx ty::List<ty::PolyExistentialPredicate<'tcx>>,
     ) -> Result<Self::DynExistential, Self::Error> {
         // Generate the main trait ref, including associated types.
         let mut first = true;
@@ -1076,8 +1080,8 @@ pub trait PrettyPrinter<'tcx>:
                 let mut resugared = false;
 
                 // Special-case `Fn(...) -> ...` and re-sugar it.
-                let fn_trait_kind = cx.tcx().fn_trait_kind_from_lang_item(principal.def_id);
-                if !cx.tcx().sess.verbose() && fn_trait_kind.is_some() {
+                let fn_trait_kind = cx.tcx().fn_trait_kind_from_def_id(principal.def_id);
+                if !cx.should_print_verbose() && fn_trait_kind.is_some() {
                     if let ty::Tuple(tys) = principal.substs.type_at(0).kind() {
                         let mut projections = predicates.projection_bounds();
                         if let (Some(proj), None) = (projections.next(), projections.next()) {
@@ -1141,7 +1145,7 @@ pub trait PrettyPrinter<'tcx>:
         //
         // To avoid causing instabilities in compiletest
         // output, sort the auto-traits alphabetically.
-        auto_traits.sort_by_cached_key(|did| self.tcx().def_path_str(*did));
+        auto_traits.sort_by_cached_key(|did| with_no_trimmed_paths!(self.tcx().def_path_str(*did)));
 
         for def_id in auto_traits {
             if !first {
@@ -1185,7 +1189,7 @@ pub trait PrettyPrinter<'tcx>:
     ) -> Result<Self::Const, Self::Error> {
         define_scoped_cx!(self);
 
-        if !NO_VERBOSE_CONSTANTS.with(|flag| flag.get()) && self.tcx().sess.verbose() {
+        if self.should_print_verbose() {
             p!(write("Const({:?}: {:?})", ct.kind(), ct.ty()));
             return Ok(self);
         }
@@ -1244,6 +1248,9 @@ pub trait PrettyPrinter<'tcx>:
                 self.pretty_print_bound_var(debruijn, bound_var)?
             }
             ty::ConstKind::Placeholder(placeholder) => p!(write("Placeholder({:?})", placeholder)),
+            // FIXME(generic_const_exprs):
+            // write out some legible representation of an abstract const?
+            ty::ConstKind::Expr(_) => p!("[Const Expr]"),
             ty::ConstKind::Error(_) => p!("[const error]"),
         };
         Ok(self)
@@ -1420,7 +1427,7 @@ pub trait PrettyPrinter<'tcx>:
     ) -> Result<Self::Const, Self::Error> {
         define_scoped_cx!(self);
 
-        if !NO_VERBOSE_CONSTANTS.with(|flag| flag.get()) && self.tcx().sess.verbose() {
+        if self.should_print_verbose() {
             p!(write("ValTree({:?}: ", valtree), print(ty), ")");
             return Ok(self);
         }
@@ -1461,8 +1468,7 @@ pub trait PrettyPrinter<'tcx>:
             }
             // Aggregates, printed as array/tuple/struct/variant construction syntax.
             (ty::ValTree::Branch(_), ty::Array(..) | ty::Tuple(..) | ty::Adt(..)) => {
-                let contents =
-                    self.tcx().destructure_const(ty::Const::from_value(self.tcx(), valtree, ty));
+                let contents = self.tcx().destructure_const(self.tcx().mk_const(valtree, ty));
                 let fields = contents.fields.iter().copied();
                 match *ty.kind() {
                     ty::Array(..) => {
@@ -1490,12 +1496,12 @@ pub trait PrettyPrinter<'tcx>:
                             contents.variant.expect("destructed const of adt without variant idx");
                         let variant_def = &def.variant(variant_idx);
                         p!(print_value_path(variant_def.def_id, substs));
-                        match variant_def.ctor_kind {
-                            CtorKind::Const => {}
-                            CtorKind::Fn => {
+                        match variant_def.ctor_kind() {
+                            Some(CtorKind::Const) => {}
+                            Some(CtorKind::Fn) => {
                                 p!("(", comma_sep(fields), ")");
                             }
-                            CtorKind::Fictive => {
+                            None => {
                                 p!(" {{ ");
                                 let mut first = true;
                                 for (field_def, field) in iter::zip(&variant_def.fields, fields) {
@@ -1564,6 +1570,10 @@ pub trait PrettyPrinter<'tcx>:
             Ok(cx)
         })
     }
+
+    fn should_print_verbose(&self) -> bool {
+        self.tcx().sess.verbose()
+    }
 }
 
 // HACK(eddyb) boxed to avoid moving around a large struct by-value.
@@ -1583,6 +1593,8 @@ pub struct FmtPrinterData<'a, 'tcx> {
     region_index: usize,
     binder_depth: usize,
     printed_type_count: usize,
+    type_length_limit: Limit,
+    truncated: bool,
 
     pub region_highlight_mode: RegionHighlightMode<'tcx>,
 
@@ -1605,6 +1617,10 @@ impl DerefMut for FmtPrinter<'_, '_> {
 
 impl<'a, 'tcx> FmtPrinter<'a, 'tcx> {
     pub fn new(tcx: TyCtxt<'tcx>, ns: Namespace) -> Self {
+        Self::new_with_limit(tcx, ns, tcx.type_length_limit())
+    }
+
+    pub fn new_with_limit(tcx: TyCtxt<'tcx>, ns: Namespace, type_length_limit: Limit) -> Self {
         FmtPrinter(Box::new(FmtPrinterData {
             tcx,
             // Estimated reasonable capacity to allocate upfront based on a few
@@ -1617,6 +1633,8 @@ impl<'a, 'tcx> FmtPrinter<'a, 'tcx> {
             region_index: 0,
             binder_depth: 0,
             printed_type_count: 0,
+            type_length_limit,
+            truncated: false,
             region_highlight_mode: RegionHighlightMode::new(tcx),
             ty_infer_name_resolver: None,
             const_infer_name_resolver: None,
@@ -1659,6 +1677,12 @@ impl<'t> TyCtxt<'t> {
         debug!("def_path_str: def_id={:?}, ns={:?}", def_id, ns);
         FmtPrinter::new(self, ns).print_def_path(def_id, substs).unwrap().into_buffer()
     }
+
+    pub fn value_path_str_with_substs(self, def_id: DefId, substs: &'t [GenericArg<'t>]) -> String {
+        let ns = guess_def_namespace(self, def_id);
+        debug!("value_path_str: def_id={:?}, ns={:?}", def_id, ns);
+        FmtPrinter::new(self, ns).print_value_path(def_id, substs).unwrap().into_buffer()
+    }
 }
 
 impl fmt::Write for FmtPrinter<'_, '_> {
@@ -1745,11 +1769,11 @@ impl<'tcx> Printer<'tcx> for FmtPrinter<'_, 'tcx> {
     }
 
     fn print_type(mut self, ty: Ty<'tcx>) -> Result<Self::Type, Self::Error> {
-        let type_length_limit = self.tcx.type_length_limit();
-        if type_length_limit.value_within_limit(self.printed_type_count) {
+        if self.type_length_limit.value_within_limit(self.printed_type_count) {
             self.printed_type_count += 1;
             self.pretty_print_type(ty)
         } else {
+            self.truncated = true;
             write!(self, "...")?;
             Ok(self)
         }
@@ -1757,7 +1781,7 @@ impl<'tcx> Printer<'tcx> for FmtPrinter<'_, 'tcx> {
 
     fn print_dyn_existential(
         self,
-        predicates: &'tcx ty::List<ty::Binder<'tcx, ty::ExistentialPredicate<'tcx>>>,
+        predicates: &'tcx ty::List<ty::PolyExistentialPredicate<'tcx>>,
     ) -> Result<Self::DynExistential, Self::Error> {
         self.pretty_print_dyn_existential(predicates)
     }
@@ -1839,7 +1863,7 @@ impl<'tcx> Printer<'tcx> for FmtPrinter<'_, 'tcx> {
             }
         }
 
-        let verbose = self.tcx.sess.verbose();
+        let verbose = self.should_print_verbose();
         disambiguated_data.fmt_maybe_verbose(&mut self, verbose)?;
 
         self.empty_path = false;
@@ -1940,24 +1964,20 @@ impl<'tcx> PrettyPrinter<'tcx> for FmtPrinter<'_, 'tcx> {
             return true;
         }
 
-        if self.tcx.sess.verbose() {
+        if self.should_print_verbose() {
             return true;
         }
 
         let identify_regions = self.tcx.sess.opts.unstable_opts.identify_regions;
 
         match *region {
-            ty::ReEarlyBound(ref data) => {
-                data.name != kw::Empty && data.name != kw::UnderscoreLifetime
-            }
+            ty::ReEarlyBound(ref data) => data.has_name(),
 
             ty::ReLateBound(_, ty::BoundRegion { kind: br, .. })
             | ty::ReFree(ty::FreeRegion { bound_region: br, .. })
             | ty::RePlaceholder(ty::Placeholder { name: br, .. }) => {
-                if let ty::BrNamed(_, name) = br {
-                    if name != kw::Empty && name != kw::UnderscoreLifetime {
-                        return true;
-                    }
+                if br.is_named() {
+                    return true;
                 }
 
                 if let Some((region, _)) = highlight.highlight_bound_region {
@@ -2012,7 +2032,7 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
             return Ok(self);
         }
 
-        if self.tcx.sess.verbose() {
+        if self.should_print_verbose() {
             p!(write("{:?}", region));
             return Ok(self);
         }
@@ -2033,11 +2053,9 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
             ty::ReLateBound(_, ty::BoundRegion { kind: br, .. })
             | ty::ReFree(ty::FreeRegion { bound_region: br, .. })
             | ty::RePlaceholder(ty::Placeholder { name: br, .. }) => {
-                if let ty::BrNamed(_, name) = br {
-                    if name != kw::Empty && name != kw::UnderscoreLifetime {
-                        p!(write("{}", name));
-                        return Ok(self);
-                    }
+                if let ty::BrNamed(_, name) = br && br.is_named() {
+                    p!(write("{}", name));
+                    return Ok(self);
                 }
 
                 if let Some((region, counter)) = highlight.highlight_bound_region {
@@ -2115,7 +2133,7 @@ impl<'a, 'tcx> ty::TypeFolder<'tcx> for RegionFolder<'a, 'tcx> {
                 // If this is an anonymous placeholder, don't rename. Otherwise, in some
                 // async fns, we get a `for<'r> Send` bound
                 match kind {
-                    ty::BrAnon(_) | ty::BrEnv => r,
+                    ty::BrAnon(..) | ty::BrEnv => r,
                     _ => {
                         // Index doesn't matter, since this is just for naming and these never get bound
                         let br = ty::BoundRegion { var: ty::BoundVar::from_u32(0), kind };
@@ -2188,11 +2206,9 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
 
         define_scoped_cx!(self);
 
-        let possible_names =
-            ('a'..='z').rev().map(|s| Symbol::intern(&format!("'{s}"))).collect::<Vec<_>>();
+        let possible_names = ('a'..='z').rev().map(|s| Symbol::intern(&format!("'{s}")));
 
         let mut available_names = possible_names
-            .into_iter()
             .filter(|name| !self.used_region_names.contains(&name))
             .collect::<Vec<_>>();
         debug!(?available_names);
@@ -2218,47 +2234,13 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
         // aren't named. Eventually, we might just want this as the default, but
         // this is not *quite* right and changes the ordering of some output
         // anyways.
-        let (new_value, map) = if self.tcx().sess.verbose() {
-            let regions: Vec<_> = value
-                .bound_vars()
-                .into_iter()
-                .map(|var| {
-                    let ty::BoundVariableKind::Region(var) = var else {
-                    // This doesn't really matter because it doesn't get used,
-                    // it's just an empty value
-                    return ty::BrAnon(0);
-                };
-                    match var {
-                        ty::BrAnon(_) | ty::BrEnv => {
-                            start_or_continue(&mut self, "for<", ", ");
-                            let name = next_name(&self);
-                            debug!(?name);
-                            do_continue(&mut self, name);
-                            ty::BrNamed(CRATE_DEF_ID.to_def_id(), name)
-                        }
-                        ty::BrNamed(def_id, kw::UnderscoreLifetime) => {
-                            start_or_continue(&mut self, "for<", ", ");
-                            let name = next_name(&self);
-                            do_continue(&mut self, name);
-                            ty::BrNamed(def_id, name)
-                        }
-                        ty::BrNamed(def_id, name) => {
-                            start_or_continue(&mut self, "for<", ", ");
-                            do_continue(&mut self, name);
-                            ty::BrNamed(def_id, name)
-                        }
-                    }
-                })
-                .collect();
+        let (new_value, map) = if self.should_print_verbose() {
+            for var in value.bound_vars().iter() {
+                start_or_continue(&mut self, "for<", ", ");
+                write!(self, "{:?}", var)?;
+            }
             start_or_continue(&mut self, "", "> ");
-
-            self.tcx.replace_late_bound_regions(value.clone(), |br| {
-                let kind = regions[br.var.as_usize()];
-                self.tcx.mk_region(ty::ReLateBound(
-                    ty::INNERMOST,
-                    ty::BoundRegion { var: br.var, kind },
-                ))
-            })
+            (value.clone().skip_binder(), BTreeMap::default())
         } else {
             let tcx = self.tcx;
 
@@ -2271,7 +2253,7 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
                             binder_level_idx: ty::DebruijnIndex,
                             br: ty::BoundRegion| {
                 let (name, kind) = match br.kind {
-                    ty::BrAnon(_) | ty::BrEnv => {
+                    ty::BrAnon(..) | ty::BrEnv => {
                         let name = next_name(&self);
 
                         if let Some(lt_idx) = lifetime_idx {
@@ -2286,7 +2268,7 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
 
                         (name, ty::BrNamed(CRATE_DEF_ID.to_def_id(), name))
                     }
-                    ty::BrNamed(def_id, kw::UnderscoreLifetime) => {
+                    ty::BrNamed(def_id, kw::UnderscoreLifetime | kw::Empty) => {
                         let name = next_name(&self);
 
                         if let Some(lt_idx) = lifetime_idx {
@@ -2551,12 +2533,12 @@ pub struct PrintClosureAsImpl<'tcx> {
 forward_display_to_print! {
     ty::Region<'tcx>,
     Ty<'tcx>,
-    &'tcx ty::List<ty::Binder<'tcx, ty::ExistentialPredicate<'tcx>>>,
+    &'tcx ty::List<ty::PolyExistentialPredicate<'tcx>>,
     ty::Const<'tcx>,
 
     // HACK(eddyb) these are exhaustive instead of generic,
     // because `for<'tcx>` isn't possible yet.
-    ty::Binder<'tcx, ty::ExistentialPredicate<'tcx>>,
+    ty::PolyExistentialPredicate<'tcx>,
     ty::Binder<'tcx, ty::TraitRef<'tcx>>,
     ty::Binder<'tcx, ty::ExistentialTraitRef<'tcx>>,
     ty::Binder<'tcx, TraitRefPrintOnlyTraitPath<'tcx>>,
@@ -2698,14 +2680,14 @@ define_print_and_forward_display! {
 
     ty::PredicateKind<'tcx> {
         match *self {
-            ty::PredicateKind::Trait(ref data) => {
+            ty::PredicateKind::Clause(ty::Clause::Trait(ref data)) => {
                 p!(print(data))
             }
             ty::PredicateKind::Subtype(predicate) => p!(print(predicate)),
             ty::PredicateKind::Coerce(predicate) => p!(print(predicate)),
-            ty::PredicateKind::RegionOutlives(predicate) => p!(print(predicate)),
-            ty::PredicateKind::TypeOutlives(predicate) => p!(print(predicate)),
-            ty::PredicateKind::Projection(predicate) => p!(print(predicate)),
+            ty::PredicateKind::Clause(ty::Clause::RegionOutlives(predicate)) => p!(print(predicate)),
+            ty::PredicateKind::Clause(ty::Clause::TypeOutlives(predicate)) => p!(print(predicate)),
+            ty::PredicateKind::Clause(ty::Clause::Projection(predicate)) => p!(print(predicate)),
             ty::PredicateKind::WellFormed(arg) => p!(print(arg), " well-formed"),
             ty::PredicateKind::ObjectSafe(trait_def_id) => {
                 p!("the trait `", print_def_path(trait_def_id, &[]), "` is object-safe")
@@ -2724,6 +2706,7 @@ define_print_and_forward_display! {
             ty::PredicateKind::TypeWellFormedFromEnv(ty) => {
                 p!("the type `", print(ty), "` is found in the environment")
             }
+            ty::PredicateKind::Ambiguous => p!("ambiguous"),
         }
     }
 
diff --git a/compiler/rustc_middle/src/ty/query.rs b/compiler/rustc_middle/src/ty/query.rs
index ec90590ad..642900d3a 100644
--- a/compiler/rustc_middle/src/ty/query.rs
+++ b/compiler/rustc_middle/src/ty/query.rs
@@ -15,6 +15,7 @@ use crate::mir::interpret::{
 };
 use crate::mir::interpret::{LitToConstError, LitToConstInput};
 use crate::mir::mono::CodegenUnit;
+use crate::query::Key;
 use crate::thir;
 use crate::traits::query::{
     CanonicalPredicateGoal, CanonicalProjectionGoal, CanonicalTyGoal,
@@ -27,6 +28,7 @@ use crate::traits::query::{
 };
 use crate::traits::specialization_graph;
 use crate::traits::{self, ImplSource};
+use crate::ty::context::TyCtxtFeed;
 use crate::ty::fast_reject::SimplifiedType;
 use crate::ty::layout::TyAndLayout;
 use crate::ty::subst::{GenericArg, SubstsRef};
@@ -121,10 +123,10 @@ macro_rules! query_helper_param_ty {
 
 macro_rules! query_storage {
     ([][$K:ty, $V:ty]) => {
-        <DefaultCacheSelector as CacheSelector<$K, $V>>::Cache
+        <<$K as Key>::CacheSelector as CacheSelector<'tcx, $V>>::Cache
     };
     ([(arena_cache) $($rest:tt)*][$K:ty, $V:ty]) => {
-        <ArenaCacheSelector<'tcx> as CacheSelector<$K, $V>>::Cache
+        <<$K as Key>::CacheSelector as CacheSelector<'tcx, $V>>::ArenaCache
     };
     ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
         query_storage!([$($modifiers)*][$($args)*])
@@ -275,13 +277,16 @@ macro_rules! define_callbacks {
 
         impl Default for Providers {
             fn default() -> Self {
+                use crate::query::Key;
+
                 Providers {
                     $($name: |_, key| bug!(
-                        "`tcx.{}({:?})` is not supported for external or local crate;\n
-                        hint: Queries can be either made to the local crate, or the external crate. This error means you tried to use it for one that's not supported (likely the local crate).\n
+                        "`tcx.{}({:?})` is not supported for {} crate;\n
+                        hint: Queries can be either made to the local crate, or the external crate. This error means you tried to use it for one that's not supported.\n
                         If that's not the case, {} was likely never assigned to a provider function.\n",
                         stringify!($name),
                         key,
+                        if key.query_crate_is_local() { "local" } else { "external" },
                         stringify!($name),
                     ),)*
                 }
@@ -323,6 +328,56 @@ macro_rules! define_callbacks {
     };
 }
 
+macro_rules! hash_result {
+    ([]) => {{
+        Some(dep_graph::hash_result)
+    }};
+    ([(no_hash) $($rest:tt)*]) => {{
+        None
+    }};
+    ([$other:tt $($modifiers:tt)*]) => {
+        hash_result!([$($modifiers)*])
+    };
+}
+
+macro_rules! define_feedable {
+    ($($(#[$attr:meta])* [$($modifiers:tt)*] fn $name:ident($($K:tt)*) -> $V:ty,)*) => {
+        $(impl<'tcx, K: IntoQueryParam<$($K)*> + Copy> TyCtxtFeed<'tcx, K> {
+            $(#[$attr])*
+            #[inline(always)]
+            pub fn $name(self, value: $V) -> query_stored::$name<'tcx> {
+                let key = self.key().into_query_param();
+                opt_remap_env_constness!([$($modifiers)*][key]);
+
+                let tcx = self.tcx;
+                let cache = &tcx.query_caches.$name;
+
+                let cached = try_get_cached(tcx, cache, &key, copy);
+
+                match cached {
+                    Ok(old) => {
+                        bug!(
+                            "Trying to feed an already recorded value for query {} key={key:?}:\nold value: {old:?}\nnew value: {value:?}",
+                            stringify!($name),
+                        );
+                    }
+                    Err(()) => (),
+                }
+
+                let dep_node = dep_graph::DepNode::construct(tcx, dep_graph::DepKind::$name, &key);
+                let dep_node_index = tcx.dep_graph.with_feed_task(
+                    dep_node,
+                    tcx,
+                    key,
+                    &value,
+                    hash_result!([$($modifiers)*]),
+                );
+                cache.complete(key, value, dep_node_index)
+            }
+        })*
+    }
+}
+
 // Each of these queries corresponds to a function pointer field in the
 // `Providers` struct for requesting a value of that type, and a method
 // on `tcx: TyCtxt` (and `tcx.at(span)`) for doing that request in a way
@@ -336,6 +391,7 @@ macro_rules! define_callbacks {
 // as they will raise an fatal error on query cycles instead.
 
 rustc_query_append! { define_callbacks! }
+rustc_feedable_queries! { define_feedable! }
 
 mod sealed {
     use super::{DefId, LocalDefId, OwnerId};
diff --git a/compiler/rustc_middle/src/ty/relate.rs b/compiler/rustc_middle/src/ty/relate.rs
index b25b4bd4f..c759fb6d5 100644
--- a/compiler/rustc_middle/src/ty/relate.rs
+++ b/compiler/rustc_middle/src/ty/relate.rs
@@ -5,7 +5,7 @@
 //! subtyping, type equality, etc.
 
 use crate::ty::error::{ExpectedFound, TypeError};
-use crate::ty::{self, ImplSubject, Term, TermKind, Ty, TyCtxt, TypeFoldable};
+use crate::ty::{self, Expr, ImplSubject, Term, TermKind, Ty, TyCtxt, TypeFoldable};
 use crate::ty::{GenericArg, GenericArgKind, SubstsRef};
 use rustc_hir as ast;
 use rustc_hir::def_id::DefId;
@@ -23,6 +23,8 @@ pub enum Cause {
 pub trait TypeRelation<'tcx>: Sized {
     fn tcx(&self) -> TyCtxt<'tcx>;
 
+    fn intercrate(&self) -> bool;
+
     fn param_env(&self) -> ty::ParamEnv<'tcx>;
 
     /// Returns a static string we can use for printouts.
@@ -32,6 +34,9 @@ pub trait TypeRelation<'tcx>: Sized {
     /// relation. Just affects error messages.
     fn a_is_expected(&self) -> bool;
 
+    /// Used during coherence. If called, must emit an always-ambiguous obligation.
+    fn mark_ambiguous(&mut self);
+
     fn with_cause<F, R>(&mut self, _cause: Cause, f: F) -> R
     where
         F: FnOnce(&mut Self) -> R,
@@ -60,7 +65,7 @@ pub trait TypeRelation<'tcx>: Sized {
 
         let tcx = self.tcx();
         let opt_variances = tcx.variances_of(item_def_id);
-        relate_substs_with_variances(self, item_def_id, opt_variances, a_subst, b_subst)
+        relate_substs_with_variances(self, item_def_id, opt_variances, a_subst, b_subst, true)
     }
 
     /// Switch variance for the purpose of relating `a` and `b`.
@@ -151,13 +156,14 @@ pub fn relate_substs_with_variances<'tcx, R: TypeRelation<'tcx>>(
     variances: &[ty::Variance],
     a_subst: SubstsRef<'tcx>,
     b_subst: SubstsRef<'tcx>,
+    fetch_ty_for_diag: bool,
 ) -> RelateResult<'tcx, SubstsRef<'tcx>> {
     let tcx = relation.tcx();
 
     let mut cached_ty = None;
     let params = iter::zip(a_subst, b_subst).enumerate().map(|(i, (a, b))| {
         let variance = variances[i];
-        let variance_info = if variance == ty::Invariant {
+        let variance_info = if variance == ty::Invariant && fetch_ty_for_diag {
             let ty =
                 *cached_ty.get_or_insert_with(|| tcx.bound_type_of(ty_def_id).subst(tcx, a_subst));
             ty::VarianceDiagInfo::Invariant { ty, param_index: i.try_into().unwrap() }
@@ -561,8 +567,23 @@ pub fn super_relate_tys<'tcx, R: TypeRelation<'tcx>>(
         (&ty::Opaque(a_def_id, a_substs), &ty::Opaque(b_def_id, b_substs))
             if a_def_id == b_def_id =>
         {
-            let substs = relate_substs(relation, a_substs, b_substs)?;
-            Ok(tcx.mk_opaque(a_def_id, substs))
+            if relation.intercrate() {
+                // During coherence, opaque types should be treated as equal to each other, even if their generic params
+                // differ, as they could resolve to the same hidden type, even for different generic params.
+                relation.mark_ambiguous();
+                Ok(a)
+            } else {
+                let opt_variances = tcx.variances_of(a_def_id);
+                let substs = relate_substs_with_variances(
+                    relation,
+                    a_def_id,
+                    opt_variances,
+                    a_substs,
+                    b_substs,
+                    false, // do not fetch `type_of(a_def_id)`, as it will cause a cycle
+                )?;
+                Ok(tcx.mk_opaque(a_def_id, substs))
+            }
         }
 
         _ => Err(TypeError::Sorts(expected_found(relation, a, b))),
@@ -592,7 +613,10 @@ pub fn super_relate_consts<'tcx, R: TypeRelation<'tcx>>(
     if a_ty != b_ty {
         relation.tcx().sess.delay_span_bug(
             DUMMY_SP,
-            &format!("cannot relate constants of different types: {} != {}", a_ty, b_ty),
+            &format!(
+                "cannot relate constants ({:?}, {:?}) of different types: {} != {}",
+                a, b, a_ty, b_ty
+            ),
         );
     }
 
@@ -602,11 +626,16 @@ pub fn super_relate_consts<'tcx, R: TypeRelation<'tcx>>(
     // an unnormalized (i.e. unevaluated) const in the param-env.
     // FIXME(generic_const_exprs): Once we always lazily unify unevaluated constants
     // these `eval` calls can be removed.
-    if !relation.tcx().features().generic_const_exprs {
+    if !tcx.features().generic_const_exprs {
         a = a.eval(tcx, relation.param_env());
         b = b.eval(tcx, relation.param_env());
     }
 
+    if tcx.features().generic_const_exprs {
+        a = tcx.expand_abstract_consts(a);
+        b = tcx.expand_abstract_consts(b);
+    }
+
     // Currently, the values that can be unified are primitive types,
     // and those that derive both `PartialEq` and `Eq`, corresponding
     // to structural-match types.
@@ -623,33 +652,69 @@ pub fn super_relate_consts<'tcx, R: TypeRelation<'tcx>>(
         (ty::ConstKind::Placeholder(p1), ty::ConstKind::Placeholder(p2)) => p1 == p2,
         (ty::ConstKind::Value(a_val), ty::ConstKind::Value(b_val)) => a_val == b_val,
 
-        (ty::ConstKind::Unevaluated(au), ty::ConstKind::Unevaluated(bu))
-            if tcx.features().generic_const_exprs =>
-        {
-            tcx.try_unify_abstract_consts(relation.param_env().and((au, bu)))
-        }
-
         // While this is slightly incorrect, it shouldn't matter for `min_const_generics`
         // and is the better alternative to waiting until `generic_const_exprs` can
         // be stabilized.
         (ty::ConstKind::Unevaluated(au), ty::ConstKind::Unevaluated(bu)) if au.def == bu.def => {
+            assert_eq!(a.ty(), b.ty());
             let substs = relation.relate_with_variance(
                 ty::Variance::Invariant,
                 ty::VarianceDiagInfo::default(),
                 au.substs,
                 bu.substs,
             )?;
-            return Ok(tcx.mk_const(ty::ConstS {
-                kind: ty::ConstKind::Unevaluated(ty::UnevaluatedConst { def: au.def, substs }),
-                ty: a.ty(),
-            }));
+            return Ok(tcx.mk_const(ty::UnevaluatedConst { def: au.def, substs }, a.ty()));
+        }
+        // Before calling relate on exprs, it is necessary to ensure that the nested consts
+        // have identical types.
+        (ty::ConstKind::Expr(ae), ty::ConstKind::Expr(be)) => {
+            let r = relation;
+
+            // FIXME(generic_const_exprs): is it possible to relate two consts which are not identical
+            // exprs? Should we care about that?
+            let expr = match (ae, be) {
+                (Expr::Binop(a_op, al, ar), Expr::Binop(b_op, bl, br))
+                    if a_op == b_op && al.ty() == bl.ty() && ar.ty() == br.ty() =>
+                {
+                    Expr::Binop(a_op, r.consts(al, bl)?, r.consts(ar, br)?)
+                }
+                (Expr::UnOp(a_op, av), Expr::UnOp(b_op, bv))
+                    if a_op == b_op && av.ty() == bv.ty() =>
+                {
+                    Expr::UnOp(a_op, r.consts(av, bv)?)
+                }
+                (Expr::Cast(ak, av, at), Expr::Cast(bk, bv, bt))
+                    if ak == bk && av.ty() == bv.ty() =>
+                {
+                    Expr::Cast(ak, r.consts(av, bv)?, r.tys(at, bt)?)
+                }
+                (Expr::FunctionCall(af, aa), Expr::FunctionCall(bf, ba))
+                    if aa.len() == ba.len()
+                        && af.ty() == bf.ty()
+                        && aa
+                            .iter()
+                            .zip(ba.iter())
+                            .all(|(a_arg, b_arg)| a_arg.ty() == b_arg.ty()) =>
+                {
+                    let func = r.consts(af, bf)?;
+                    let mut related_args = Vec::with_capacity(aa.len());
+                    for (a_arg, b_arg) in aa.iter().zip(ba.iter()) {
+                        related_args.push(r.consts(a_arg, b_arg)?);
+                    }
+                    let related_args = tcx.mk_const_list(related_args.iter());
+                    Expr::FunctionCall(func, related_args)
+                }
+                _ => return Err(TypeError::ConstMismatch(expected_found(r, a, b))),
+            };
+            let kind = ty::ConstKind::Expr(expr);
+            return Ok(tcx.mk_const(kind, a.ty()));
         }
         _ => false,
     };
     if is_match { Ok(a) } else { Err(TypeError::ConstMismatch(expected_found(relation, a, b))) }
 }
 
-impl<'tcx> Relate<'tcx> for &'tcx ty::List<ty::Binder<'tcx, ty::ExistentialPredicate<'tcx>>> {
+impl<'tcx> Relate<'tcx> for &'tcx ty::List<ty::PolyExistentialPredicate<'tcx>> {
     fn relate<R: TypeRelation<'tcx>>(
         relation: &mut R,
         a: Self,
diff --git a/compiler/rustc_middle/src/ty/structural_impls.rs b/compiler/rustc_middle/src/ty/structural_impls.rs
index 2cad333e3..9f70b4f1f 100644
--- a/compiler/rustc_middle/src/ty/structural_impls.rs
+++ b/compiler/rustc_middle/src/ty/structural_impls.rs
@@ -68,7 +68,7 @@ impl<'tcx> fmt::Debug for ty::adjustment::Adjustment<'tcx> {
 impl fmt::Debug for ty::BoundRegionKind {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match *self {
-            ty::BrAnon(n) => write!(f, "BrAnon({:?})", n),
+            ty::BrAnon(n, span) => write!(f, "BrAnon({n:?}, {span:?})"),
             ty::BrNamed(did, name) => {
                 if did.is_crate_root() {
                     write!(f, "BrNamed({})", name)
@@ -150,15 +150,23 @@ impl<'tcx> fmt::Debug for ty::Predicate<'tcx> {
     }
 }
 
+impl<'tcx> fmt::Debug for ty::Clause<'tcx> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match *self {
+            ty::Clause::Trait(ref a) => a.fmt(f),
+            ty::Clause::RegionOutlives(ref pair) => pair.fmt(f),
+            ty::Clause::TypeOutlives(ref pair) => pair.fmt(f),
+            ty::Clause::Projection(ref pair) => pair.fmt(f),
+        }
+    }
+}
+
 impl<'tcx> fmt::Debug for ty::PredicateKind<'tcx> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match *self {
-            ty::PredicateKind::Trait(ref a) => a.fmt(f),
+            ty::PredicateKind::Clause(ref a) => a.fmt(f),
             ty::PredicateKind::Subtype(ref pair) => pair.fmt(f),
             ty::PredicateKind::Coerce(ref pair) => pair.fmt(f),
-            ty::PredicateKind::RegionOutlives(ref pair) => pair.fmt(f),
-            ty::PredicateKind::TypeOutlives(ref pair) => pair.fmt(f),
-            ty::PredicateKind::Projection(ref pair) => pair.fmt(f),
             ty::PredicateKind::WellFormed(data) => write!(f, "WellFormed({:?})", data),
             ty::PredicateKind::ObjectSafe(trait_def_id) => {
                 write!(f, "ObjectSafe({:?})", trait_def_id)
@@ -173,6 +181,7 @@ impl<'tcx> fmt::Debug for ty::PredicateKind<'tcx> {
             ty::PredicateKind::TypeWellFormedFromEnv(ty) => {
                 write!(f, "TypeWellFormedFromEnv({:?})", ty)
             }
+            ty::PredicateKind::Ambiguous => write!(f, "Ambiguous"),
         }
     }
 }
@@ -240,7 +249,6 @@ TrivialTypeTraversalAndLiftImpls! {
     Field,
     interpret::Scalar,
     rustc_target::abi::Size,
-    ty::DelaySpanBugEmitted,
     rustc_type_ir::DebruijnIndex,
     ty::BoundVar,
     ty::Placeholder<ty::BoundVar>,
@@ -587,12 +595,18 @@ impl<'tcx, T: TypeVisitable<'tcx>> TypeSuperVisitable<'tcx> for ty::Binder<'tcx,
     }
 }
 
-impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::List<ty::Binder<'tcx, ty::ExistentialPredicate<'tcx>>> {
+impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::List<ty::PolyExistentialPredicate<'tcx>> {
     fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
         ty::util::fold_list(self, folder, |tcx, v| tcx.intern_poly_existential_predicates(v))
     }
 }
 
+impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::List<ty::Const<'tcx>> {
+    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
+        ty::util::fold_list(self, folder, |tcx, v| tcx.mk_const_list(v.iter()))
+    }
+}
+
 impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::List<ProjectionKind> {
     fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
         ty::util::fold_list(self, folder, |tcx, v| tcx.intern_projs(v))
@@ -806,7 +820,7 @@ impl<'tcx> TypeSuperFoldable<'tcx> for ty::Const<'tcx> {
         let ty = self.ty().try_fold_with(folder)?;
         let kind = self.kind().try_fold_with(folder)?;
         if ty != self.ty() || kind != self.kind() {
-            Ok(folder.tcx().mk_const(ty::ConstS { ty, kind }))
+            Ok(folder.tcx().mk_const(kind, ty))
         } else {
             Ok(self)
         }
diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs
index cf420bafe..9cbda95a4 100644
--- a/compiler/rustc_middle/src/ty/sty.rs
+++ b/compiler/rustc_middle/src/ty/sty.rs
@@ -17,9 +17,11 @@ use rustc_data_structures::captures::Captures;
 use rustc_data_structures::intern::Interned;
 use rustc_hir as hir;
 use rustc_hir::def_id::DefId;
+use rustc_hir::LangItem;
 use rustc_index::vec::Idx;
 use rustc_macros::HashStable;
 use rustc_span::symbol::{kw, sym, Symbol};
+use rustc_span::Span;
 use rustc_target::abi::VariantIdx;
 use rustc_target::spec::abi;
 use std::borrow::Cow;
@@ -58,7 +60,7 @@ pub struct FreeRegion {
 #[derive(HashStable)]
 pub enum BoundRegionKind {
     /// An anonymous region parameter for a given fn (&T)
-    BrAnon(u32),
+    BrAnon(u32, Option<Span>),
 
     /// Named region parameters for functions (a in &'a T)
     ///
@@ -81,7 +83,9 @@ pub struct BoundRegion {
 impl BoundRegionKind {
     pub fn is_named(&self) -> bool {
         match *self {
-            BoundRegionKind::BrNamed(_, name) => name != kw::UnderscoreLifetime,
+            BoundRegionKind::BrNamed(_, name) => {
+                name != kw::UnderscoreLifetime && name != kw::Empty
+            }
             _ => false,
         }
     }
@@ -702,7 +706,9 @@ impl<'tcx> ExistentialPredicate<'tcx> {
     }
 }
 
-impl<'tcx> Binder<'tcx, ExistentialPredicate<'tcx>> {
+pub type PolyExistentialPredicate<'tcx> = Binder<'tcx, ExistentialPredicate<'tcx>>;
+
+impl<'tcx> PolyExistentialPredicate<'tcx> {
     /// Given an existential predicate like `?Self: PartialEq<u32>` (e.g., derived from `dyn PartialEq<u32>`),
     /// and a concrete type `self_ty`, returns a full predicate where the existentially quantified variable `?Self`
     /// has been replaced with `self_ty` (e.g., `self_ty: PartialEq<u32>`, in our example).
@@ -716,17 +722,23 @@ impl<'tcx> Binder<'tcx, ExistentialPredicate<'tcx>> {
                 self.rebind(p.with_self_ty(tcx, self_ty)).to_predicate(tcx)
             }
             ExistentialPredicate::AutoTrait(did) => {
-                let trait_ref = self.rebind(ty::TraitRef {
-                    def_id: did,
-                    substs: tcx.mk_substs_trait(self_ty, &[]),
-                });
-                trait_ref.without_const().to_predicate(tcx)
+                let generics = tcx.generics_of(did);
+                let trait_ref = if generics.params.len() == 1 {
+                    tcx.mk_trait_ref(did, [self_ty])
+                } else {
+                    // If this is an ill-formed auto trait, then synthesize
+                    // new error substs for the missing generics.
+                    let err_substs =
+                        ty::InternalSubsts::extend_with_error(tcx, did, &[self_ty.into()]);
+                    tcx.mk_trait_ref(did, err_substs)
+                };
+                self.rebind(trait_ref).without_const().to_predicate(tcx)
             }
         }
     }
 }
 
-impl<'tcx> List<ty::Binder<'tcx, ExistentialPredicate<'tcx>>> {
+impl<'tcx> List<ty::PolyExistentialPredicate<'tcx>> {
     /// Returns the "principal `DefId`" of this set of existential predicates.
     ///
     /// A Rust trait object type consists (in addition to a lifetime bound)
@@ -811,6 +823,13 @@ impl<'tcx> TraitRef<'tcx> {
         TraitRef { def_id, substs }
     }
 
+    pub fn with_self_type(self, tcx: TyCtxt<'tcx>, self_ty: Ty<'tcx>) -> Self {
+        tcx.mk_trait_ref(
+            self.def_id,
+            [self_ty.into()].into_iter().chain(self.substs.iter().skip(1)),
+        )
+    }
+
     /// Returns a `TraitRef` of the form `P0: Foo<P1..Pn>` where `Pi`
     /// are the parameters defined on trait.
     pub fn identity(tcx: TyCtxt<'tcx>, def_id: DefId) -> Binder<'tcx, TraitRef<'tcx>> {
@@ -845,23 +864,6 @@ impl<'tcx> PolyTraitRef<'tcx> {
     pub fn def_id(&self) -> DefId {
         self.skip_binder().def_id
     }
-
-    pub fn to_poly_trait_predicate(&self) -> ty::PolyTraitPredicate<'tcx> {
-        self.map_bound(|trait_ref| ty::TraitPredicate {
-            trait_ref,
-            constness: ty::BoundConstness::NotConst,
-            polarity: ty::ImplPolarity::Positive,
-        })
-    }
-
-    /// Same as [`PolyTraitRef::to_poly_trait_predicate`] but sets a negative polarity instead.
-    pub fn to_poly_trait_predicate_negative_polarity(&self) -> ty::PolyTraitPredicate<'tcx> {
-        self.map_bound(|trait_ref| ty::TraitPredicate {
-            trait_ref,
-            constness: ty::BoundConstness::NotConst,
-            polarity: ty::ImplPolarity::Negative,
-        })
-    }
 }
 
 impl rustc_errors::IntoDiagnosticArg for PolyTraitRef<'_> {
@@ -906,7 +908,7 @@ impl<'tcx> ExistentialTraitRef<'tcx> {
         // otherwise the escaping vars would be captured by the binder
         // debug_assert!(!self_ty.has_escaping_bound_vars());
 
-        ty::TraitRef { def_id: self.def_id, substs: tcx.mk_substs_trait(self_ty, self.substs) }
+        tcx.mk_trait_ref(self.def_id, [self_ty.into()].into_iter().chain(self.substs.iter()))
     }
 }
 
@@ -1282,6 +1284,12 @@ impl<'tcx> ParamTy {
     pub fn to_ty(self, tcx: TyCtxt<'tcx>) -> Ty<'tcx> {
         tcx.mk_ty_param(self.index, self.name)
     }
+
+    pub fn span_from_generics(&self, tcx: TyCtxt<'tcx>, item_with_generics: DefId) -> Span {
+        let generics = tcx.generics_of(item_with_generics);
+        let type_param = generics.type_param(self, tcx);
+        tcx.def_span(type_param.def_id)
+    }
 }
 
 #[derive(Copy, Clone, Hash, TyEncodable, TyDecodable, Eq, PartialEq, Ord, PartialOrd)]
@@ -1603,7 +1611,7 @@ impl<'tcx> Region<'tcx> {
 impl<'tcx> Ty<'tcx> {
     #[inline(always)]
     pub fn kind(self) -> &'tcx TyKind<'tcx> {
-        &self.0.0.kind
+        &self.0.0
     }
 
     #[inline(always)]
@@ -2095,7 +2103,7 @@ impl<'tcx> Ty<'tcx> {
 
             ty::Str | ty::Slice(_) => (tcx.types.usize, false),
             ty::Dynamic(..) => {
-                let dyn_metadata = tcx.lang_items().dyn_metadata().unwrap();
+                let dyn_metadata = tcx.require_lang_item(LangItem::DynMetadata, None);
                 (tcx.bound_type_of(dyn_metadata).subst(tcx, &[tail.into()]), false)
             },
 
@@ -2117,8 +2125,7 @@ impl<'tcx> Ty<'tcx> {
     /// parameter. This is kind of a phantom type, except that the
     /// most convenient thing for us to are the integral types. This
     /// function converts such a special type into the closure
-    /// kind. To go the other way, use
-    /// `tcx.closure_kind_ty(closure_kind)`.
+    /// kind. To go the other way, use `closure_kind.to_ty(tcx)`.
     ///
     /// Note that during type checking, we use an inference variable
     /// to represent the closure kind, because it has not yet been
@@ -2244,7 +2251,7 @@ impl<'tcx> Ty<'tcx> {
         }
     }
 
-    // If `self` is a primitive, return its [`Symbol`].
+    /// If `self` is a primitive, return its [`Symbol`].
     pub fn primitive_symbol(self) -> Option<Symbol> {
         match self.kind() {
             ty::Bool => Some(sym::bool),
diff --git a/compiler/rustc_middle/src/ty/subst.rs b/compiler/rustc_middle/src/ty/subst.rs
index 0660e9b79..a1b084a5e 100644
--- a/compiler/rustc_middle/src/ty/subst.rs
+++ b/compiler/rustc_middle/src/ty/subst.rs
@@ -6,11 +6,11 @@ use crate::ty::sty::{ClosureSubsts, GeneratorSubsts, InlineConstSubsts};
 use crate::ty::visit::{TypeVisitable, TypeVisitor};
 use crate::ty::{self, Lift, List, ParamConst, Ty, TyCtxt};
 
-use rustc_data_structures::captures::Captures;
-use rustc_data_structures::intern::{Interned, WithStableHash};
+use rustc_data_structures::intern::Interned;
 use rustc_hir::def_id::DefId;
 use rustc_macros::HashStable;
 use rustc_serialize::{self, Decodable, Encodable};
+use rustc_type_ir::WithCachedTypeInfo;
 use smallvec::SmallVec;
 
 use core::intrinsics;
@@ -19,7 +19,7 @@ use std::fmt;
 use std::marker::PhantomData;
 use std::mem;
 use std::num::NonZeroUsize;
-use std::ops::ControlFlow;
+use std::ops::{ControlFlow, Deref};
 use std::slice;
 
 /// An entity in the Rust type system, which can be one of
@@ -85,7 +85,7 @@ impl<'tcx> GenericArgKind<'tcx> {
             GenericArgKind::Type(ty) => {
                 // Ensure we can use the tag bits.
                 assert_eq!(mem::align_of_val(&*ty.0.0) & TAG_MASK, 0);
-                (TYPE_TAG, ty.0.0 as *const WithStableHash<ty::TyS<'tcx>> as usize)
+                (TYPE_TAG, ty.0.0 as *const WithCachedTypeInfo<ty::TyKind<'tcx>> as usize)
             }
             GenericArgKind::Const(ct) => {
                 // Ensure we can use the tag bits.
@@ -141,6 +141,15 @@ impl<'tcx> From<ty::Const<'tcx>> for GenericArg<'tcx> {
     }
 }
 
+impl<'tcx> From<ty::Term<'tcx>> for GenericArg<'tcx> {
+    fn from(value: ty::Term<'tcx>) -> Self {
+        match value.unpack() {
+            ty::TermKind::Ty(t) => t.into(),
+            ty::TermKind::Const(c) => c.into(),
+        }
+    }
+}
+
 impl<'tcx> GenericArg<'tcx> {
     #[inline]
     pub fn unpack(self) -> GenericArgKind<'tcx> {
@@ -154,7 +163,7 @@ impl<'tcx> GenericArg<'tcx> {
                     &*((ptr & !TAG_MASK) as *const ty::RegionKind<'tcx>),
                 ))),
                 TYPE_TAG => GenericArgKind::Type(Ty(Interned::new_unchecked(
-                    &*((ptr & !TAG_MASK) as *const WithStableHash<ty::TyS<'tcx>>),
+                    &*((ptr & !TAG_MASK) as *const WithCachedTypeInfo<ty::TyKind<'tcx>>),
                 ))),
                 CONST_TAG => GenericArgKind::Const(ty::Const(Interned::new_unchecked(
                     &*((ptr & !TAG_MASK) as *const ty::ConstS<'tcx>),
@@ -344,6 +353,22 @@ impl<'tcx> InternalSubsts<'tcx> {
         }
     }
 
+    // Extend an `original_substs` list to the full number of substs expected by `def_id`,
+    // filling in the missing parameters with error ty/ct or 'static regions.
+    pub fn extend_with_error(
+        tcx: TyCtxt<'tcx>,
+        def_id: DefId,
+        original_substs: &[GenericArg<'tcx>],
+    ) -> SubstsRef<'tcx> {
+        ty::InternalSubsts::for_item(tcx, def_id, |def, substs| {
+            if let Some(subst) = original_substs.get(def.index as usize) {
+                *subst
+            } else {
+                def.to_error(tcx, substs)
+            }
+        })
+    }
+
     #[inline]
     pub fn types(&'tcx self) -> impl DoubleEndedIterator<Item = Ty<'tcx>> + 'tcx {
         self.iter()
@@ -506,6 +531,9 @@ impl<'tcx, T: TypeVisitable<'tcx>> TypeVisitable<'tcx> for &'tcx ty::List<T> {
     }
 }
 
+/// Similar to [`super::Binder`] except that it tracks early bound generics, i.e. `struct Foo<T>(T)`
+/// needs `T` substituted immediately. This type primarily exists to avoid forgetting to call
+/// `subst`.
 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
 #[derive(Encodable, Decodable, HashStable)]
 pub struct EarlyBinder<T>(pub T);
@@ -559,25 +587,94 @@ impl<T, U> EarlyBinder<(T, U)> {
     }
 }
 
-impl<'tcx, 's, T: IntoIterator<Item = I>, I: TypeFoldable<'tcx>> EarlyBinder<T> {
+impl<'tcx, 's, I: IntoIterator> EarlyBinder<I>
+where
+    I::Item: TypeFoldable<'tcx>,
+{
     pub fn subst_iter(
         self,
         tcx: TyCtxt<'tcx>,
         substs: &'s [GenericArg<'tcx>],
-    ) -> impl Iterator<Item = I> + Captures<'s> + Captures<'tcx> {
-        self.0.into_iter().map(move |t| EarlyBinder(t).subst(tcx, substs))
+    ) -> SubstIter<'s, 'tcx, I> {
+        SubstIter { it: self.0.into_iter(), tcx, substs }
+    }
+}
+
+pub struct SubstIter<'s, 'tcx, I: IntoIterator> {
+    it: I::IntoIter,
+    tcx: TyCtxt<'tcx>,
+    substs: &'s [GenericArg<'tcx>],
+}
+
+impl<'tcx, I: IntoIterator> Iterator for SubstIter<'_, 'tcx, I>
+where
+    I::Item: TypeFoldable<'tcx>,
+{
+    type Item = I::Item;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        Some(EarlyBinder(self.it.next()?).subst(self.tcx, self.substs))
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        self.it.size_hint()
     }
 }
 
-impl<'tcx, 's, 'a, T: IntoIterator<Item = &'a I>, I: Copy + TypeFoldable<'tcx> + 'a>
-    EarlyBinder<T>
+impl<'tcx, I: IntoIterator> DoubleEndedIterator for SubstIter<'_, 'tcx, I>
+where
+    I::IntoIter: DoubleEndedIterator,
+    I::Item: TypeFoldable<'tcx>,
+{
+    fn next_back(&mut self) -> Option<Self::Item> {
+        Some(EarlyBinder(self.it.next_back()?).subst(self.tcx, self.substs))
+    }
+}
+
+impl<'tcx, 's, I: IntoIterator> EarlyBinder<I>
+where
+    I::Item: Deref,
+    <I::Item as Deref>::Target: Copy + TypeFoldable<'tcx>,
 {
     pub fn subst_iter_copied(
         self,
         tcx: TyCtxt<'tcx>,
         substs: &'s [GenericArg<'tcx>],
-    ) -> impl Iterator<Item = I> + Captures<'s> + Captures<'tcx> + Captures<'a> {
-        self.0.into_iter().map(move |t| EarlyBinder(*t).subst(tcx, substs))
+    ) -> SubstIterCopied<'s, 'tcx, I> {
+        SubstIterCopied { it: self.0.into_iter(), tcx, substs }
+    }
+}
+
+pub struct SubstIterCopied<'a, 'tcx, I: IntoIterator> {
+    it: I::IntoIter,
+    tcx: TyCtxt<'tcx>,
+    substs: &'a [GenericArg<'tcx>],
+}
+
+impl<'tcx, I: IntoIterator> Iterator for SubstIterCopied<'_, 'tcx, I>
+where
+    I::Item: Deref,
+    <I::Item as Deref>::Target: Copy + TypeFoldable<'tcx>,
+{
+    type Item = <I::Item as Deref>::Target;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        Some(EarlyBinder(*self.it.next()?).subst(self.tcx, self.substs))
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        self.it.size_hint()
+    }
+}
+
+impl<'tcx, I: IntoIterator> DoubleEndedIterator for SubstIterCopied<'_, 'tcx, I>
+where
+    I::IntoIter: DoubleEndedIterator,
+    I::Item: Deref,
+    <I::Item as Deref>::Target: Copy + TypeFoldable<'tcx>,
+{
+    fn next_back(&mut self) -> Option<Self::Item> {
+        Some(EarlyBinder(*self.it.next_back()?).subst(self.tcx, self.substs))
     }
 }
 
@@ -597,6 +694,10 @@ impl<T: Iterator> Iterator for EarlyBinderIter<T> {
     fn next(&mut self) -> Option<Self::Item> {
         self.t.next().map(|i| EarlyBinder(i))
     }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        self.t.size_hint()
+    }
 }
 
 impl<'tcx, T: TypeFoldable<'tcx>> ty::EarlyBinder<T> {
diff --git a/compiler/rustc_middle/src/ty/trait_def.rs b/compiler/rustc_middle/src/ty/trait_def.rs
index ac79949fc..b38a5fbf2 100644
--- a/compiler/rustc_middle/src/ty/trait_def.rs
+++ b/compiler/rustc_middle/src/ty/trait_def.rs
@@ -211,7 +211,7 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 }
 
-// Query provider for `trait_impls_of`.
+/// Query provider for `trait_impls_of`.
 pub(super) fn trait_impls_of_provider(tcx: TyCtxt<'_>, trait_id: DefId) -> TraitImpls {
     let mut impls = TraitImpls::default();
 
@@ -255,7 +255,7 @@ pub(super) fn trait_impls_of_provider(tcx: TyCtxt<'_>, trait_id: DefId) -> Trait
     impls
 }
 
-// Query provider for `incoherent_impls`.
+/// Query provider for `incoherent_impls`.
 pub(super) fn incoherent_impls_provider(tcx: TyCtxt<'_>, simp: SimplifiedType) -> &[DefId] {
     let mut impls = Vec::new();
 
diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs
index f72e236ed..47c1ce807 100644
--- a/compiler/rustc_middle/src/ty/util.rs
+++ b/compiler/rustc_middle/src/ty/util.rs
@@ -8,8 +8,6 @@ use crate::ty::{
 };
 use crate::ty::{GenericArgKind, SubstsRef};
 use rustc_apfloat::Float as _;
-use rustc_ast as ast;
-use rustc_attr::{self as attr, SignedInt, UnsignedInt};
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_errors::ErrorGuaranteed;
@@ -19,7 +17,7 @@ use rustc_hir::def_id::DefId;
 use rustc_index::bit_set::GrowableBitSet;
 use rustc_macros::HashStable;
 use rustc_span::{sym, DUMMY_SP};
-use rustc_target::abi::{Integer, Size, TargetDataLayout};
+use rustc_target::abi::{Integer, IntegerType, Size, TargetDataLayout};
 use rustc_target::spec::abi::Abi;
 use smallvec::SmallVec;
 use std::{fmt, iter};
@@ -104,21 +102,12 @@ pub trait IntTypeExt {
     fn initial_discriminant<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Discr<'tcx>;
 }
 
-impl IntTypeExt for attr::IntType {
+impl IntTypeExt for IntegerType {
     fn to_ty<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Ty<'tcx> {
-        match *self {
-            SignedInt(ast::IntTy::I8) => tcx.types.i8,
-            SignedInt(ast::IntTy::I16) => tcx.types.i16,
-            SignedInt(ast::IntTy::I32) => tcx.types.i32,
-            SignedInt(ast::IntTy::I64) => tcx.types.i64,
-            SignedInt(ast::IntTy::I128) => tcx.types.i128,
-            SignedInt(ast::IntTy::Isize) => tcx.types.isize,
-            UnsignedInt(ast::UintTy::U8) => tcx.types.u8,
-            UnsignedInt(ast::UintTy::U16) => tcx.types.u16,
-            UnsignedInt(ast::UintTy::U32) => tcx.types.u32,
-            UnsignedInt(ast::UintTy::U64) => tcx.types.u64,
-            UnsignedInt(ast::UintTy::U128) => tcx.types.u128,
-            UnsignedInt(ast::UintTy::Usize) => tcx.types.usize,
+        match self {
+            IntegerType::Pointer(true) => tcx.types.isize,
+            IntegerType::Pointer(false) => tcx.types.usize,
+            IntegerType::Fixed(i, s) => i.to_ty(tcx, *s),
         }
     }
 
@@ -1219,11 +1208,11 @@ pub fn is_trivially_const_drop<'tcx>(ty: Ty<'tcx>) -> bool {
     }
 }
 
-// Does the equivalent of
-// ```
-// let v = self.iter().map(|p| p.fold_with(folder)).collect::<SmallVec<[_; 8]>>();
-// folder.tcx().intern_*(&v)
-// ```
+/// Does the equivalent of
+/// ```ignore (ilustrative)
+/// let v = self.iter().map(|p| p.fold_with(folder)).collect::<SmallVec<[_; 8]>>();
+/// folder.tcx().intern_*(&v)
+/// ```
 pub fn fold_list<'tcx, F, T>(
     list: &'tcx ty::List<T>,
     folder: &mut F,
@@ -1259,9 +1248,9 @@ where
 #[derive(Copy, Clone, Debug, HashStable, TyEncodable, TyDecodable)]
 pub struct AlwaysRequiresDrop;
 
-/// Normalizes all opaque types in the given value, replacing them
+/// Reveals all opaque types in the given value, replacing them
 /// with their underlying types.
-pub fn normalize_opaque_types<'tcx>(
+pub fn reveal_opaque_types_in_bounds<'tcx>(
     tcx: TyCtxt<'tcx>,
     val: &'tcx ty::List<ty::Predicate<'tcx>>,
 ) -> &'tcx ty::List<ty::Predicate<'tcx>> {
@@ -1298,7 +1287,7 @@ pub fn is_intrinsic(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
 
 pub fn provide(providers: &mut ty::query::Providers) {
     *providers = ty::query::Providers {
-        normalize_opaque_types,
+        reveal_opaque_types_in_bounds,
         is_doc_hidden,
         is_doc_notable_trait,
         is_intrinsic,
diff --git a/compiler/rustc_middle/src/ty/visit.rs b/compiler/rustc_middle/src/ty/visit.rs
index c09f71f9a..4cdfd9e59 100644
--- a/compiler/rustc_middle/src/ty/visit.rs
+++ b/compiler/rustc_middle/src/ty/visit.rs
@@ -72,12 +72,18 @@ pub trait TypeVisitable<'tcx>: fmt::Debug + Clone {
         self.visit_with(&mut HasEscapingVarsVisitor { outer_index: binder }).is_break()
     }
 
-    /// Returns `true` if this `self` has any regions that escape `binder` (and
+    /// Returns `true` if this type has any regions that escape `binder` (and
     /// hence are not bound by it).
     fn has_vars_bound_above(&self, binder: ty::DebruijnIndex) -> bool {
         self.has_vars_bound_at_or_above(binder.shifted_in(1))
     }
 
+    /// Return `true` if this type has regions that are not a part of the type.
+    /// For example, `for<'a> fn(&'a i32)` return `false`, while `fn(&'a i32)`
+    /// would return `true`. The latter can occur when traversing through the
+    /// former.
+    ///
+    /// See [`HasEscapingVarsVisitor`] for more information.
     fn has_escaping_bound_vars(&self) -> bool {
         self.has_vars_bound_at_or_above(ty::INNERMOST)
     }
@@ -95,11 +101,15 @@ pub trait TypeVisitable<'tcx>: fmt::Debug + Clone {
     fn references_error(&self) -> bool {
         self.has_type_flags(TypeFlags::HAS_ERROR)
     }
-    fn error_reported(&self) -> Option<ErrorGuaranteed> {
+    fn error_reported(&self) -> Result<(), ErrorGuaranteed> {
         if self.references_error() {
-            Some(ErrorGuaranteed::unchecked_claim_error_was_emitted())
+            if let Some(reported) = ty::tls::with(|tcx| tcx.sess.is_compilation_going_to_fail()) {
+                Err(reported)
+            } else {
+                bug!("expect tcx.sess.is_compilation_going_to_fail return `Some`");
+            }
         } else {
-            None
+            Ok(())
         }
     }
     fn has_non_region_param(&self) -> bool {
diff --git a/compiler/rustc_middle/src/ty/vtable.rs b/compiler/rustc_middle/src/ty/vtable.rs
index 5ca51c25a..6eae94511 100644
--- a/compiler/rustc_middle/src/ty/vtable.rs
+++ b/compiler/rustc_middle/src/ty/vtable.rs
@@ -66,7 +66,7 @@ pub(super) fn vtable_allocation_provider<'tcx>(
     let layout = tcx
         .layout_of(ty::ParamEnv::reveal_all().and(ty))
         .expect("failed to build vtable representation");
-    assert!(!layout.is_unsized(), "can't create a vtable for an unsized type");
+    assert!(layout.is_sized(), "can't create a vtable for an unsized type");
     let size = layout.size.bytes();
     let align = layout.align.abi.bytes();
 
diff --git a/compiler/rustc_middle/src/ty/walk.rs b/compiler/rustc_middle/src/ty/walk.rs
index 91db9698c..4fab5abe9 100644
--- a/compiler/rustc_middle/src/ty/walk.rs
+++ b/compiler/rustc_middle/src/ty/walk.rs
@@ -214,6 +214,24 @@ fn push_inner<'tcx>(stack: &mut TypeWalkerStack<'tcx>, parent: GenericArg<'tcx>)
                 | ty::ConstKind::Value(_)
                 | ty::ConstKind::Error(_) => {}
 
+                ty::ConstKind::Expr(expr) => match expr {
+                    ty::Expr::UnOp(_, v) => push_inner(stack, v.into()),
+                    ty::Expr::Binop(_, l, r) => {
+                        push_inner(stack, r.into());
+                        push_inner(stack, l.into())
+                    }
+                    ty::Expr::FunctionCall(func, args) => {
+                        for a in args.iter().rev() {
+                            push_inner(stack, a.into());
+                        }
+                        push_inner(stack, func.into());
+                    }
+                    ty::Expr::Cast(_, c, t) => {
+                        push_inner(stack, t.into());
+                        push_inner(stack, c.into());
+                    }
+                },
+
                 ty::ConstKind::Unevaluated(ct) => {
                     stack.extend(ct.substs.iter().rev());
                 }
diff --git a/compiler/rustc_middle/src/values.rs b/compiler/rustc_middle/src/values.rs
index f4b4c3fb0..70b98e59a 100644
--- a/compiler/rustc_middle/src/values.rs
+++ b/compiler/rustc_middle/src/values.rs
@@ -32,13 +32,23 @@ impl<'tcx> Value<TyCtxt<'tcx>> for ty::SymbolName<'_> {
 }
 
 impl<'tcx> Value<TyCtxt<'tcx>> for ty::Binder<'_, ty::FnSig<'_>> {
-    fn from_cycle_error(tcx: TyCtxt<'tcx>, _: &[QueryInfo]) -> Self {
+    fn from_cycle_error(tcx: TyCtxt<'tcx>, stack: &[QueryInfo]) -> Self {
         let err = tcx.ty_error();
-        // FIXME(compiler-errors): It would be nice if we could get the
-        // query key, so we could at least generate a fn signature that
-        // has the right arity.
+
+        let arity = if let Some(frame) = stack.get(0)
+            && frame.query.name == "fn_sig"
+            && let Some(def_id) = frame.query.def_id
+            && let Some(node) = tcx.hir().get_if_local(def_id)
+            && let Some(sig) = node.fn_sig()
+        {
+            sig.decl.inputs.len() + sig.decl.implicit_self.has_implicit_self() as usize
+        } else {
+            tcx.sess.abort_if_errors();
+            unreachable!()
+        };
+
         let fn_sig = ty::Binder::dummy(tcx.mk_fn_sig(
-            [].into_iter(),
+            std::iter::repeat(err).take(arity),
             err,
             false,
             rustc_hir::Unsafety::Normal,
@@ -185,7 +195,8 @@ fn find_item_ty_spans(
                 });
                 if check_params && let Some(args) = path.segments.last().unwrap().args {
                     let params_in_repr = tcx.params_in_repr(def_id);
-                    for (i, arg) in args.args.iter().enumerate() {
+                    // the domain size check is needed because the HIR may not be well-formed at this point
+                    for (i, arg) in args.args.iter().enumerate().take(params_in_repr.domain_size()) {
                         if let hir::GenericArg::Type(ty) = arg && params_in_repr.contains(i as u32) {
                             find_item_ty_spans(tcx, ty, needle, spans, seen_representable);
                         }