Merging upstream version 1.71.1+dfsg1.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

90 files changed, 3175 insertions, 1822 deletions

diff --git a/compiler/rustc_middle/Cargo.toml b/compiler/rustc_middle/Cargo.toml
index 5b2ec9029..7c56af1da 100644
--- a/compiler/rustc_middle/Cargo.toml
+++ b/compiler/rustc_middle/Cargo.toml
@@ -11,6 +11,8 @@ chalk-ir = "0.87.0"
 derive_more = "0.99.17"
 either = "1.5.0"
 gsgdt = "0.1.2"
+field-offset = "0.3.5"
+measureme = "10.0.0"
 polonius-engine = "0.13.0"
 rustc_apfloat = { path = "../rustc_apfloat" }
 rustc_arena = { path = "../rustc_arena" }
@@ -21,6 +23,7 @@ rustc_errors = { path = "../rustc_errors" }
 # Used for intra-doc links
 rustc_error_messages = { path = "../rustc_error_messages" }
 rustc_feature = { path = "../rustc_feature" }
+rustc_fluent_macro = { path = "../rustc_fluent_macro" }
 rustc_graphviz = { path = "../rustc_graphviz" }
 rustc_hir = { path = "../rustc_hir" }
 rustc_index = { path = "../rustc_index" }
diff --git a/compiler/rustc_middle/messages.ftl b/compiler/rustc_middle/messages.ftl
index bd9d89dee..3d581daa9 100644
--- a/compiler/rustc_middle/messages.ftl
+++ b/compiler/rustc_middle/messages.ftl
@@ -1,40 +1,45 @@
+middle_cannot_be_normalized =
+    unable to determine layout for `{$ty}` because `{$failure_ty}` cannot be normalized
+
+middle_conflict_types =
+    this expression supplies two conflicting concrete types for the same opaque type
+
+middle_const_eval_non_int =
+    constant evaluation of enum discriminant resulted in non-integer
+
+middle_const_not_used_in_type_alias =
+    const parameter `{$ct}` is part of concrete type but not used in parameter list for the `impl Trait` type alias
+
+middle_cycle =
+    a cycle occurred during layout computation
+
 middle_drop_check_overflow =
     overflow while adding drop-check rules for {$ty}
     .note = overflowed on {$overflow_ty}
 
+middle_limit_invalid =
+    `limit` must be a non-negative integer
+    .label = {$error_str}
+
 middle_opaque_hidden_type_mismatch =
     concrete type differs from previous defining opaque type use
     .label = expected `{$self_ty}`, got `{$other_ty}`
 
-middle_conflict_types =
-    this expression supplies two conflicting concrete types for the same opaque type
-
 middle_previous_use_here =
     previous use here
 
-middle_limit_invalid =
-    `limit` must be a non-negative integer
-    .label = {$error_str}
-
 middle_recursion_limit_reached =
     reached the recursion limit finding the struct tail for `{$ty}`
     .help = consider increasing the recursion limit by adding a `#![recursion_limit = "{$suggested_limit}"]`
 
-middle_const_eval_non_int =
-    constant evaluation of enum discriminant resulted in non-integer
+middle_requires_lang_item = requires `{$name}` lang_item
+
+middle_strict_coherence_needs_negative_coherence =
+    to use `strict_coherence` on this trait, the `with_negative_coherence` feature must be enabled
+    .label = due to this attribute
 
 middle_unknown_layout =
     the type `{$ty}` has an unknown layout
 
 middle_values_too_big =
     values of the type `{$ty}` are too big for the current architecture
-
-middle_cannot_be_normalized =
-    unable to determine layout for `{$ty}` because `{$failure_ty}` cannot be normalized
-
-middle_strict_coherence_needs_negative_coherence =
-    to use `strict_coherence` on this trait, the `with_negative_coherence` feature must be enabled
-    .label = due to this attribute
-
-middle_const_not_used_in_type_alias =
-    const parameter `{$ct}` is part of concrete type but not used in parameter list for the `impl Trait` type alias
diff --git a/compiler/rustc_middle/src/arena.rs b/compiler/rustc_middle/src/arena.rs
index dd1e254f4..a149a61ec 100644
--- a/compiler/rustc_middle/src/arena.rs
+++ b/compiler/rustc_middle/src/arena.rs
@@ -17,13 +17,13 @@ macro_rules! arena_types {
             [decode] mir: rustc_middle::mir::Body<'tcx>,
             [] steal_promoted:
                 rustc_data_structures::steal::Steal<
-                    rustc_index::vec::IndexVec<
+                    rustc_index::IndexVec<
                         rustc_middle::mir::Promoted,
                         rustc_middle::mir::Body<'tcx>
                     >
                 >,
             [decode] promoted:
-                rustc_index::vec::IndexVec<
+                rustc_index::IndexVec<
                     rustc_middle::mir::Promoted,
                     rustc_middle::mir::Body<'tcx>
                 >,
@@ -114,9 +114,14 @@ macro_rules! arena_types {
 
             [] dep_kind: rustc_middle::dep_graph::DepKindStruct<'tcx>,
 
-            [decode] trait_impl_trait_tys: rustc_data_structures::fx::FxHashMap<rustc_hir::def_id::DefId, rustc_middle::ty::Ty<'tcx>>,
+            [decode] trait_impl_trait_tys:
+                rustc_data_structures::fx::FxHashMap<
+                    rustc_hir::def_id::DefId,
+                    rustc_middle::ty::EarlyBinder<rustc_middle::ty::Ty<'tcx>>
+                >,
             [] bit_set_u32: rustc_index::bit_set::BitSet<u32>,
             [] external_constraints: rustc_middle::traits::solve::ExternalConstraintsData<'tcx>,
+            [] predefined_opaques_in_body: rustc_middle::traits::solve::PredefinedOpaquesData<'tcx>,
             [decode] doc_link_resolutions: rustc_hir::def::DocLinkResMap,
             [] closure_kind_origin: (rustc_span::Span, rustc_middle::hir::place::Place<'tcx>),
             [] mod_child: rustc_middle::metadata::ModChild,
diff --git a/compiler/rustc_middle/src/dep_graph/dep_node.rs b/compiler/rustc_middle/src/dep_graph/dep_node.rs
index 865bb70af..2dc5b8969 100644
--- a/compiler/rustc_middle/src/dep_graph/dep_node.rs
+++ b/compiler/rustc_middle/src/dep_graph/dep_node.rs
@@ -357,20 +357,20 @@ impl<'tcx> DepNodeParams<TyCtxt<'tcx>> for HirId {
         Fingerprint::new(
             // `owner` is local, so is completely defined by the local hash
             def_path_hash.local_hash(),
-            local_id.as_u32().into(),
+            local_id.as_u32() as u64,
         )
     }
 
     #[inline(always)]
     fn to_debug_str(&self, tcx: TyCtxt<'tcx>) -> String {
         let HirId { owner, local_id } = *self;
-        format!("{}.{}", tcx.def_path_str(owner.to_def_id()), local_id.as_u32())
+        format!("{}.{}", tcx.def_path_str(owner), local_id.as_u32())
     }
 
     #[inline(always)]
     fn recover(tcx: TyCtxt<'tcx>, dep_node: &DepNode) -> Option<Self> {
         if tcx.fingerprint_style(dep_node.kind) == FingerprintStyle::HirId {
-            let (local_hash, local_id) = Fingerprint::from(dep_node.hash).as_value();
+            let (local_hash, local_id) = Fingerprint::from(dep_node.hash).split();
             let def_path_hash = DefPathHash::new(tcx.sess.local_stable_crate_id(), local_hash);
             let def_id = tcx
                 .def_path_hash_to_def_id(def_path_hash, &mut || {
@@ -378,6 +378,7 @@ impl<'tcx> DepNodeParams<TyCtxt<'tcx>> for HirId {
                 })
                 .expect_local();
             let local_id = local_id
+                .as_u64()
                 .try_into()
                 .unwrap_or_else(|_| panic!("local id should be u32, found {:?}", local_id));
             Some(HirId { owner: OwnerId { def_id }, local_id: ItemLocalId::from_u32(local_id) })
diff --git a/compiler/rustc_middle/src/error.rs b/compiler/rustc_middle/src/error.rs
index dc4aa1864..046186d27 100644
--- a/compiler/rustc_middle/src/error.rs
+++ b/compiler/rustc_middle/src/error.rs
@@ -1,5 +1,5 @@
 use rustc_macros::Diagnostic;
-use rustc_span::Span;
+use rustc_span::{Span, Symbol};
 
 use crate::ty::Ty;
 
@@ -74,6 +74,14 @@ pub(crate) struct StrictCoherenceNeedsNegativeCoherence {
 }
 
 #[derive(Diagnostic)]
+#[diag(middle_requires_lang_item)]
+pub(crate) struct RequiresLangItem {
+    #[primary_span]
+    pub span: Option<Span>,
+    pub name: Symbol,
+}
+
+#[derive(Diagnostic)]
 #[diag(middle_const_not_used_in_type_alias)]
 pub(super) struct ConstNotUsedTraitAlias {
     pub ct: String,
diff --git a/compiler/rustc_middle/src/hir/map/mod.rs b/compiler/rustc_middle/src/hir/map/mod.rs
index e551c76f8..d1ddc8fc1 100644
--- a/compiler/rustc_middle/src/hir/map/mod.rs
+++ b/compiler/rustc_middle/src/hir/map/mod.rs
@@ -1,17 +1,18 @@
 use crate::hir::{ModuleItems, Owner};
+use crate::middle::debugger_visualizer::DebuggerVisualizerFile;
 use crate::query::LocalCrate;
 use crate::ty::TyCtxt;
 use rustc_ast as ast;
 use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::svh::Svh;
-use rustc_data_structures::sync::{par_for_each_in, Send, Sync};
+use rustc_data_structures::sync::{par_for_each_in, DynSend, DynSync};
 use rustc_hir::def::{DefKind, Res};
 use rustc_hir::def_id::{DefId, LocalDefId, CRATE_DEF_ID, LOCAL_CRATE};
 use rustc_hir::definitions::{DefKey, DefPath, DefPathData, DefPathHash};
 use rustc_hir::intravisit::{self, Visitor};
 use rustc_hir::*;
-use rustc_index::vec::Idx;
+use rustc_index::Idx;
 use rustc_middle::hir::nested_filter;
 use rustc_span::def_id::StableCrateId;
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
@@ -150,11 +151,6 @@ impl<'hir> Map<'hir> {
         self.tcx.hir_module_items(module).items()
     }
 
-    #[inline]
-    pub fn par_for_each_item(self, f: impl Fn(ItemId) + Sync + Send) {
-        par_for_each_in(&self.tcx.hir_crate_items(()).items[..], |id| f(*id));
-    }
-
     pub fn def_key(self, def_id: LocalDefId) -> DefKey {
         // Accessing the DefKey is ok, since it is part of DefPathHash.
         self.tcx.definitions_untracked().def_key(def_id)
@@ -414,7 +410,7 @@ impl<'hir> Map<'hir> {
     /// item (possibly associated), a closure, or a `hir::AnonConst`.
     pub fn body_owner(self, BodyId { hir_id }: BodyId) -> HirId {
         let parent = self.parent_id(hir_id);
-        assert!(self.find(parent).map_or(false, |n| is_body_owner(n, hir_id)), "{hir_id:?}");
+        assert!(self.find(parent).is_some_and(|n| is_body_owner(n, hir_id)), "{hir_id:?}");
         parent
     }
 
@@ -502,7 +498,7 @@ impl<'hir> Map<'hir> {
     }
 
     #[inline]
-    pub fn par_body_owners(self, f: impl Fn(LocalDefId) + Sync + Send) {
+    pub fn par_body_owners(self, f: impl Fn(LocalDefId) + DynSend + DynSync) {
         par_for_each_in(&self.tcx.hir_crate_items(()).body_owners[..], |&def_id| f(def_id));
     }
 
@@ -640,7 +636,7 @@ impl<'hir> Map<'hir> {
     }
 
     #[inline]
-    pub fn par_for_each_module(self, f: impl Fn(LocalDefId) + Sync + Send) {
+    pub fn par_for_each_module(self, f: impl Fn(LocalDefId) + DynSend + DynSync) {
         let crate_items = self.tcx.hir_crate_items(());
         par_for_each_in(&crate_items.submodules[..], |module| f(module.def_id))
     }
@@ -1170,11 +1166,26 @@ pub(super) fn crate_hash(tcx: TyCtxt<'_>, _: LocalCrate) -> Svh {
 
     source_file_names.sort_unstable();
 
+    // We have to take care of debugger visualizers explicitly. The HIR (and
+    // thus `hir_body_hash`) contains the #[debugger_visualizer] attributes but
+    // these attributes only store the file path to the visualizer file, not
+    // their content. Yet that content is exported into crate metadata, so any
+    // changes to it need to be reflected in the crate hash.
+    let debugger_visualizers: Vec<_> = tcx
+        .debugger_visualizers(LOCAL_CRATE)
+        .iter()
+        // We ignore the path to the visualizer file since it's not going to be
+        // encoded in crate metadata and we already hash the full contents of
+        // the file.
+        .map(DebuggerVisualizerFile::path_erased)
+        .collect();
+
     let crate_hash: Fingerprint = tcx.with_stable_hashing_context(|mut hcx| {
         let mut stable_hasher = StableHasher::new();
         hir_body_hash.hash_stable(&mut hcx, &mut stable_hasher);
         upstream_crates.hash_stable(&mut hcx, &mut stable_hasher);
         source_file_names.hash_stable(&mut hcx, &mut stable_hasher);
+        debugger_visualizers.hash_stable(&mut hcx, &mut stable_hasher);
         if tcx.sess.opts.incremental_relative_spans() {
             let definitions = tcx.definitions_untracked();
             let mut owner_spans: Vec<_> = krate
@@ -1199,7 +1210,7 @@ pub(super) fn crate_hash(tcx: TyCtxt<'_>, _: LocalCrate) -> Svh {
         stable_hasher.finish()
     });
 
-    Svh::new(crate_hash.to_smaller_hash())
+    Svh::new(crate_hash)
 }
 
 fn upstream_crates(tcx: TyCtxt<'_>) -> Vec<(StableCrateId, Svh)> {
@@ -1217,7 +1228,7 @@ fn upstream_crates(tcx: TyCtxt<'_>) -> Vec<(StableCrateId, Svh)> {
 }
 
 fn hir_id_to_string(map: Map<'_>, id: HirId) -> String {
-    let path_str = |def_id: LocalDefId| map.tcx.def_path_str(def_id.to_def_id());
+    let path_str = |def_id: LocalDefId| map.tcx.def_path_str(def_id);
 
     let span_str = || map.tcx.sess.source_map().span_to_snippet(map.span(id)).unwrap_or_default();
     let node_str = |prefix| format!("{id} ({prefix} `{}`)", span_str());
diff --git a/compiler/rustc_middle/src/hir/mod.rs b/compiler/rustc_middle/src/hir/mod.rs
index 7770a5e47..45a07fdd2 100644
--- a/compiler/rustc_middle/src/hir/mod.rs
+++ b/compiler/rustc_middle/src/hir/mod.rs
@@ -6,10 +6,11 @@ pub mod map;
 pub mod nested_filter;
 pub mod place;
 
-use crate::ty::query::Providers;
+use crate::query::Providers;
 use crate::ty::{EarlyBinder, ImplSubject, TyCtxt};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
-use rustc_data_structures::sync::{par_for_each_in, Send, Sync};
+use rustc_data_structures::sync::{par_for_each_in, DynSend, DynSync};
+use rustc_hir::def::DefKind;
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::*;
 use rustc_query_system::ich::StableHashingContext;
@@ -77,19 +78,19 @@ impl ModuleItems {
         self.owners().map(|id| id.def_id)
     }
 
-    pub fn par_items(&self, f: impl Fn(ItemId) + Send + Sync) {
+    pub fn par_items(&self, f: impl Fn(ItemId) + DynSend + DynSync) {
         par_for_each_in(&self.items[..], |&id| f(id))
     }
 
-    pub fn par_trait_items(&self, f: impl Fn(TraitItemId) + Send + Sync) {
+    pub fn par_trait_items(&self, f: impl Fn(TraitItemId) + DynSend + DynSync) {
         par_for_each_in(&self.trait_items[..], |&id| f(id))
     }
 
-    pub fn par_impl_items(&self, f: impl Fn(ImplItemId) + Send + Sync) {
+    pub fn par_impl_items(&self, f: impl Fn(ImplItemId) + DynSend + DynSync) {
         par_for_each_in(&self.impl_items[..], |&id| f(id))
     }
 
-    pub fn par_foreign_items(&self, f: impl Fn(ForeignItemId) + Send + Sync) {
+    pub fn par_foreign_items(&self, f: impl Fn(ForeignItemId) + DynSend + DynSync) {
         par_for_each_in(&self.foreign_items[..], |&id| f(id))
     }
 }
@@ -110,6 +111,12 @@ impl<'tcx> TyCtxt<'tcx> {
             None => self.type_of(def_id).map_bound(ImplSubject::Inherent),
         }
     }
+
+    /// Returns `true` if this is a foreign item (i.e., linked via `extern { ... }`).
+    pub fn is_foreign_item(self, def_id: impl Into<DefId>) -> bool {
+        self.opt_parent(def_id.into())
+            .is_some_and(|parent| matches!(self.def_kind(parent), DefKind::ForeignMod))
+    }
 }
 
 pub fn provide(providers: &mut Providers) {
diff --git a/compiler/rustc_middle/src/hir/place.rs b/compiler/rustc_middle/src/hir/place.rs
index 80b4c964c..8a22de931 100644
--- a/compiler/rustc_middle/src/hir/place.rs
+++ b/compiler/rustc_middle/src/hir/place.rs
@@ -66,7 +66,6 @@ pub struct Place<'tcx> {
 ///
 /// This is an HIR version of [`rustc_middle::mir::Place`].
 #[derive(Clone, Debug, PartialEq, Eq, Hash, TyEncodable, TyDecodable, HashStable)]
-#[derive(TypeFoldable, TypeVisitable)]
 pub struct PlaceWithHirId<'tcx> {
     /// `HirId` of the expression or pattern producing this value.
     pub hir_id: HirId,
diff --git a/compiler/rustc_middle/src/infer/canonical.rs b/compiler/rustc_middle/src/infer/canonical.rs
index b5b712c36..561713149 100644
--- a/compiler/rustc_middle/src/infer/canonical.rs
+++ b/compiler/rustc_middle/src/infer/canonical.rs
@@ -280,7 +280,7 @@ pub struct QueryResponse<'tcx, R> {
     /// should get its hidden type inferred. So we bubble the opaque type
     /// and the type it was compared against upwards and let the query caller
     /// handle it.
-    pub opaque_types: Vec<(Ty<'tcx>, Ty<'tcx>)>,
+    pub opaque_types: Vec<(ty::OpaqueTypeKey<'tcx>, Ty<'tcx>)>,
     pub value: R,
 }
 
@@ -348,14 +348,6 @@ impl<'tcx, R> Canonical<'tcx, QueryResponse<'tcx, R>> {
     }
 }
 
-impl<'tcx, R> Canonical<'tcx, ty::ParamEnvAnd<'tcx, R>> {
-    #[inline]
-    pub fn without_const(mut self) -> Self {
-        self.value = self.value.without_const();
-        self
-    }
-}
-
 impl<'tcx, V> Canonical<'tcx, V> {
     /// Allows you to map the `value` of a canonical while keeping the
     /// same set of bound variables.
@@ -400,10 +392,8 @@ pub type QueryOutlivesConstraint<'tcx> =
     (ty::OutlivesPredicate<GenericArg<'tcx>, Region<'tcx>>, ConstraintCategory<'tcx>);
 
 TrivialTypeTraversalAndLiftImpls! {
-    for <'tcx> {
-        crate::infer::canonical::Certainty,
-        crate::infer::canonical::CanonicalTyVarKind,
-    }
+    crate::infer::canonical::Certainty,
+    crate::infer::canonical::CanonicalTyVarKind,
 }
 
 impl<'tcx> CanonicalVarValues<'tcx> {
diff --git a/compiler/rustc_middle/src/lib.rs b/compiler/rustc_middle/src/lib.rs
index b4edb02f6..22ee2a8c5 100644
--- a/compiler/rustc_middle/src/lib.rs
+++ b/compiler/rustc_middle/src/lib.rs
@@ -59,6 +59,8 @@
 #![feature(result_option_inspect)]
 #![feature(const_option)]
 #![feature(trait_alias)]
+#![feature(ptr_alignment_type)]
+#![feature(macro_metavar_expr)]
 #![recursion_limit = "512"]
 #![allow(rustc::potential_query_instability)]
 
@@ -74,7 +76,7 @@ extern crate tracing;
 extern crate smallvec;
 
 use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage};
-use rustc_macros::fluent_messages;
+use rustc_fluent_macro::fluent_messages;
 
 #[cfg(test)]
 mod tests;
@@ -83,12 +85,7 @@ mod tests;
 mod macros;
 
 #[macro_use]
-pub mod query;
-
-#[macro_use]
 pub mod arena;
-#[macro_use]
-pub mod dep_graph;
 pub(crate) mod error;
 pub mod hir;
 pub mod infer;
@@ -99,12 +96,13 @@ pub mod mir;
 pub mod thir;
 pub mod traits;
 pub mod ty;
+pub mod util;
 mod values;
 
-pub mod util {
-    pub mod bug;
-    pub mod common;
-}
+#[macro_use]
+pub mod query;
+#[macro_use]
+pub mod dep_graph;
 
 // Allows macros to refer to this crate as `::rustc_middle`
 extern crate self as rustc_middle;
diff --git a/compiler/rustc_middle/src/lint.rs b/compiler/rustc_middle/src/lint.rs
index c61de97d5..14343ac11 100644
--- a/compiler/rustc_middle/src/lint.rs
+++ b/compiler/rustc_middle/src/lint.rs
@@ -231,19 +231,19 @@ pub fn explain_lint_level_source(
     let name = lint.name_lower();
     match src {
         LintLevelSource::Default => {
-            err.note_once(&format!("`#[{}({})]` on by default", level.as_str(), name));
+            err.note_once(format!("`#[{}({})]` on by default", level.as_str(), name));
         }
         LintLevelSource::CommandLine(lint_flag_val, orig_level) => {
             let flag = orig_level.to_cmd_flag();
             let hyphen_case_lint_name = name.replace('_', "-");
             if lint_flag_val.as_str() == name {
-                err.note_once(&format!(
+                err.note_once(format!(
                     "requested on the command line with `{} {}`",
                     flag, hyphen_case_lint_name
                 ));
             } else {
                 let hyphen_case_flag_val = lint_flag_val.as_str().replace('_', "-");
-                err.note_once(&format!(
+                err.note_once(format!(
                     "`{} {}` implied by `{} {}`",
                     flag, hyphen_case_lint_name, flag, hyphen_case_flag_val
                 ));
@@ -256,7 +256,7 @@ pub fn explain_lint_level_source(
             err.span_note_once(span, "the lint level is defined here");
             if lint_attr_name.as_str() != name {
                 let level_str = level.as_str();
-                err.note_once(&format!(
+                err.note_once(format!(
                     "`#[{}({})]` implied by `#[{}({})]`",
                     level_str, name, level_str, lint_attr_name
                 ));
@@ -444,12 +444,12 @@ pub fn struct_lint_level(
             };
 
             if future_incompatible.explain_reason {
-                err.warn(&explanation);
+                err.warn(explanation);
             }
             if !future_incompatible.reference.is_empty() {
                 let citation =
                     format!("for more information, see {}", future_incompatible.reference);
-                err.note(&citation);
+                err.note(citation);
             }
         }
 
@@ -468,8 +468,7 @@ pub fn struct_lint_level(
 pub fn in_external_macro(sess: &Session, span: Span) -> bool {
     let expn_data = span.ctxt().outer_expn_data();
     match expn_data.kind {
-        ExpnKind::Inlined
-        | ExpnKind::Root
+        ExpnKind::Root
         | ExpnKind::Desugaring(
             DesugaringKind::ForLoop | DesugaringKind::WhileLoop | DesugaringKind::OpaqueTy,
         ) => false,
diff --git a/compiler/rustc_middle/src/macros.rs b/compiler/rustc_middle/src/macros.rs
index 89014f62d..cd1c6c330 100644
--- a/compiler/rustc_middle/src/macros.rs
+++ b/compiler/rustc_middle/src/macros.rs
@@ -43,34 +43,26 @@ macro_rules! span_bug {
 
 #[macro_export]
 macro_rules! CloneLiftImpls {
-    (for <$tcx:lifetime> { $($ty:ty,)+ }) => {
+    ($($ty:ty,)+) => {
         $(
-            impl<$tcx> $crate::ty::Lift<$tcx> for $ty {
+            impl<'tcx> $crate::ty::Lift<'tcx> for $ty {
                 type Lifted = Self;
-                fn lift_to_tcx(self, _: $crate::ty::TyCtxt<$tcx>) -> Option<Self> {
+                fn lift_to_tcx(self, _: $crate::ty::TyCtxt<'tcx>) -> Option<Self> {
                     Some(self)
                 }
             }
         )+
     };
-
-    ($($ty:ty,)+) => {
-        CloneLiftImpls! {
-            for <'tcx> {
-                $($ty,)+
-            }
-        }
-    };
 }
 
 /// Used for types that are `Copy` and which **do not care arena
 /// allocated data** (i.e., don't need to be folded).
 #[macro_export]
 macro_rules! TrivialTypeTraversalImpls {
-    (for <$tcx:lifetime> { $($ty:ty,)+ }) => {
+    ($($ty:ty,)+) => {
         $(
-            impl<$tcx> $crate::ty::fold::TypeFoldable<$crate::ty::TyCtxt<$tcx>> for $ty {
-                fn try_fold_with<F: $crate::ty::fold::FallibleTypeFolder<$crate::ty::TyCtxt<$tcx>>>(
+            impl<'tcx> $crate::ty::fold::TypeFoldable<$crate::ty::TyCtxt<'tcx>> for $ty {
+                fn try_fold_with<F: $crate::ty::fold::FallibleTypeFolder<$crate::ty::TyCtxt<'tcx>>>(
                     self,
                     _: &mut F,
                 ) -> ::std::result::Result<Self, F::Error> {
@@ -78,7 +70,7 @@ macro_rules! TrivialTypeTraversalImpls {
                 }
 
                 #[inline]
-                fn fold_with<F: $crate::ty::fold::TypeFolder<$crate::ty::TyCtxt<$tcx>>>(
+                fn fold_with<F: $crate::ty::fold::TypeFolder<$crate::ty::TyCtxt<'tcx>>>(
                     self,
                     _: &mut F,
                 ) -> Self {
@@ -86,9 +78,9 @@ macro_rules! TrivialTypeTraversalImpls {
                 }
             }
 
-            impl<$tcx> $crate::ty::visit::TypeVisitable<$crate::ty::TyCtxt<$tcx>> for $ty {
+            impl<'tcx> $crate::ty::visit::TypeVisitable<$crate::ty::TyCtxt<'tcx>> for $ty {
                 #[inline]
-                fn visit_with<F: $crate::ty::visit::TypeVisitor<$crate::ty::TyCtxt<$tcx>>>(
+                fn visit_with<F: $crate::ty::visit::TypeVisitor<$crate::ty::TyCtxt<'tcx>>>(
                     &self,
                     _: &mut F)
                     -> ::std::ops::ControlFlow<F::BreakTy>
@@ -98,14 +90,6 @@ macro_rules! TrivialTypeTraversalImpls {
             }
         )+
     };
-
-    ($($ty:ty,)+) => {
-        TrivialTypeTraversalImpls! {
-            for<'tcx> {
-                $($ty,)+
-            }
-        }
-    };
 }
 
 #[macro_export]
diff --git a/compiler/rustc_middle/src/metadata.rs b/compiler/rustc_middle/src/metadata.rs
index f3170e0ec..674402cb4 100644
--- a/compiler/rustc_middle/src/metadata.rs
+++ b/compiler/rustc_middle/src/metadata.rs
@@ -4,7 +4,6 @@ use rustc_hir::def::Res;
 use rustc_macros::HashStable;
 use rustc_span::def_id::DefId;
 use rustc_span::symbol::Ident;
-use rustc_span::Span;
 use smallvec::SmallVec;
 
 /// A simplified version of `ImportKind` from resolve.
@@ -41,8 +40,6 @@ pub struct ModChild {
     pub res: Res<!>,
     /// Visibility of the item.
     pub vis: ty::Visibility<DefId>,
-    /// Span of the item.
-    pub span: Span,
     /// Reexport chain linking this module child to its original reexported item.
     /// Empty if the module child is a proper item.
     pub reexport_chain: SmallVec<[Reexport; 2]>,
diff --git a/compiler/rustc_middle/src/middle/debugger_visualizer.rs b/compiler/rustc_middle/src/middle/debugger_visualizer.rs
new file mode 100644
index 000000000..a0497d805
--- /dev/null
+++ b/compiler/rustc_middle/src/middle/debugger_visualizer.rs
@@ -0,0 +1,38 @@
+use rustc_data_structures::sync::Lrc;
+use std::path::PathBuf;
+
+#[derive(HashStable)]
+#[derive(Copy, PartialEq, PartialOrd, Clone, Ord, Eq, Hash, Debug, Encodable, Decodable)]
+pub enum DebuggerVisualizerType {
+    Natvis,
+    GdbPrettyPrinter,
+}
+
+/// A single debugger visualizer file.
+#[derive(HashStable)]
+#[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Encodable, Decodable)]
+pub struct DebuggerVisualizerFile {
+    /// The complete debugger visualizer source.
+    pub src: Lrc<[u8]>,
+    /// Indicates which visualizer type this targets.
+    pub visualizer_type: DebuggerVisualizerType,
+    /// The file path to the visualizer file. This is used for reporting
+    /// visualizer files in dep-info. Before it is written to crate metadata,
+    /// the path is erased to `None`, so as not to emit potentially privacy
+    /// sensitive data.
+    pub path: Option<PathBuf>,
+}
+
+impl DebuggerVisualizerFile {
+    pub fn new(src: Lrc<[u8]>, visualizer_type: DebuggerVisualizerType, path: PathBuf) -> Self {
+        DebuggerVisualizerFile { src, visualizer_type, path: Some(path) }
+    }
+
+    pub fn path_erased(&self) -> Self {
+        DebuggerVisualizerFile {
+            src: self.src.clone(),
+            visualizer_type: self.visualizer_type,
+            path: None,
+        }
+    }
+}
diff --git a/compiler/rustc_middle/src/middle/exported_symbols.rs b/compiler/rustc_middle/src/middle/exported_symbols.rs
index c0c0fd07b..9041da9a0 100644
--- a/compiler/rustc_middle/src/middle/exported_symbols.rs
+++ b/compiler/rustc_middle/src/middle/exported_symbols.rs
@@ -72,6 +72,6 @@ pub fn metadata_symbol_name(tcx: TyCtxt<'_>) -> String {
     format!(
         "rust_metadata_{}_{:08x}",
         tcx.crate_name(LOCAL_CRATE),
-        tcx.sess.local_stable_crate_id().to_u64(),
+        tcx.sess.local_stable_crate_id(),
     )
 }
diff --git a/compiler/rustc_middle/src/middle/lang_items.rs b/compiler/rustc_middle/src/middle/lang_items.rs
index 343ea1f00..9a633e04c 100644
--- a/compiler/rustc_middle/src/middle/lang_items.rs
+++ b/compiler/rustc_middle/src/middle/lang_items.rs
@@ -18,12 +18,8 @@ impl<'tcx> TyCtxt<'tcx> {
     /// Returns the `DefId` for a given `LangItem`.
     /// If not found, fatally aborts compilation.
     pub fn require_lang_item(self, lang_item: LangItem, span: Option<Span>) -> DefId {
-        self.lang_items().require(lang_item).unwrap_or_else(|err| {
-            if let Some(span) = span {
-                self.sess.span_fatal(span, err.to_string())
-            } else {
-                self.sess.fatal(err.to_string())
-            }
+        self.lang_items().get(lang_item).unwrap_or_else(|| {
+            self.sess.emit_fatal(crate::error::RequiresLangItem { span, name: lang_item.name() });
         })
     }
 
diff --git a/compiler/rustc_middle/src/middle/limits.rs b/compiler/rustc_middle/src/middle/limits.rs
index 12aef66bc..bd859d4d6 100644
--- a/compiler/rustc_middle/src/middle/limits.rs
+++ b/compiler/rustc_middle/src/middle/limits.rs
@@ -11,7 +11,7 @@
 
 use crate::bug;
 use crate::error::LimitInvalid;
-use crate::ty;
+use crate::query::Providers;
 use rustc_ast::Attribute;
 use rustc_session::Session;
 use rustc_session::{Limit, Limits};
@@ -19,7 +19,7 @@ use rustc_span::symbol::{sym, Symbol};
 
 use std::num::IntErrorKind;
 
-pub fn provide(providers: &mut ty::query::Providers) {
+pub fn provide(providers: &mut Providers) {
     providers.limits = |tcx, ()| Limits {
         recursion_limit: get_recursion_limit(tcx.hir().krate_attrs(), tcx.sess),
         move_size_limit: get_limit(
diff --git a/compiler/rustc_middle/src/middle/mod.rs b/compiler/rustc_middle/src/middle/mod.rs
index 9c25f3009..85c5af9ca 100644
--- a/compiler/rustc_middle/src/middle/mod.rs
+++ b/compiler/rustc_middle/src/middle/mod.rs
@@ -1,4 +1,5 @@
 pub mod codegen_fn_attrs;
+pub mod debugger_visualizer;
 pub mod dependency_format;
 pub mod exported_symbols;
 pub mod lang_items;
@@ -32,6 +33,6 @@ pub mod region;
 pub mod resolve_bound_vars;
 pub mod stability;
 
-pub fn provide(providers: &mut crate::ty::query::Providers) {
+pub fn provide(providers: &mut crate::query::Providers) {
     limits::provide(providers);
 }
diff --git a/compiler/rustc_middle/src/middle/privacy.rs b/compiler/rustc_middle/src/middle/privacy.rs
index 967fed687..f45cf788d 100644
--- a/compiler/rustc_middle/src/middle/privacy.rs
+++ b/compiler/rustc_middle/src/middle/privacy.rs
@@ -64,7 +64,7 @@ impl EffectiveVisibility {
         self.at_level(level).is_public()
     }
 
-    pub fn from_vis(vis: Visibility) -> EffectiveVisibility {
+    pub const fn from_vis(vis: Visibility) -> EffectiveVisibility {
         EffectiveVisibility {
             direct: vis,
             reexported: vis,
@@ -72,6 +72,18 @@ impl EffectiveVisibility {
             reachable_through_impl_trait: vis,
         }
     }
+
+    #[must_use]
+    pub fn min(mut self, lhs: EffectiveVisibility, tcx: TyCtxt<'_>) -> Self {
+        for l in Level::all_levels() {
+            let rhs_vis = self.at_level_mut(l);
+            let lhs_vis = *lhs.at_level(l);
+            if rhs_vis.is_at_least(lhs_vis, tcx) {
+                *rhs_vis = lhs_vis;
+            };
+        }
+        self
+    }
 }
 
 /// Holds a map of effective visibilities for reachable HIR nodes.
@@ -82,8 +94,7 @@ pub struct EffectiveVisibilities<Id = LocalDefId> {
 
 impl EffectiveVisibilities {
     pub fn is_public_at_level(&self, id: LocalDefId, level: Level) -> bool {
-        self.effective_vis(id)
-            .map_or(false, |effective_vis| effective_vis.is_public_at_level(level))
+        self.effective_vis(id).is_some_and(|effective_vis| effective_vis.is_public_at_level(level))
     }
 
     /// See `Level::Reachable`.
@@ -137,24 +148,6 @@ impl EffectiveVisibilities {
         };
     }
 
-    pub fn set_public_at_level(
-        &mut self,
-        id: LocalDefId,
-        lazy_private_vis: impl FnOnce() -> Visibility,
-        level: Level,
-    ) {
-        let mut effective_vis = self
-            .effective_vis(id)
-            .copied()
-            .unwrap_or_else(|| EffectiveVisibility::from_vis(lazy_private_vis()));
-        for l in Level::all_levels() {
-            if l <= level {
-                *effective_vis.at_level_mut(l) = Visibility::Public;
-            }
-        }
-        self.map.insert(id, effective_vis);
-    }
-
     pub fn check_invariants(&self, tcx: TyCtxt<'_>, early: bool) {
         if !cfg!(debug_assertions) {
             return;
@@ -219,7 +212,7 @@ impl<Id: Eq + Hash> EffectiveVisibilities<Id> {
     pub fn update(
         &mut self,
         id: Id,
-        nominal_vis: Visibility,
+        nominal_vis: Option<Visibility>,
         lazy_private_vis: impl FnOnce() -> Visibility,
         inherited_effective_vis: EffectiveVisibility,
         level: Level,
@@ -243,12 +236,11 @@ impl<Id: Eq + Hash> EffectiveVisibilities<Id> {
                 if !(inherited_effective_vis_at_prev_level == inherited_effective_vis_at_level
                     && level != l)
                 {
-                    calculated_effective_vis =
-                        if nominal_vis.is_at_least(inherited_effective_vis_at_level, tcx) {
-                            inherited_effective_vis_at_level
-                        } else {
-                            nominal_vis
-                        };
+                    calculated_effective_vis = if let Some(nominal_vis) = nominal_vis && !nominal_vis.is_at_least(inherited_effective_vis_at_level, tcx) {
+                        nominal_vis
+                    } else {
+                        inherited_effective_vis_at_level
+                    }
                 }
                 // effective visibility can't be decreased at next update call for the
                 // same id
diff --git a/compiler/rustc_middle/src/middle/region.rs b/compiler/rustc_middle/src/middle/region.rs
index 94ca38c0e..10712e146 100644
--- a/compiler/rustc_middle/src/middle/region.rs
+++ b/compiler/rustc_middle/src/middle/region.rs
@@ -203,7 +203,7 @@ impl Scope {
 pub type ScopeDepth = u32;
 
 /// The region scope tree encodes information about region relationships.
-#[derive(TyEncodable, TyDecodable, Default, Debug)]
+#[derive(Default, Debug)]
 pub struct ScopeTree {
     /// If not empty, this body is the root of this region hierarchy.
     pub root_body: Option<hir::HirId>,
@@ -317,13 +317,13 @@ pub struct ScopeTree {
 /// candidates in general). In constants, the `lifetime` field is None
 /// to indicate that certain expressions escape into 'static and
 /// should have no local cleanup scope.
-#[derive(Debug, Copy, Clone, TyEncodable, TyDecodable, HashStable)]
+#[derive(Debug, Copy, Clone, HashStable)]
 pub enum RvalueCandidateType {
     Borrow { target: hir::ItemLocalId, lifetime: Option<Scope> },
     Pattern { target: hir::ItemLocalId, lifetime: Option<Scope> },
 }
 
-#[derive(Debug, Copy, Clone, TyEncodable, TyDecodable, HashStable)]
+#[derive(Debug, Copy, Clone, HashStable)]
 pub struct YieldData {
     /// The `Span` of the yield.
     pub span: Span,
diff --git a/compiler/rustc_middle/src/middle/stability.rs b/compiler/rustc_middle/src/middle/stability.rs
index b61f7806b..6354c0aab 100644
--- a/compiler/rustc_middle/src/middle/stability.rs
+++ b/compiler/rustc_middle/src/middle/stability.rs
@@ -115,8 +115,8 @@ pub fn report_unstable(
         soft_handler(SOFT_UNSTABLE, span, &msg)
     } else {
         let mut err =
-            feature_err_issue(&sess.parse_sess, feature, span, GateIssue::Library(issue), &msg);
-        if let Some((inner_types, ref msg, sugg, applicability)) = suggestion {
+            feature_err_issue(&sess.parse_sess, feature, span, GateIssue::Library(issue), msg);
+        if let Some((inner_types, msg, sugg, applicability)) = suggestion {
             err.span_suggestion(inner_types, msg, sugg, applicability);
         }
         err.emit();
@@ -170,7 +170,7 @@ pub fn deprecation_suggestion(
     if let Some(suggestion) = suggestion {
         diag.span_suggestion_verbose(
             span,
-            &format!("replace the use of the deprecated {}", kind),
+            format!("replace the use of the deprecated {}", kind),
             suggestion,
             Applicability::MachineApplicable,
         );
@@ -375,7 +375,7 @@ impl<'tcx> TyCtxt<'tcx> {
                 let parent_def_id = self.hir().get_parent_item(id);
                 let skip = self
                     .lookup_deprecation_entry(parent_def_id.to_def_id())
-                    .map_or(false, |parent_depr| parent_depr.same_origin(&depr_entry));
+                    .is_some_and(|parent_depr| parent_depr.same_origin(&depr_entry));
 
                 // #[deprecated] doesn't emit a notice if we're not on the
                 // topmost deprecation. For example, if a struct is deprecated,
@@ -599,7 +599,7 @@ impl<'tcx> TyCtxt<'tcx> {
             |span, def_id| {
                 // The API could be uncallable for other reasons, for example when a private module
                 // was referenced.
-                self.sess.delay_span_bug(span, &format!("encountered unmarked API: {:?}", def_id));
+                self.sess.delay_span_bug(span, format!("encountered unmarked API: {:?}", def_id));
             },
         )
     }
diff --git a/compiler/rustc_middle/src/mir/basic_blocks.rs b/compiler/rustc_middle/src/mir/basic_blocks.rs
index 3fb468379..9d70dbfa0 100644
--- a/compiler/rustc_middle/src/mir/basic_blocks.rs
+++ b/compiler/rustc_middle/src/mir/basic_blocks.rs
@@ -6,7 +6,7 @@ use rustc_data_structures::graph;
 use rustc_data_structures::graph::dominators::{dominators, Dominators};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::sync::OnceCell;
-use rustc_index::vec::{IndexSlice, IndexVec};
+use rustc_index::{IndexSlice, IndexVec};
 use rustc_serialize::{Decodable, Decoder, Encodable, Encoder};
 use smallvec::SmallVec;
 
@@ -27,6 +27,7 @@ struct Cache {
     switch_sources: OnceCell<SwitchSources>,
     is_cyclic: OnceCell<bool>,
     postorder: OnceCell<Vec<BasicBlock>>,
+    dominators: OnceCell<Dominators<BasicBlock>>,
 }
 
 impl<'tcx> BasicBlocks<'tcx> {
@@ -41,8 +42,8 @@ impl<'tcx> BasicBlocks<'tcx> {
         *self.cache.is_cyclic.get_or_init(|| graph::is_cyclic(self))
     }
 
-    pub fn dominators(&self) -> Dominators<BasicBlock> {
-        dominators(&self)
+    pub fn dominators(&self) -> &Dominators<BasicBlock> {
+        self.cache.dominators.get_or_init(|| dominators(self))
     }
 
     /// Returns predecessors for each basic block.
diff --git a/compiler/rustc_middle/src/mir/graphviz.rs b/compiler/rustc_middle/src/mir/graphviz.rs
index cf6d46e1e..2de73db3a 100644
--- a/compiler/rustc_middle/src/mir/graphviz.rs
+++ b/compiler/rustc_middle/src/mir/graphviz.rs
@@ -16,19 +16,16 @@ where
 {
     let def_ids = dump_mir_def_ids(tcx, single);
 
-    let mirs =
-        def_ids
-            .iter()
-            .flat_map(|def_id| {
-                if tcx.is_const_fn_raw(*def_id) {
-                    vec![tcx.optimized_mir(*def_id), tcx.mir_for_ctfe(*def_id)]
-                } else {
-                    vec![tcx.instance_mir(ty::InstanceDef::Item(ty::WithOptConstParam::unknown(
-                        *def_id,
-                    )))]
-                }
-            })
-            .collect::<Vec<_>>();
+    let mirs = def_ids
+        .iter()
+        .flat_map(|def_id| {
+            if tcx.is_const_fn_raw(*def_id) {
+                vec![tcx.optimized_mir(*def_id), tcx.mir_for_ctfe(*def_id)]
+            } else {
+                vec![tcx.instance_mir(ty::InstanceDef::Item(*def_id))]
+            }
+        })
+        .collect::<Vec<_>>();
 
     let use_subgraphs = mirs.len() > 1;
     if use_subgraphs {
diff --git a/compiler/rustc_middle/src/mir/interpret/allocation/init_mask.rs b/compiler/rustc_middle/src/mir/interpret/allocation/init_mask.rs
index dcb56a175..d4dd56a42 100644
--- a/compiler/rustc_middle/src/mir/interpret/allocation/init_mask.rs
+++ b/compiler/rustc_middle/src/mir/interpret/allocation/init_mask.rs
@@ -5,7 +5,9 @@ use std::hash;
 use std::iter;
 use std::ops::Range;
 
+use rustc_serialize::{Decodable, Encodable};
 use rustc_target::abi::Size;
+use rustc_type_ir::{TyDecoder, TyEncoder};
 
 use super::AllocRange;
 
@@ -182,11 +184,39 @@ impl InitMask {
 /// The actual materialized blocks of the bitmask, when we can't keep the `InitMask` lazy.
 // Note: for performance reasons when interning, some of the fields can be partially
 // hashed. (see the `Hash` impl below for more details), so the impl is not derived.
-#[derive(Clone, Debug, Eq, PartialEq, TyEncodable, TyDecodable, HashStable)]
+#[derive(Clone, Debug, Eq, PartialEq, HashStable)]
 struct InitMaskMaterialized {
     blocks: Vec<Block>,
 }
 
+// `Block` is a `u64`, but it is a bitmask not a numeric value. If we were to just derive
+// Encodable and Decodable we would apply varint encoding to the bitmasks, which is slower
+// and also produces more output when the high bits of each `u64` are occupied.
+// Note: There is probably a remaining optimization for masks that do not use an entire
+// `Block`.
+impl<E: TyEncoder> Encodable<E> for InitMaskMaterialized {
+    fn encode(&self, encoder: &mut E) {
+        encoder.emit_usize(self.blocks.len());
+        for block in &self.blocks {
+            encoder.emit_raw_bytes(&block.to_le_bytes());
+        }
+    }
+}
+
+// This implementation is deliberately not derived, see the matching `Encodable` impl.
+impl<D: TyDecoder> Decodable<D> for InitMaskMaterialized {
+    fn decode(decoder: &mut D) -> Self {
+        let num_blocks = decoder.read_usize();
+        let mut blocks = Vec::with_capacity(num_blocks);
+        for _ in 0..num_blocks {
+            let bytes = decoder.read_raw_bytes(8);
+            let block = u64::from_le_bytes(bytes.try_into().unwrap());
+            blocks.push(block);
+        }
+        InitMaskMaterialized { blocks }
+    }
+}
+
 // Const allocations are only hashed for interning. However, they can be large, making the hashing
 // expensive especially since it uses `FxHash`: it's better suited to short keys, not potentially
 // big buffers like the allocation's init mask. We can partially hash some fields when they're
diff --git a/compiler/rustc_middle/src/mir/interpret/error.rs b/compiler/rustc_middle/src/mir/interpret/error.rs
index c5137cf06..055d8e9a3 100644
--- a/compiler/rustc_middle/src/mir/interpret/error.rs
+++ b/compiler/rustc_middle/src/mir/interpret/error.rs
@@ -1,7 +1,8 @@
 use super::{AllocId, AllocRange, ConstAlloc, Pointer, Scalar};
 
 use crate::mir::interpret::ConstValue;
-use crate::ty::{layout, query::TyCtxtAt, tls, Ty, ValTree};
+use crate::query::TyCtxtAt;
+use crate::ty::{layout, tls, Ty, ValTree};
 
 use rustc_data_structures::sync::Lock;
 use rustc_errors::{pluralize, struct_span_err, DiagnosticBuilder, ErrorGuaranteed};
@@ -15,15 +16,49 @@ use std::{any::Any, backtrace::Backtrace, fmt};
 pub enum ErrorHandled {
     /// Already reported an error for this evaluation, and the compilation is
     /// *guaranteed* to fail. Warnings/lints *must not* produce `Reported`.
-    Reported(ErrorGuaranteed),
+    Reported(ReportedErrorInfo),
     /// Don't emit an error, the evaluation failed because the MIR was generic
     /// and the substs didn't fully monomorphize it.
     TooGeneric,
 }
 
 impl From<ErrorGuaranteed> for ErrorHandled {
-    fn from(err: ErrorGuaranteed) -> ErrorHandled {
-        ErrorHandled::Reported(err)
+    #[inline]
+    fn from(error: ErrorGuaranteed) -> ErrorHandled {
+        ErrorHandled::Reported(error.into())
+    }
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, HashStable, TyEncodable, TyDecodable)]
+pub struct ReportedErrorInfo {
+    error: ErrorGuaranteed,
+    is_tainted_by_errors: bool,
+}
+
+impl ReportedErrorInfo {
+    #[inline]
+    pub fn tainted_by_errors(error: ErrorGuaranteed) -> ReportedErrorInfo {
+        ReportedErrorInfo { is_tainted_by_errors: true, error }
+    }
+
+    /// Returns true if evaluation failed because MIR was tainted by errors.
+    #[inline]
+    pub fn is_tainted_by_errors(self) -> bool {
+        self.is_tainted_by_errors
+    }
+}
+
+impl From<ErrorGuaranteed> for ReportedErrorInfo {
+    #[inline]
+    fn from(error: ErrorGuaranteed) -> ReportedErrorInfo {
+        ReportedErrorInfo { is_tainted_by_errors: false, error }
+    }
+}
+
+impl Into<ErrorGuaranteed> for ReportedErrorInfo {
+    #[inline]
+    fn into(self) -> ErrorGuaranteed {
+        self.error
     }
 }
 
@@ -89,7 +124,7 @@ fn print_backtrace(backtrace: &Backtrace) {
 
 impl From<ErrorGuaranteed> for InterpErrorInfo<'_> {
     fn from(err: ErrorGuaranteed) -> Self {
-        InterpError::InvalidProgram(InvalidProgramInfo::AlreadyReported(err)).into()
+        InterpError::InvalidProgram(InvalidProgramInfo::AlreadyReported(err.into())).into()
     }
 }
 
@@ -125,7 +160,7 @@ pub enum InvalidProgramInfo<'tcx> {
     /// Resolution can fail if we are in a too generic context.
     TooGeneric,
     /// Abort in case errors are already reported.
-    AlreadyReported(ErrorGuaranteed),
+    AlreadyReported(ReportedErrorInfo),
     /// An error occurred during layout computation.
     Layout(layout::LayoutError<'tcx>),
     /// An error occurred during FnAbi computation: the passed --target lacks FFI support
@@ -134,6 +169,9 @@ pub enum InvalidProgramInfo<'tcx> {
     FnAbiAdjustForForeignAbi(call::AdjustForForeignAbiError),
     /// SizeOf of unsized type was requested.
     SizeOfUnsizedType(Ty<'tcx>),
+    /// An unsized local was accessed without having been initialized.
+    /// This is not meaningful as we can't even have backing memory for such locals.
+    UninitUnsizedLocal,
 }
 
 impl fmt::Display for InvalidProgramInfo<'_> {
@@ -141,7 +179,7 @@ impl fmt::Display for InvalidProgramInfo<'_> {
         use InvalidProgramInfo::*;
         match self {
             TooGeneric => write!(f, "encountered overly generic constant"),
-            AlreadyReported(ErrorGuaranteed { .. }) => {
+            AlreadyReported(_) => {
                 write!(
                     f,
                     "an error has already been reported elsewhere (this should not usually be printed)"
@@ -150,6 +188,7 @@ impl fmt::Display for InvalidProgramInfo<'_> {
             Layout(ref err) => write!(f, "{err}"),
             FnAbiAdjustForForeignAbi(ref err) => write!(f, "{err}"),
             SizeOfUnsizedType(ty) => write!(f, "size_of called on unsized type `{ty}`"),
+            UninitUnsizedLocal => write!(f, "unsized local is used while uninitialized"),
         }
     }
 }
diff --git a/compiler/rustc_middle/src/mir/interpret/mod.rs b/compiler/rustc_middle/src/mir/interpret/mod.rs
index 1f8b650e3..3620385fa 100644
--- a/compiler/rustc_middle/src/mir/interpret/mod.rs
+++ b/compiler/rustc_middle/src/mir/interpret/mod.rs
@@ -120,8 +120,8 @@ use crate::ty::{self, Instance, Ty, TyCtxt};
 pub use self::error::{
     struct_error, CheckInAllocMsg, ErrorHandled, EvalToAllocationRawResult, EvalToConstValueResult,
     EvalToValTreeResult, InterpError, InterpErrorInfo, InterpResult, InvalidProgramInfo,
-    MachineStopType, ResourceExhaustionInfo, ScalarSizeMismatch, UndefinedBehaviorInfo,
-    UninitBytesAccess, UnsupportedOpInfo,
+    MachineStopType, ReportedErrorInfo, ResourceExhaustionInfo, ScalarSizeMismatch,
+    UndefinedBehaviorInfo, UninitBytesAccess, UnsupportedOpInfo,
 };
 
 pub use self::value::{get_slice_bytes, ConstAlloc, ConstValue, Scalar};
@@ -227,7 +227,9 @@ pub fn specialized_encode_alloc_id<'tcx, E: TyEncoder<I = TyCtxt<'tcx>>>(
             // References to statics doesn't need to know about their allocations,
             // just about its `DefId`.
             AllocDiscriminant::Static.encode(encoder);
-            did.encode(encoder);
+            // Cannot use `did.encode(encoder)` because of a bug around
+            // specializations and method calls.
+            Encodable::<E>::encode(&did, encoder);
         }
     }
 }
diff --git a/compiler/rustc_middle/src/mir/interpret/pointer.rs b/compiler/rustc_middle/src/mir/interpret/pointer.rs
index 60927eed8..65d049193 100644
--- a/compiler/rustc_middle/src/mir/interpret/pointer.rs
+++ b/compiler/rustc_middle/src/mir/interpret/pointer.rs
@@ -102,7 +102,7 @@ impl<T: HasDataLayout> PointerArithmetic for T {}
 /// This trait abstracts over the kind of provenance that is associated with a `Pointer`. It is
 /// mostly opaque; the `Machine` trait extends it with some more operations that also have access to
 /// some global state.
-/// The `Debug` rendering is used to distplay bare provenance, and for the default impl of `fmt`.
+/// The `Debug` rendering is used to display bare provenance, and for the default impl of `fmt`.
 pub trait Provenance: Copy + fmt::Debug {
     /// Says whether the `offset` field of `Pointer`s with this provenance is the actual physical address.
     /// - If `false`, the offset *must* be relative. This means the bytes representing a pointer are
diff --git a/compiler/rustc_middle/src/mir/interpret/queries.rs b/compiler/rustc_middle/src/mir/interpret/queries.rs
index 856d821a5..f53dc8cb0 100644
--- a/compiler/rustc_middle/src/mir/interpret/queries.rs
+++ b/compiler/rustc_middle/src/mir/interpret/queries.rs
@@ -1,9 +1,10 @@
 use super::{ErrorHandled, EvalToConstValueResult, EvalToValTreeResult, GlobalId};
 
 use crate::mir;
+use crate::query::{TyCtxtAt, TyCtxtEnsure};
 use crate::ty::subst::InternalSubsts;
 use crate::ty::visit::TypeVisitableExt;
-use crate::ty::{self, query::TyCtxtAt, query::TyCtxtEnsure, TyCtxt};
+use crate::ty::{self, TyCtxt};
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::DefId;
 use rustc_session::lint;
@@ -42,7 +43,7 @@ impl<'tcx> TyCtxt<'tcx> {
         span: Option<Span>,
     ) -> EvalToConstValueResult<'tcx> {
         // Cannot resolve `Unevaluated` constants that contain inference
-        // variables. We reject those here since `resolve_opt_const_arg`
+        // variables. We reject those here since `resolve`
         // would fail otherwise.
         //
         // When trying to evaluate constants containing inference variables,
@@ -51,7 +52,7 @@ impl<'tcx> TyCtxt<'tcx> {
             bug!("did not expect inference variables here");
         }
 
-        match ty::Instance::resolve_opt_const_arg(
+        match ty::Instance::resolve(
             self, param_env,
             // FIXME: maybe have a separate version for resolving mir::UnevaluatedConst?
             ct.def, ct.substs,
@@ -61,7 +62,7 @@ impl<'tcx> TyCtxt<'tcx> {
                 self.const_eval_global_id(param_env, cid, span)
             }
             Ok(None) => Err(ErrorHandled::TooGeneric),
-            Err(error_reported) => Err(ErrorHandled::Reported(error_reported)),
+            Err(err) => Err(ErrorHandled::Reported(err.into())),
         }
     }
 
@@ -73,7 +74,7 @@ impl<'tcx> TyCtxt<'tcx> {
         span: Option<Span>,
     ) -> EvalToValTreeResult<'tcx> {
         // Cannot resolve `Unevaluated` constants that contain inference
-        // variables. We reject those here since `resolve_opt_const_arg`
+        // variables. We reject those here since `resolve`
         // would fail otherwise.
         //
         // When trying to evaluate constants containing inference variables,
@@ -82,7 +83,7 @@ impl<'tcx> TyCtxt<'tcx> {
             bug!("did not expect inference variables here");
         }
 
-        match ty::Instance::resolve_opt_const_arg(self, param_env, ct.def, ct.substs) {
+        match ty::Instance::resolve(self, param_env, ct.def, ct.substs) {
             Ok(Some(instance)) => {
                 let cid = GlobalId { instance, promoted: None };
                 self.const_eval_global_id_for_typeck(param_env, cid, span).inspect(|_| {
@@ -94,14 +95,14 @@ impl<'tcx> TyCtxt<'tcx> {
                     // used generic parameters is a bug of evaluation, so checking for it
                     // here does feel somewhat sensible.
                     if !self.features().generic_const_exprs && ct.substs.has_non_region_param() {
-                        assert!(matches!(self.def_kind(ct.def.did), DefKind::AnonConst));
-                        let mir_body = self.mir_for_ctfe_opt_const_arg(ct.def);
+                        assert!(matches!(self.def_kind(ct.def), DefKind::AnonConst));
+                        let mir_body = self.mir_for_ctfe(ct.def);
                         if mir_body.is_polymorphic {
-                            let Some(local_def_id) = ct.def.did.as_local() else { return };
+                            let Some(local_def_id) = ct.def.as_local() else { return };
                             self.struct_span_lint_hir(
                                 lint::builtin::CONST_EVALUATABLE_UNCHECKED,
                                 self.hir().local_def_id_to_hir_id(local_def_id),
-                                self.def_span(ct.def.did),
+                                self.def_span(ct.def),
                                 "cannot use constants which depend on generic parameters in types",
                                 |err| err,
                             )
@@ -110,7 +111,7 @@ impl<'tcx> TyCtxt<'tcx> {
                 })
             }
             Ok(None) => Err(ErrorHandled::TooGeneric),
-            Err(error_reported) => Err(ErrorHandled::Reported(error_reported)),
+            Err(err) => Err(ErrorHandled::Reported(err.into())),
         }
     }
 
diff --git a/compiler/rustc_middle/src/mir/mod.rs b/compiler/rustc_middle/src/mir/mod.rs
index 2ea8602af..5c71910a9 100644
--- a/compiler/rustc_middle/src/mir/mod.rs
+++ b/compiler/rustc_middle/src/mir/mod.rs
@@ -9,7 +9,7 @@ use crate::mir::visit::MirVisitable;
 use crate::ty::codec::{TyDecoder, TyEncoder};
 use crate::ty::fold::{FallibleTypeFolder, TypeFoldable};
 use crate::ty::print::{FmtPrinter, Printer};
-use crate::ty::visit::{TypeVisitable, TypeVisitableExt, TypeVisitor};
+use crate::ty::visit::TypeVisitableExt;
 use crate::ty::{self, List, Ty, TyCtxt};
 use crate::ty::{AdtDef, InstanceDef, ScalarInt, UserTypeAnnotationIndex};
 use crate::ty::{GenericArg, InternalSubsts, SubstsRef};
@@ -27,7 +27,7 @@ use polonius_engine::Atom;
 pub use rustc_ast::Mutability;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_data_structures::graph::dominators::Dominators;
-use rustc_index::vec::{Idx, IndexSlice, IndexVec};
+use rustc_index::{Idx, IndexSlice, IndexVec};
 use rustc_serialize::{Decodable, Encodable};
 use rustc_span::symbol::Symbol;
 use rustc_span::{Span, DUMMY_SP};
@@ -36,7 +36,7 @@ use either::Either;
 
 use std::borrow::Cow;
 use std::fmt::{self, Debug, Display, Formatter, Write};
-use std::ops::{ControlFlow, Index, IndexMut};
+use std::ops::{Index, IndexMut};
 use std::{iter, mem};
 
 pub use self::query::*;
@@ -101,7 +101,7 @@ impl<'tcx> HasLocalDecls<'tcx> for Body<'tcx> {
 /// pass will be named after the type, and it will consist of a main
 /// loop that goes over each available MIR and applies `run_pass`.
 pub trait MirPass<'tcx> {
-    fn name(&self) -> &str {
+    fn name(&self) -> &'static str {
         let name = std::any::type_name::<Self>();
         if let Some((_, tail)) = name.rsplit_once(':') { tail } else { name }
     }
@@ -191,20 +191,13 @@ pub struct MirSource<'tcx> {
 
 impl<'tcx> MirSource<'tcx> {
     pub fn item(def_id: DefId) -> Self {
-        MirSource {
-            instance: InstanceDef::Item(ty::WithOptConstParam::unknown(def_id)),
-            promoted: None,
-        }
+        MirSource { instance: InstanceDef::Item(def_id), promoted: None }
     }
 
     pub fn from_instance(instance: InstanceDef<'tcx>) -> Self {
         MirSource { instance, promoted: None }
     }
 
-    pub fn with_opt_param(self) -> ty::WithOptConstParam<DefId> {
-        self.instance.with_opt_param()
-    }
-
     #[inline]
     pub fn def_id(&self) -> DefId {
         self.instance.def_id()
@@ -714,9 +707,7 @@ pub enum BindingForm<'tcx> {
 }
 
 TrivialTypeTraversalAndLiftImpls! {
-    for<'tcx> {
-        BindingForm<'tcx>,
-    }
+    BindingForm<'tcx>,
 }
 
 mod binding_form_impl {
@@ -1120,6 +1111,10 @@ pub struct VarDebugInfo<'tcx> {
     /// originated from (starting from 1). Note, if MIR inlining is enabled, then this is the
     /// argument number in the original function before it was inlined.
     pub argument_index: Option<u16>,
+
+    /// The data represents `name` dereferenced `references` times,
+    /// and not the direct value.
+    pub references: u8,
 }
 
 ///////////////////////////////////////////////////////////////////////////
@@ -1533,6 +1528,19 @@ impl<V, T> ProjectionElem<V, T> {
         }
     }
 
+    /// Returns `true` if the target of this projection always refers to the same memory region
+    /// whatever the state of the program.
+    pub fn is_stable_offset(&self) -> bool {
+        match self {
+            Self::Deref | Self::Index(_) => false,
+            Self::Field(_, _)
+            | Self::OpaqueCast(_)
+            | Self::ConstantIndex { .. }
+            | Self::Subslice { .. }
+            | Self::Downcast(_, _) => true,
+        }
+    }
+
     /// Returns `true` if this is a `Downcast` projection with the given `VariantIdx`.
     pub fn is_downcast_to(&self, v: VariantIdx) -> bool {
         matches!(*self, Self::Downcast(_, x) if x == v)
@@ -1546,8 +1554,11 @@ impl<V, T> ProjectionElem<V, T> {
     /// Returns `true` if this is accepted inside `VarDebugInfoContents::Place`.
     pub fn can_use_in_debuginfo(&self) -> bool {
         match self {
-            Self::Deref | Self::Downcast(_, _) | Self::Field(_, _) => true,
-            Self::ConstantIndex { .. }
+            Self::ConstantIndex { from_end: false, .. }
+            | Self::Deref
+            | Self::Downcast(_, _)
+            | Self::Field(_, _) => true,
+            Self::ConstantIndex { from_end: true, .. }
             | Self::Index(_)
             | Self::OpaqueCast(_)
             | Self::Subslice { .. } => false,
@@ -1635,18 +1646,7 @@ impl<'tcx> Place<'tcx> {
             return self;
         }
 
-        let mut v: Vec<PlaceElem<'tcx>>;
-
-        let new_projections = if self.projection.is_empty() {
-            more_projections
-        } else {
-            v = Vec::with_capacity(self.projection.len() + more_projections.len());
-            v.extend(self.projection);
-            v.extend(more_projections);
-            &v
-        };
-
-        Place { local: self.local, projection: tcx.mk_place_elems(new_projections) }
+        self.as_ref().project_deeper(more_projections, tcx)
     }
 }
 
@@ -1717,6 +1717,27 @@ impl<'tcx> PlaceRef<'tcx> {
             (base, *proj)
         })
     }
+
+    /// Generates a new place by appending `more_projections` to the existing ones
+    /// and interning the result.
+    pub fn project_deeper(
+        self,
+        more_projections: &[PlaceElem<'tcx>],
+        tcx: TyCtxt<'tcx>,
+    ) -> Place<'tcx> {
+        let mut v: Vec<PlaceElem<'tcx>>;
+
+        let new_projections = if self.projection.is_empty() {
+            more_projections
+        } else {
+            v = Vec::with_capacity(self.projection.len() + more_projections.len());
+            v.extend(self.projection);
+            v.extend(more_projections);
+            &v
+        };
+
+        Place { local: self.local, projection: tcx.mk_place_elems(new_projections) }
+    }
 }
 
 impl Debug for Place<'_> {
@@ -2050,7 +2071,11 @@ impl<'tcx> Debug for Rvalue<'tcx> {
             }
             UnaryOp(ref op, ref a) => write!(fmt, "{:?}({:?})", op, a),
             Discriminant(ref place) => write!(fmt, "discriminant({:?})", place),
-            NullaryOp(ref op, ref t) => write!(fmt, "{:?}({:?})", op, t),
+            NullaryOp(ref op, ref t) => match op {
+                NullOp::SizeOf => write!(fmt, "SizeOf({:?})", t),
+                NullOp::AlignOf => write!(fmt, "AlignOf({:?})", t),
+                NullOp::OffsetOf(fields) => write!(fmt, "OffsetOf({:?}, {:?})", t, fields),
+            },
             ThreadLocalRef(did) => ty::tls::with(|tcx| {
                 let muta = tcx.static_mutability(did).unwrap().prefix_str();
                 write!(fmt, "&/*tls*/ {}{}", muta, tcx.def_path_str(did))
@@ -2305,7 +2330,7 @@ impl<'tcx> ConstantKind<'tcx> {
                 if let Some(val) = c.kind().try_eval_for_mir(tcx, param_env) {
                     match val {
                         Ok(val) => Self::Val(val, c.ty()),
-                        Err(_) => Self::Ty(tcx.const_error(self.ty())),
+                        Err(guar) => Self::Ty(tcx.const_error(self.ty(), guar)),
                     }
                 } else {
                     self
@@ -2317,9 +2342,7 @@ impl<'tcx> ConstantKind<'tcx> {
                 match tcx.const_eval_resolve(param_env, uneval, None) {
                     Ok(val) => Self::Val(val, ty),
                     Err(ErrorHandled::TooGeneric) => self,
-                    Err(ErrorHandled::Reported(guar)) => {
-                        Self::Ty(tcx.const_error_with_guaranteed(ty, guar))
-                    }
+                    Err(ErrorHandled::Reported(guar)) => Self::Ty(tcx.const_error(ty, guar.into())),
                 }
             }
         }
@@ -2438,16 +2461,6 @@ impl<'tcx> ConstantKind<'tcx> {
         Self::Val(val, ty)
     }
 
-    /// Literals are converted to `ConstantKindVal`, const generic parameters are eagerly
-    /// converted to a constant, everything else becomes `Unevaluated`.
-    pub fn from_anon_const(
-        tcx: TyCtxt<'tcx>,
-        def_id: LocalDefId,
-        param_env: ty::ParamEnv<'tcx>,
-    ) -> Self {
-        Self::from_opt_const_arg_anon_const(tcx, ty::WithOptConstParam::unknown(def_id), param_env)
-    }
-
     #[instrument(skip(tcx), level = "debug", ret)]
     pub fn from_inline_const(tcx: TyCtxt<'tcx>, def_id: LocalDefId) -> Self {
         let hir_id = tcx.hir().local_def_id_to_hir_id(def_id);
@@ -2487,28 +2500,25 @@ impl<'tcx> ConstantKind<'tcx> {
             ty::InlineConstSubsts::new(tcx, ty::InlineConstSubstsParts { parent_substs, ty })
                 .substs;
 
-        let uneval = UnevaluatedConst {
-            def: ty::WithOptConstParam::unknown(def_id).to_global(),
-            substs,
-            promoted: None,
-        };
+        let uneval = UnevaluatedConst { def: def_id.to_def_id(), substs, promoted: None };
         debug_assert!(!uneval.has_free_regions());
 
         Self::Unevaluated(uneval, ty)
     }
 
+    /// Literals are converted to `ConstantKindVal`, const generic parameters are eagerly
+    /// converted to a constant, everything else becomes `Unevaluated`.
     #[instrument(skip(tcx), level = "debug", ret)]
-    fn from_opt_const_arg_anon_const(
+    pub fn from_anon_const(
         tcx: TyCtxt<'tcx>,
-        def: ty::WithOptConstParam<LocalDefId>,
+        def: LocalDefId,
         param_env: ty::ParamEnv<'tcx>,
     ) -> Self {
-        let body_id = match tcx.hir().get_by_def_id(def.did) {
+        let body_id = match tcx.hir().get_by_def_id(def) {
             hir::Node::AnonConst(ac) => ac.body,
-            _ => span_bug!(
-                tcx.def_span(def.did.to_def_id()),
-                "from_anon_const can only process anonymous constants"
-            ),
+            _ => {
+                span_bug!(tcx.def_span(def), "from_anon_const can only process anonymous constants")
+            }
         };
 
         let expr = &tcx.hir().body(body_id).value;
@@ -2524,7 +2534,7 @@ impl<'tcx> ConstantKind<'tcx> {
         };
         debug!("expr.kind: {:?}", expr.kind);
 
-        let ty = tcx.type_of(def.def_id_for_type_of()).subst_identity();
+        let ty = tcx.type_of(def).subst_identity();
         debug!(?ty);
 
         // FIXME(const_generics): We currently have to special case parameters because `min_const_generics`
@@ -2551,7 +2561,7 @@ impl<'tcx> ConstantKind<'tcx> {
             _ => {}
         }
 
-        let hir_id = tcx.hir().local_def_id_to_hir_id(def.did);
+        let hir_id = tcx.hir().local_def_id_to_hir_id(def);
         let parent_substs = if let Some(parent_hir_id) = tcx.hir().opt_parent_id(hir_id)
             && let Some(parent_did) = parent_hir_id.as_owner()
         {
@@ -2561,15 +2571,14 @@ impl<'tcx> ConstantKind<'tcx> {
         };
         debug!(?parent_substs);
 
-        let did = def.did.to_def_id();
+        let did = def.to_def_id();
         let child_substs = InternalSubsts::identity_for_item(tcx, did);
         let substs =
             tcx.mk_substs_from_iter(parent_substs.into_iter().chain(child_substs.into_iter()));
         debug!(?substs);
 
-        let hir_id = tcx.hir().local_def_id_to_hir_id(def.did);
-        let span = tcx.hir().span(hir_id);
-        let uneval = UnevaluatedConst::new(def.to_global(), substs);
+        let span = tcx.def_span(def);
+        let uneval = UnevaluatedConst::new(did, substs);
         debug!(?span, ?param_env);
 
         match tcx.const_eval_resolve(param_env, uneval, Some(span)) {
@@ -2583,8 +2592,8 @@ impl<'tcx> ConstantKind<'tcx> {
                 // new unevaluated const and error hard later in codegen
                 Self::Unevaluated(
                     UnevaluatedConst {
-                        def: def.to_global(),
-                        substs: InternalSubsts::identity_for_item(tcx, def.did),
+                        def: did,
+                        substs: InternalSubsts::identity_for_item(tcx, did),
                         promoted: None,
                     },
                     ty,
@@ -2609,7 +2618,7 @@ impl<'tcx> ConstantKind<'tcx> {
 #[derive(Copy, Clone, Debug, Eq, PartialEq, PartialOrd, Ord, TyEncodable, TyDecodable, Lift)]
 #[derive(Hash, HashStable, TypeFoldable, TypeVisitable)]
 pub struct UnevaluatedConst<'tcx> {
-    pub def: ty::WithOptConstParam<DefId>,
+    pub def: DefId,
     pub substs: SubstsRef<'tcx>,
     pub promoted: Option<Promoted>,
 }
@@ -2625,10 +2634,7 @@ impl<'tcx> UnevaluatedConst<'tcx> {
 
 impl<'tcx> UnevaluatedConst<'tcx> {
     #[inline]
-    pub fn new(
-        def: ty::WithOptConstParam<DefId>,
-        substs: SubstsRef<'tcx>,
-    ) -> UnevaluatedConst<'tcx> {
+    pub fn new(def: DefId, substs: SubstsRef<'tcx>) -> UnevaluatedConst<'tcx> {
         UnevaluatedConst { def, substs, promoted: Default::default() }
     }
 }
@@ -2744,13 +2750,12 @@ impl<'tcx> UserTypeProjections {
 ///   `field[0]` (aka `.0`), indicating that the type of `s` is
 ///   determined by finding the type of the `.0` field from `T`.
 #[derive(Clone, Debug, TyEncodable, TyDecodable, Hash, HashStable, PartialEq)]
+#[derive(TypeFoldable, TypeVisitable)]
 pub struct UserTypeProjection {
     pub base: UserTypeAnnotationIndex,
     pub projs: Vec<ProjectionKind>,
 }
 
-impl Copy for ProjectionKind {}
-
 impl UserTypeProjection {
     pub(crate) fn index(mut self) -> Self {
         self.projs.push(ProjectionElem::Index(()));
@@ -2787,28 +2792,6 @@ impl UserTypeProjection {
     }
 }
 
-impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for UserTypeProjection {
-    fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
-        self,
-        folder: &mut F,
-    ) -> Result<Self, F::Error> {
-        Ok(UserTypeProjection {
-            base: self.base.try_fold_with(folder)?,
-            projs: self.projs.try_fold_with(folder)?,
-        })
-    }
-}
-
-impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for UserTypeProjection {
-    fn visit_with<Vs: TypeVisitor<TyCtxt<'tcx>>>(
-        &self,
-        visitor: &mut Vs,
-    ) -> ControlFlow<Vs::BreakTy> {
-        self.base.visit_with(visitor)
-        // Note: there's nothing in `self.proj` to visit.
-    }
-}
-
 rustc_index::newtype_index! {
     #[derive(HashStable)]
     #[debug_format = "promoted[{}]"]
@@ -3116,11 +3099,13 @@ mod size_asserts {
     use super::*;
     use rustc_data_structures::static_assert_size;
     // tidy-alphabetical-start
-    static_assert_size!(BasicBlockData<'_>, 144);
+    static_assert_size!(BasicBlockData<'_>, 136);
     static_assert_size!(LocalDecl<'_>, 40);
+    static_assert_size!(SourceScopeData<'_>, 72);
     static_assert_size!(Statement<'_>, 32);
     static_assert_size!(StatementKind<'_>, 16);
-    static_assert_size!(Terminator<'_>, 112);
-    static_assert_size!(TerminatorKind<'_>, 96);
+    static_assert_size!(Terminator<'_>, 104);
+    static_assert_size!(TerminatorKind<'_>, 88);
+    static_assert_size!(VarDebugInfo<'_>, 80);
     // tidy-alphabetical-end
 }
diff --git a/compiler/rustc_middle/src/mir/mono.rs b/compiler/rustc_middle/src/mir/mono.rs
index f592f1515..f31b343c9 100644
--- a/compiler/rustc_middle/src/mir/mono.rs
+++ b/compiler/rustc_middle/src/mir/mono.rs
@@ -4,10 +4,10 @@ use rustc_attr::InlineAttr;
 use rustc_data_structures::base_n;
 use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::fx::FxHashMap;
-use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
+use rustc_data_structures::stable_hasher::{Hash128, HashStable, StableHasher};
 use rustc_hir::def_id::{CrateNum, DefId, LOCAL_CRATE};
 use rustc_hir::ItemId;
-use rustc_index::vec::Idx;
+use rustc_index::Idx;
 use rustc_query_system::ich::StableHashingContext;
 use rustc_session::config::OptLevel;
 use rustc_span::source_map::Span;
@@ -313,8 +313,8 @@ impl<'tcx> CodegenUnit<'tcx> {
         // avoid collisions and is still reasonably short for filenames.
         let mut hasher = StableHasher::new();
         human_readable_name.hash(&mut hasher);
-        let hash: u128 = hasher.finish();
-        let hash = hash & ((1u128 << 80) - 1);
+        let hash: Hash128 = hasher.finish();
+        let hash = hash.as_u128() & ((1u128 << 80) - 1);
         base_n::encode(hash, base_n::CASE_INSENSITIVE)
     }
 
@@ -334,10 +334,7 @@ impl<'tcx> CodegenUnit<'tcx> {
     }
 
     pub fn modify_size_estimate(&mut self, delta: usize) {
-        assert!(self.size_estimate.is_some());
-        if let Some(size_estimate) = self.size_estimate {
-            self.size_estimate = Some(size_estimate + delta);
-        }
+        *self.size_estimate.as_mut().unwrap() += delta;
     }
 
     pub fn contains_item(&self, item: &MonoItem<'tcx>) -> bool {
@@ -373,7 +370,7 @@ impl<'tcx> CodegenUnit<'tcx> {
                             // instances into account. The others don't matter for
                             // the codegen tests and can even make item order
                             // unstable.
-                            InstanceDef::Item(def) => def.did.as_local().map(Idx::index),
+                            InstanceDef::Item(def) => def.as_local().map(Idx::index),
                             InstanceDef::VTableShim(..)
                             | InstanceDef::ReifyShim(..)
                             | InstanceDef::Intrinsic(..)
@@ -505,22 +502,13 @@ impl<'tcx> CodegenUnitNameBuilder<'tcx> {
             // instantiating stuff for upstream crates.
             let local_crate_id = if cnum != LOCAL_CRATE {
                 let local_stable_crate_id = tcx.sess.local_stable_crate_id();
-                format!(
-                    "-in-{}.{:08x}",
-                    tcx.crate_name(LOCAL_CRATE),
-                    local_stable_crate_id.to_u64() as u32,
-                )
+                format!("-in-{}.{:08x}", tcx.crate_name(LOCAL_CRATE), local_stable_crate_id)
             } else {
                 String::new()
             };
 
             let stable_crate_id = tcx.sess.local_stable_crate_id();
-            format!(
-                "{}.{:08x}{}",
-                tcx.crate_name(cnum),
-                stable_crate_id.to_u64() as u32,
-                local_crate_id,
-            )
+            format!("{}.{:08x}{}", tcx.crate_name(cnum), stable_crate_id, local_crate_id)
         });
 
         write!(cgu_name, "{}", crate_prefix).unwrap();
diff --git a/compiler/rustc_middle/src/mir/patch.rs b/compiler/rustc_middle/src/mir/patch.rs
index f62853c3e..c4c3341f8 100644
--- a/compiler/rustc_middle/src/mir/patch.rs
+++ b/compiler/rustc_middle/src/mir/patch.rs
@@ -1,4 +1,4 @@
-use rustc_index::vec::{Idx, IndexVec};
+use rustc_index::{Idx, IndexVec};
 use rustc_middle::mir::*;
 use rustc_middle::ty::Ty;
 use rustc_span::Span;
diff --git a/compiler/rustc_middle/src/mir/pretty.rs b/compiler/rustc_middle/src/mir/pretty.rs
index 7e5195359..62c3d8cf2 100644
--- a/compiler/rustc_middle/src/mir/pretty.rs
+++ b/compiler/rustc_middle/src/mir/pretty.rs
@@ -10,7 +10,7 @@ use super::spanview::write_mir_fn_spanview;
 use either::Either;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_hir::def_id::DefId;
-use rustc_index::vec::Idx;
+use rustc_index::Idx;
 use rustc_middle::mir::interpret::{
     alloc_range, read_target_uint, AllocBytes, AllocId, Allocation, ConstAllocation, ConstValue,
     GlobalAlloc, Pointer, Provenance,
@@ -298,8 +298,7 @@ pub fn write_mir_pretty<'tcx>(
             // are shared between mir_for_ctfe and optimized_mir
             write_mir_fn(tcx, tcx.mir_for_ctfe(def_id), &mut |_, _| Ok(()), w)?;
         } else {
-            let instance_mir =
-                tcx.instance_mir(ty::InstanceDef::Item(ty::WithOptConstParam::unknown(def_id)));
+            let instance_mir = tcx.instance_mir(ty::InstanceDef::Item(def_id));
             render_body(w, instance_mir)?;
         }
     }
@@ -464,11 +463,7 @@ impl<'tcx> Visitor<'tcx> for ExtraComments<'tcx> {
                 ConstantKind::Ty(ct) => match ct.kind() {
                     ty::ConstKind::Param(p) => format!("Param({})", p),
                     ty::ConstKind::Unevaluated(uv) => {
-                        format!(
-                            "Unevaluated({}, {:?})",
-                            self.tcx.def_path_str(uv.def.did),
-                            uv.substs,
-                        )
+                        format!("Unevaluated({}, {:?})", self.tcx.def_path_str(uv.def), uv.substs,)
                     }
                     ty::ConstKind::Value(val) => format!("Value({})", fmt_valtree(&val)),
                     ty::ConstKind::Error(_) => "Error".to_string(),
@@ -481,7 +476,7 @@ impl<'tcx> Visitor<'tcx> for ExtraComments<'tcx> {
                 ConstantKind::Unevaluated(uv, _) => {
                     format!(
                         "Unevaluated({}, {:?}, {:?})",
-                        self.tcx.def_path_str(uv.def.did),
+                        self.tcx.def_path_str(uv.def),
                         uv.substs,
                         uv.promoted,
                     )
@@ -556,8 +551,13 @@ fn write_scope_tree(
         }
 
         let indented_debug_info = format!(
-            "{0:1$}debug {2} => {3:?};",
-            INDENT, indent, var_debug_info.name, var_debug_info.value,
+            "{0:1$}debug {2} => {3:&<4$}{5:?};",
+            INDENT,
+            indent,
+            var_debug_info.name,
+            "",
+            var_debug_info.references as usize,
+            var_debug_info.value,
         );
 
         writeln!(
diff --git a/compiler/rustc_middle/src/mir/query.rs b/compiler/rustc_middle/src/mir/query.rs
index cfdf1dcf5..53fd2dd23 100644
--- a/compiler/rustc_middle/src/mir/query.rs
+++ b/compiler/rustc_middle/src/mir/query.rs
@@ -1,14 +1,14 @@
 //! Values computed by queries that use MIR.
 
-use crate::mir::{Body, ConstantKind, Promoted};
+use crate::mir::ConstantKind;
 use crate::ty::{self, OpaqueHiddenType, Ty, TyCtxt};
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_data_structures::unord::UnordSet;
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir as hir;
-use rustc_hir::def_id::{DefId, LocalDefId};
+use rustc_hir::def_id::LocalDefId;
 use rustc_index::bit_set::BitMatrix;
-use rustc_index::vec::{Idx, IndexVec};
+use rustc_index::{Idx, IndexVec};
 use rustc_span::Span;
 use rustc_target::abi::{FieldIdx, VariantIdx};
 use smallvec::SmallVec;
@@ -454,42 +454,3 @@ pub struct CoverageInfo {
     /// The total number of coverage region counter expressions added to the MIR `Body`.
     pub num_expressions: u32,
 }
-
-/// Shims which make dealing with `WithOptConstParam` easier.
-///
-/// For more information on why this is needed, consider looking
-/// at the docs for `WithOptConstParam` itself.
-impl<'tcx> TyCtxt<'tcx> {
-    #[inline]
-    pub fn mir_const_qualif_opt_const_arg(
-        self,
-        def: ty::WithOptConstParam<LocalDefId>,
-    ) -> ConstQualifs {
-        if let Some(param_did) = def.const_param_did {
-            self.mir_const_qualif_const_arg((def.did, param_did))
-        } else {
-            self.mir_const_qualif(def.did)
-        }
-    }
-
-    #[inline]
-    pub fn promoted_mir_opt_const_arg(
-        self,
-        def: ty::WithOptConstParam<DefId>,
-    ) -> &'tcx IndexVec<Promoted, Body<'tcx>> {
-        if let Some((did, param_did)) = def.as_const_arg() {
-            self.promoted_mir_of_const_arg((did, param_did))
-        } else {
-            self.promoted_mir(def.did)
-        }
-    }
-
-    #[inline]
-    pub fn mir_for_ctfe_opt_const_arg(self, def: ty::WithOptConstParam<DefId>) -> &'tcx Body<'tcx> {
-        if let Some((did, param_did)) = def.as_const_arg() {
-            self.mir_for_ctfe_of_const_arg((did, param_did))
-        } else {
-            self.mir_for_ctfe(def.did)
-        }
-    }
-}
diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs
index 93800d484..3e474c1d3 100644
--- a/compiler/rustc_middle/src/mir/syntax.rs
+++ b/compiler/rustc_middle/src/mir/syntax.rs
@@ -16,7 +16,7 @@ use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
 use rustc_hir::def_id::DefId;
 use rustc_hir::{self as hir};
 use rustc_hir::{self, GeneratorKind};
-use rustc_index::vec::IndexVec;
+use rustc_index::IndexVec;
 use rustc_target::abi::{FieldIdx, VariantIdx};
 
 use rustc_ast::Mutability;
@@ -256,7 +256,7 @@ pub enum StatementKind<'tcx> {
     /// **Needs clarification**: The implication of the above idea would be that assignment implies
     /// that the resulting value is initialized. I believe we could commit to this separately from
     /// committing to whatever part of the memory model we would need to decide on to make the above
-    /// paragragh precise. Do we want to?
+    /// paragraph precise. Do we want to?
     ///
     /// Assignments in which the types of the place and rvalue differ are not well-formed.
     ///
@@ -336,9 +336,8 @@ pub enum StatementKind<'tcx> {
     /// This is especially useful for `let _ = PLACE;` bindings that desugar to a single
     /// `PlaceMention(PLACE)`.
     ///
-    /// When executed at runtime this is a nop.
-    ///
-    /// Disallowed after drop elaboration.
+    /// When executed at runtime, this computes the given place, but then discards
+    /// it without doing a load. It is UB if the place is not pointing to live memory.
     PlaceMention(Box<Place<'tcx>>),
 
     /// Encodes a user's type ascription. These need to be preserved
@@ -609,7 +608,11 @@ pub enum TerminatorKind<'tcx> {
     /// > The drop glue is executed if, among all statements executed within this `Body`, an assignment to
     /// > the place or one of its "parents" occurred more recently than a move out of it. This does not
     /// > consider indirect assignments.
-    Drop { place: Place<'tcx>, target: BasicBlock, unwind: UnwindAction },
+    ///
+    /// The `replace` flag indicates whether this terminator was created as part of an assignment.
+    /// This should only be used for diagnostic purposes, and does not have any operational
+    /// meaning.
+    Drop { place: Place<'tcx>, target: BasicBlock, unwind: UnwindAction, replace: bool },
 
     /// Roughly speaking, evaluates the `func` operand and the arguments, and starts execution of
     /// the referred to function. The operand types must match the argument types of the function.
@@ -657,7 +660,7 @@ pub enum TerminatorKind<'tcx> {
     Assert {
         cond: Operand<'tcx>,
         expected: bool,
-        msg: AssertMessage<'tcx>,
+        msg: Box<AssertMessage<'tcx>>,
         target: BasicBlock,
         unwind: UnwindAction,
     },
@@ -755,6 +758,29 @@ pub enum TerminatorKind<'tcx> {
     },
 }
 
+impl TerminatorKind<'_> {
+    /// Returns a simple string representation of a `TerminatorKind` variant, independent of any
+    /// values it might hold (e.g. `TerminatorKind::Call` always returns `"Call"`).
+    pub const fn name(&self) -> &'static str {
+        match self {
+            TerminatorKind::Goto { .. } => "Goto",
+            TerminatorKind::SwitchInt { .. } => "SwitchInt",
+            TerminatorKind::Resume => "Resume",
+            TerminatorKind::Terminate => "Terminate",
+            TerminatorKind::Return => "Return",
+            TerminatorKind::Unreachable => "Unreachable",
+            TerminatorKind::Drop { .. } => "Drop",
+            TerminatorKind::Call { .. } => "Call",
+            TerminatorKind::Assert { .. } => "Assert",
+            TerminatorKind::Yield { .. } => "Yield",
+            TerminatorKind::GeneratorDrop => "GeneratorDrop",
+            TerminatorKind::FalseEdge { .. } => "FalseEdge",
+            TerminatorKind::FalseUnwind { .. } => "FalseUnwind",
+            TerminatorKind::InlineAsm { .. } => "InlineAsm",
+        }
+    }
+}
+
 /// Action to be taken when a stack unwind happens.
 #[derive(Copy, Clone, Debug, PartialEq, Eq, TyEncodable, TyDecodable, Hash, HashStable)]
 #[derive(TypeFoldable, TypeVisitable)]
@@ -1002,7 +1028,7 @@ pub type PlaceElem<'tcx> = ProjectionElem<Local, Ty<'tcx>>;
 /// This is what is implemented in miri today. Are these the semantics we want for MIR? Is this
 /// something we can even decide without knowing more about Rust's memory model?
 ///
-/// **Needs clarifiation:** Is loading a place that has its variant index set well-formed? Miri
+/// **Needs clarification:** Is loading a place that has its variant index set well-formed? Miri
 /// currently implements it, but it seems like this may be something to check against in the
 /// validator.
 #[derive(Clone, PartialEq, TyEncodable, TyDecodable, Hash, HashStable, TypeFoldable, TypeVisitable)]
@@ -1120,7 +1146,7 @@ pub enum Rvalue<'tcx> {
     CheckedBinaryOp(BinOp, Box<(Operand<'tcx>, Operand<'tcx>)>),
 
     /// Computes a value as described by the operation.
-    NullaryOp(NullOp, Ty<'tcx>),
+    NullaryOp(NullOp<'tcx>, Ty<'tcx>),
 
     /// Exactly like `BinaryOp`, but less operands.
     ///
@@ -1216,12 +1242,14 @@ pub enum AggregateKind<'tcx> {
     Generator(DefId, SubstsRef<'tcx>, hir::Movability),
 }
 
-#[derive(Copy, Clone, Debug, PartialEq, Eq, TyEncodable, TyDecodable, Hash, HashStable)]
-pub enum NullOp {
+#[derive(Clone, Debug, PartialEq, Eq, TyEncodable, TyDecodable, Hash, HashStable)]
+pub enum NullOp<'tcx> {
     /// Returns the size of a value of that type
     SizeOf,
     /// Returns the minimum alignment of a type
     AlignOf,
+    /// Returns the offset of a field
+    OffsetOf(&'tcx List<FieldIdx>),
 }
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
diff --git a/compiler/rustc_middle/src/mir/tcx.rs b/compiler/rustc_middle/src/mir/tcx.rs
index 4f00abf7f..5ca824134 100644
--- a/compiler/rustc_middle/src/mir/tcx.rs
+++ b/compiler/rustc_middle/src/mir/tcx.rs
@@ -188,7 +188,9 @@ impl<'tcx> Rvalue<'tcx> {
             }
             Rvalue::UnaryOp(UnOp::Not | UnOp::Neg, ref operand) => operand.ty(local_decls, tcx),
             Rvalue::Discriminant(ref place) => place.ty(local_decls, tcx).ty.discriminant_ty(tcx),
-            Rvalue::NullaryOp(NullOp::SizeOf | NullOp::AlignOf, _) => tcx.types.usize,
+            Rvalue::NullaryOp(NullOp::SizeOf | NullOp::AlignOf | NullOp::OffsetOf(..), _) => {
+                tcx.types.usize
+            }
             Rvalue::Aggregate(ref ak, ref ops) => match **ak {
                 AggregateKind::Array(ty) => tcx.mk_array(ty, ops.len() as u64),
                 AggregateKind::Tuple => {
diff --git a/compiler/rustc_middle/src/mir/type_foldable.rs b/compiler/rustc_middle/src/mir/type_foldable.rs
index 988158321..06874741b 100644
--- a/compiler/rustc_middle/src/mir/type_foldable.rs
+++ b/compiler/rustc_middle/src/mir/type_foldable.rs
@@ -16,7 +16,6 @@ TrivialTypeTraversalAndLiftImpls! {
     UserTypeAnnotationIndex,
     BorrowKind,
     CastKind,
-    NullOp,
     hir::Movability,
     BasicBlock,
     SwitchTargets,
@@ -25,9 +24,8 @@ TrivialTypeTraversalAndLiftImpls! {
 }
 
 TrivialTypeTraversalImpls! {
-    for <'tcx> {
-        ConstValue<'tcx>,
-    }
+    ConstValue<'tcx>,
+    NullOp<'tcx>,
 }
 
 impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for &'tcx [InlineAsmTemplatePiece] {
diff --git a/compiler/rustc_middle/src/mir/visit.rs b/compiler/rustc_middle/src/mir/visit.rs
index caa5edc32..8d44e929a 100644
--- a/compiler/rustc_middle/src/mir/visit.rs
+++ b/compiler/rustc_middle/src/mir/visit.rs
@@ -64,7 +64,7 @@
 
 use crate::mir::*;
 use crate::ty::subst::SubstsRef;
-use crate::ty::{CanonicalUserTypeAnnotation, Ty};
+use crate::ty::{self, CanonicalUserTypeAnnotation, Ty};
 use rustc_span::Span;
 
 macro_rules! make_mir_visitor {
@@ -192,6 +192,14 @@ macro_rules! make_mir_visitor {
                 self.super_constant(constant, location);
             }
 
+            fn visit_ty_const(
+                &mut self,
+                ct: $( & $mutability)? ty::Const<'tcx>,
+                location: Location,
+            ) {
+                self.super_ty_const(ct, location);
+            }
+
             fn visit_span(
                 &mut self,
                 span: $(& $mutability)? Span,
@@ -410,7 +418,7 @@ macro_rules! make_mir_visitor {
                     StatementKind::PlaceMention(place) => {
                         self.visit_place(
                             place,
-                            PlaceContext::NonUse(NonUseContext::PlaceMention),
+                            PlaceContext::NonMutatingUse(NonMutatingUseContext::PlaceMention),
                             location
                         );
                     }
@@ -496,6 +504,7 @@ macro_rules! make_mir_visitor {
                         place,
                         target: _,
                         unwind: _,
+                        replace: _,
                     } => {
                         self.visit_place(
                             place,
@@ -625,8 +634,9 @@ macro_rules! make_mir_visitor {
                         self.visit_operand(operand, location);
                     }
 
-                    Rvalue::Repeat(value, _) => {
+                    Rvalue::Repeat(value, ct) => {
                         self.visit_operand(value, location);
+                        self.visit_ty_const($(&$mutability)? *ct, location);
                     }
 
                     Rvalue::ThreadLocalRef(_) => {}
@@ -773,12 +783,12 @@ macro_rules! make_mir_visitor {
 
             fn super_ascribe_user_ty(&mut self,
                                      place: & $($mutability)? Place<'tcx>,
-                                     _variance: $(& $mutability)? ty::Variance,
+                                     variance: $(& $mutability)? ty::Variance,
                                      user_ty: & $($mutability)? UserTypeProjection,
                                      location: Location) {
                 self.visit_place(
                     place,
-                    PlaceContext::NonUse(NonUseContext::AscribeUserTy),
+                    PlaceContext::NonUse(NonUseContext::AscribeUserTy($(* &$mutability *)? variance)),
                     location
                 );
                 self.visit_user_type_projection(user_ty);
@@ -833,6 +843,7 @@ macro_rules! make_mir_visitor {
                     source_info,
                     value,
                     argument_index: _,
+                    references: _,
                 } = var_debug_info;
 
                 self.visit_source_info(source_info);
@@ -871,19 +882,26 @@ macro_rules! make_mir_visitor {
             ) {
                 let Constant {
                     span,
-                    user_ty,
+                    user_ty: _, // no visit method for this
                     literal,
                 } = constant;
 
                 self.visit_span($(& $mutability)? *span);
-                drop(user_ty); // no visit method for this
                 match literal {
-                    ConstantKind::Ty(_) => {}
+                    ConstantKind::Ty(ct) => self.visit_ty_const($(&$mutability)? *ct, location),
                     ConstantKind::Val(_, ty) => self.visit_ty($(& $mutability)? *ty, TyContext::Location(location)),
                     ConstantKind::Unevaluated(_, ty) => self.visit_ty($(& $mutability)? *ty, TyContext::Location(location)),
                 }
             }
 
+            fn super_ty_const(
+                &mut self,
+                _ct: $(& $mutability)? ty::Const<'tcx>,
+                _location: Location,
+            ) {
+
+            }
+
             fn super_span(&mut self, _span: $(& $mutability)? Span) {
             }
 
@@ -1249,6 +1267,11 @@ pub enum NonMutatingUseContext {
     ShallowBorrow,
     /// AddressOf for *const pointer.
     AddressOf,
+    /// PlaceMention statement.
+    ///
+    /// This statement is executed as a check that the `Place` is live without reading from it,
+    /// so it must be considered as a non-mutating use.
+    PlaceMention,
     /// Used as base for another place, e.g., `x` in `x.y`. Will not mutate the place.
     /// For example, the projection `x.y` is not marked as a mutation in these cases:
     /// ```ignore (illustrative)
@@ -1296,11 +1319,9 @@ pub enum NonUseContext {
     /// Ending a storage live range.
     StorageDead,
     /// User type annotation assertions for NLL.
-    AscribeUserTy,
+    AscribeUserTy(ty::Variance),
     /// The data of a user variable, for debug info.
     VarDebugInfo,
-    /// PlaceMention statement.
-    PlaceMention,
 }
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
diff --git a/compiler/rustc_middle/src/query/erase.rs b/compiler/rustc_middle/src/query/erase.rs
index 7d9aea022..fd02a1613 100644
--- a/compiler/rustc_middle/src/query/erase.rs
+++ b/compiler/rustc_middle/src/query/erase.rs
@@ -28,7 +28,7 @@ pub fn erase<T: EraseType>(src: T) -> Erase<T> {
     };
 
     Erased::<<T as EraseType>::Result> {
-        // SAFETY: Is it safe to transmute to MaybeUninit for types with the same sizes.
+        // SAFETY: It is safe to transmute to MaybeUninit for types with the same sizes.
         data: unsafe { transmute_copy(&src) },
     }
 }
@@ -82,9 +82,10 @@ impl EraseType for Result<Option<ty::Instance<'_>>, rustc_errors::ErrorGuarantee
         [u8; size_of::<Result<Option<ty::Instance<'static>>, rustc_errors::ErrorGuaranteed>>()];
 }
 
-impl EraseType for Result<Option<ty::Const<'_>>, rustc_errors::ErrorGuaranteed> {
-    type Result =
-        [u8; size_of::<Result<Option<ty::Const<'static>>, rustc_errors::ErrorGuaranteed>>()];
+impl EraseType for Result<Option<ty::EarlyBinder<ty::Const<'_>>>, rustc_errors::ErrorGuaranteed> {
+    type Result = [u8; size_of::<
+        Result<Option<ty::EarlyBinder<ty::Const<'static>>>, rustc_errors::ErrorGuaranteed>,
+    >()];
 }
 
 impl EraseType for Result<ty::GenericArg<'_>, traits::query::NoSolution> {
@@ -171,6 +172,10 @@ impl EraseType for ty::Binder<'_, ty::FnSig<'_>> {
     type Result = [u8; size_of::<ty::Binder<'static, ty::FnSig<'static>>>()];
 }
 
+impl EraseType for ty::Binder<'_, &'_ ty::List<Ty<'_>>> {
+    type Result = [u8; size_of::<ty::Binder<'static, &'static ty::List<Ty<'static>>>>()];
+}
+
 impl<T0, T1> EraseType for (&'_ T0, &'_ T1) {
     type Result = [u8; size_of::<(&'static (), &'static ())>()];
 }
diff --git a/compiler/rustc_middle/src/query/keys.rs b/compiler/rustc_middle/src/query/keys.rs
index 23b28ac5c..fa62b7f32 100644
--- a/compiler/rustc_middle/src/query/keys.rs
+++ b/compiler/rustc_middle/src/query/keys.rs
@@ -174,14 +174,6 @@ impl AsLocalKey for DefId {
     }
 }
 
-impl Key for ty::WithOptConstParam<LocalDefId> {
-    type CacheSelector = DefaultCacheSelector<Self>;
-
-    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
-        self.did.default_span(tcx)
-    }
-}
-
 impl Key for SimplifiedType {
     type CacheSelector = DefaultCacheSelector<Self>;
 
@@ -313,7 +305,7 @@ impl<'tcx> Key for (ty::UnevaluatedConst<'tcx>, ty::UnevaluatedConst<'tcx>) {
     type CacheSelector = DefaultCacheSelector<Self>;
 
     fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
-        (self.0).def.did.default_span(tcx)
+        (self.0).def.default_span(tcx)
     }
 }
 
diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs
index 7a5a16035..1528be42f 100644
--- a/compiler/rustc_middle/src/query/mod.rs
+++ b/compiler/rustc_middle/src/query/mod.rs
@@ -4,12 +4,98 @@
 //! ["Queries: demand-driven compilation"](https://rustc-dev-guide.rust-lang.org/query.html).
 //! This chapter includes instructions for adding new queries.
 
-use crate::ty::{self, print::describe_as_module, TyCtxt};
+#![allow(unused_parens)]
+
+use crate::dep_graph;
+use crate::dep_graph::DepKind;
+use crate::infer::canonical::{self, Canonical};
+use crate::lint::LintExpectation;
+use crate::metadata::ModChild;
+use crate::middle::codegen_fn_attrs::CodegenFnAttrs;
+use crate::middle::debugger_visualizer::DebuggerVisualizerFile;
+use crate::middle::exported_symbols::{ExportedSymbol, SymbolExportInfo};
+use crate::middle::lib_features::LibFeatures;
+use crate::middle::privacy::EffectiveVisibilities;
+use crate::middle::resolve_bound_vars::{ObjectLifetimeDefault, ResolveBoundVars, ResolvedArg};
+use crate::middle::stability::{self, DeprecationEntry};
+use crate::mir;
+use crate::mir::interpret::GlobalId;
+use crate::mir::interpret::{
+    ConstValue, EvalToAllocationRawResult, EvalToConstValueResult, EvalToValTreeResult,
+};
+use crate::mir::interpret::{LitToConstError, LitToConstInput};
+use crate::mir::mono::CodegenUnit;
+use crate::query::erase::{erase, restore, Erase};
+use crate::query::plumbing::{query_ensure, query_get_at, DynamicQuery};
+use crate::thir;
+use crate::traits::query::{
+    CanonicalPredicateGoal, CanonicalProjectionGoal, CanonicalTyGoal,
+    CanonicalTypeOpAscribeUserTypeGoal, CanonicalTypeOpEqGoal, CanonicalTypeOpNormalizeGoal,
+    CanonicalTypeOpProvePredicateGoal, CanonicalTypeOpSubtypeGoal, NoSolution,
+};
+use crate::traits::query::{
+    DropckConstraint, DropckOutlivesResult, MethodAutoderefStepsResult, NormalizationResult,
+    OutlivesBound,
+};
+use crate::traits::specialization_graph;
+use crate::traits::{
+    CanonicalChalkEnvironmentAndGoal, CodegenObligationError, EvaluationResult, ImplSource,
+    ObjectSafetyViolation, ObligationCause, OverflowError, WellFormedLoc,
+};
+use crate::ty::fast_reject::SimplifiedType;
+use crate::ty::layout::ValidityRequirement;
+use crate::ty::subst::{GenericArg, SubstsRef};
+use crate::ty::util::AlwaysRequiresDrop;
+use crate::ty::GeneratorDiagnosticData;
+use crate::ty::TyCtxtFeed;
+use crate::ty::{
+    self, print::describe_as_module, CrateInherentImpls, ParamEnvAnd, Ty, TyCtxt,
+    UnusedGenericParams,
+};
+use rustc_arena::TypedArena;
+use rustc_ast as ast;
+use rustc_ast::expand::allocator::AllocatorKind;
+use rustc_attr as attr;
+use rustc_data_structures::fingerprint::Fingerprint;
+use rustc_data_structures::fx::{FxHashMap, FxIndexMap, FxIndexSet};
+use rustc_data_structures::steal::Steal;
+use rustc_data_structures::svh::Svh;
+use rustc_data_structures::sync::Lrc;
+use rustc_data_structures::sync::WorkerLocal;
+use rustc_data_structures::unord::UnordSet;
+use rustc_errors::ErrorGuaranteed;
+use rustc_hir as hir;
+use rustc_hir::def::{DefKind, DocLinkResMap};
+use rustc_hir::def_id::{
+    CrateNum, DefId, DefIdMap, DefIdSet, LocalDefId, LocalDefIdMap, LocalDefIdSet,
+};
+use rustc_hir::lang_items::{LangItem, LanguageItems};
+use rustc_hir::{Crate, ItemLocalId, TraitCandidate};
+use rustc_index::IndexVec;
+use rustc_query_system::ich::StableHashingContext;
+use rustc_query_system::query::{try_get_cached, CacheSelector, QueryCache, QueryMode, QueryState};
+use rustc_session::config::{EntryFnType, OptLevel, OutputFilenames, SymbolManglingVersion};
+use rustc_session::cstore::{CrateDepKind, CrateSource};
+use rustc_session::cstore::{ExternCrate, ForeignModule, LinkagePreference, NativeLib};
+use rustc_session::lint::LintExpectationId;
+use rustc_session::Limits;
 use rustc_span::def_id::LOCAL_CRATE;
+use rustc_span::symbol::Symbol;
+use rustc_span::{Span, DUMMY_SP};
+use rustc_target::abi;
+use rustc_target::spec::PanicStrategy;
+use std::mem;
+use std::ops::Deref;
+use std::path::PathBuf;
+use std::sync::Arc;
 
 pub mod erase;
 mod keys;
 pub use keys::{AsLocalKey, Key, LocalCrate};
+pub mod on_disk_cache;
+#[macro_use]
+pub mod plumbing;
+pub use plumbing::{IntoQueryParam, TyCtxtAt, TyCtxtEnsure, TyCtxtEnsureWithValue};
 
 // Each of these queries corresponds to a function pointer field in the
 // `Providers` struct for requesting a value of that type, and a method
@@ -83,7 +169,7 @@ rustc_queries! {
     /// Avoid calling this query directly.
     query hir_module_items(key: LocalDefId) -> &'tcx rustc_middle::hir::ModuleItems {
         arena_cache
-        desc { |tcx| "getting HIR module items in `{}`", tcx.def_path_str(key.to_def_id()) }
+        desc { |tcx| "getting HIR module items in `{}`", tcx.def_path_str(key) }
         cache_on_disk_if { true }
     }
 
@@ -92,14 +178,14 @@ rustc_queries! {
     /// This can be conveniently accessed by methods on `tcx.hir()`.
     /// Avoid calling this query directly.
     query hir_owner(key: hir::OwnerId) -> Option<crate::hir::Owner<'tcx>> {
-        desc { |tcx| "getting HIR owner of `{}`", tcx.def_path_str(key.to_def_id()) }
+        desc { |tcx| "getting HIR owner of `{}`", tcx.def_path_str(key) }
     }
 
     /// Gives access to the HIR ID for the given `LocalDefId` owner `key` if any.
     ///
     /// Definitions that were generated with no HIR, would be fed to return `None`.
     query opt_local_def_id_to_hir_id(key: LocalDefId) -> Option<hir::HirId>{
-        desc { |tcx| "getting HIR ID of `{}`", tcx.def_path_str(key.to_def_id()) }
+        desc { |tcx| "getting HIR ID of `{}`", tcx.def_path_str(key) }
         feedable
     }
 
@@ -108,7 +194,7 @@ rustc_queries! {
     /// This can be conveniently accessed by methods on `tcx.hir()`.
     /// Avoid calling this query directly.
     query hir_owner_parent(key: hir::OwnerId) -> hir::HirId {
-        desc { |tcx| "getting HIR parent of `{}`", tcx.def_path_str(key.to_def_id()) }
+        desc { |tcx| "getting HIR parent of `{}`", tcx.def_path_str(key) }
     }
 
     /// Gives access to the HIR nodes and bodies inside the HIR owner `key`.
@@ -116,7 +202,7 @@ rustc_queries! {
     /// This can be conveniently accessed by methods on `tcx.hir()`.
     /// Avoid calling this query directly.
     query hir_owner_nodes(key: hir::OwnerId) -> hir::MaybeOwner<&'tcx hir::OwnerNodes<'tcx>> {
-        desc { |tcx| "getting HIR owner items in `{}`", tcx.def_path_str(key.to_def_id()) }
+        desc { |tcx| "getting HIR owner items in `{}`", tcx.def_path_str(key) }
     }
 
     /// Gives access to the HIR attributes inside the HIR owner `key`.
@@ -124,30 +210,7 @@ rustc_queries! {
     /// This can be conveniently accessed by methods on `tcx.hir()`.
     /// Avoid calling this query directly.
     query hir_attrs(key: hir::OwnerId) -> &'tcx hir::AttributeMap<'tcx> {
-        desc { |tcx| "getting HIR owner attributes in `{}`", tcx.def_path_str(key.to_def_id()) }
-    }
-
-    /// Computes the `DefId` of the corresponding const parameter in case the `key` is a
-    /// const argument and returns `None` otherwise.
-    ///
-    /// ```ignore (incomplete)
-    /// let a = foo::<7>();
-    /// //            ^ Calling `opt_const_param_of` for this argument,
-    ///
-    /// fn foo<const N: usize>()
-    /// //           ^ returns this `DefId`.
-    ///
-    /// fn bar() {
-    /// // ^ While calling `opt_const_param_of` for other bodies returns `None`.
-    /// }
-    /// ```
-    // It looks like caching this query on disk actually slightly
-    // worsened performance in #74376.
-    //
-    // Once const generics are more prevalently used, we might want to
-    // consider only caching calls returning `Some`.
-    query opt_const_param_of(key: LocalDefId) -> Option<DefId> {
-        desc { |tcx| "computing the optional const parameter of `{}`", tcx.def_path_str(key.to_def_id()) }
+        desc { |tcx| "getting HIR owner attributes in `{}`", tcx.def_path_str(key) }
     }
 
     /// Given the def_id of a const-generic parameter, computes the associated default const
@@ -181,7 +244,7 @@ rustc_queries! {
     }
 
     query collect_return_position_impl_trait_in_trait_tys(key: DefId)
-        -> Result<&'tcx FxHashMap<DefId, Ty<'tcx>>, ErrorGuaranteed>
+        -> Result<&'tcx FxHashMap<DefId, ty::EarlyBinder<Ty<'tcx>>>, ErrorGuaranteed>
     {
         desc { "comparing an impl and trait method signature, inferring any hidden `impl Trait` types in the process" }
         cache_on_disk_if { key.is_local() }
@@ -258,6 +321,15 @@ rustc_queries! {
         cache_on_disk_if { key.is_local() }
     }
 
+    query opaque_types_defined_by(
+        key: LocalDefId
+    ) -> &'tcx [LocalDefId] {
+        desc {
+            |tcx| "computing the opaque types defined by `{}`",
+            tcx.def_path_str(key.to_def_id())
+        }
+    }
+
     /// Returns the list of bounds that can be used for
     /// `SelectionCandidate::ProjectionCandidate(_)` and
     /// `ProjectionTyCandidate::TraitDef`.
@@ -274,7 +346,7 @@ rustc_queries! {
     /// `key` is the `DefId` of the associated type or opaque type.
     ///
     /// Bounds from the parent (e.g. with nested impl trait) are not included.
-    query explicit_item_bounds(key: DefId) -> &'tcx [(ty::Predicate<'tcx>, Span)] {
+    query explicit_item_bounds(key: DefId) -> ty::EarlyBinder<&'tcx [(ty::Predicate<'tcx>, Span)]> {
         desc { |tcx| "finding item bounds for `{}`", tcx.def_path_str(key) }
         cache_on_disk_if { key.is_local() }
         separate_provide_extern
@@ -318,7 +390,7 @@ rustc_queries! {
     query shallow_lint_levels_on(key: hir::OwnerId) -> &'tcx rustc_middle::lint::ShallowLintLevelMap {
         eval_always // fetches `resolutions`
         arena_cache
-        desc { |tcx| "looking up lint levels for `{}`", tcx.def_path_str(key.to_def_id()) }
+        desc { |tcx| "looking up lint levels for `{}`", tcx.def_path_str(key) }
     }
 
     query lint_expectations(_: ()) -> &'tcx Vec<(LintExpectationId, LintExpectation)> {
@@ -328,7 +400,7 @@ rustc_queries! {
 
     query parent_module_from_def_id(key: LocalDefId) -> LocalDefId {
         eval_always
-        desc { |tcx| "getting the parent module of `{}`", tcx.def_path_str(key.to_def_id()) }
+        desc { |tcx| "getting the parent module of `{}`", tcx.def_path_str(key) }
     }
 
     query expn_that_defined(key: DefId) -> rustc_span::ExpnId {
@@ -344,7 +416,7 @@ rustc_queries! {
 
     /// Checks whether a type is representable or infinitely sized
     query representability(_: LocalDefId) -> rustc_middle::ty::Representability {
-        desc { "checking if `{}` is representable", tcx.def_path_str(key.to_def_id()) }
+        desc { "checking if `{}` is representable", tcx.def_path_str(key) }
         // infinitely sized types will cause a cycle
         cycle_delay_bug
         // we don't want recursive representability calls to be forced with
@@ -369,26 +441,24 @@ rustc_queries! {
     }
 
     /// Fetch the THIR for a given body. If typeck for that body failed, returns an empty `Thir`.
-    query thir_body(key: ty::WithOptConstParam<LocalDefId>)
-        -> Result<(&'tcx Steal<thir::Thir<'tcx>>, thir::ExprId), ErrorGuaranteed>
-    {
+    query thir_body(key: LocalDefId) -> Result<(&'tcx Steal<thir::Thir<'tcx>>, thir::ExprId), ErrorGuaranteed> {
         // Perf tests revealed that hashing THIR is inefficient (see #85729).
         no_hash
-        desc { |tcx| "building THIR for `{}`", tcx.def_path_str(key.did.to_def_id()) }
+        desc { |tcx| "building THIR for `{}`", tcx.def_path_str(key) }
     }
 
     /// Create a THIR tree for debugging.
-    query thir_tree(key: ty::WithOptConstParam<LocalDefId>) -> &'tcx String {
+    query thir_tree(key: LocalDefId) -> &'tcx String {
         no_hash
         arena_cache
-        desc { |tcx| "constructing THIR tree for `{}`", tcx.def_path_str(key.did.to_def_id()) }
+        desc { |tcx| "constructing THIR tree for `{}`", tcx.def_path_str(key) }
     }
 
     /// Create a list-like THIR representation for debugging.
-    query thir_flat(key: ty::WithOptConstParam<LocalDefId>) -> &'tcx String {
+    query thir_flat(key: LocalDefId) -> &'tcx String {
         no_hash
         arena_cache
-        desc { |tcx| "constructing flat THIR representation for `{}`", tcx.def_path_str(key.did.to_def_id()) }
+        desc { |tcx| "constructing flat THIR representation for `{}`", tcx.def_path_str(key) }
     }
 
     /// Set of all the `DefId`s in this crate that have MIR associated with
@@ -407,59 +477,35 @@ rustc_queries! {
         cache_on_disk_if { key.is_local() }
         separate_provide_extern
     }
-    query mir_const_qualif_const_arg(
-        key: (LocalDefId, DefId)
-    ) -> mir::ConstQualifs {
-        desc {
-            |tcx| "const checking the const argument `{}`",
-            tcx.def_path_str(key.0.to_def_id())
-        }
-    }
 
     /// Fetch the MIR for a given `DefId` right after it's built - this includes
     /// unreachable code.
-    query mir_built(key: ty::WithOptConstParam<LocalDefId>) -> &'tcx Steal<mir::Body<'tcx>> {
-        desc { |tcx| "building MIR for `{}`", tcx.def_path_str(key.did.to_def_id()) }
+    query mir_built(key: LocalDefId) -> &'tcx Steal<mir::Body<'tcx>> {
+        desc { |tcx| "building MIR for `{}`", tcx.def_path_str(key) }
     }
 
     /// Fetch the MIR for a given `DefId` up till the point where it is
     /// ready for const qualification.
     ///
     /// See the README for the `mir` module for details.
-    query mir_const(key: ty::WithOptConstParam<LocalDefId>) -> &'tcx Steal<mir::Body<'tcx>> {
-        desc {
-            |tcx| "preparing {}`{}` for borrow checking",
-            if key.const_param_did.is_some() { "the const argument " } else { "" },
-            tcx.def_path_str(key.did.to_def_id()),
-        }
+    query mir_const(key: LocalDefId) -> &'tcx Steal<mir::Body<'tcx>> {
+        desc { |tcx| "preparing `{}` for borrow checking", tcx.def_path_str(key) }
         no_hash
     }
 
     /// Try to build an abstract representation of the given constant.
     query thir_abstract_const(
         key: DefId
-    ) -> Result<Option<ty::Const<'tcx>>, ErrorGuaranteed> {
+    ) -> Result<Option<ty::EarlyBinder<ty::Const<'tcx>>>, ErrorGuaranteed> {
         desc {
             |tcx| "building an abstract representation for `{}`", tcx.def_path_str(key),
         }
         separate_provide_extern
     }
-    /// Try to build an abstract representation of the given constant.
-    query thir_abstract_const_of_const_arg(
-        key: (LocalDefId, DefId)
-    ) -> Result<Option<ty::Const<'tcx>>, ErrorGuaranteed> {
-        desc {
-            |tcx|
-            "building an abstract representation for the const argument `{}`",
-            tcx.def_path_str(key.0.to_def_id()),
-        }
-    }
 
-    query mir_drops_elaborated_and_const_checked(
-        key: ty::WithOptConstParam<LocalDefId>
-    ) -> &'tcx Steal<mir::Body<'tcx>> {
+    query mir_drops_elaborated_and_const_checked(key: LocalDefId) -> &'tcx Steal<mir::Body<'tcx>> {
         no_hash
-        desc { |tcx| "elaborating drops for `{}`", tcx.def_path_str(key.did.to_def_id()) }
+        desc { |tcx| "elaborating drops for `{}`", tcx.def_path_str(key) }
     }
 
     query mir_for_ctfe(
@@ -470,34 +516,22 @@ rustc_queries! {
         separate_provide_extern
     }
 
-    query mir_for_ctfe_of_const_arg(key: (LocalDefId, DefId)) -> &'tcx mir::Body<'tcx> {
-        desc {
-            |tcx| "caching MIR for CTFE of the const argument `{}`",
-            tcx.def_path_str(key.0.to_def_id())
-        }
-    }
-
-    query mir_promoted(key: ty::WithOptConstParam<LocalDefId>) ->
-        (
-            &'tcx Steal<mir::Body<'tcx>>,
-            &'tcx Steal<IndexVec<mir::Promoted, mir::Body<'tcx>>>
-        ) {
+    query mir_promoted(key: LocalDefId) -> (
+        &'tcx Steal<mir::Body<'tcx>>,
+        &'tcx Steal<IndexVec<mir::Promoted, mir::Body<'tcx>>>
+    ) {
         no_hash
-        desc {
-            |tcx| "processing MIR for {}`{}`",
-            if key.const_param_did.is_some() { "the const argument " } else { "" },
-            tcx.def_path_str(key.did.to_def_id()),
-        }
+        desc { |tcx| "promoting constants in MIR for `{}`", tcx.def_path_str(key) }
     }
 
     query closure_typeinfo(key: LocalDefId) -> ty::ClosureTypeInfo<'tcx> {
         desc {
             |tcx| "finding symbols for captures of closure `{}`",
-            tcx.def_path_str(key.to_def_id())
+            tcx.def_path_str(key)
         }
     }
 
-    query mir_generator_witnesses(key: DefId) -> &'tcx mir::GeneratorLayout<'tcx> {
+    query mir_generator_witnesses(key: DefId) -> &'tcx Option<mir::GeneratorLayout<'tcx>> {
         arena_cache
         desc { |tcx| "generator witness types for `{}`", tcx.def_path_str(key) }
         cache_on_disk_if { key.is_local() }
@@ -505,7 +539,7 @@ rustc_queries! {
     }
 
     query check_generator_obligations(key: LocalDefId) {
-        desc { |tcx| "verify auto trait bounds for generator interior type `{}`", tcx.def_path_str(key.to_def_id()) }
+        desc { |tcx| "verify auto trait bounds for generator interior type `{}`", tcx.def_path_str(key) }
     }
 
     /// MIR after our optimization passes have run. This is MIR that is ready
@@ -544,14 +578,6 @@ rustc_queries! {
         cache_on_disk_if { key.is_local() }
         separate_provide_extern
     }
-    query promoted_mir_of_const_arg(
-        key: (LocalDefId, DefId)
-    ) -> &'tcx IndexVec<mir::Promoted, mir::Body<'tcx>> {
-        desc {
-            |tcx| "optimizing promoted MIR for the const argument `{}`",
-            tcx.def_path_str(key.0.to_def_id()),
-        }
-    }
 
     /// Erases regions from `ty` to yield a new type.
     /// Normally you would just use `tcx.erase_regions(value)`,
@@ -595,7 +621,7 @@ rustc_queries! {
     /// `explicit_predicates_of` and `explicit_item_bounds` will then take
     /// the appropriate subsets of the predicates here.
     query trait_explicit_predicates_and_bounds(key: LocalDefId) -> ty::GenericPredicates<'tcx> {
-        desc { |tcx| "computing explicit predicates of trait `{}`", tcx.def_path_str(key.to_def_id()) }
+        desc { |tcx| "computing explicit predicates of trait `{}`", tcx.def_path_str(key) }
     }
 
     /// Returns the predicates written explicitly by the user.
@@ -637,7 +663,7 @@ rustc_queries! {
     /// returns the full set of predicates. If `Some<Ident>`, then the query returns only the
     /// subset of super-predicates that reference traits that define the given associated type.
     /// This is used to avoid cycles in resolving types like `T::Item`.
-    query super_predicates_that_define_assoc_type(key: (DefId, rustc_span::symbol::Ident)) -> ty::GenericPredicates<'tcx> {
+    query super_predicates_that_define_assoc_item(key: (DefId, rustc_span::symbol::Ident)) -> ty::GenericPredicates<'tcx> {
         desc { |tcx| "computing the super traits of `{}` with associated type name `{}`",
             tcx.def_path_str(key.0),
             key.1
@@ -685,13 +711,11 @@ rustc_queries! {
     /// `is_const_fn` function. Consider using `is_const_fn` or `is_const_fn_raw` instead.
     query constness(key: DefId) -> hir::Constness {
         desc { |tcx| "checking if item is const: `{}`", tcx.def_path_str(key) }
-        cache_on_disk_if { key.is_local() }
         separate_provide_extern
     }
 
     query asyncness(key: DefId) -> hir::IsAsync {
         desc { |tcx| "checking if the function is async: `{}`", tcx.def_path_str(key) }
-        cache_on_disk_if { key.is_local() }
         separate_provide_extern
     }
 
@@ -706,17 +730,9 @@ rustc_queries! {
         desc { |tcx| "checking if item is promotable: `{}`", tcx.def_path_str(key) }
     }
 
-    /// Returns `true` if this is a foreign item (i.e., linked via `extern { ... }`).
-    query is_foreign_item(key: DefId) -> bool {
-        desc { |tcx| "checking if `{}` is a foreign item", tcx.def_path_str(key) }
-        cache_on_disk_if { key.is_local() }
-        separate_provide_extern
-    }
-
     /// Returns `Some(generator_kind)` if the node pointed to by `def_id` is a generator.
     query generator_kind(def_id: DefId) -> Option<hir::GeneratorKind> {
         desc { |tcx| "looking up generator kind of `{}`", tcx.def_path_str(def_id) }
-        cache_on_disk_if { def_id.is_local() }
         separate_provide_extern
     }
 
@@ -739,9 +755,10 @@ rustc_queries! {
         desc { "computing the inferred outlives predicates for items in this crate" }
     }
 
-    /// Maps from an impl/trait `DefId` to a list of the `DefId`s of its items.
+    /// Maps from an impl/trait or struct/variant `DefId`
+    /// to a list of the `DefId`s of its associated items or fields.
     query associated_item_def_ids(key: DefId) -> &'tcx [DefId] {
-        desc { |tcx| "collecting associated items of `{}`", tcx.def_path_str(key) }
+        desc { |tcx| "collecting associated items or fields of `{}`", tcx.def_path_str(key) }
         cache_on_disk_if { key.is_local() }
         separate_provide_extern
     }
@@ -814,7 +831,6 @@ rustc_queries! {
     }
     query impl_polarity(impl_id: DefId) -> ty::ImplPolarity {
         desc { |tcx| "computing implementation polarity of `{}`", tcx.def_path_str(impl_id) }
-        cache_on_disk_if { impl_id.is_local() }
         separate_provide_extern
     }
 
@@ -837,28 +853,16 @@ rustc_queries! {
 
     /// The result of unsafety-checking this `LocalDefId`.
     query unsafety_check_result(key: LocalDefId) -> &'tcx mir::UnsafetyCheckResult {
-        desc { |tcx| "unsafety-checking `{}`", tcx.def_path_str(key.to_def_id()) }
+        desc { |tcx| "unsafety-checking `{}`", tcx.def_path_str(key) }
         cache_on_disk_if { true }
     }
-    query unsafety_check_result_for_const_arg(key: (LocalDefId, DefId)) -> &'tcx mir::UnsafetyCheckResult {
-        desc {
-            |tcx| "unsafety-checking the const argument `{}`",
-            tcx.def_path_str(key.0.to_def_id())
-        }
-    }
 
     /// Unsafety-check this `LocalDefId` with THIR unsafeck. This should be
     /// used with `-Zthir-unsafeck`.
     query thir_check_unsafety(key: LocalDefId) {
-        desc { |tcx| "unsafety-checking `{}`", tcx.def_path_str(key.to_def_id()) }
+        desc { |tcx| "unsafety-checking `{}`", tcx.def_path_str(key) }
         cache_on_disk_if { true }
     }
-    query thir_check_unsafety_for_const_arg(key: (LocalDefId, DefId)) {
-        desc {
-            |tcx| "unsafety-checking the const argument `{}`",
-            tcx.def_path_str(key.0.to_def_id())
-        }
-    }
 
     /// Returns the types assumed to be well formed while "inside" of the given item.
     ///
@@ -913,7 +917,7 @@ rustc_queries! {
         desc { |tcx| "checking privacy in {}", describe_as_module(key, tcx) }
     }
 
-    query check_liveness(key: DefId) {
+    query check_liveness(key: LocalDefId) {
         desc { |tcx| "checking liveness of variables in `{}`", tcx.def_path_str(key) }
     }
 
@@ -952,29 +956,16 @@ rustc_queries! {
         separate_provide_extern
     }
 
-    query typeck_item_bodies(_: ()) -> () {
-        desc { "type-checking all item bodies" }
-    }
-
     query typeck(key: LocalDefId) -> &'tcx ty::TypeckResults<'tcx> {
-        desc { |tcx| "type-checking `{}`", tcx.def_path_str(key.to_def_id()) }
-        cache_on_disk_if { true }
-    }
-    query typeck_const_arg(
-        key: (LocalDefId, DefId)
-    ) -> &'tcx ty::TypeckResults<'tcx> {
-        desc {
-            |tcx| "type-checking the const argument `{}`",
-            tcx.def_path_str(key.0.to_def_id()),
-        }
+        desc { |tcx| "type-checking `{}`", tcx.def_path_str(key) }
+        cache_on_disk_if(tcx) { !tcx.is_typeck_child(key.to_def_id()) }
     }
     query diagnostic_only_typeck(key: LocalDefId) -> &'tcx ty::TypeckResults<'tcx> {
-        desc { |tcx| "type-checking `{}`", tcx.def_path_str(key.to_def_id()) }
-        cache_on_disk_if { true }
+        desc { |tcx| "type-checking `{}`", tcx.def_path_str(key) }
     }
 
     query used_trait_imports(key: LocalDefId) -> &'tcx UnordSet<LocalDefId> {
-        desc { |tcx| "finding used_trait_imports `{}`", tcx.def_path_str(key.to_def_id()) }
+        desc { |tcx| "finding used_trait_imports `{}`", tcx.def_path_str(key) }
         cache_on_disk_if { true }
     }
 
@@ -989,15 +980,9 @@ rustc_queries! {
     /// Borrow-checks the function body. If this is a closure, returns
     /// additional requirements that the closure's creator must verify.
     query mir_borrowck(key: LocalDefId) -> &'tcx mir::BorrowCheckResult<'tcx> {
-        desc { |tcx| "borrow-checking `{}`", tcx.def_path_str(key.to_def_id()) }
+        desc { |tcx| "borrow-checking `{}`", tcx.def_path_str(key) }
         cache_on_disk_if(tcx) { tcx.is_typeck_child(key.to_def_id()) }
     }
-    query mir_borrowck_const_arg(key: (LocalDefId, DefId)) -> &'tcx mir::BorrowCheckResult<'tcx> {
-        desc {
-            |tcx| "borrow-checking the const argument`{}`",
-            tcx.def_path_str(key.0.to_def_id())
-        }
-    }
 
     /// Gets a complete map from all types to their inherent impls.
     /// Not meant to be used directly outside of coherence.
@@ -1017,7 +1002,7 @@ rustc_queries! {
     query orphan_check_impl(key: LocalDefId) -> Result<(), ErrorGuaranteed> {
         desc { |tcx|
             "checking whether impl `{}` follows the orphan rules",
-            tcx.def_path_str(key.to_def_id()),
+            tcx.def_path_str(key),
         }
     }
 
@@ -1029,7 +1014,7 @@ rustc_queries! {
         desc { |tcx|
             "computing if `{}` (transitively) calls `{}`",
             key.0,
-            tcx.def_path_str(key.1.to_def_id()),
+            tcx.def_path_str(key.1),
         }
     }
 
@@ -1094,7 +1079,6 @@ rustc_queries! {
         key: ty::ParamEnvAnd<'tcx, mir::ConstantKind<'tcx>>
     ) -> Option<mir::DestructuredConstant<'tcx>> {
         desc { "destructuring MIR constant"}
-        remap_env_constness
     }
 
     /// Dereference a constant reference or raw pointer and turn the result into a constant
@@ -1103,7 +1087,6 @@ rustc_queries! {
         key: ty::ParamEnvAnd<'tcx, mir::ConstantKind<'tcx>>
     ) -> mir::ConstantKind<'tcx> {
         desc { "dereferencing MIR constant" }
-        remap_env_constness
     }
 
     query const_caller_location(key: (rustc_span::Symbol, u32, u32)) -> ConstValue<'tcx> {
@@ -1121,8 +1104,8 @@ rustc_queries! {
         desc { "converting literal to mir constant" }
     }
 
-    query check_match(key: LocalDefId) {
-        desc { |tcx| "match-checking `{}`", tcx.def_path_str(key.to_def_id()) }
+    query check_match(key: LocalDefId) -> Result<(), rustc_errors::ErrorGuaranteed> {
+        desc { |tcx| "match-checking `{}`", tcx.def_path_str(key) }
         cache_on_disk_if { true }
     }
 
@@ -1243,7 +1226,6 @@ rustc_queries! {
 
     query fn_arg_names(def_id: DefId) -> &'tcx [rustc_span::symbol::Ident] {
         desc { |tcx| "looking up function parameter names for `{}`", tcx.def_path_str(def_id) }
-        cache_on_disk_if { def_id.is_local() }
         separate_provide_extern
     }
     /// Gets the rendered value of the specified constant or associated constant.
@@ -1251,12 +1233,10 @@ rustc_queries! {
     query rendered_const(def_id: DefId) -> &'tcx String {
         arena_cache
         desc { |tcx| "rendering constant initializer of `{}`", tcx.def_path_str(def_id) }
-        cache_on_disk_if { def_id.is_local() }
         separate_provide_extern
     }
     query impl_parent(def_id: DefId) -> Option<DefId> {
         desc { |tcx| "computing specialization parent impl of `{}`", tcx.def_path_str(def_id) }
-        cache_on_disk_if { def_id.is_local() }
         separate_provide_extern
     }
 
@@ -1296,7 +1276,7 @@ rustc_queries! {
 
     query codegen_select_candidate(
         key: (ty::ParamEnv<'tcx>, ty::PolyTraitRef<'tcx>)
-    ) -> Result<&'tcx ImplSource<'tcx, ()>, traits::CodegenObligationError> {
+    ) -> Result<&'tcx ImplSource<'tcx, ()>, CodegenObligationError> {
         cache_on_disk_if { true }
         desc { |tcx| "computing candidate for `{}`", key.1 }
     }
@@ -1317,7 +1297,7 @@ rustc_queries! {
         desc { |tcx| "building specialization graph of trait `{}`", tcx.def_path_str(trait_id) }
         cache_on_disk_if { true }
     }
-    query object_safety_violations(trait_id: DefId) -> &'tcx [traits::ObjectSafetyViolation] {
+    query object_safety_violations(trait_id: DefId) -> &'tcx [ObjectSafetyViolation] {
         desc { |tcx| "determining object safety of trait `{}`", tcx.def_path_str(trait_id) }
     }
     query check_is_object_safe(trait_id: DefId) -> bool {
@@ -1346,32 +1326,26 @@ rustc_queries! {
     /// `ty.is_copy()`, etc, since that will prune the environment where possible.
     query is_copy_raw(env: ty::ParamEnvAnd<'tcx, Ty<'tcx>>) -> bool {
         desc { "computing whether `{}` is `Copy`", env.value }
-        remap_env_constness
     }
     /// Query backing `Ty::is_sized`.
     query is_sized_raw(env: ty::ParamEnvAnd<'tcx, Ty<'tcx>>) -> bool {
         desc { "computing whether `{}` is `Sized`", env.value }
-        remap_env_constness
     }
     /// Query backing `Ty::is_freeze`.
     query is_freeze_raw(env: ty::ParamEnvAnd<'tcx, Ty<'tcx>>) -> bool {
         desc { "computing whether `{}` is freeze", env.value }
-        remap_env_constness
     }
     /// Query backing `Ty::is_unpin`.
     query is_unpin_raw(env: ty::ParamEnvAnd<'tcx, Ty<'tcx>>) -> bool {
         desc { "computing whether `{}` is `Unpin`", env.value }
-        remap_env_constness
     }
     /// Query backing `Ty::needs_drop`.
     query needs_drop_raw(env: ty::ParamEnvAnd<'tcx, Ty<'tcx>>) -> bool {
         desc { "computing whether `{}` needs drop", env.value }
-        remap_env_constness
     }
     /// Query backing `Ty::has_significant_drop_raw`.
     query has_significant_drop_raw(env: ty::ParamEnvAnd<'tcx, Ty<'tcx>>) -> bool {
         desc { "computing whether `{}` has a significant drop", env.value }
-        remap_env_constness
     }
 
     /// Query backing `Ty::is_structural_eq_shallow`.
@@ -1411,7 +1385,6 @@ rustc_queries! {
     ) -> Result<ty::layout::TyAndLayout<'tcx>, ty::layout::LayoutError<'tcx>> {
         depth_limit
         desc { "computing layout of `{}`", key.value }
-        remap_env_constness
     }
 
     /// Compute a `FnAbi` suitable for indirect calls, i.e. to `fn` pointers.
@@ -1422,7 +1395,6 @@ rustc_queries! {
         key: ty::ParamEnvAnd<'tcx, (ty::PolyFnSig<'tcx>, &'tcx ty::List<Ty<'tcx>>)>
     ) -> Result<&'tcx abi::call::FnAbi<'tcx, Ty<'tcx>>, ty::layout::FnAbiError<'tcx>> {
         desc { "computing call ABI of `{}` function pointers", key.value.0 }
-        remap_env_constness
     }
 
     /// Compute a `FnAbi` suitable for declaring/defining an `fn` instance, and for
@@ -1434,7 +1406,6 @@ rustc_queries! {
         key: ty::ParamEnvAnd<'tcx, (ty::Instance<'tcx>, &'tcx ty::List<Ty<'tcx>>)>
     ) -> Result<&'tcx abi::call::FnAbi<'tcx, Ty<'tcx>>, ty::layout::FnAbiError<'tcx>> {
         desc { "computing call ABI of `{}`", key.value.0 }
-        remap_env_constness
     }
 
     query dylib_dependency_formats(_: CrateNum)
@@ -1478,7 +1449,7 @@ rustc_queries! {
         separate_provide_extern
     }
     query has_ffi_unwind_calls(key: LocalDefId) -> bool {
-        desc { |tcx| "checking if `{}` contains FFI-unwind calls", tcx.def_path_str(key.to_def_id()) }
+        desc { |tcx| "checking if `{}` contains FFI-unwind calls", tcx.def_path_str(key) }
         cache_on_disk_if { true }
     }
     query required_panic_strategy(_: CrateNum) -> Option<PanicStrategy> {
@@ -1518,13 +1489,12 @@ rustc_queries! {
 
     query impl_defaultness(def_id: DefId) -> hir::Defaultness {
         desc { |tcx| "looking up whether `{}` is a default impl", tcx.def_path_str(def_id) }
-        cache_on_disk_if { def_id.is_local() }
         separate_provide_extern
         feedable
     }
 
     query check_well_formed(key: hir::OwnerId) -> () {
-        desc { |tcx| "checking that `{}` is well-formed", tcx.def_path_str(key.to_def_id()) }
+        desc { |tcx| "checking that `{}` is well-formed", tcx.def_path_str(key) }
     }
 
     // The `DefId`s of all non-generic functions and statics in the given crate
@@ -1553,7 +1523,7 @@ rustc_queries! {
     query is_unreachable_local_definition(def_id: LocalDefId) -> bool {
         desc { |tcx|
             "checking whether `{}` is reachable from outside the crate",
-            tcx.def_path_str(def_id.to_def_id()),
+            tcx.def_path_str(def_id),
         }
     }
 
@@ -1747,7 +1717,7 @@ rustc_queries! {
         separate_provide_extern
     }
     query extern_mod_stmt_cnum(def_id: LocalDefId) -> Option<CrateNum> {
-        desc { |tcx| "computing crate imported by `{}`", tcx.def_path_str(def_id.to_def_id()) }
+        desc { |tcx| "computing crate imported by `{}`", tcx.def_path_str(def_id) }
     }
 
     query lib_features(_: ()) -> &'tcx LibFeatures {
@@ -1818,12 +1788,18 @@ rustc_queries! {
         desc { "looking at the source for a crate" }
         separate_provide_extern
     }
+
     /// Returns the debugger visualizers defined for this crate.
-    query debugger_visualizers(_: CrateNum) -> &'tcx Vec<rustc_span::DebuggerVisualizerFile> {
+    /// NOTE: This query has to be marked `eval_always` because it reads data
+    ///       directly from disk that is not tracked anywhere else. I.e. it
+    ///       represents a genuine input to the query system.
+    query debugger_visualizers(_: CrateNum) -> &'tcx Vec<DebuggerVisualizerFile> {
         arena_cache
         desc { "looking up the debugger visualizers for this crate" }
         separate_provide_extern
+        eval_always
     }
+
     query postorder_cnums(_: ()) -> &'tcx [CrateNum] {
         eval_always
         desc { "generating a postorder list of CrateNums" }
@@ -1851,7 +1827,7 @@ rustc_queries! {
         desc { "fetching potentially unused trait imports" }
     }
     query names_imported_by_glob_use(def_id: LocalDefId) -> &'tcx UnordSet<Symbol> {
-        desc { |tcx| "finding names imported by glob use for `{}`", tcx.def_path_str(def_id.to_def_id()) }
+        desc { |tcx| "finding names imported by glob use for `{}`", tcx.def_path_str(def_id) }
     }
 
     query stability_index(_: ()) -> &'tcx stability::Index {
@@ -1865,8 +1841,7 @@ rustc_queries! {
     }
 
     /// A list of all traits in a crate, used by rustdoc and error reporting.
-    /// NOTE: Not named just `traits` due to a naming conflict.
-    query traits_in_crate(_: CrateNum) -> &'tcx [DefId] {
+    query traits(_: CrateNum) -> &'tcx [DefId] {
         desc { "fetching all traits in a crate" }
         separate_provide_extern
     }
@@ -1937,7 +1912,16 @@ rustc_queries! {
         NoSolution,
     > {
         desc { "normalizing `{}`", goal.value.value }
-        remap_env_constness
+    }
+
+    /// Do not call this query directly: invoke `normalize` instead.
+    query normalize_inherent_projection_ty(
+        goal: CanonicalProjectionGoal<'tcx>
+    ) -> Result<
+        &'tcx Canonical<'tcx, canonical::QueryResponse<'tcx, NormalizationResult<'tcx>>>,
+        NoSolution,
+    > {
+        desc { "normalizing `{}`", goal.value.value }
     }
 
     /// Do not call this query directly: invoke `try_normalize_erasing_regions` instead.
@@ -1945,7 +1929,6 @@ rustc_queries! {
         goal: ParamEnvAnd<'tcx, GenericArg<'tcx>>
     ) -> Result<GenericArg<'tcx>, NoSolution> {
         desc { "normalizing `{}`", goal.value }
-        remap_env_constness
     }
 
     query implied_outlives_bounds(
@@ -1955,7 +1938,6 @@ rustc_queries! {
         NoSolution,
     > {
         desc { "computing implied outlives bounds for `{}`", goal.value.value }
-        remap_env_constness
     }
 
     /// Do not call this query directly:
@@ -1967,19 +1949,18 @@ rustc_queries! {
         NoSolution,
     > {
         desc { "computing dropck types for `{}`", goal.value.value }
-        remap_env_constness
     }
 
     /// Do not call this query directly: invoke `infcx.predicate_may_hold()` or
     /// `infcx.predicate_must_hold()` instead.
     query evaluate_obligation(
         goal: CanonicalPredicateGoal<'tcx>
-    ) -> Result<traits::EvaluationResult, traits::OverflowError> {
+    ) -> Result<EvaluationResult, OverflowError> {
         desc { "evaluating trait selection obligation `{}`", goal.value.value }
     }
 
     query evaluate_goal(
-        goal: traits::CanonicalChalkEnvironmentAndGoal<'tcx>
+        goal: CanonicalChalkEnvironmentAndGoal<'tcx>
     ) -> Result<
         &'tcx Canonical<'tcx, canonical::QueryResponse<'tcx, ()>>,
         NoSolution
@@ -1995,7 +1976,6 @@ rustc_queries! {
         NoSolution,
     > {
         desc { "evaluating `type_op_ascribe_user_type` `{:?}`", goal.value.value }
-        remap_env_constness
     }
 
     /// Do not call this query directly: part of the `Eq` type-op
@@ -2006,7 +1986,6 @@ rustc_queries! {
         NoSolution,
     > {
         desc { "evaluating `type_op_eq` `{:?}`", goal.value.value }
-        remap_env_constness
     }
 
     /// Do not call this query directly: part of the `Subtype` type-op
@@ -2017,7 +1996,6 @@ rustc_queries! {
         NoSolution,
     > {
         desc { "evaluating `type_op_subtype` `{:?}`", goal.value.value }
-        remap_env_constness
     }
 
     /// Do not call this query directly: part of the `ProvePredicate` type-op
@@ -2038,7 +2016,6 @@ rustc_queries! {
         NoSolution,
     > {
         desc { "normalizing `{}`", goal.value.value.value }
-        remap_env_constness
     }
 
     /// Do not call this query directly: part of the `Normalize` type-op
@@ -2049,7 +2026,6 @@ rustc_queries! {
         NoSolution,
     > {
         desc { "normalizing `{:?}`", goal.value.value.value }
-        remap_env_constness
     }
 
     /// Do not call this query directly: part of the `Normalize` type-op
@@ -2060,7 +2036,6 @@ rustc_queries! {
         NoSolution,
     > {
         desc { "normalizing `{:?}`", goal.value.value.value }
-        remap_env_constness
     }
 
     /// Do not call this query directly: part of the `Normalize` type-op
@@ -2071,7 +2046,6 @@ rustc_queries! {
         NoSolution,
     > {
         desc { "normalizing `{:?}`", goal.value.value.value }
-        remap_env_constness
     }
 
     query subst_and_check_impossible_predicates(key: (DefId, SubstsRef<'tcx>)) -> bool {
@@ -2093,7 +2067,6 @@ rustc_queries! {
         goal: CanonicalTyGoal<'tcx>
     ) -> MethodAutoderefStepsResult<'tcx> {
         desc { "computing autoderef types for `{}`", goal.value.value }
-        remap_env_constness
     }
 
     query supported_target_features(_: CrateNum) -> &'tcx FxHashMap<String, Option<Symbol>> {
@@ -2138,17 +2111,6 @@ rustc_queries! {
         key: ty::ParamEnvAnd<'tcx, (DefId, SubstsRef<'tcx>)>
     ) -> Result<Option<ty::Instance<'tcx>>, ErrorGuaranteed> {
         desc { "resolving instance `{}`", ty::Instance::new(key.value.0, key.value.1) }
-        remap_env_constness
-    }
-
-    query resolve_instance_of_const_arg(
-        key: ty::ParamEnvAnd<'tcx, (LocalDefId, DefId, SubstsRef<'tcx>)>
-    ) -> Result<Option<ty::Instance<'tcx>>, ErrorGuaranteed> {
-        desc {
-            "resolving instance of the const argument `{}`",
-            ty::Instance::new(key.value.0.to_def_id(), key.value.2),
-        }
-        remap_env_constness
     }
 
     query reveal_opaque_types_in_bounds(key: &'tcx ty::List<ty::Predicate<'tcx>>) -> &'tcx ty::List<ty::Predicate<'tcx>> {
@@ -2168,8 +2130,8 @@ rustc_queries! {
     /// all of the cases that the normal `ty::Ty`-based wfcheck does. This is fine,
     /// because the `ty::Ty`-based wfcheck is always run.
     query diagnostic_hir_wf_check(
-        key: (ty::Predicate<'tcx>, traits::WellFormedLoc)
-    ) -> &'tcx Option<traits::ObligationCause<'tcx>> {
+        key: (ty::Predicate<'tcx>, WellFormedLoc)
+    ) -> &'tcx Option<ObligationCause<'tcx>> {
         arena_cache
         eval_always
         no_hash
@@ -2197,7 +2159,7 @@ rustc_queries! {
     query compare_impl_const(
         key: (LocalDefId, DefId)
     ) -> Result<(), ErrorGuaranteed> {
-        desc { |tcx| "checking assoc const `{}` has the same type as trait item", tcx.def_path_str(key.0.to_def_id()) }
+        desc { |tcx| "checking assoc const `{}` has the same type as trait item", tcx.def_path_str(key.0) }
     }
 
     query deduced_param_attrs(def_id: DefId) -> &'tcx [ty::DeducedParamAttrs] {
@@ -2224,3 +2186,6 @@ rustc_queries! {
         desc { "check whether two const param are definitely not equal to eachother"}
     }
 }
+
+rustc_query_append! { define_callbacks! }
+rustc_feedable_queries! { define_feedable! }
diff --git a/compiler/rustc_middle/src/query/on_disk_cache.rs b/compiler/rustc_middle/src/query/on_disk_cache.rs
new file mode 100644
index 000000000..220118ae5
--- /dev/null
+++ b/compiler/rustc_middle/src/query/on_disk_cache.rs
@@ -0,0 +1,1042 @@
+use rustc_data_structures::fx::{FxHashMap, FxIndexSet};
+use rustc_data_structures::memmap::Mmap;
+use rustc_data_structures::stable_hasher::Hash64;
+use rustc_data_structures::sync::{HashMapExt, Lock, Lrc, RwLock};
+use rustc_data_structures::unhash::UnhashMap;
+use rustc_data_structures::unord::UnordSet;
+use rustc_hir::def_id::{CrateNum, DefId, DefIndex, LocalDefId, StableCrateId, LOCAL_CRATE};
+use rustc_hir::definitions::DefPathHash;
+use rustc_index::{Idx, IndexVec};
+use rustc_middle::dep_graph::{DepNodeIndex, SerializedDepNodeIndex};
+use rustc_middle::mir::interpret::{AllocDecodingSession, AllocDecodingState};
+use rustc_middle::mir::{self, interpret};
+use rustc_middle::ty::codec::{RefDecodable, TyDecoder, TyEncoder};
+use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_query_system::query::QuerySideEffects;
+use rustc_serialize::{
+    opaque::{FileEncodeResult, FileEncoder, IntEncodedWithFixedSize, MemDecoder},
+    Decodable, Decoder, Encodable, Encoder,
+};
+use rustc_session::Session;
+use rustc_span::hygiene::{
+    ExpnId, HygieneDecodeContext, HygieneEncodeContext, SyntaxContext, SyntaxContextData,
+};
+use rustc_span::source_map::{SourceMap, StableSourceFileId};
+use rustc_span::{BytePos, ExpnData, ExpnHash, Pos, SourceFile, Span};
+use rustc_span::{CachingSourceMapView, Symbol};
+use std::collections::hash_map::Entry;
+use std::io;
+use std::mem;
+
+const TAG_FILE_FOOTER: u128 = 0xC0FFEE_C0FFEE_C0FFEE_C0FFEE_C0FFEE;
+
+// A normal span encoded with both location information and a `SyntaxContext`
+const TAG_FULL_SPAN: u8 = 0;
+// A partial span with no location information, encoded only with a `SyntaxContext`
+const TAG_PARTIAL_SPAN: u8 = 1;
+const TAG_RELATIVE_SPAN: u8 = 2;
+
+const TAG_SYNTAX_CONTEXT: u8 = 0;
+const TAG_EXPN_DATA: u8 = 1;
+
+// Tags for encoding Symbol's
+const SYMBOL_STR: u8 = 0;
+const SYMBOL_OFFSET: u8 = 1;
+const SYMBOL_PREINTERNED: u8 = 2;
+
+/// Provides an interface to incremental compilation data cached from the
+/// previous compilation session. This data will eventually include the results
+/// of a few selected queries (like `typeck` and `mir_optimized`) and
+/// any side effects that have been emitted during a query.
+pub struct OnDiskCache<'sess> {
+    // The complete cache data in serialized form.
+    serialized_data: RwLock<Option<Mmap>>,
+
+    // Collects all `QuerySideEffects` created during the current compilation
+    // session.
+    current_side_effects: Lock<FxHashMap<DepNodeIndex, QuerySideEffects>>,
+
+    source_map: &'sess SourceMap,
+    file_index_to_stable_id: FxHashMap<SourceFileIndex, EncodedSourceFileId>,
+
+    // Caches that are populated lazily during decoding.
+    file_index_to_file: Lock<FxHashMap<SourceFileIndex, Lrc<SourceFile>>>,
+
+    // A map from dep-node to the position of the cached query result in
+    // `serialized_data`.
+    query_result_index: FxHashMap<SerializedDepNodeIndex, AbsoluteBytePos>,
+
+    // A map from dep-node to the position of any associated `QuerySideEffects` in
+    // `serialized_data`.
+    prev_side_effects_index: FxHashMap<SerializedDepNodeIndex, AbsoluteBytePos>,
+
+    alloc_decoding_state: AllocDecodingState,
+
+    // A map from syntax context ids to the position of their associated
+    // `SyntaxContextData`. We use a `u32` instead of a `SyntaxContext`
+    // to represent the fact that we are storing *encoded* ids. When we decode
+    // a `SyntaxContext`, a new id will be allocated from the global `HygieneData`,
+    // which will almost certainly be different than the serialized id.
+    syntax_contexts: FxHashMap<u32, AbsoluteBytePos>,
+    // A map from the `DefPathHash` of an `ExpnId` to the position
+    // of their associated `ExpnData`. Ideally, we would store a `DefId`,
+    // but we need to decode this before we've constructed a `TyCtxt` (which
+    // makes it difficult to decode a `DefId`).
+
+    // Note that these `DefPathHashes` correspond to both local and foreign
+    // `ExpnData` (e.g `ExpnData.krate` may not be `LOCAL_CRATE`). Alternatively,
+    // we could look up the `ExpnData` from the metadata of foreign crates,
+    // but it seemed easier to have `OnDiskCache` be independent of the `CStore`.
+    expn_data: UnhashMap<ExpnHash, AbsoluteBytePos>,
+    // Additional information used when decoding hygiene data.
+    hygiene_context: HygieneDecodeContext,
+    // Maps `ExpnHash`es to their raw value from the *previous*
+    // compilation session. This is used as an initial 'guess' when
+    // we try to map an `ExpnHash` to its value in the current
+    // compilation session.
+    foreign_expn_data: UnhashMap<ExpnHash, u32>,
+}
+
+// This type is used only for serialization and deserialization.
+#[derive(Encodable, Decodable)]
+struct Footer {
+    file_index_to_stable_id: FxHashMap<SourceFileIndex, EncodedSourceFileId>,
+    query_result_index: EncodedDepNodeIndex,
+    side_effects_index: EncodedDepNodeIndex,
+    // The location of all allocations.
+    interpret_alloc_index: Vec<u32>,
+    // See `OnDiskCache.syntax_contexts`
+    syntax_contexts: FxHashMap<u32, AbsoluteBytePos>,
+    // See `OnDiskCache.expn_data`
+    expn_data: UnhashMap<ExpnHash, AbsoluteBytePos>,
+    foreign_expn_data: UnhashMap<ExpnHash, u32>,
+}
+
+pub type EncodedDepNodeIndex = Vec<(SerializedDepNodeIndex, AbsoluteBytePos)>;
+
+#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, Encodable, Decodable)]
+struct SourceFileIndex(u32);
+
+#[derive(Copy, Clone, Debug, Hash, Eq, PartialEq, Encodable, Decodable)]
+pub struct AbsoluteBytePos(u64);
+
+impl AbsoluteBytePos {
+    #[inline]
+    pub fn new(pos: usize) -> AbsoluteBytePos {
+        AbsoluteBytePos(pos.try_into().expect("Incremental cache file size overflowed u64."))
+    }
+
+    #[inline]
+    fn to_usize(self) -> usize {
+        self.0 as usize
+    }
+}
+
+/// An `EncodedSourceFileId` is the same as a `StableSourceFileId` except that
+/// the source crate is represented as a [StableCrateId] instead of as a
+/// `CrateNum`. This way `EncodedSourceFileId` can be encoded and decoded
+/// without any additional context, i.e. with a simple `opaque::Decoder` (which
+/// is the only thing available when decoding the cache's [Footer].
+#[derive(Encodable, Decodable, Clone, Debug)]
+struct EncodedSourceFileId {
+    file_name_hash: Hash64,
+    stable_crate_id: StableCrateId,
+}
+
+impl EncodedSourceFileId {
+    #[inline]
+    fn translate(&self, tcx: TyCtxt<'_>) -> StableSourceFileId {
+        let cnum = tcx.stable_crate_id_to_crate_num(self.stable_crate_id);
+        StableSourceFileId { file_name_hash: self.file_name_hash, cnum }
+    }
+
+    #[inline]
+    fn new(tcx: TyCtxt<'_>, file: &SourceFile) -> EncodedSourceFileId {
+        let source_file_id = StableSourceFileId::new(file);
+        EncodedSourceFileId {
+            file_name_hash: source_file_id.file_name_hash,
+            stable_crate_id: tcx.stable_crate_id(source_file_id.cnum),
+        }
+    }
+}
+
+impl<'sess> OnDiskCache<'sess> {
+    /// Creates a new `OnDiskCache` instance from the serialized data in `data`.
+    pub fn new(sess: &'sess Session, data: Mmap, start_pos: usize) -> Self {
+        debug_assert!(sess.opts.incremental.is_some());
+
+        // Wrap in a scope so we can borrow `data`.
+        let footer: Footer = {
+            let mut decoder = MemDecoder::new(&data, start_pos);
+
+            // Decode the *position* of the footer, which can be found in the
+            // last 8 bytes of the file.
+            let footer_pos = decoder
+                .with_position(decoder.len() - IntEncodedWithFixedSize::ENCODED_SIZE, |decoder| {
+                    IntEncodedWithFixedSize::decode(decoder).0 as usize
+                });
+            // Decode the file footer, which contains all the lookup tables, etc.
+            decoder.with_position(footer_pos, |decoder| decode_tagged(decoder, TAG_FILE_FOOTER))
+        };
+
+        Self {
+            serialized_data: RwLock::new(Some(data)),
+            file_index_to_stable_id: footer.file_index_to_stable_id,
+            file_index_to_file: Default::default(),
+            source_map: sess.source_map(),
+            current_side_effects: Default::default(),
+            query_result_index: footer.query_result_index.into_iter().collect(),
+            prev_side_effects_index: footer.side_effects_index.into_iter().collect(),
+            alloc_decoding_state: AllocDecodingState::new(footer.interpret_alloc_index),
+            syntax_contexts: footer.syntax_contexts,
+            expn_data: footer.expn_data,
+            foreign_expn_data: footer.foreign_expn_data,
+            hygiene_context: Default::default(),
+        }
+    }
+
+    pub fn new_empty(source_map: &'sess SourceMap) -> Self {
+        Self {
+            serialized_data: RwLock::new(None),
+            file_index_to_stable_id: Default::default(),
+            file_index_to_file: Default::default(),
+            source_map,
+            current_side_effects: Default::default(),
+            query_result_index: Default::default(),
+            prev_side_effects_index: Default::default(),
+            alloc_decoding_state: AllocDecodingState::new(Vec::new()),
+            syntax_contexts: FxHashMap::default(),
+            expn_data: UnhashMap::default(),
+            foreign_expn_data: UnhashMap::default(),
+            hygiene_context: Default::default(),
+        }
+    }
+
+    /// Execute all cache promotions and release the serialized backing Mmap.
+    ///
+    /// Cache promotions require invoking queries, which needs to read the serialized data.
+    /// In order to serialize the new on-disk cache, the former on-disk cache file needs to be
+    /// deleted, hence we won't be able to refer to its memmapped data.
+    pub fn drop_serialized_data(&self, tcx: TyCtxt<'_>) {
+        // Load everything into memory so we can write it out to the on-disk
+        // cache. The vast majority of cacheable query results should already
+        // be in memory, so this should be a cheap operation.
+        // Do this *before* we clone 'latest_foreign_def_path_hashes', since
+        // loading existing queries may cause us to create new DepNodes, which
+        // may in turn end up invoking `store_foreign_def_id_hash`
+        tcx.dep_graph.exec_cache_promotions(tcx);
+
+        *self.serialized_data.write() = None;
+    }
+
+    pub fn serialize(&self, tcx: TyCtxt<'_>, encoder: FileEncoder) -> FileEncodeResult {
+        // Serializing the `DepGraph` should not modify it.
+        tcx.dep_graph.with_ignore(|| {
+            // Allocate `SourceFileIndex`es.
+            let (file_to_file_index, file_index_to_stable_id) = {
+                let files = tcx.sess.source_map().files();
+                let mut file_to_file_index =
+                    FxHashMap::with_capacity_and_hasher(files.len(), Default::default());
+                let mut file_index_to_stable_id =
+                    FxHashMap::with_capacity_and_hasher(files.len(), Default::default());
+
+                for (index, file) in files.iter().enumerate() {
+                    let index = SourceFileIndex(index as u32);
+                    let file_ptr: *const SourceFile = &**file as *const _;
+                    file_to_file_index.insert(file_ptr, index);
+                    let source_file_id = EncodedSourceFileId::new(tcx, &file);
+                    file_index_to_stable_id.insert(index, source_file_id);
+                }
+
+                (file_to_file_index, file_index_to_stable_id)
+            };
+
+            let hygiene_encode_context = HygieneEncodeContext::default();
+
+            let mut encoder = CacheEncoder {
+                tcx,
+                encoder,
+                type_shorthands: Default::default(),
+                predicate_shorthands: Default::default(),
+                interpret_allocs: Default::default(),
+                source_map: CachingSourceMapView::new(tcx.sess.source_map()),
+                file_to_file_index,
+                hygiene_context: &hygiene_encode_context,
+                symbol_table: Default::default(),
+            };
+
+            // Encode query results.
+            let mut query_result_index = EncodedDepNodeIndex::new();
+
+            tcx.sess.time("encode_query_results", || {
+                let enc = &mut encoder;
+                let qri = &mut query_result_index;
+                (tcx.query_system.fns.encode_query_results)(tcx, enc, qri);
+            });
+
+            // Encode side effects.
+            let side_effects_index: EncodedDepNodeIndex = self
+                .current_side_effects
+                .borrow()
+                .iter()
+                .map(|(dep_node_index, side_effects)| {
+                    let pos = AbsoluteBytePos::new(encoder.position());
+                    let dep_node_index = SerializedDepNodeIndex::new(dep_node_index.index());
+                    encoder.encode_tagged(dep_node_index, side_effects);
+
+                    (dep_node_index, pos)
+                })
+                .collect();
+
+            let interpret_alloc_index = {
+                let mut interpret_alloc_index = Vec::new();
+                let mut n = 0;
+                loop {
+                    let new_n = encoder.interpret_allocs.len();
+                    // If we have found new IDs, serialize those too.
+                    if n == new_n {
+                        // Otherwise, abort.
+                        break;
+                    }
+                    interpret_alloc_index.reserve(new_n - n);
+                    for idx in n..new_n {
+                        let id = encoder.interpret_allocs[idx];
+                        let pos: u32 = encoder.position().try_into().unwrap();
+                        interpret_alloc_index.push(pos);
+                        interpret::specialized_encode_alloc_id(&mut encoder, tcx, id);
+                    }
+                    n = new_n;
+                }
+                interpret_alloc_index
+            };
+
+            let mut syntax_contexts = FxHashMap::default();
+            let mut expn_data = UnhashMap::default();
+            let mut foreign_expn_data = UnhashMap::default();
+
+            // Encode all hygiene data (`SyntaxContextData` and `ExpnData`) from the current
+            // session.
+
+            hygiene_encode_context.encode(
+                &mut encoder,
+                |encoder, index, ctxt_data| {
+                    let pos = AbsoluteBytePos::new(encoder.position());
+                    encoder.encode_tagged(TAG_SYNTAX_CONTEXT, ctxt_data);
+                    syntax_contexts.insert(index, pos);
+                },
+                |encoder, expn_id, data, hash| {
+                    if expn_id.krate == LOCAL_CRATE {
+                        let pos = AbsoluteBytePos::new(encoder.position());
+                        encoder.encode_tagged(TAG_EXPN_DATA, data);
+                        expn_data.insert(hash, pos);
+                    } else {
+                        foreign_expn_data.insert(hash, expn_id.local_id.as_u32());
+                    }
+                },
+            );
+
+            // Encode the file footer.
+            let footer_pos = encoder.position() as u64;
+            encoder.encode_tagged(
+                TAG_FILE_FOOTER,
+                &Footer {
+                    file_index_to_stable_id,
+                    query_result_index,
+                    side_effects_index,
+                    interpret_alloc_index,
+                    syntax_contexts,
+                    expn_data,
+                    foreign_expn_data,
+                },
+            );
+
+            // Encode the position of the footer as the last 8 bytes of the
+            // file so we know where to look for it.
+            IntEncodedWithFixedSize(footer_pos).encode(&mut encoder.encoder);
+
+            // DO NOT WRITE ANYTHING TO THE ENCODER AFTER THIS POINT! The address
+            // of the footer must be the last thing in the data stream.
+
+            encoder.finish()
+        })
+    }
+
+    /// Loads a `QuerySideEffects` created during the previous compilation session.
+    pub fn load_side_effects(
+        &self,
+        tcx: TyCtxt<'_>,
+        dep_node_index: SerializedDepNodeIndex,
+    ) -> QuerySideEffects {
+        let side_effects: Option<QuerySideEffects> =
+            self.load_indexed(tcx, dep_node_index, &self.prev_side_effects_index);
+
+        side_effects.unwrap_or_default()
+    }
+
+    /// Stores a `QuerySideEffects` emitted during the current compilation session.
+    /// Anything stored like this will be available via `load_side_effects` in
+    /// the next compilation session.
+    pub fn store_side_effects(&self, dep_node_index: DepNodeIndex, side_effects: QuerySideEffects) {
+        let mut current_side_effects = self.current_side_effects.borrow_mut();
+        let prev = current_side_effects.insert(dep_node_index, side_effects);
+        debug_assert!(prev.is_none());
+    }
+
+    /// Return whether the cached query result can be decoded.
+    #[inline]
+    pub fn loadable_from_disk(&self, dep_node_index: SerializedDepNodeIndex) -> bool {
+        self.query_result_index.contains_key(&dep_node_index)
+        // with_decoder is infallible, so we can stop here
+    }
+
+    /// Returns the cached query result if there is something in the cache for
+    /// the given `SerializedDepNodeIndex`; otherwise returns `None`.
+    pub fn try_load_query_result<'tcx, T>(
+        &self,
+        tcx: TyCtxt<'tcx>,
+        dep_node_index: SerializedDepNodeIndex,
+    ) -> Option<T>
+    where
+        T: for<'a> Decodable<CacheDecoder<'a, 'tcx>>,
+    {
+        let opt_value = self.load_indexed(tcx, dep_node_index, &self.query_result_index);
+        debug_assert_eq!(opt_value.is_some(), self.loadable_from_disk(dep_node_index));
+        opt_value
+    }
+
+    /// Stores side effect emitted during computation of an anonymous query.
+    /// Since many anonymous queries can share the same `DepNode`, we aggregate
+    /// them -- as opposed to regular queries where we assume that there is a
+    /// 1:1 relationship between query-key and `DepNode`.
+    pub fn store_side_effects_for_anon_node(
+        &self,
+        dep_node_index: DepNodeIndex,
+        side_effects: QuerySideEffects,
+    ) {
+        let mut current_side_effects = self.current_side_effects.borrow_mut();
+
+        let x = current_side_effects.entry(dep_node_index).or_default();
+        x.append(side_effects);
+    }
+
+    fn load_indexed<'tcx, T>(
+        &self,
+        tcx: TyCtxt<'tcx>,
+        dep_node_index: SerializedDepNodeIndex,
+        index: &FxHashMap<SerializedDepNodeIndex, AbsoluteBytePos>,
+    ) -> Option<T>
+    where
+        T: for<'a> Decodable<CacheDecoder<'a, 'tcx>>,
+    {
+        let pos = index.get(&dep_node_index).cloned()?;
+        let value = self.with_decoder(tcx, pos, |decoder| decode_tagged(decoder, dep_node_index));
+        Some(value)
+    }
+
+    fn with_decoder<'a, 'tcx, T, F: for<'s> FnOnce(&mut CacheDecoder<'s, 'tcx>) -> T>(
+        &'sess self,
+        tcx: TyCtxt<'tcx>,
+        pos: AbsoluteBytePos,
+        f: F,
+    ) -> T
+    where
+        T: Decodable<CacheDecoder<'a, 'tcx>>,
+    {
+        let serialized_data = self.serialized_data.read();
+        let mut decoder = CacheDecoder {
+            tcx,
+            opaque: MemDecoder::new(serialized_data.as_deref().unwrap_or(&[]), pos.to_usize()),
+            source_map: self.source_map,
+            file_index_to_file: &self.file_index_to_file,
+            file_index_to_stable_id: &self.file_index_to_stable_id,
+            alloc_decoding_session: self.alloc_decoding_state.new_decoding_session(),
+            syntax_contexts: &self.syntax_contexts,
+            expn_data: &self.expn_data,
+            foreign_expn_data: &self.foreign_expn_data,
+            hygiene_context: &self.hygiene_context,
+        };
+        f(&mut decoder)
+    }
+}
+
+//- DECODING -------------------------------------------------------------------
+
+/// A decoder that can read from the incremental compilation cache. It is similar to the one
+/// we use for crate metadata decoding in that it can rebase spans and eventually
+/// will also handle things that contain `Ty` instances.
+pub struct CacheDecoder<'a, 'tcx> {
+    tcx: TyCtxt<'tcx>,
+    opaque: MemDecoder<'a>,
+    source_map: &'a SourceMap,
+    file_index_to_file: &'a Lock<FxHashMap<SourceFileIndex, Lrc<SourceFile>>>,
+    file_index_to_stable_id: &'a FxHashMap<SourceFileIndex, EncodedSourceFileId>,
+    alloc_decoding_session: AllocDecodingSession<'a>,
+    syntax_contexts: &'a FxHashMap<u32, AbsoluteBytePos>,
+    expn_data: &'a UnhashMap<ExpnHash, AbsoluteBytePos>,
+    foreign_expn_data: &'a UnhashMap<ExpnHash, u32>,
+    hygiene_context: &'a HygieneDecodeContext,
+}
+
+impl<'a, 'tcx> CacheDecoder<'a, 'tcx> {
+    #[inline]
+    fn file_index_to_file(&self, index: SourceFileIndex) -> Lrc<SourceFile> {
+        let CacheDecoder {
+            tcx,
+            ref file_index_to_file,
+            ref file_index_to_stable_id,
+            ref source_map,
+            ..
+        } = *self;
+
+        file_index_to_file
+            .borrow_mut()
+            .entry(index)
+            .or_insert_with(|| {
+                let stable_id = file_index_to_stable_id[&index].translate(tcx);
+
+                // If this `SourceFile` is from a foreign crate, then make sure
+                // that we've imported all of the source files from that crate.
+                // This has usually already been done during macro invocation.
+                // However, when encoding query results like `TypeckResults`,
+                // we might encode an `AdtDef` for a foreign type (because it
+                // was referenced in the body of the function). There is no guarantee
+                // that we will load the source files from that crate during macro
+                // expansion, so we use `import_source_files` to ensure that the foreign
+                // source files are actually imported before we call `source_file_by_stable_id`.
+                if stable_id.cnum != LOCAL_CRATE {
+                    self.tcx.cstore_untracked().import_source_files(self.tcx.sess, stable_id.cnum);
+                }
+
+                source_map
+                    .source_file_by_stable_id(stable_id)
+                    .expect("failed to lookup `SourceFile` in new context")
+            })
+            .clone()
+    }
+}
+
+// Decodes something that was encoded with `encode_tagged()` and verify that the
+// tag matches and the correct amount of bytes was read.
+fn decode_tagged<D, T, V>(decoder: &mut D, expected_tag: T) -> V
+where
+    T: Decodable<D> + Eq + std::fmt::Debug,
+    V: Decodable<D>,
+    D: Decoder,
+{
+    let start_pos = decoder.position();
+
+    let actual_tag = T::decode(decoder);
+    assert_eq!(actual_tag, expected_tag);
+    let value = V::decode(decoder);
+    let end_pos = decoder.position();
+
+    let expected_len: u64 = Decodable::decode(decoder);
+    assert_eq!((end_pos - start_pos) as u64, expected_len);
+
+    value
+}
+
+impl<'a, 'tcx> TyDecoder for CacheDecoder<'a, 'tcx> {
+    type I = TyCtxt<'tcx>;
+    const CLEAR_CROSS_CRATE: bool = false;
+
+    #[inline]
+    fn interner(&self) -> TyCtxt<'tcx> {
+        self.tcx
+    }
+
+    fn cached_ty_for_shorthand<F>(&mut self, shorthand: usize, or_insert_with: F) -> Ty<'tcx>
+    where
+        F: FnOnce(&mut Self) -> Ty<'tcx>,
+    {
+        let tcx = self.tcx;
+
+        let cache_key = ty::CReaderCacheKey { cnum: None, pos: shorthand };
+
+        if let Some(&ty) = tcx.ty_rcache.borrow().get(&cache_key) {
+            return ty;
+        }
+
+        let ty = or_insert_with(self);
+        // This may overwrite the entry, but it should overwrite with the same value.
+        tcx.ty_rcache.borrow_mut().insert_same(cache_key, ty);
+        ty
+    }
+
+    fn with_position<F, R>(&mut self, pos: usize, f: F) -> R
+    where
+        F: FnOnce(&mut Self) -> R,
+    {
+        debug_assert!(pos < self.opaque.len());
+
+        let new_opaque = MemDecoder::new(self.opaque.data(), pos);
+        let old_opaque = mem::replace(&mut self.opaque, new_opaque);
+        let r = f(self);
+        self.opaque = old_opaque;
+        r
+    }
+
+    fn decode_alloc_id(&mut self) -> interpret::AllocId {
+        let alloc_decoding_session = self.alloc_decoding_session;
+        alloc_decoding_session.decode_alloc_id(self)
+    }
+}
+
+rustc_middle::implement_ty_decoder!(CacheDecoder<'a, 'tcx>);
+
+// This ensures that the `Decodable<opaque::Decoder>::decode` specialization for `Vec<u8>` is used
+// when a `CacheDecoder` is passed to `Decodable::decode`. Unfortunately, we have to manually opt
+// into specializations this way, given how `CacheDecoder` and the decoding traits currently work.
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for Vec<u8> {
+    fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self {
+        Decodable::decode(&mut d.opaque)
+    }
+}
+
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for SyntaxContext {
+    fn decode(decoder: &mut CacheDecoder<'a, 'tcx>) -> Self {
+        let syntax_contexts = decoder.syntax_contexts;
+        rustc_span::hygiene::decode_syntax_context(decoder, decoder.hygiene_context, |this, id| {
+            // This closure is invoked if we haven't already decoded the data for the `SyntaxContext` we are deserializing.
+            // We look up the position of the associated `SyntaxData` and decode it.
+            let pos = syntax_contexts.get(&id).unwrap();
+            this.with_position(pos.to_usize(), |decoder| {
+                let data: SyntaxContextData = decode_tagged(decoder, TAG_SYNTAX_CONTEXT);
+                data
+            })
+        })
+    }
+}
+
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for ExpnId {
+    fn decode(decoder: &mut CacheDecoder<'a, 'tcx>) -> Self {
+        let hash = ExpnHash::decode(decoder);
+        if hash.is_root() {
+            return ExpnId::root();
+        }
+
+        if let Some(expn_id) = ExpnId::from_hash(hash) {
+            return expn_id;
+        }
+
+        let krate = decoder.tcx.stable_crate_id_to_crate_num(hash.stable_crate_id());
+
+        let expn_id = if krate == LOCAL_CRATE {
+            // We look up the position of the associated `ExpnData` and decode it.
+            let pos = decoder
+                .expn_data
+                .get(&hash)
+                .unwrap_or_else(|| panic!("Bad hash {:?} (map {:?})", hash, decoder.expn_data));
+
+            let data: ExpnData = decoder
+                .with_position(pos.to_usize(), |decoder| decode_tagged(decoder, TAG_EXPN_DATA));
+            let expn_id = rustc_span::hygiene::register_local_expn_id(data, hash);
+
+            #[cfg(debug_assertions)]
+            {
+                use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
+                let local_hash = decoder.tcx.with_stable_hashing_context(|mut hcx| {
+                    let mut hasher = StableHasher::new();
+                    expn_id.expn_data().hash_stable(&mut hcx, &mut hasher);
+                    hasher.finish()
+                });
+                debug_assert_eq!(hash.local_hash(), local_hash);
+            }
+
+            expn_id
+        } else {
+            let index_guess = decoder.foreign_expn_data[&hash];
+            decoder.tcx.cstore_untracked().expn_hash_to_expn_id(
+                decoder.tcx.sess,
+                krate,
+                index_guess,
+                hash,
+            )
+        };
+
+        debug_assert_eq!(expn_id.krate, krate);
+        expn_id
+    }
+}
+
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for Span {
+    fn decode(decoder: &mut CacheDecoder<'a, 'tcx>) -> Self {
+        let ctxt = SyntaxContext::decode(decoder);
+        let parent = Option::<LocalDefId>::decode(decoder);
+        let tag: u8 = Decodable::decode(decoder);
+
+        if tag == TAG_PARTIAL_SPAN {
+            return Span::new(BytePos(0), BytePos(0), ctxt, parent);
+        } else if tag == TAG_RELATIVE_SPAN {
+            let dlo = u32::decode(decoder);
+            let dto = u32::decode(decoder);
+
+            let enclosing = decoder.tcx.source_span_untracked(parent.unwrap()).data_untracked();
+            let span = Span::new(
+                enclosing.lo + BytePos::from_u32(dlo),
+                enclosing.lo + BytePos::from_u32(dto),
+                ctxt,
+                parent,
+            );
+
+            return span;
+        } else {
+            debug_assert_eq!(tag, TAG_FULL_SPAN);
+        }
+
+        let file_lo_index = SourceFileIndex::decode(decoder);
+        let line_lo = usize::decode(decoder);
+        let col_lo = BytePos::decode(decoder);
+        let len = BytePos::decode(decoder);
+
+        let file_lo = decoder.file_index_to_file(file_lo_index);
+        let lo = file_lo.lines(|lines| lines[line_lo - 1] + col_lo);
+        let hi = lo + len;
+
+        Span::new(lo, hi, ctxt, parent)
+    }
+}
+
+// copy&paste impl from rustc_metadata
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for Symbol {
+    #[inline]
+    fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self {
+        let tag = d.read_u8();
+
+        match tag {
+            SYMBOL_STR => {
+                let s = d.read_str();
+                Symbol::intern(s)
+            }
+            SYMBOL_OFFSET => {
+                // read str offset
+                let pos = d.read_usize();
+
+                // move to str offset and read
+                d.opaque.with_position(pos, |d| {
+                    let s = d.read_str();
+                    Symbol::intern(s)
+                })
+            }
+            SYMBOL_PREINTERNED => {
+                let symbol_index = d.read_u32();
+                Symbol::new_from_decoded(symbol_index)
+            }
+            _ => unreachable!(),
+        }
+    }
+}
+
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for CrateNum {
+    #[inline]
+    fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self {
+        let stable_id = StableCrateId::decode(d);
+        let cnum = d.tcx.stable_crate_id_to_crate_num(stable_id);
+        cnum
+    }
+}
+
+// This impl makes sure that we get a runtime error when we try decode a
+// `DefIndex` that is not contained in a `DefId`. Such a case would be problematic
+// because we would not know how to transform the `DefIndex` to the current
+// context.
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for DefIndex {
+    fn decode(_d: &mut CacheDecoder<'a, 'tcx>) -> DefIndex {
+        panic!("trying to decode `DefIndex` outside the context of a `DefId`")
+    }
+}
+
+// Both the `CrateNum` and the `DefIndex` of a `DefId` can change in between two
+// compilation sessions. We use the `DefPathHash`, which is stable across
+// sessions, to map the old `DefId` to the new one.
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for DefId {
+    #[inline]
+    fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self {
+        // Load the `DefPathHash` which is was we encoded the `DefId` as.
+        let def_path_hash = DefPathHash::decode(d);
+
+        // Using the `DefPathHash`, we can lookup the new `DefId`.
+        // Subtle: We only encode a `DefId` as part of a query result.
+        // If we get to this point, then all of the query inputs were green,
+        // which means that the definition with this hash is guaranteed to
+        // still exist in the current compilation session.
+        d.tcx.def_path_hash_to_def_id(def_path_hash, &mut || {
+            panic!("Failed to convert DefPathHash {def_path_hash:?}")
+        })
+    }
+}
+
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for &'tcx UnordSet<LocalDefId> {
+    #[inline]
+    fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self {
+        RefDecodable::decode(d)
+    }
+}
+
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>>
+    for &'tcx FxHashMap<DefId, ty::EarlyBinder<Ty<'tcx>>>
+{
+    #[inline]
+    fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self {
+        RefDecodable::decode(d)
+    }
+}
+
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>>
+    for &'tcx IndexVec<mir::Promoted, mir::Body<'tcx>>
+{
+    #[inline]
+    fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self {
+        RefDecodable::decode(d)
+    }
+}
+
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for &'tcx [(ty::Predicate<'tcx>, Span)] {
+    #[inline]
+    fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self {
+        RefDecodable::decode(d)
+    }
+}
+
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for &'tcx [(ty::Clause<'tcx>, Span)] {
+    #[inline]
+    fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self {
+        RefDecodable::decode(d)
+    }
+}
+
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for &'tcx [rustc_ast::InlineAsmTemplatePiece] {
+    #[inline]
+    fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self {
+        RefDecodable::decode(d)
+    }
+}
+
+macro_rules! impl_ref_decoder {
+    (<$tcx:tt> $($ty:ty,)*) => {
+        $(impl<'a, $tcx> Decodable<CacheDecoder<'a, $tcx>> for &$tcx [$ty] {
+            #[inline]
+            fn decode(d: &mut CacheDecoder<'a, $tcx>) -> Self {
+                RefDecodable::decode(d)
+            }
+        })*
+    };
+}
+
+impl_ref_decoder! {<'tcx>
+    Span,
+    rustc_ast::Attribute,
+    rustc_span::symbol::Ident,
+    ty::Variance,
+    rustc_span::def_id::DefId,
+    rustc_span::def_id::LocalDefId,
+    (rustc_middle::middle::exported_symbols::ExportedSymbol<'tcx>, rustc_middle::middle::exported_symbols::SymbolExportInfo),
+    ty::DeducedParamAttrs,
+}
+
+//- ENCODING -------------------------------------------------------------------
+
+/// An encoder that can write to the incremental compilation cache.
+pub struct CacheEncoder<'a, 'tcx> {
+    tcx: TyCtxt<'tcx>,
+    encoder: FileEncoder,
+    type_shorthands: FxHashMap<Ty<'tcx>, usize>,
+    predicate_shorthands: FxHashMap<ty::PredicateKind<'tcx>, usize>,
+    interpret_allocs: FxIndexSet<interpret::AllocId>,
+    source_map: CachingSourceMapView<'tcx>,
+    file_to_file_index: FxHashMap<*const SourceFile, SourceFileIndex>,
+    hygiene_context: &'a HygieneEncodeContext,
+    symbol_table: FxHashMap<Symbol, usize>,
+}
+
+impl<'a, 'tcx> CacheEncoder<'a, 'tcx> {
+    #[inline]
+    fn source_file_index(&mut self, source_file: Lrc<SourceFile>) -> SourceFileIndex {
+        self.file_to_file_index[&(&*source_file as *const SourceFile)]
+    }
+
+    /// Encode something with additional information that allows to do some
+    /// sanity checks when decoding the data again. This method will first
+    /// encode the specified tag, then the given value, then the number of
+    /// bytes taken up by tag and value. On decoding, we can then verify that
+    /// we get the expected tag and read the expected number of bytes.
+    pub fn encode_tagged<T: Encodable<Self>, V: Encodable<Self>>(&mut self, tag: T, value: &V) {
+        let start_pos = self.position();
+
+        tag.encode(self);
+        value.encode(self);
+
+        let end_pos = self.position();
+        ((end_pos - start_pos) as u64).encode(self);
+    }
+
+    #[inline]
+    fn finish(self) -> Result<usize, io::Error> {
+        self.encoder.finish()
+    }
+}
+
+impl<'a, 'tcx> Encodable<CacheEncoder<'a, 'tcx>> for SyntaxContext {
+    fn encode(&self, s: &mut CacheEncoder<'a, 'tcx>) {
+        rustc_span::hygiene::raw_encode_syntax_context(*self, s.hygiene_context, s);
+    }
+}
+
+impl<'a, 'tcx> Encodable<CacheEncoder<'a, 'tcx>> for ExpnId {
+    fn encode(&self, s: &mut CacheEncoder<'a, 'tcx>) {
+        s.hygiene_context.schedule_expn_data_for_encoding(*self);
+        self.expn_hash().encode(s);
+    }
+}
+
+impl<'a, 'tcx> Encodable<CacheEncoder<'a, 'tcx>> for Span {
+    fn encode(&self, s: &mut CacheEncoder<'a, 'tcx>) {
+        let span_data = self.data_untracked();
+        span_data.ctxt.encode(s);
+        span_data.parent.encode(s);
+
+        if span_data.is_dummy() {
+            return TAG_PARTIAL_SPAN.encode(s);
+        }
+
+        if let Some(parent) = span_data.parent {
+            let enclosing = s.tcx.source_span(parent).data_untracked();
+            if enclosing.contains(span_data) {
+                TAG_RELATIVE_SPAN.encode(s);
+                (span_data.lo - enclosing.lo).to_u32().encode(s);
+                (span_data.hi - enclosing.lo).to_u32().encode(s);
+                return;
+            }
+        }
+
+        let pos = s.source_map.byte_pos_to_line_and_col(span_data.lo);
+        let partial_span = match &pos {
+            Some((file_lo, _, _)) => !file_lo.contains(span_data.hi),
+            None => true,
+        };
+
+        if partial_span {
+            return TAG_PARTIAL_SPAN.encode(s);
+        }
+
+        let (file_lo, line_lo, col_lo) = pos.unwrap();
+
+        let len = span_data.hi - span_data.lo;
+
+        let source_file_index = s.source_file_index(file_lo);
+
+        TAG_FULL_SPAN.encode(s);
+        source_file_index.encode(s);
+        line_lo.encode(s);
+        col_lo.encode(s);
+        len.encode(s);
+    }
+}
+
+// copy&paste impl from rustc_metadata
+impl<'a, 'tcx> Encodable<CacheEncoder<'a, 'tcx>> for Symbol {
+    fn encode(&self, s: &mut CacheEncoder<'a, 'tcx>) {
+        // if symbol preinterned, emit tag and symbol index
+        if self.is_preinterned() {
+            s.encoder.emit_u8(SYMBOL_PREINTERNED);
+            s.encoder.emit_u32(self.as_u32());
+        } else {
+            // otherwise write it as string or as offset to it
+            match s.symbol_table.entry(*self) {
+                Entry::Vacant(o) => {
+                    s.encoder.emit_u8(SYMBOL_STR);
+                    let pos = s.encoder.position();
+                    o.insert(pos);
+                    s.emit_str(self.as_str());
+                }
+                Entry::Occupied(o) => {
+                    let x = *o.get();
+                    s.emit_u8(SYMBOL_OFFSET);
+                    s.emit_usize(x);
+                }
+            }
+        }
+    }
+}
+
+impl<'a, 'tcx> TyEncoder for CacheEncoder<'a, 'tcx> {
+    type I = TyCtxt<'tcx>;
+    const CLEAR_CROSS_CRATE: bool = false;
+
+    #[inline]
+    fn position(&self) -> usize {
+        self.encoder.position()
+    }
+    #[inline]
+    fn type_shorthands(&mut self) -> &mut FxHashMap<Ty<'tcx>, usize> {
+        &mut self.type_shorthands
+    }
+    #[inline]
+    fn predicate_shorthands(&mut self) -> &mut FxHashMap<ty::PredicateKind<'tcx>, usize> {
+        &mut self.predicate_shorthands
+    }
+    #[inline]
+    fn encode_alloc_id(&mut self, alloc_id: &interpret::AllocId) {
+        let (index, _) = self.interpret_allocs.insert_full(*alloc_id);
+
+        index.encode(self);
+    }
+}
+
+impl<'a, 'tcx> Encodable<CacheEncoder<'a, 'tcx>> for CrateNum {
+    #[inline]
+    fn encode(&self, s: &mut CacheEncoder<'a, 'tcx>) {
+        s.tcx.stable_crate_id(*self).encode(s);
+    }
+}
+
+impl<'a, 'tcx> Encodable<CacheEncoder<'a, 'tcx>> for DefId {
+    #[inline]
+    fn encode(&self, s: &mut CacheEncoder<'a, 'tcx>) {
+        s.tcx.def_path_hash(*self).encode(s);
+    }
+}
+
+impl<'a, 'tcx> Encodable<CacheEncoder<'a, 'tcx>> for DefIndex {
+    fn encode(&self, _: &mut CacheEncoder<'a, 'tcx>) {
+        bug!("encoding `DefIndex` without context");
+    }
+}
+
+macro_rules! encoder_methods {
+    ($($name:ident($ty:ty);)*) => {
+        #[inline]
+        $(fn $name(&mut self, value: $ty) {
+            self.encoder.$name(value)
+        })*
+    }
+}
+
+impl<'a, 'tcx> Encoder for CacheEncoder<'a, 'tcx> {
+    encoder_methods! {
+        emit_usize(usize);
+        emit_u128(u128);
+        emit_u64(u64);
+        emit_u32(u32);
+        emit_u16(u16);
+        emit_u8(u8);
+
+        emit_isize(isize);
+        emit_i128(i128);
+        emit_i64(i64);
+        emit_i32(i32);
+        emit_i16(i16);
+
+        emit_raw_bytes(&[u8]);
+    }
+}
+
+// This ensures that the `Encodable<opaque::FileEncoder>::encode` specialization for byte slices
+// is used when a `CacheEncoder` having an `opaque::FileEncoder` is passed to `Encodable::encode`.
+// Unfortunately, we have to manually opt into specializations this way, given how `CacheEncoder`
+// and the encoding traits currently work.
+impl<'a, 'tcx> Encodable<CacheEncoder<'a, 'tcx>> for [u8] {
+    fn encode(&self, e: &mut CacheEncoder<'a, 'tcx>) {
+        self.encode(&mut e.encoder);
+    }
+}
diff --git a/compiler/rustc_middle/src/ty/query.rs b/compiler/rustc_middle/src/query/plumbing.rs
index fa9fea723..97edfc2fc 100644
--- a/compiler/rustc_middle/src/ty/query.rs
+++ b/compiler/rustc_middle/src/query/plumbing.rs
@@ -1,87 +1,88 @@
-#![allow(unused_parens)]
-
 use crate::dep_graph;
-use crate::infer::canonical::{self, Canonical};
-use crate::lint::LintExpectation;
-use crate::metadata::ModChild;
-use crate::middle::codegen_fn_attrs::CodegenFnAttrs;
-use crate::middle::exported_symbols::{ExportedSymbol, SymbolExportInfo};
-use crate::middle::lib_features::LibFeatures;
-use crate::middle::privacy::EffectiveVisibilities;
-use crate::middle::resolve_bound_vars::{ObjectLifetimeDefault, ResolveBoundVars, ResolvedArg};
-use crate::middle::stability::{self, DeprecationEntry};
-use crate::mir;
-use crate::mir::interpret::GlobalId;
-use crate::mir::interpret::{
-    ConstValue, EvalToAllocationRawResult, EvalToConstValueResult, EvalToValTreeResult,
-};
-use crate::mir::interpret::{LitToConstError, LitToConstInput};
-use crate::mir::mono::CodegenUnit;
-use crate::query::erase::{erase, restore, Erase};
-use crate::query::{AsLocalKey, Key};
-use crate::thir;
-use crate::traits::query::{
-    CanonicalPredicateGoal, CanonicalProjectionGoal, CanonicalTyGoal,
-    CanonicalTypeOpAscribeUserTypeGoal, CanonicalTypeOpEqGoal, CanonicalTypeOpNormalizeGoal,
-    CanonicalTypeOpProvePredicateGoal, CanonicalTypeOpSubtypeGoal, NoSolution,
-};
-use crate::traits::query::{
-    DropckConstraint, DropckOutlivesResult, MethodAutoderefStepsResult, NormalizationResult,
-    OutlivesBound,
-};
-use crate::traits::specialization_graph;
-use crate::traits::{self, ImplSource};
-use crate::ty::context::TyCtxtFeed;
-use crate::ty::fast_reject::SimplifiedType;
-use crate::ty::layout::ValidityRequirement;
-use crate::ty::subst::{GenericArg, SubstsRef};
-use crate::ty::util::AlwaysRequiresDrop;
-use crate::ty::GeneratorDiagnosticData;
-use crate::ty::{self, CrateInherentImpls, ParamEnvAnd, Ty, TyCtxt, UnusedGenericParams};
-use rustc_arena::TypedArena;
-use rustc_ast as ast;
-use rustc_ast::expand::allocator::AllocatorKind;
-use rustc_attr as attr;
-use rustc_data_structures::fx::{FxHashMap, FxIndexMap, FxIndexSet};
-use rustc_data_structures::steal::Steal;
-use rustc_data_structures::svh::Svh;
-use rustc_data_structures::sync::Lrc;
-use rustc_data_structures::sync::WorkerLocal;
-use rustc_data_structures::unord::UnordSet;
-use rustc_errors::ErrorGuaranteed;
-use rustc_hir as hir;
-use rustc_hir::def::{DefKind, DocLinkResMap};
-use rustc_hir::def_id::{
-    CrateNum, DefId, DefIdMap, DefIdSet, LocalDefId, LocalDefIdMap, LocalDefIdSet,
+use crate::dep_graph::DepKind;
+use crate::query::on_disk_cache::CacheEncoder;
+use crate::query::on_disk_cache::EncodedDepNodeIndex;
+use crate::query::on_disk_cache::OnDiskCache;
+use crate::query::{
+    DynamicQueries, ExternProviders, Providers, QueryArenas, QueryCaches, QueryEngine, QueryStates,
 };
+use crate::ty::TyCtxt;
+use field_offset::FieldOffset;
+use measureme::StringId;
+use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::sync::AtomicU64;
+use rustc_hir::def::DefKind;
+use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::hir_id::OwnerId;
-use rustc_hir::lang_items::{LangItem, LanguageItems};
-use rustc_hir::{Crate, ItemLocalId, TraitCandidate};
-use rustc_index::vec::IndexVec;
+use rustc_query_system::dep_graph::DepNodeIndex;
+use rustc_query_system::dep_graph::SerializedDepNodeIndex;
 pub(crate) use rustc_query_system::query::QueryJobId;
 use rustc_query_system::query::*;
-use rustc_session::config::{EntryFnType, OptLevel, OutputFilenames, SymbolManglingVersion};
-use rustc_session::cstore::{CrateDepKind, CrateSource};
-use rustc_session::cstore::{ExternCrate, ForeignModule, LinkagePreference, NativeLib};
-use rustc_session::lint::LintExpectationId;
-use rustc_session::Limits;
-use rustc_span::symbol::Symbol;
+use rustc_query_system::HandleCycleError;
 use rustc_span::{Span, DUMMY_SP};
-use rustc_target::abi;
-use rustc_target::spec::PanicStrategy;
-
-use std::marker::PhantomData;
-use std::mem;
 use std::ops::Deref;
-use std::path::PathBuf;
-use std::sync::Arc;
 
-#[derive(Default)]
+pub struct QueryKeyStringCache {
+    pub def_id_cache: FxHashMap<DefId, StringId>,
+}
+
+impl QueryKeyStringCache {
+    pub fn new() -> QueryKeyStringCache {
+        QueryKeyStringCache { def_id_cache: Default::default() }
+    }
+}
+
+pub struct DynamicQuery<'tcx, C: QueryCache> {
+    pub name: &'static str,
+    pub eval_always: bool,
+    pub dep_kind: DepKind,
+    pub handle_cycle_error: HandleCycleError,
+    pub query_state: FieldOffset<QueryStates<'tcx>, QueryState<C::Key, DepKind>>,
+    pub query_cache: FieldOffset<QueryCaches<'tcx>, C>,
+    pub cache_on_disk: fn(tcx: TyCtxt<'tcx>, key: &C::Key) -> bool,
+    pub execute_query: fn(tcx: TyCtxt<'tcx>, k: C::Key) -> C::Value,
+    pub compute: fn(tcx: TyCtxt<'tcx>, key: C::Key) -> C::Value,
+    pub can_load_from_disk: bool,
+    pub try_load_from_disk: fn(
+        tcx: TyCtxt<'tcx>,
+        key: &C::Key,
+        prev_index: SerializedDepNodeIndex,
+        index: DepNodeIndex,
+    ) -> Option<C::Value>,
+    pub loadable_from_disk:
+        fn(tcx: TyCtxt<'tcx>, key: &C::Key, index: SerializedDepNodeIndex) -> bool,
+    pub hash_result: HashResult<C::Value>,
+    pub value_from_cycle_error: fn(tcx: TyCtxt<'tcx>, cycle: &[QueryInfo<DepKind>]) -> C::Value,
+    pub format_value: fn(&C::Value) -> String,
+}
+
+pub struct QuerySystemFns<'tcx> {
+    pub engine: QueryEngine,
+    pub local_providers: Providers,
+    pub extern_providers: ExternProviders,
+    pub encode_query_results: fn(
+        tcx: TyCtxt<'tcx>,
+        encoder: &mut CacheEncoder<'_, 'tcx>,
+        query_result_index: &mut EncodedDepNodeIndex,
+    ),
+    pub try_mark_green: fn(tcx: TyCtxt<'tcx>, dep_node: &dep_graph::DepNode) -> bool,
+}
+
 pub struct QuerySystem<'tcx> {
+    pub states: QueryStates<'tcx>,
     pub arenas: QueryArenas<'tcx>,
     pub caches: QueryCaches<'tcx>,
-    // Since we erase query value types we tell the typesystem about them with `PhantomData`.
-    _phantom_values: QueryPhantomValues<'tcx>,
+    pub dynamic_queries: DynamicQueries<'tcx>,
+
+    /// This provides access to the incremental compilation on-disk cache for query results.
+    /// Do not access this directly. It is only meant to be used by
+    /// `DepGraph::try_mark_green()` and the query infrastructure.
+    /// This is `None` if we are not incremental compilation mode
+    pub on_disk_cache: Option<OnDiskCache<'tcx>>,
+
+    pub fns: QuerySystemFns<'tcx>,
+
+    pub jobs: AtomicU64,
 }
 
 #[derive(Copy, Clone)]
@@ -133,7 +134,41 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     pub fn try_mark_green(self, dep_node: &dep_graph::DepNode) -> bool {
-        self.queries.try_mark_green(self, dep_node)
+        (self.query_system.fns.try_mark_green)(self, dep_node)
+    }
+}
+
+#[inline]
+pub fn query_get_at<'tcx, Cache>(
+    tcx: TyCtxt<'tcx>,
+    execute_query: fn(TyCtxt<'tcx>, Span, Cache::Key, QueryMode) -> Option<Cache::Value>,
+    query_cache: &Cache,
+    span: Span,
+    key: Cache::Key,
+) -> Cache::Value
+where
+    Cache: QueryCache,
+{
+    let key = key.into_query_param();
+    match try_get_cached(tcx, query_cache, &key) {
+        Some(value) => value,
+        None => execute_query(tcx, span, key, QueryMode::Get).unwrap(),
+    }
+}
+
+#[inline]
+pub fn query_ensure<'tcx, Cache>(
+    tcx: TyCtxt<'tcx>,
+    execute_query: fn(TyCtxt<'tcx>, Span, Cache::Key, QueryMode) -> Option<Cache::Value>,
+    query_cache: &Cache,
+    key: Cache::Key,
+    check_cache: bool,
+) where
+    Cache: QueryCache,
+{
+    let key = key.into_query_param();
+    if try_get_cached(tcx, query_cache, &key).is_none() {
+        execute_query(tcx, DUMMY_SP, key, QueryMode::Ensure { check_cache });
     }
 }
 
@@ -176,8 +211,8 @@ macro_rules! separate_provide_extern_decl {
     ([(separate_provide_extern) $($rest:tt)*][$name:ident]) => {
         for<'tcx> fn(
             TyCtxt<'tcx>,
-            query_keys::$name<'tcx>,
-        ) -> query_provided::$name<'tcx>
+            queries::$name::Key<'tcx>,
+        ) -> queries::$name::ProvidedValue<'tcx>
     };
     ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
         separate_provide_extern_decl!([$($modifiers)*][$($args)*])
@@ -202,75 +237,42 @@ macro_rules! separate_provide_extern_default {
     };
 }
 
-macro_rules! opt_remap_env_constness {
-    ([][$name:ident]) => {};
-    ([(remap_env_constness) $($rest:tt)*][$name:ident]) => {
-        let $name = $name.without_const();
-    };
-    ([$other:tt $($modifiers:tt)*][$name:ident]) => {
-        opt_remap_env_constness!([$($modifiers)*][$name])
-    };
-}
-
 macro_rules! define_callbacks {
     (
      $($(#[$attr:meta])*
         [$($modifiers:tt)*] fn $name:ident($($K:tt)*) -> $V:ty,)*) => {
 
-        // HACK(eddyb) this is like the `impl QueryConfig for queries::$name`
-        // below, but using type aliases instead of associated types, to bypass
-        // the limitations around normalizing under HRTB - for example, this:
-        // `for<'tcx> fn(...) -> <queries::$name<'tcx> as QueryConfig<TyCtxt<'tcx>>>::Value`
-        // doesn't currently normalize to `for<'tcx> fn(...) -> query_values::$name<'tcx>`.
-        // This is primarily used by the `provide!` macro in `rustc_metadata`.
-        #[allow(nonstandard_style, unused_lifetimes)]
-        pub mod query_keys {
-            use super::*;
-
-            $(pub type $name<'tcx> = $($K)*;)*
-        }
-        #[allow(nonstandard_style, unused_lifetimes)]
-        pub mod query_keys_local {
-            use super::*;
-
-            $(pub type $name<'tcx> = local_key_if_separate_extern!([$($modifiers)*] $($K)*);)*
-        }
-        #[allow(nonstandard_style, unused_lifetimes)]
-        pub mod query_values {
-            use super::*;
-
-            $(pub type $name<'tcx> = $V;)*
-        }
-
-        /// This module specifies the type returned from query providers and the type used for
-        /// decoding. For regular queries this is the declared returned type `V`, but
-        /// `arena_cache` will use `<V as Deref>::Target` instead.
-        #[allow(nonstandard_style, unused_lifetimes)]
-        pub mod query_provided {
-            use super::*;
-
-            $(
-                pub type $name<'tcx> = query_if_arena!([$($modifiers)*] (<$V as Deref>::Target) ($V));
-            )*
-        }
-
-        /// This module has a function per query which takes a `query_provided` value and coverts
-        /// it to a regular `V` value by allocating it on an arena if the query has the
-        /// `arena_cache` modifier. This will happen when computing the query using a provider or
-        /// decoding a stored result.
-        #[allow(nonstandard_style, unused_lifetimes)]
-        pub mod query_provided_to_value {
-            use super::*;
-
-            $(
+        #[allow(unused_lifetimes)]
+        pub mod queries {
+            $(pub mod $name {
+                use super::super::*;
+
+                pub type Key<'tcx> = $($K)*;
+                pub type Value<'tcx> = $V;
+
+                pub type LocalKey<'tcx> = local_key_if_separate_extern!([$($modifiers)*] $($K)*);
+
+                /// This type alias specifies the type returned from query providers and the type
+                /// used for decoding. For regular queries this is the declared returned type `V`,
+                /// but `arena_cache` will use `<V as Deref>::Target` instead.
+                pub type ProvidedValue<'tcx> = query_if_arena!(
+                    [$($modifiers)*]
+                    (<$V as Deref>::Target)
+                    ($V)
+                );
+
+                /// This function takes `ProvidedValue` and coverts it to an erased `Value` by
+                /// allocating it on an arena if the query has the `arena_cache` modifier. The
+                /// value is then erased and returned. This will happen when computing the query
+                /// using a provider or decoding a stored result.
                 #[inline(always)]
-                pub fn $name<'tcx>(
+                pub fn provided_to_erased<'tcx>(
                     _tcx: TyCtxt<'tcx>,
-                    value: query_provided::$name<'tcx>,
-                ) -> Erase<query_values::$name<'tcx>> {
+                    value: ProvidedValue<'tcx>,
+                ) -> Erase<Value<'tcx>> {
                     erase(query_if_arena!([$($modifiers)*]
                         {
-                            if mem::needs_drop::<query_provided::$name<'tcx>>() {
+                            if mem::needs_drop::<ProvidedValue<'tcx>>() {
                                 &*_tcx.query_system.arenas.$name.alloc(value)
                             } else {
                                 &*_tcx.arena.dropless.alloc(value)
@@ -279,46 +281,40 @@ macro_rules! define_callbacks {
                         (value)
                     ))
                 }
-            )*
-        }
-        #[allow(nonstandard_style, unused_lifetimes)]
-        pub mod query_storage {
-            use super::*;
 
-            $(
-                pub type $name<'tcx> = <<$($K)* as Key>::CacheSelector as CacheSelector<'tcx, Erase<$V>>>::Cache;
-            )*
-        }
+                pub type Storage<'tcx> = <
+                    <$($K)* as keys::Key>::CacheSelector as CacheSelector<'tcx, Erase<$V>>
+                >::Cache;
 
-        $(
-            // Ensure that keys grow no larger than 64 bytes
-            #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
-            const _: () = {
-                if mem::size_of::<query_keys::$name<'static>>() > 64 {
-                    panic!("{}", concat!(
-                        "the query `",
-                        stringify!($name),
-                        "` has a key type `",
-                        stringify!($($K)*),
-                        "` that is too large"
-                    ));
-                }
-            };
-
-            // Ensure that values grow no larger than 64 bytes
-            #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
-            const _: () = {
-                if mem::size_of::<query_values::$name<'static>>() > 64 {
-                    panic!("{}", concat!(
-                        "the query `",
-                        stringify!($name),
-                        "` has a value type `",
-                        stringify!($V),
-                        "` that is too large"
-                    ));
-                }
-            };
-        )*
+                // Ensure that keys grow no larger than 64 bytes
+                #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
+                const _: () = {
+                    if mem::size_of::<Key<'static>>() > 64 {
+                        panic!("{}", concat!(
+                            "the query `",
+                            stringify!($name),
+                            "` has a key type `",
+                            stringify!($($K)*),
+                            "` that is too large"
+                        ));
+                    }
+                };
+
+                // Ensure that values grow no larger than 64 bytes
+                #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
+                const _: () = {
+                    if mem::size_of::<Value<'static>>() > 64 {
+                        panic!("{}", concat!(
+                            "the query `",
+                            stringify!($name),
+                            "` has a value type `",
+                            stringify!($V),
+                            "` that is too large"
+                        ));
+                    }
+                };
+            })*
+        }
 
         pub struct QueryArenas<'tcx> {
             $($(#[$attr])* pub $name: query_if_arena!([$($modifiers)*]
@@ -339,31 +335,21 @@ macro_rules! define_callbacks {
         }
 
         #[derive(Default)]
-        pub struct QueryPhantomValues<'tcx> {
-            $($(#[$attr])* pub $name: PhantomData<query_values::$name<'tcx>>,)*
-        }
-
-        #[derive(Default)]
         pub struct QueryCaches<'tcx> {
-            $($(#[$attr])* pub $name: query_storage::$name<'tcx>,)*
+            $($(#[$attr])* pub $name: queries::$name::Storage<'tcx>,)*
         }
 
         impl<'tcx> TyCtxtEnsure<'tcx> {
             $($(#[$attr])*
             #[inline(always)]
             pub fn $name(self, key: query_helper_param_ty!($($K)*)) {
-                let key = key.into_query_param();
-                opt_remap_env_constness!([$($modifiers)*][key]);
-
-                match try_get_cached(self.tcx, &self.tcx.query_system.caches.$name, &key) {
-                    Some(_) => return,
-                    None => self.tcx.queries.$name(
-                        self.tcx,
-                        DUMMY_SP,
-                        key,
-                        QueryMode::Ensure { check_cache: false },
-                    ),
-                };
+                query_ensure(
+                    self.tcx,
+                    self.tcx.query_system.fns.engine.$name,
+                    &self.tcx.query_system.caches.$name,
+                    key.into_query_param(),
+                    false,
+                );
             })*
         }
 
@@ -371,18 +357,13 @@ macro_rules! define_callbacks {
             $($(#[$attr])*
             #[inline(always)]
             pub fn $name(self, key: query_helper_param_ty!($($K)*)) {
-                let key = key.into_query_param();
-                opt_remap_env_constness!([$($modifiers)*][key]);
-
-                match try_get_cached(self.tcx, &self.tcx.query_system.caches.$name, &key) {
-                    Some(_) => return,
-                    None => self.tcx.queries.$name(
-                        self.tcx,
-                        DUMMY_SP,
-                        key,
-                        QueryMode::Ensure { check_cache: true },
-                    ),
-                };
+                query_ensure(
+                    self.tcx,
+                    self.tcx.query_system.fns.engine.$name,
+                    &self.tcx.query_system.caches.$name,
+                    key.into_query_param(),
+                    true,
+                );
             })*
         }
 
@@ -401,21 +382,34 @@ macro_rules! define_callbacks {
             #[inline(always)]
             pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> $V
             {
-                let key = key.into_query_param();
-                opt_remap_env_constness!([$($modifiers)*][key]);
-
-                restore::<$V>(match try_get_cached(self.tcx, &self.tcx.query_system.caches.$name, &key) {
-                    Some(value) => value,
-                    None => self.tcx.queries.$name(self.tcx, self.span, key, QueryMode::Get).unwrap(),
-                })
+                restore::<$V>(query_get_at(
+                    self.tcx,
+                    self.tcx.query_system.fns.engine.$name,
+                    &self.tcx.query_system.caches.$name,
+                    self.span,
+                    key.into_query_param(),
+                ))
             })*
         }
 
+        pub struct DynamicQueries<'tcx> {
+            $(
+                pub $name: DynamicQuery<'tcx, queries::$name::Storage<'tcx>>,
+            )*
+        }
+
+        #[derive(Default)]
+        pub struct QueryStates<'tcx> {
+            $(
+                pub $name: QueryState<$($K)*, DepKind>,
+            )*
+        }
+
         pub struct Providers {
             $(pub $name: for<'tcx> fn(
                 TyCtxt<'tcx>,
-                query_keys_local::$name<'tcx>,
-            ) -> query_provided::$name<'tcx>,)*
+                queries::$name::LocalKey<'tcx>,
+            ) -> queries::$name::ProvidedValue<'tcx>,)*
         }
 
         pub struct ExternProviders {
@@ -456,19 +450,13 @@ macro_rules! define_callbacks {
             fn clone(&self) -> Self { *self }
         }
 
-        pub trait QueryEngine<'tcx>: rustc_data_structures::sync::Sync {
-            fn as_any(&'tcx self) -> &'tcx dyn std::any::Any;
-
-            fn try_mark_green(&'tcx self, tcx: TyCtxt<'tcx>, dep_node: &dep_graph::DepNode) -> bool;
-
-            $($(#[$attr])*
-            fn $name(
-                &'tcx self,
-                tcx: TyCtxt<'tcx>,
-                span: Span,
-                key: query_keys::$name<'tcx>,
-                mode: QueryMode,
-            ) -> Option<Erase<$V>>;)*
+        pub struct QueryEngine {
+            $(pub $name: for<'tcx> fn(
+                TyCtxt<'tcx>,
+                Span,
+                queries::$name::Key<'tcx>,
+                QueryMode,
+            ) -> Option<Erase<$V>>,)*
         }
     };
 }
@@ -490,22 +478,41 @@ macro_rules! define_feedable {
         $(impl<'tcx, K: IntoQueryParam<$($K)*> + Copy> TyCtxtFeed<'tcx, K> {
             $(#[$attr])*
             #[inline(always)]
-            pub fn $name(self, value: query_provided::$name<'tcx>) -> $V {
+            pub fn $name(self, value: queries::$name::ProvidedValue<'tcx>) {
                 let key = self.key().into_query_param();
-                opt_remap_env_constness!([$($modifiers)*][key]);
 
                 let tcx = self.tcx;
-                let erased = query_provided_to_value::$name(tcx, value);
+                let erased = queries::$name::provided_to_erased(tcx, value);
                 let value = restore::<$V>(erased);
                 let cache = &tcx.query_system.caches.$name;
 
+                let hasher: Option<fn(&mut StableHashingContext<'_>, &_) -> _> = hash_result!([$($modifiers)*]);
                 match try_get_cached(tcx, cache, &key) {
                     Some(old) => {
                         let old = restore::<$V>(old);
-                        bug!(
-                            "Trying to feed an already recorded value for query {} key={key:?}:\nold value: {old:?}\nnew value: {value:?}",
-                            stringify!($name),
-                        )
+                        if let Some(hasher) = hasher {
+                            let (value_hash, old_hash): (Fingerprint, Fingerprint) = tcx.with_stable_hashing_context(|mut hcx|
+                                (hasher(&mut hcx, &value), hasher(&mut hcx, &old))
+                            );
+                            if old_hash != value_hash {
+                                // We have an inconsistency. This can happen if one of the two
+                                // results is tainted by errors. In this case, delay a bug to
+                                // ensure compilation is doomed, and keep the `old` value.
+                                tcx.sess.delay_span_bug(DUMMY_SP, format!(
+                                    "Trying to feed an already recorded value for query {} key={key:?}:\n\
+                                    old value: {old:?}\nnew value: {value:?}",
+                                    stringify!($name),
+                                ));
+                            }
+                        } else {
+                            // The query is `no_hash`, so we have no way to perform a sanity check.
+                            // If feeding the same value multiple times needs to be supported,
+                            // the query should not be marked `no_hash`.
+                            bug!(
+                                "Trying to feed an already recorded value for query {} key={key:?}:\nold value: {old:?}\nnew value: {value:?}",
+                                stringify!($name),
+                            )
+                        }
                     }
                     None => {
                         let dep_node = dep_graph::DepNode::construct(tcx, dep_graph::DepKind::$name, &key);
@@ -517,7 +524,6 @@ macro_rules! define_feedable {
                             hash_result!([$($modifiers)*]),
                         );
                         cache.complete(key, erased, dep_node_index);
-                        value
                     }
                 }
             }
@@ -537,9 +543,6 @@ macro_rules! define_feedable {
 // Queries marked with `fatal_cycle` do not need the latter implementation,
 // as they will raise an fatal error on query cycles instead.
 
-rustc_query_append! { define_callbacks! }
-rustc_feedable_queries! { define_feedable! }
-
 mod sealed {
     use super::{DefId, LocalDefId, OwnerId};
 
@@ -587,7 +590,7 @@ mod sealed {
     }
 }
 
-use sealed::IntoQueryParam;
+pub use sealed::IntoQueryParam;
 
 impl<'tcx> TyCtxt<'tcx> {
     pub fn def_kind(self, def_id: impl IntoQueryParam<DefId>) -> DefKind {
diff --git a/compiler/rustc_middle/src/thir.rs b/compiler/rustc_middle/src/thir.rs
index 7d79a13d3..813e109c4 100644
--- a/compiler/rustc_middle/src/thir.rs
+++ b/compiler/rustc_middle/src/thir.rs
@@ -14,13 +14,13 @@ use rustc_hir as hir;
 use rustc_hir::def_id::DefId;
 use rustc_hir::RangeEnd;
 use rustc_index::newtype_index;
-use rustc_index::vec::IndexVec;
+use rustc_index::IndexVec;
 use rustc_middle::middle::region;
 use rustc_middle::mir::interpret::AllocId;
 use rustc_middle::mir::{self, BinOp, BorrowKind, FakeReadCause, Mutability, UnOp};
 use rustc_middle::ty::adjustment::PointerCast;
 use rustc_middle::ty::subst::SubstsRef;
-use rustc_middle::ty::{self, AdtDef, FnSig, Ty, UpvarSubsts};
+use rustc_middle::ty::{self, AdtDef, FnSig, List, Ty, UpvarSubsts};
 use rustc_middle::ty::{CanonicalUserType, CanonicalUserTypeAnnotation};
 use rustc_span::def_id::LocalDefId;
 use rustc_span::{sym, Span, Symbol, DUMMY_SP};
@@ -234,7 +234,6 @@ pub enum StmtKind<'tcx> {
 }
 
 #[derive(Clone, Debug, Copy, PartialEq, Eq, Hash, HashStable, TyEncodable, TyDecodable)]
-#[derive(TypeFoldable, TypeVisitable)]
 pub struct LocalVarId(pub hir::HirId);
 
 /// A THIR expression.
@@ -481,6 +480,11 @@ pub enum ExprKind<'tcx> {
     },
     /// Inline assembly, i.e. `asm!()`.
     InlineAsm(Box<InlineAsmExpr<'tcx>>),
+    /// Field offset (`offset_of!`)
+    OffsetOf {
+        container: Ty<'tcx>,
+        fields: &'tcx List<FieldIdx>,
+    },
     /// An expression taking a reference to a thread local.
     ThreadLocalRef(DefId),
     /// A `yield` expression.
@@ -929,8 +933,8 @@ mod size_asserts {
     static_assert_size!(Block, 56);
     static_assert_size!(Expr<'_>, 64);
     static_assert_size!(ExprKind<'_>, 40);
-    static_assert_size!(Pat<'_>, 72);
-    static_assert_size!(PatKind<'_>, 56);
+    static_assert_size!(Pat<'_>, 64);
+    static_assert_size!(PatKind<'_>, 48);
     static_assert_size!(Stmt<'_>, 56);
     static_assert_size!(StmtKind<'_>, 48);
     // tidy-alphabetical-end
diff --git a/compiler/rustc_middle/src/thir/visit.rs b/compiler/rustc_middle/src/thir/visit.rs
index 5614528c4..5c7ec31cf 100644
--- a/compiler/rustc_middle/src/thir/visit.rs
+++ b/compiler/rustc_middle/src/thir/visit.rs
@@ -160,6 +160,7 @@ pub fn walk_expr<'a, 'tcx: 'a, V: Visitor<'a, 'tcx>>(visitor: &mut V, expr: &Exp
                 }
             }
         }
+        OffsetOf { container: _, fields: _ } => {}
         ThreadLocalRef(_) => {}
         Yield { value } => visitor.visit_expr(&visitor.thir()[value]),
     }
diff --git a/compiler/rustc_middle/src/traits/mod.rs b/compiler/rustc_middle/src/traits/mod.rs
index 91f07389f..0a903a769 100644
--- a/compiler/rustc_middle/src/traits/mod.rs
+++ b/compiler/rustc_middle/src/traits/mod.rs
@@ -199,7 +199,7 @@ impl<'tcx> ObligationCause<'tcx> {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, Eq, Hash, Lift, HashStable, TyEncodable, TyDecodable)]
+#[derive(Clone, Debug, PartialEq, Eq, Lift, HashStable, TyEncodable, TyDecodable)]
 #[derive(TypeVisitable, TypeFoldable)]
 pub struct UnifyReceiverContext<'tcx> {
     pub assoc_item: ty::AssocItem,
@@ -207,7 +207,7 @@ pub struct UnifyReceiverContext<'tcx> {
     pub substs: SubstsRef<'tcx>,
 }
 
-#[derive(Clone, PartialEq, Eq, Hash, Lift, Default, HashStable)]
+#[derive(Clone, PartialEq, Eq, Lift, Default, HashStable)]
 #[derive(TypeVisitable, TypeFoldable, TyEncodable, TyDecodable)]
 pub struct InternedObligationCauseCode<'tcx> {
     /// `None` for `ObligationCauseCode::MiscObligation` (a common case, occurs ~60% of
@@ -243,7 +243,7 @@ impl<'tcx> std::ops::Deref for InternedObligationCauseCode<'tcx> {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, Eq, Hash, Lift, HashStable, TyEncodable, TyDecodable)]
+#[derive(Clone, Debug, PartialEq, Eq, Lift, HashStable, TyEncodable, TyDecodable)]
 #[derive(TypeVisitable, TypeFoldable)]
 pub enum ObligationCauseCode<'tcx> {
     /// Not well classified or should be obvious from the span.
@@ -281,9 +281,6 @@ pub enum ObligationCauseCode<'tcx> {
     /// A type like `Box<Foo<'a> + 'b>` is WF only if `'b: 'a`.
     ObjectTypeBound(Ty<'tcx>, ty::Region<'tcx>),
 
-    /// Obligation incurred due to an object cast.
-    ObjectCastObligation(/* Concrete type */ Ty<'tcx>, /* Object type */ Ty<'tcx>),
-
     /// Obligation incurred due to a coercion.
     Coercion {
         source: Ty<'tcx>,
@@ -444,6 +441,10 @@ pub enum ObligationCauseCode<'tcx> {
     AscribeUserTypeProvePredicate(Span),
 
     RustCall,
+
+    /// Obligations to prove that a `std::ops::Drop` impl is not stronger than
+    /// the ADT it's being implemented for.
+    DropImpl,
 }
 
 /// The 'location' at which we try to perform HIR-based wf checking.
@@ -468,7 +469,7 @@ pub enum WellFormedLoc {
     },
 }
 
-#[derive(Clone, Debug, PartialEq, Eq, Hash, Lift, HashStable, TyEncodable, TyDecodable)]
+#[derive(Clone, Debug, PartialEq, Eq, Lift, HashStable, TyEncodable, TyDecodable)]
 #[derive(TypeVisitable, TypeFoldable)]
 pub struct ImplDerivedObligationCause<'tcx> {
     pub derived: DerivedObligationCause<'tcx>,
@@ -529,7 +530,7 @@ impl<'tcx> ty::Lift<'tcx> for StatementAsExpression {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, Eq, Hash, Lift, HashStable, TyEncodable, TyDecodable)]
+#[derive(Clone, Debug, PartialEq, Eq, Lift, HashStable, TyEncodable, TyDecodable)]
 #[derive(TypeVisitable, TypeFoldable)]
 pub struct MatchExpressionArmCause<'tcx> {
     pub arm_block_id: Option<hir::HirId>,
@@ -545,7 +546,7 @@ pub struct MatchExpressionArmCause<'tcx> {
     pub opt_suggest_box_span: Option<Span>,
 }
 
-#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
 #[derive(Lift, TypeFoldable, TypeVisitable, HashStable, TyEncodable, TyDecodable)]
 pub struct IfExpressionCause<'tcx> {
     pub then_id: hir::HirId,
@@ -556,7 +557,7 @@ pub struct IfExpressionCause<'tcx> {
     pub opt_suggest_box_span: Option<Span>,
 }
 
-#[derive(Clone, Debug, PartialEq, Eq, Hash, Lift, HashStable, TyEncodable, TyDecodable)]
+#[derive(Clone, Debug, PartialEq, Eq, Lift, HashStable, TyEncodable, TyDecodable)]
 #[derive(TypeVisitable, TypeFoldable)]
 pub struct DerivedObligationCause<'tcx> {
     /// The trait predicate of the parent obligation that led to the
@@ -569,18 +570,14 @@ pub struct DerivedObligationCause<'tcx> {
     pub parent_code: InternedObligationCauseCode<'tcx>,
 }
 
-#[derive(Clone, Debug, TypeFoldable, TypeVisitable, Lift)]
+#[derive(Clone, Debug, TypeVisitable, Lift)]
 pub enum SelectionError<'tcx> {
     /// The trait is not implemented.
     Unimplemented,
     /// After a closure impl has selected, its "outputs" were evaluated
     /// (which for closures includes the "input" type params) and they
     /// didn't resolve. See `confirm_poly_trait_refs` for more.
-    OutputTypeParameterMismatch(
-        ty::PolyTraitRef<'tcx>,
-        ty::PolyTraitRef<'tcx>,
-        ty::error::TypeError<'tcx>,
-    ),
+    OutputTypeParameterMismatch(Box<SelectionOutputTypeParameterMismatch<'tcx>>),
     /// The trait pointed by `DefId` is not object safe.
     TraitNotObjectSafe(DefId),
     /// A given constant couldn't be evaluated.
@@ -592,6 +589,13 @@ pub enum SelectionError<'tcx> {
     ErrorReporting,
 }
 
+#[derive(Clone, Debug, TypeVisitable, Lift)]
+pub struct SelectionOutputTypeParameterMismatch<'tcx> {
+    pub found_trait_ref: ty::PolyTraitRef<'tcx>,
+    pub expected_trait_ref: ty::PolyTraitRef<'tcx>,
+    pub terr: ty::error::TypeError<'tcx>,
+}
+
 /// When performing resolution, it is typically the case that there
 /// can be one of three outcomes:
 ///
@@ -1027,10 +1031,7 @@ impl ObjectSafetyViolation {
             ) => {
                 err.span_suggestion(
                     *span,
-                    &format!(
-                        "consider changing method `{}`'s `self` parameter to be `&self`",
-                        name
-                    ),
+                    format!("consider changing method `{}`'s `self` parameter to be `&self`", name),
                     "&Self",
                     Applicability::MachineApplicable,
                 );
@@ -1038,7 +1039,7 @@ impl ObjectSafetyViolation {
             ObjectSafetyViolation::AssocConst(name, _)
             | ObjectSafetyViolation::GAT(name, _)
             | ObjectSafetyViolation::Method(name, ..) => {
-                err.help(&format!("consider moving `{}` to another trait", name));
+                err.help(format!("consider moving `{}` to another trait", name));
             }
         }
     }
@@ -1107,3 +1108,14 @@ pub enum CodegenObligationError {
     Unimplemented,
     FulfillmentError,
 }
+
+#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, TypeFoldable, TypeVisitable)]
+pub enum DefiningAnchor {
+    /// `DefId` of the item.
+    Bind(LocalDefId),
+    /// When opaque types are not resolved, we `Bubble` up, meaning
+    /// return the opaque/hidden type pair from query, for caller of query to handle it.
+    Bubble,
+    /// Used to catch type mismatch errors when handling opaque types.
+    Error,
+}
diff --git a/compiler/rustc_middle/src/traits/query.rs b/compiler/rustc_middle/src/traits/query.rs
index c4f871875..eae5a280e 100644
--- a/compiler/rustc_middle/src/traits/query.rs
+++ b/compiler/rustc_middle/src/traits/query.rs
@@ -95,8 +95,6 @@ pub type CanonicalTypeOpNormalizeGoal<'tcx, T> =
 #[derive(Copy, Clone, Debug, HashStable, PartialEq, Eq)]
 pub struct NoSolution;
 
-pub type Fallible<T> = Result<T, NoSolution>;
-
 impl<'tcx> From<TypeError<'tcx>> for NoSolution {
     fn from(_: TypeError<'tcx>) -> NoSolution {
         NoSolution
diff --git a/compiler/rustc_middle/src/traits/select.rs b/compiler/rustc_middle/src/traits/select.rs
index 1cc9fd526..f2dda003b 100644
--- a/compiler/rustc_middle/src/traits/select.rs
+++ b/compiler/rustc_middle/src/traits/select.rs
@@ -103,7 +103,7 @@ pub type EvaluationCache<'tcx> = Cache<
 /// required for associated types to work in default impls, as the bounds
 /// are visible both as projection bounds and as where-clauses from the
 /// parameter environment.
-#[derive(PartialEq, Eq, Debug, Clone, TypeFoldable, TypeVisitable)]
+#[derive(PartialEq, Eq, Debug, Clone, TypeVisitable)]
 pub enum SelectionCandidate<'tcx> {
     /// A builtin implementation for some specific traits, used in cases
     /// where we cannot rely an ordinary library implementations.
diff --git a/compiler/rustc_middle/src/traits/solve.rs b/compiler/rustc_middle/src/traits/solve.rs
index fef2be133..2c5b64a59 100644
--- a/compiler/rustc_middle/src/traits/solve.rs
+++ b/compiler/rustc_middle/src/traits/solve.rs
@@ -5,13 +5,13 @@ use rustc_query_system::cache::Cache;
 
 use crate::infer::canonical::{CanonicalVarValues, QueryRegionConstraints};
 use crate::traits::query::NoSolution;
-use crate::traits::Canonical;
+use crate::traits::{Canonical, DefiningAnchor};
 use crate::ty::{
     self, FallibleTypeFolder, ToPredicate, Ty, TyCtxt, TypeFoldable, TypeFolder, TypeVisitable,
     TypeVisitor,
 };
 
-pub type EvaluationCache<'tcx> = Cache<CanonicalGoal<'tcx>, QueryResult<'tcx>>;
+pub type EvaluationCache<'tcx> = Cache<CanonicalInput<'tcx>, QueryResult<'tcx>>;
 
 /// A goal is a statement, i.e. `predicate`, we want to prove
 /// given some assumptions, i.e. `param_env`.
@@ -96,7 +96,31 @@ pub enum MaybeCause {
     Overflow,
 }
 
-pub type CanonicalGoal<'tcx, T = ty::Predicate<'tcx>> = Canonical<'tcx, Goal<'tcx, T>>;
+#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, TypeFoldable, TypeVisitable)]
+pub struct QueryInput<'tcx, T> {
+    pub goal: Goal<'tcx, T>,
+    pub anchor: DefiningAnchor,
+    pub predefined_opaques_in_body: PredefinedOpaques<'tcx>,
+}
+
+/// Additional constraints returned on success.
+#[derive(Debug, PartialEq, Eq, Clone, Hash, Default)]
+pub struct PredefinedOpaquesData<'tcx> {
+    pub opaque_types: Vec<(ty::OpaqueTypeKey<'tcx>, Ty<'tcx>)>,
+}
+
+#[derive(Debug, PartialEq, Eq, Copy, Clone, Hash)]
+pub struct PredefinedOpaques<'tcx>(pub(crate) Interned<'tcx, PredefinedOpaquesData<'tcx>>);
+
+impl<'tcx> std::ops::Deref for PredefinedOpaques<'tcx> {
+    type Target = PredefinedOpaquesData<'tcx>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+pub type CanonicalInput<'tcx, T = ty::Predicate<'tcx>> = Canonical<'tcx, QueryInput<'tcx, T>>;
 
 pub type CanonicalResponse<'tcx> = Canonical<'tcx, Response<'tcx>>;
 
@@ -120,11 +144,11 @@ impl<'tcx> std::ops::Deref for ExternalConstraints<'tcx> {
 }
 
 /// Additional constraints returned on success.
-#[derive(Debug, PartialEq, Eq, Clone, Hash, Default, TypeFoldable, TypeVisitable)]
+#[derive(Debug, PartialEq, Eq, Clone, Hash, Default)]
 pub struct ExternalConstraintsData<'tcx> {
     // FIXME: implement this.
     pub region_constraints: QueryRegionConstraints<'tcx>,
-    pub opaque_types: Vec<(Ty<'tcx>, Ty<'tcx>)>,
+    pub opaque_types: Vec<(ty::OpaqueTypeKey<'tcx>, Ty<'tcx>)>,
 }
 
 // FIXME: Having to clone `region_constraints` for folding feels bad and
@@ -165,3 +189,40 @@ impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for ExternalConstraints<'tcx> {
         ControlFlow::Continue(())
     }
 }
+
+// FIXME: Having to clone `region_constraints` for folding feels bad and
+// probably isn't great wrt performance.
+//
+// Not sure how to fix this, maybe we should also intern `opaque_types` and
+// `region_constraints` here or something.
+impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for PredefinedOpaques<'tcx> {
+    fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
+        self,
+        folder: &mut F,
+    ) -> Result<Self, F::Error> {
+        Ok(FallibleTypeFolder::interner(folder).mk_predefined_opaques_in_body(
+            PredefinedOpaquesData {
+                opaque_types: self
+                    .opaque_types
+                    .iter()
+                    .map(|opaque| opaque.try_fold_with(folder))
+                    .collect::<Result<_, F::Error>>()?,
+            },
+        ))
+    }
+
+    fn fold_with<F: TypeFolder<TyCtxt<'tcx>>>(self, folder: &mut F) -> Self {
+        TypeFolder::interner(folder).mk_predefined_opaques_in_body(PredefinedOpaquesData {
+            opaque_types: self.opaque_types.iter().map(|opaque| opaque.fold_with(folder)).collect(),
+        })
+    }
+}
+
+impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for PredefinedOpaques<'tcx> {
+    fn visit_with<V: TypeVisitor<TyCtxt<'tcx>>>(
+        &self,
+        visitor: &mut V,
+    ) -> std::ops::ControlFlow<V::BreakTy> {
+        self.opaque_types.visit_with(visitor)
+    }
+}
diff --git a/compiler/rustc_middle/src/ty/_match.rs b/compiler/rustc_middle/src/ty/_match.rs
index 468c2c818..cbc68fde9 100644
--- a/compiler/rustc_middle/src/ty/_match.rs
+++ b/compiler/rustc_middle/src/ty/_match.rs
@@ -83,7 +83,7 @@ impl<'tcx> TypeRelation<'tcx> for Match<'tcx> {
 
             (&ty::Error(guar), _) | (_, &ty::Error(guar)) => Ok(self.tcx().ty_error(guar)),
 
-            _ => relate::super_relate_tys(self, a, b),
+            _ => relate::structurally_relate_tys(self, a, b),
         }
     }
 
@@ -109,7 +109,7 @@ impl<'tcx> TypeRelation<'tcx> for Match<'tcx> {
             _ => {}
         }
 
-        relate::super_relate_consts(self, a, b)
+        relate::structurally_relate_consts(self, a, b)
     }
 
     fn binders<T>(
diff --git a/compiler/rustc_middle/src/ty/abstract_const.rs b/compiler/rustc_middle/src/ty/abstract_const.rs
index f889ce827..a39631da9 100644
--- a/compiler/rustc_middle/src/ty/abstract_const.rs
+++ b/compiler/rustc_middle/src/ty/abstract_const.rs
@@ -1,10 +1,9 @@
-//! A subset of a mir body used for const evaluatability checking.
+//! A subset of a mir body used for const evaluability checking.
 use crate::ty::{
     self, Const, EarlyBinder, Ty, TyCtxt, TypeFoldable, TypeFolder, TypeSuperFoldable,
     TypeVisitableExt,
 };
 use rustc_errors::ErrorGuaranteed;
-use rustc_hir::def_id::DefId;
 
 #[derive(Hash, Debug, Clone, Copy, Ord, PartialOrd, PartialEq, Eq)]
 #[derive(TyDecodable, TyEncodable, HashStable, TypeVisitable, TypeFoldable)]
@@ -35,19 +34,6 @@ TrivialTypeTraversalAndLiftImpls! {
 pub type BoundAbstractConst<'tcx> = Result<Option<EarlyBinder<ty::Const<'tcx>>>, ErrorGuaranteed>;
 
 impl<'tcx> TyCtxt<'tcx> {
-    /// Returns a const without substs applied
-    pub fn bound_abstract_const(
-        self,
-        uv: ty::WithOptConstParam<DefId>,
-    ) -> BoundAbstractConst<'tcx> {
-        let ac = if let Some((did, param_did)) = uv.as_const_arg() {
-            self.thir_abstract_const_of_const_arg((did, param_did))
-        } else {
-            self.thir_abstract_const(uv.did)
-        };
-        Ok(ac?.map(|ac| EarlyBinder(ac)))
-    }
-
     pub fn expand_abstract_consts<T: TypeFoldable<TyCtxt<'tcx>>>(self, ac: T) -> T {
         struct Expander<'tcx> {
             tcx: TyCtxt<'tcx>,
@@ -66,11 +52,12 @@ impl<'tcx> TyCtxt<'tcx> {
             }
             fn fold_const(&mut self, c: Const<'tcx>) -> Const<'tcx> {
                 let ct = match c.kind() {
-                    ty::ConstKind::Unevaluated(uv) => match self.tcx.bound_abstract_const(uv.def) {
-                        Err(e) => self.tcx.const_error_with_guaranteed(c.ty(), e),
+                    ty::ConstKind::Unevaluated(uv) => match self.tcx.thir_abstract_const(uv.def) {
+                        Err(e) => self.tcx.const_error(c.ty(), e),
                         Ok(Some(bac)) => {
                             let substs = self.tcx.erase_regions(uv.substs);
-                            bac.subst(self.tcx, substs)
+                            let bac = bac.subst(self.tcx, substs);
+                            return bac.fold_with(self);
                         }
                         Ok(None) => c,
                     },
diff --git a/compiler/rustc_middle/src/ty/adt.rs b/compiler/rustc_middle/src/ty/adt.rs
index 3a03c0901..7c5c030c2 100644
--- a/compiler/rustc_middle/src/ty/adt.rs
+++ b/compiler/rustc_middle/src/ty/adt.rs
@@ -10,7 +10,7 @@ use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_hir as hir;
 use rustc_hir::def::{CtorKind, DefKind, Res};
 use rustc_hir::def_id::DefId;
-use rustc_index::vec::{IndexSlice, IndexVec};
+use rustc_index::{IndexSlice, IndexVec};
 use rustc_query_system::ich::StableHashingContext;
 use rustc_session::DataTypeKind;
 use rustc_span::symbol::sym;
@@ -26,7 +26,7 @@ use super::{Destructor, FieldDef, GenericPredicates, Ty, TyCtxt, VariantDef, Var
 
 bitflags! {
     #[derive(HashStable, TyEncodable, TyDecodable)]
-    pub struct AdtFlags: u32 {
+    pub struct AdtFlags: u16 {
         const NO_ADT_FLAGS        = 0;
         /// Indicates whether the ADT is an enum.
         const IS_ENUM             = 1 << 0;
@@ -111,12 +111,30 @@ impl Ord for AdtDefData {
     }
 }
 
-/// There should be only one AdtDef for each `did`, therefore
-/// it is fine to implement `PartialEq` only based on `did`.
 impl PartialEq for AdtDefData {
     #[inline]
     fn eq(&self, other: &Self) -> bool {
-        self.did == other.did
+        // There should be only one `AdtDefData` for each `def_id`, therefore
+        // it is fine to implement `PartialEq` only based on `def_id`.
+        //
+        // Below, we exhaustively destructure `self` and `other` so that if the
+        // definition of `AdtDefData` changes, a compile-error will be produced,
+        // reminding us to revisit this assumption.
+
+        let Self { did: self_def_id, variants: _, flags: _, repr: _ } = self;
+        let Self { did: other_def_id, variants: _, flags: _, repr: _ } = other;
+
+        let res = self_def_id == other_def_id;
+
+        // Double check that implicit assumption detailed above.
+        if cfg!(debug_assertions) && res {
+            let deep = self.flags == other.flags
+                && self.repr == other.repr
+                && self.variants == other.variants;
+            assert!(deep, "AdtDefData for the same def-id has differing data");
+        }
+
+        res
     }
 }
 
@@ -188,7 +206,7 @@ impl<'tcx> AdtDef<'tcx> {
     }
 }
 
-#[derive(Copy, Clone, Debug, Eq, PartialEq, Hash, HashStable, TyEncodable, TyDecodable)]
+#[derive(Copy, Clone, Debug, Eq, PartialEq, HashStable, TyEncodable, TyDecodable)]
 pub enum AdtKind {
     Struct,
     Union,
diff --git a/compiler/rustc_middle/src/ty/closure.rs b/compiler/rustc_middle/src/ty/closure.rs
index f29bf92b0..be7b2b7ec 100644
--- a/compiler/rustc_middle/src/ty/closure.rs
+++ b/compiler/rustc_middle/src/ty/closure.rs
@@ -5,6 +5,7 @@ use crate::{mir, ty};
 
 use std::fmt::Write;
 
+use crate::query::Providers;
 use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::{self as hir, LangItem};
@@ -457,6 +458,6 @@ impl BorrowKind {
     }
 }
 
-pub fn provide(providers: &mut ty::query::Providers) {
-    *providers = ty::query::Providers { closure_typeinfo, ..*providers }
+pub fn provide(providers: &mut Providers) {
+    *providers = Providers { closure_typeinfo, ..*providers }
 }
diff --git a/compiler/rustc_middle/src/ty/codec.rs b/compiler/rustc_middle/src/ty/codec.rs
index 8ef4a46a7..7fc75674d 100644
--- a/compiler/rustc_middle/src/ty/codec.rs
+++ b/compiler/rustc_middle/src/ty/codec.rs
@@ -19,6 +19,7 @@ use rustc_data_structures::fx::FxHashMap;
 use rustc_middle::ty::TyCtxt;
 use rustc_serialize::{Decodable, Encodable};
 use rustc_span::Span;
+use rustc_target::abi::FieldIdx;
 pub use rustc_type_ir::{TyDecoder, TyEncoder};
 use std::hash::Hash;
 use std::intrinsics;
@@ -401,6 +402,15 @@ impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D> for ty::List<ty
     }
 }
 
+impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D> for ty::List<FieldIdx> {
+    fn decode(decoder: &mut D) -> &'tcx Self {
+        let len = decoder.read_usize();
+        decoder
+            .interner()
+            .mk_fields_from_iter((0..len).map::<FieldIdx, _>(|_| Decodable::decode(decoder)))
+    }
+}
+
 impl_decodable_via_ref! {
     &'tcx ty::TypeckResults<'tcx>,
     &'tcx ty::List<Ty<'tcx>>,
@@ -412,6 +422,7 @@ impl_decodable_via_ref! {
     &'tcx mir::coverage::CodeRegion,
     &'tcx ty::List<ty::BoundVariableKind>,
     &'tcx ty::List<ty::Predicate<'tcx>>,
+    &'tcx ty::List<FieldIdx>,
 }
 
 #[macro_export]
@@ -489,36 +500,40 @@ impl_arena_copy_decoder! {<'tcx>
 macro_rules! implement_ty_decoder {
     ($DecoderName:ident <$($typaram:tt),*>) => {
         mod __ty_decoder_impl {
-            use std::borrow::Cow;
             use rustc_serialize::Decoder;
 
             use super::$DecoderName;
 
             impl<$($typaram ),*> Decoder for $DecoderName<$($typaram),*> {
                 $crate::__impl_decoder_methods! {
+                    read_usize -> usize;
                     read_u128 -> u128;
                     read_u64 -> u64;
                     read_u32 -> u32;
                     read_u16 -> u16;
                     read_u8 -> u8;
-                    read_usize -> usize;
 
+                    read_isize -> isize;
                     read_i128 -> i128;
                     read_i64 -> i64;
                     read_i32 -> i32;
                     read_i16 -> i16;
-                    read_i8 -> i8;
-                    read_isize -> isize;
-
-                    read_bool -> bool;
-                    read_char -> char;
-                    read_str -> &str;
                 }
 
                 #[inline]
                 fn read_raw_bytes(&mut self, len: usize) -> &[u8] {
                     self.opaque.read_raw_bytes(len)
                 }
+
+                #[inline]
+                fn peek_byte(&self) -> u8 {
+                    self.opaque.peek_byte()
+                }
+
+                #[inline]
+                fn position(&self) -> usize {
+                    self.opaque.position()
+                }
             }
         }
     }
diff --git a/compiler/rustc_middle/src/ty/consts.rs b/compiler/rustc_middle/src/ty/consts.rs
index bcedae233..1a4bd1481 100644
--- a/compiler/rustc_middle/src/ty/consts.rs
+++ b/compiler/rustc_middle/src/ty/consts.rs
@@ -6,7 +6,6 @@ use rustc_hir as hir;
 use rustc_hir::def::{DefKind, Res};
 use rustc_hir::def_id::LocalDefId;
 use rustc_macros::HashStable;
-use std::fmt;
 
 mod int;
 mod kind;
@@ -21,15 +20,6 @@ pub use valtree::*;
 #[rustc_pass_by_value]
 pub struct Const<'tcx>(pub(super) Interned<'tcx, ConstData<'tcx>>);
 
-impl<'tcx> fmt::Debug for Const<'tcx> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        // This reflects what `Const` looked liked before `Interned` was
-        // introduced. We print it like this to avoid having to update expected
-        // output in a lot of tests.
-        write!(f, "Const {{ ty: {:?}, kind: {:?} }}", self.ty(), self.kind())
-    }
-}
-
 /// Typed constant value.
 #[derive(PartialEq, Eq, PartialOrd, Ord, Hash, HashStable, TyEncodable, TyDecodable)]
 pub struct ConstData<'tcx> {
@@ -53,19 +43,12 @@ impl<'tcx> Const<'tcx> {
 
     /// Literals and const generic parameters are eagerly converted to a constant, everything else
     /// becomes `Unevaluated`.
-    pub fn from_anon_const(tcx: TyCtxt<'tcx>, def_id: LocalDefId) -> Self {
-        Self::from_opt_const_arg_anon_const(tcx, ty::WithOptConstParam::unknown(def_id))
-    }
-
     #[instrument(skip(tcx), level = "debug")]
-    pub fn from_opt_const_arg_anon_const(
-        tcx: TyCtxt<'tcx>,
-        def: ty::WithOptConstParam<LocalDefId>,
-    ) -> Self {
-        let body_id = match tcx.hir().get_by_def_id(def.did) {
+    pub fn from_anon_const(tcx: TyCtxt<'tcx>, def: LocalDefId) -> Self {
+        let body_id = match tcx.hir().get_by_def_id(def) {
             hir::Node::AnonConst(ac) => ac.body,
             _ => span_bug!(
-                tcx.def_span(def.did.to_def_id()),
+                tcx.def_span(def.to_def_id()),
                 "from_anon_const can only process anonymous constants"
             ),
         };
@@ -73,17 +56,14 @@ impl<'tcx> Const<'tcx> {
         let expr = &tcx.hir().body(body_id).value;
         debug!(?expr);
 
-        let ty = tcx
-            .type_of(def.def_id_for_type_of())
-            .no_bound_vars()
-            .expect("const parameter types cannot be generic");
+        let ty = tcx.type_of(def).no_bound_vars().expect("const parameter types cannot be generic");
 
         match Self::try_eval_lit_or_param(tcx, ty, expr) {
             Some(v) => v,
             None => tcx.mk_const(
                 ty::UnevaluatedConst {
-                    def: def.to_global(),
-                    substs: InternalSubsts::identity_for_item(tcx, def.did),
+                    def: def.to_def_id(),
+                    substs: InternalSubsts::identity_for_item(tcx, def.to_def_id()),
                 },
                 ty,
             ),
@@ -124,7 +104,7 @@ impl<'tcx> Const<'tcx> {
                 Err(e) => {
                     tcx.sess.delay_span_bug(
                         expr.span,
-                        &format!("Const::from_anon_const: couldn't lit_to_const {:?}", e),
+                        format!("Const::from_anon_const: couldn't lit_to_const {:?}", e),
                     );
                 }
             }
@@ -152,9 +132,7 @@ impl<'tcx> Const<'tcx> {
                         ty::ConstKind::Bound(debruijn, ty::BoundVar::from_u32(index)),
                         param_ty,
                     )),
-                    Some(rbv::ResolvedArg::Error(guar)) => {
-                        Some(tcx.const_error_with_guaranteed(param_ty, guar))
-                    }
+                    Some(rbv::ResolvedArg::Error(guar)) => Some(tcx.const_error(param_ty, guar)),
                     arg => bug!("unexpected bound var resolution for {:?}: {arg:?}", expr.hir_id),
                 }
             }
@@ -238,7 +216,7 @@ impl<'tcx> Const<'tcx> {
         if let Some(val) = self.kind().try_eval_for_typeck(tcx, param_env) {
             match val {
                 Ok(val) => tcx.mk_const(val, self.ty()),
-                Err(guar) => tcx.const_error_with_guaranteed(self.ty(), guar),
+                Err(guar) => tcx.const_error(self.ty(), guar),
             }
         } else {
             // Either the constant isn't evaluatable or ValTree creation failed.
diff --git a/compiler/rustc_middle/src/ty/consts/int.rs b/compiler/rustc_middle/src/ty/consts/int.rs
index c0e557d48..d1dbc531e 100644
--- a/compiler/rustc_middle/src/ty/consts/int.rs
+++ b/compiler/rustc_middle/src/ty/consts/int.rs
@@ -141,14 +141,18 @@ impl<CTX> crate::ty::HashStable<CTX> for ScalarInt {
 
 impl<S: Encoder> Encodable<S> for ScalarInt {
     fn encode(&self, s: &mut S) {
-        s.emit_u128(self.data);
-        s.emit_u8(self.size.get());
+        let size = self.size.get();
+        s.emit_u8(size);
+        s.emit_raw_bytes(&self.data.to_le_bytes()[..size as usize]);
     }
 }
 
 impl<D: Decoder> Decodable<D> for ScalarInt {
     fn decode(d: &mut D) -> ScalarInt {
-        ScalarInt { data: d.read_u128(), size: NonZeroU8::new(d.read_u8()).unwrap() }
+        let mut data = [0u8; 16];
+        let size = d.read_u8();
+        data[..size as usize].copy_from_slice(d.read_raw_bytes(size as usize));
+        ScalarInt { data: u128::from_le_bytes(data), size: NonZeroU8::new(size).unwrap() }
     }
 }
 
diff --git a/compiler/rustc_middle/src/ty/consts/kind.rs b/compiler/rustc_middle/src/ty/consts/kind.rs
index 560caa041..1dd4f8a24 100644
--- a/compiler/rustc_middle/src/ty/consts/kind.rs
+++ b/compiler/rustc_middle/src/ty/consts/kind.rs
@@ -17,7 +17,7 @@ use super::ScalarInt;
 #[derive(Copy, Clone, Debug, Eq, PartialEq, PartialOrd, Ord, TyEncodable, TyDecodable, Lift)]
 #[derive(Hash, HashStable, TypeFoldable, TypeVisitable)]
 pub struct UnevaluatedConst<'tcx> {
-    pub def: ty::WithOptConstParam<DefId>,
+    pub def: DefId,
     pub substs: SubstsRef<'tcx>,
 }
 
@@ -36,16 +36,13 @@ impl<'tcx> UnevaluatedConst<'tcx> {
 
 impl<'tcx> UnevaluatedConst<'tcx> {
     #[inline]
-    pub fn new(
-        def: ty::WithOptConstParam<DefId>,
-        substs: SubstsRef<'tcx>,
-    ) -> UnevaluatedConst<'tcx> {
+    pub fn new(def: DefId, substs: SubstsRef<'tcx>) -> UnevaluatedConst<'tcx> {
         UnevaluatedConst { def, substs }
     }
 }
 
 /// Represents a constant in Rust.
-#[derive(Copy, Clone, Debug, Eq, PartialEq, PartialOrd, Ord, TyEncodable, TyDecodable)]
+#[derive(Copy, Clone, Eq, PartialEq, PartialOrd, Ord, TyEncodable, TyDecodable)]
 #[derive(Hash, HashStable, TypeFoldable, TypeVisitable)]
 #[derive(derive_more::From)]
 pub enum ConstKind<'tcx> {
@@ -131,7 +128,7 @@ impl<'tcx> ConstKind<'tcx> {
 }
 
 /// An inference variable for a const, for use in const generics.
-#[derive(Copy, Clone, Debug, Eq, PartialEq, PartialOrd, Ord, TyEncodable, TyDecodable, Hash)]
+#[derive(Copy, Clone, Eq, PartialEq, PartialOrd, Ord, TyEncodable, TyDecodable, Hash)]
 pub enum InferConst<'tcx> {
     /// Infer the value of the const.
     Var(ty::ConstVid<'tcx>),
@@ -224,9 +221,9 @@ impl<'tcx> ConstKind<'tcx> {
             // FIXME(eddyb, skinny121) pass `InferCtxt` into here when it's available, so that
             // we can call `infcx.const_eval_resolve` which handles inference variables.
             let param_env_and = if (param_env, unevaluated).has_non_region_infer() {
-                tcx.param_env(unevaluated.def.did).and(ty::UnevaluatedConst {
+                tcx.param_env(unevaluated.def).and(ty::UnevaluatedConst {
                     def: unevaluated.def,
-                    substs: InternalSubsts::identity_for_item(tcx, unevaluated.def.did),
+                    substs: InternalSubsts::identity_for_item(tcx, unevaluated.def),
                 })
             } else {
                 tcx.erase_regions(param_env)
@@ -248,7 +245,7 @@ impl<'tcx> ConstKind<'tcx> {
                         // can leak through `val` into the const we return.
                         Ok(val) => Some(Ok(EvalResult::ValTree(val?))),
                         Err(ErrorHandled::TooGeneric) => None,
-                        Err(ErrorHandled::Reported(e)) => Some(Err(e)),
+                        Err(ErrorHandled::Reported(e)) => Some(Err(e.into())),
                     }
                 }
                 EvalMode::Mir => {
@@ -259,7 +256,7 @@ impl<'tcx> ConstKind<'tcx> {
                         // can leak through `val` into the const we return.
                         Ok(val) => Some(Ok(EvalResult::ConstVal(val))),
                         Err(ErrorHandled::TooGeneric) => None,
-                        Err(ErrorHandled::Reported(e)) => Some(Err(e)),
+                        Err(ErrorHandled::Reported(e)) => Some(Err(e.into())),
                     }
                 }
             }
diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index 63f7cc2ee..2bde55bc4 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -13,25 +13,28 @@ use crate::middle::codegen_fn_attrs::CodegenFnAttrs;
 use crate::middle::resolve_bound_vars;
 use crate::middle::stability;
 use crate::mir::interpret::{self, Allocation, ConstAllocation};
-use crate::mir::{Body, BorrowCheckResult, Local, Place, PlaceElem, ProjectionKind, Promoted};
+use crate::mir::{Body, Local, Place, PlaceElem, ProjectionKind, Promoted};
+use crate::query::plumbing::QuerySystem;
 use crate::query::LocalCrate;
+use crate::query::Providers;
+use crate::query::{IntoQueryParam, TyCtxtAt};
 use crate::thir::Thir;
 use crate::traits;
 use crate::traits::solve;
-use crate::traits::solve::{ExternalConstraints, ExternalConstraintsData};
-use crate::ty::query::{self, TyCtxtAt};
+use crate::traits::solve::{
+    ExternalConstraints, ExternalConstraintsData, PredefinedOpaques, PredefinedOpaquesData,
+};
 use crate::ty::{
     self, AdtDef, AdtDefData, AdtKind, Binder, Const, ConstData, FloatTy, FloatVar, FloatVid,
     GenericParamDefKind, ImplPolarity, InferTy, IntTy, IntVar, IntVid, List, ParamConst, ParamTy,
     PolyExistentialPredicate, PolyFnSig, Predicate, PredicateKind, Region, RegionKind, ReprOptions,
-    TraitObjectVisitor, Ty, TyKind, TyVar, TyVid, TypeAndMut, TypeckResults, UintTy, Visibility,
+    TraitObjectVisitor, Ty, TyKind, TyVar, TyVid, TypeAndMut, UintTy, Visibility,
 };
 use crate::ty::{GenericArg, InternalSubsts, SubstsRef};
 use rustc_ast::{self as ast, attr};
 use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::intern::Interned;
-use rustc_data_structures::memmap::Mmap;
 use rustc_data_structures::profiling::SelfProfilerRef;
 use rustc_data_structures::sharded::{IntoPointer, ShardedHashMap};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
@@ -50,7 +53,7 @@ use rustc_hir::lang_items::LangItem;
 use rustc_hir::{
     Constness, ExprKind, HirId, ImplItemKind, ItemKind, Node, TraitCandidate, TraitItemKind,
 };
-use rustc_index::vec::IndexVec;
+use rustc_index::IndexVec;
 use rustc_macros::HashStable;
 use rustc_query_system::dep_graph::DepNodeIndex;
 use rustc_query_system::ich::StableHashingContext;
@@ -61,7 +64,6 @@ use rustc_session::lint::Lint;
 use rustc_session::Limit;
 use rustc_session::Session;
 use rustc_span::def_id::{DefPathHash, StableCrateId};
-use rustc_span::source_map::SourceMap;
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::{Span, DUMMY_SP};
 use rustc_target::abi::{FieldIdx, Layout, LayoutS, TargetDataLayout, VariantIdx};
@@ -82,21 +84,6 @@ use std::ops::{Bound, Deref};
 
 const TINY_CONST_EVAL_LIMIT: Limit = Limit(20);
 
-pub trait OnDiskCache<'tcx>: rustc_data_structures::sync::Sync {
-    /// Creates a new `OnDiskCache` instance from the serialized data in `data`.
-    fn new(sess: &'tcx Session, data: Mmap, start_pos: usize) -> Self
-    where
-        Self: Sized;
-
-    fn new_empty(source_map: &'tcx SourceMap) -> Self
-    where
-        Self: Sized;
-
-    fn drop_serialized_data(&self, tcx: TyCtxt<'tcx>);
-
-    fn serialize(&self, tcx: TyCtxt<'tcx>, encoder: FileEncoder) -> FileEncodeResult;
-}
-
 #[allow(rustc::usage_of_ty_tykind)]
 impl<'tcx> Interner for TyCtxt<'tcx> {
     type AdtDef = ty::AdtDef<'tcx>;
@@ -141,6 +128,7 @@ pub struct CtxtInterners<'tcx> {
     type_: InternedSet<'tcx, WithCachedTypeInfo<TyKind<'tcx>>>,
     const_lists: InternedSet<'tcx, List<ty::Const<'tcx>>>,
     substs: InternedSet<'tcx, InternalSubsts<'tcx>>,
+    type_lists: InternedSet<'tcx, List<Ty<'tcx>>>,
     canonical_var_infos: InternedSet<'tcx, List<CanonicalVarInfo<'tcx>>>,
     region: InternedSet<'tcx, RegionKind<'tcx>>,
     poly_existential_predicates: InternedSet<'tcx, List<PolyExistentialPredicate<'tcx>>>,
@@ -154,6 +142,8 @@ pub struct CtxtInterners<'tcx> {
     layout: InternedSet<'tcx, LayoutS>,
     adt_def: InternedSet<'tcx, AdtDefData>,
     external_constraints: InternedSet<'tcx, ExternalConstraintsData<'tcx>>,
+    predefined_opaques_in_body: InternedSet<'tcx, PredefinedOpaquesData<'tcx>>,
+    fields: InternedSet<'tcx, List<FieldIdx>>,
 }
 
 impl<'tcx> CtxtInterners<'tcx> {
@@ -163,6 +153,7 @@ impl<'tcx> CtxtInterners<'tcx> {
             type_: Default::default(),
             const_lists: Default::default(),
             substs: Default::default(),
+            type_lists: Default::default(),
             region: Default::default(),
             poly_existential_predicates: Default::default(),
             canonical_var_infos: Default::default(),
@@ -176,6 +167,8 @@ impl<'tcx> CtxtInterners<'tcx> {
             layout: Default::default(),
             adt_def: Default::default(),
             external_constraints: Default::default(),
+            predefined_opaques_in_body: Default::default(),
+            fields: Default::default(),
         }
     }
 
@@ -209,7 +202,7 @@ impl<'tcx> CtxtInterners<'tcx> {
     ) -> Fingerprint {
         // It's impossible to hash inference variables (and will ICE), so we don't need to try to cache them.
         // Without incremental, we rarely stable-hash types, so let's not do it proactively.
-        if flags.flags.intersects(TypeFlags::NEEDS_INFER) || sess.opts.incremental.is_none() {
+        if flags.flags.intersects(TypeFlags::HAS_INFER) || sess.opts.incremental.is_none() {
             Fingerprint::ZERO
         } else {
             let mut hasher = StableHasher::new();
@@ -447,6 +440,14 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn feed_local_crate(self) -> TyCtxtFeed<'tcx, CrateNum> {
         TyCtxtFeed { tcx: self, key: LOCAL_CRATE }
     }
+
+    /// In order to break cycles involving `AnonConst`, we need to set the expected type by side
+    /// effect. However, we do not want this as a general capability, so this interface restricts
+    /// to the only allowed case.
+    pub fn feed_anon_const_type(self, key: LocalDefId, value: ty::EarlyBinder<Ty<'tcx>>) {
+        debug_assert_eq!(self.def_kind(key), DefKind::AnonConst);
+        TyCtxtFeed { tcx: self, key }.type_of(value)
+    }
 }
 
 impl<'tcx, KEY: Copy> TyCtxtFeed<'tcx, KEY> {
@@ -496,7 +497,7 @@ pub struct GlobalCtxt<'tcx> {
     ///
     /// FIXME(Centril): consider `dyn LintStoreMarker` once
     /// we can upcast to `Any` for some additional type safety.
-    pub lint_store: Lrc<dyn Any + sync::Sync + sync::Send>,
+    pub lint_store: Lrc<dyn Any + sync::DynSync + sync::DynSend>,
 
     pub dep_graph: DepGraph,
 
@@ -513,14 +514,7 @@ pub struct GlobalCtxt<'tcx> {
 
     untracked: Untracked,
 
-    /// This provides access to the incremental compilation on-disk cache for query results.
-    /// Do not access this directly. It is only meant to be used by
-    /// `DepGraph::try_mark_green()` and the query infrastructure.
-    /// This is `None` if we are not incremental compilation mode
-    pub on_disk_cache: Option<&'tcx dyn OnDiskCache<'tcx>>,
-
-    pub queries: &'tcx dyn query::QueryEngine<'tcx>,
-    pub query_system: query::QuerySystem<'tcx>,
+    pub query_system: QuerySystem<'tcx>,
     pub(crate) query_kinds: &'tcx [DepKindStruct<'tcx>],
 
     // Internal caches for metadata decoding. No need to track deps on this.
@@ -581,28 +575,6 @@ impl<'tcx> TyCtxt<'tcx> {
         }
     }
 
-    pub fn typeck_opt_const_arg(
-        self,
-        def: ty::WithOptConstParam<LocalDefId>,
-    ) -> &'tcx TypeckResults<'tcx> {
-        if let Some(param_did) = def.const_param_did {
-            self.typeck_const_arg((def.did, param_did))
-        } else {
-            self.typeck(def.did)
-        }
-    }
-
-    pub fn mir_borrowck_opt_const_arg(
-        self,
-        def: ty::WithOptConstParam<LocalDefId>,
-    ) -> &'tcx BorrowCheckResult<'tcx> {
-        if let Some(param_did) = def.const_param_did {
-            self.mir_borrowck_const_arg((def.did, param_did))
-        } else {
-            self.mir_borrowck(def.did)
-        }
-    }
-
     pub fn alloc_steal_thir(self, thir: Thir<'tcx>) -> &'tcx Steal<Thir<'tcx>> {
         self.arena.alloc(Steal::new(thir))
     }
@@ -677,14 +649,13 @@ impl<'tcx> TyCtxt<'tcx> {
     /// reference to the context, to allow formatting values that need it.
     pub fn create_global_ctxt(
         s: &'tcx Session,
-        lint_store: Lrc<dyn Any + sync::Send + sync::Sync>,
+        lint_store: Lrc<dyn Any + sync::DynSend + sync::DynSync>,
         arena: &'tcx WorkerLocal<Arena<'tcx>>,
         hir_arena: &'tcx WorkerLocal<hir::Arena<'tcx>>,
         untracked: Untracked,
         dep_graph: DepGraph,
-        on_disk_cache: Option<&'tcx dyn OnDiskCache<'tcx>>,
-        queries: &'tcx dyn query::QueryEngine<'tcx>,
         query_kinds: &'tcx [DepKindStruct<'tcx>],
+        query_system: QuerySystem<'tcx>,
     ) -> GlobalCtxt<'tcx> {
         let data_layout = s.target.parse_data_layout().unwrap_or_else(|err| {
             s.emit_fatal(err);
@@ -706,9 +677,7 @@ impl<'tcx> TyCtxt<'tcx> {
             lifetimes: common_lifetimes,
             consts: common_consts,
             untracked,
-            on_disk_cache,
-            queries,
-            query_system: Default::default(),
+            query_system,
             query_kinds,
             ty_rcache: Default::default(),
             pred_rcache: Default::default(),
@@ -735,7 +704,11 @@ impl<'tcx> TyCtxt<'tcx> {
     /// Constructs a `TyKind::Error` type and registers a `delay_span_bug` with the given `msg` to
     /// ensure it gets used.
     #[track_caller]
-    pub fn ty_error_with_message<S: Into<MultiSpan>>(self, span: S, msg: &str) -> Ty<'tcx> {
+    pub fn ty_error_with_message<S: Into<MultiSpan>>(
+        self,
+        span: S,
+        msg: impl Into<DiagnosticMessage>,
+    ) -> Ty<'tcx> {
         let reported = self.sess.delay_span_bug(span, msg);
         self.mk_ty_from_kind(Error(reported))
     }
@@ -766,17 +739,13 @@ impl<'tcx> TyCtxt<'tcx> {
 
     /// Like [TyCtxt::ty_error] but for constants, with current `ErrorGuaranteed`
     #[track_caller]
-    pub fn const_error_with_guaranteed(
-        self,
-        ty: Ty<'tcx>,
-        reported: ErrorGuaranteed,
-    ) -> Const<'tcx> {
+    pub fn const_error(self, ty: Ty<'tcx>, reported: ErrorGuaranteed) -> Const<'tcx> {
         self.mk_const(ty::ConstKind::Error(reported), ty)
     }
 
     /// Like [TyCtxt::ty_error] but for constants.
     #[track_caller]
-    pub fn const_error(self, ty: Ty<'tcx>) -> Const<'tcx> {
+    pub fn const_error_misc(self, ty: Ty<'tcx>) -> Const<'tcx> {
         self.const_error_with_message(
             ty,
             DUMMY_SP,
@@ -834,7 +803,8 @@ impl<'tcx> TyCtxt<'tcx> {
         self.features_query(())
     }
 
-    pub fn def_key(self, id: DefId) -> rustc_hir::definitions::DefKey {
+    pub fn def_key(self, id: impl IntoQueryParam<DefId>) -> rustc_hir::definitions::DefKey {
+        let id = id.into_query_param();
         // Accessing the DefKey is ok, since it is part of DefPathHash.
         if let Some(id) = id.as_local() {
             self.definitions_untracked().def_key(id)
@@ -925,7 +895,7 @@ impl<'tcx> TyCtxt<'tcx> {
             crate_name,
             // Don't print the whole stable crate id. That's just
             // annoying in debug output.
-            stable_crate_id.to_u64() >> (8 * 6),
+            stable_crate_id.as_u64() >> (8 * 6),
             self.def_path(def_id).to_string_no_crate_verbose()
         )
     }
@@ -1046,7 +1016,7 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     pub fn serialize_query_result_cache(self, encoder: FileEncoder) -> FileEncodeResult {
-        self.on_disk_cache.as_ref().map_or(Ok(0), |c| c.serialize(self, encoder))
+        self.query_system.on_disk_cache.as_ref().map_or(Ok(0), |c| c.serialize(self, encoder))
     }
 
     /// If `true`, we should use lazy normalization for constants, otherwise
@@ -1123,11 +1093,13 @@ impl<'tcx> TyCtxt<'tcx> {
         v.0
     }
 
-    /// Given a `DefId` for an `fn`, return all the `dyn` and `impl` traits in its return type and associated alias span when type alias is used
+    /// Given a `DefId` for an `fn`, return all the `dyn` and `impl` traits in
+    /// its return type, and the associated alias span when type alias is used,
+    /// along with a span for lifetime suggestion (if there are existing generics).
     pub fn return_type_impl_or_dyn_traits_with_type_alias(
         self,
         scope_def_id: LocalDefId,
-    ) -> Option<(Vec<&'tcx hir::Ty<'tcx>>, Span)> {
+    ) -> Option<(Vec<&'tcx hir::Ty<'tcx>>, Span, Option<Span>)> {
         let hir_id = self.hir().local_def_id_to_hir_id(scope_def_id);
         let mut v = TraitObjectVisitor(vec![], self.hir());
         // when the return type is a type alias
@@ -1141,7 +1113,7 @@ impl<'tcx> TyCtxt<'tcx> {
         {
             v.visit_ty(alias_ty);
             if !v.0.is_empty() {
-                return Some((v.0, alias_generics.span));
+                return Some((v.0, alias_generics.span, alias_generics.span_for_lifetime_suggestion()));
             }
         }
         return None;
@@ -1231,7 +1203,7 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn all_traits(self) -> impl Iterator<Item = DefId> + 'tcx {
         iter::once(LOCAL_CRATE)
             .chain(self.crates(()).iter().copied())
-            .flat_map(move |cnum| self.traits_in_crate(cnum).iter().copied())
+            .flat_map(move |cnum| self.traits(cnum).iter().copied())
     }
 
     #[inline]
@@ -1278,25 +1250,6 @@ macro_rules! nop_lift {
     };
 }
 
-// Can't use the macros as we have reuse the `substs` here.
-//
-// See `mk_type_list` for more info.
-impl<'a, 'tcx> Lift<'tcx> for &'a List<Ty<'a>> {
-    type Lifted = &'tcx List<Ty<'tcx>>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        if self.is_empty() {
-            return Some(List::empty());
-        }
-
-        tcx.interners
-            .substs
-            .contains_pointer_to(&InternedInSet(self.as_substs()))
-            // SAFETY: `self` is interned and therefore valid
-            // for the entire lifetime of the `TyCtxt`.
-            .then(|| unsafe { mem::transmute::<&'a List<Ty<'a>>, &'tcx List<Ty<'tcx>>>(self) })
-    }
-}
-
 macro_rules! nop_list_lift {
     ($set:ident; $ty:ty => $lifted:ty) => {
         impl<'a, 'tcx> Lift<'tcx> for &'a List<$ty> {
@@ -1320,6 +1273,7 @@ nop_lift! {const_; Const<'a> => Const<'tcx>}
 nop_lift! {const_allocation; ConstAllocation<'a> => ConstAllocation<'tcx>}
 nop_lift! {predicate; Predicate<'a> => Predicate<'tcx>}
 
+nop_list_lift! {type_lists; Ty<'a> => Ty<'tcx>}
 nop_list_lift! {poly_existential_predicates; PolyExistentialPredicate<'a> => PolyExistentialPredicate<'tcx>}
 nop_list_lift! {predicates; Predicate<'a> => Predicate<'tcx>}
 nop_list_lift! {canonical_var_infos; CanonicalVarInfo<'a> => CanonicalVarInfo<'tcx>}
@@ -1329,9 +1283,12 @@ nop_list_lift! {bound_variable_kinds; ty::BoundVariableKind => ty::BoundVariable
 // This is the impl for `&'a InternalSubsts<'a>`.
 nop_list_lift! {substs; GenericArg<'a> => GenericArg<'tcx>}
 
-CloneLiftImpls! { for<'tcx> {
-    Constness, traits::WellFormedLoc, ImplPolarity, crate::mir::ReturnConstraint,
-} }
+CloneLiftImpls! {
+    Constness,
+    traits::WellFormedLoc,
+    ImplPolarity,
+    crate::mir::ReturnConstraint,
+}
 
 macro_rules! sty_debug_print {
     ($fmt: expr, $ctxt: expr, $($variant: ident),*) => {{
@@ -1567,6 +1524,8 @@ direct_interners! {
     adt_def: pub mk_adt_def_from_data(AdtDefData): AdtDef -> AdtDef<'tcx>,
     external_constraints: pub mk_external_constraints(ExternalConstraintsData<'tcx>):
         ExternalConstraints -> ExternalConstraints<'tcx>,
+    predefined_opaques_in_body: pub mk_predefined_opaques_in_body(PredefinedOpaquesData<'tcx>):
+        PredefinedOpaques -> PredefinedOpaques<'tcx>,
 }
 
 macro_rules! slice_interners {
@@ -1591,12 +1550,14 @@ macro_rules! slice_interners {
 slice_interners!(
     const_lists: pub mk_const_list(Const<'tcx>),
     substs: pub mk_substs(GenericArg<'tcx>),
+    type_lists: pub mk_type_list(Ty<'tcx>),
     canonical_var_infos: pub mk_canonical_var_infos(CanonicalVarInfo<'tcx>),
     poly_existential_predicates: intern_poly_existential_predicates(PolyExistentialPredicate<'tcx>),
     predicates: intern_predicates(Predicate<'tcx>),
     projs: pub mk_projs(ProjectionKind),
     place_elems: pub mk_place_elems(PlaceElem<'tcx>),
     bound_variable_kinds: pub mk_bound_variable_kinds(ty::BoundVariableKind),
+    fields: pub mk_fields(FieldIdx),
 );
 
 impl<'tcx> TyCtxt<'tcx> {
@@ -1610,11 +1571,11 @@ impl<'tcx> TyCtxt<'tcx> {
 
     /// Given the def_id of a Trait `trait_def_id` and the name of an associated item `assoc_name`
     /// returns true if the `trait_def_id` defines an associated item of name `assoc_name`.
-    pub fn trait_may_define_assoc_type(self, trait_def_id: DefId, assoc_name: Ident) -> bool {
+    pub fn trait_may_define_assoc_item(self, trait_def_id: DefId, assoc_name: Ident) -> bool {
         self.super_traits_of(trait_def_id).any(|trait_did| {
             self.associated_items(trait_did)
-                .find_by_name_and_kind(self, assoc_name, ty::AssocKind::Type, trait_did)
-                .is_some()
+                .filter_by_name_unhygienic(assoc_name.name)
+                .any(|item| self.hygienic_eq(assoc_name, item.ident(self), trait_did))
         })
     }
 
@@ -1623,7 +1584,7 @@ impl<'tcx> TyCtxt<'tcx> {
         let ty::Alias(ty::Opaque, ty::AliasTy { def_id, .. }) = ty.kind() else { return false };
         let future_trait = self.require_lang_item(LangItem::Future, None);
 
-        self.explicit_item_bounds(def_id).iter().any(|(predicate, _)| {
+        self.explicit_item_bounds(def_id).skip_binder().iter().any(|&(predicate, _)| {
             let ty::PredicateKind::Clause(ty::Clause::Trait(trait_predicate)) = predicate.kind().skip_binder() else {
                 return false;
             };
@@ -1881,7 +1842,7 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     #[inline(always)]
-    fn check_and_mk_substs(
+    pub(crate) fn check_and_mk_substs(
         self,
         _def_id: DefId,
         substs: impl IntoIterator<Item: Into<GenericArg<'tcx>>>,
@@ -1889,7 +1850,17 @@ impl<'tcx> TyCtxt<'tcx> {
         let substs = substs.into_iter().map(Into::into);
         #[cfg(debug_assertions)]
         {
-            let n = self.generics_of(_def_id).count();
+            let generics = self.generics_of(_def_id);
+
+            let n = if let DefKind::AssocTy = self.def_kind(_def_id)
+                && let DefKind::Impl { of_trait: false } = self.def_kind(self.parent(_def_id))
+            {
+                // If this is an inherent projection.
+
+                generics.params.len() + 1
+            } else {
+                generics.count()
+            };
             assert_eq!(
                 (n, Some(n)),
                 substs.size_hint(),
@@ -2050,7 +2021,7 @@ impl<'tcx> TyCtxt<'tcx> {
         debug_assert_matches!(
             (kind, self.def_kind(alias_ty.def_id)),
             (ty::Opaque, DefKind::OpaqueTy)
-                | (ty::Projection, DefKind::AssocTy)
+                | (ty::Projection | ty::Inherent, DefKind::AssocTy)
                 | (ty::Opaque | ty::Projection, DefKind::ImplTraitPlaceholder)
         );
         self.mk_ty_from_kind(Alias(kind, alias_ty))
@@ -2190,18 +2161,6 @@ impl<'tcx> TyCtxt<'tcx> {
         T::collect_and_apply(iter, |xs| self.mk_const_list(xs))
     }
 
-    pub fn mk_type_list(self, ts: &[Ty<'tcx>]) -> &'tcx List<Ty<'tcx>> {
-        // Actually intern type lists as lists of `GenericArg`s.
-        //
-        // Transmuting from `Ty<'tcx>` to `GenericArg<'tcx>` is sound
-        // as explained in `ty_slice_as_generic_arg`. With this,
-        // we guarantee that even when transmuting between `List<Ty<'tcx>>`
-        // and `List<GenericArg<'tcx>>`, the uniqueness requirement for
-        // lists is upheld.
-        let substs = self.mk_substs(ty::subst::ty_slice_as_generic_args(ts));
-        substs.try_as_type_list().unwrap()
-    }
-
     // Unlike various other `mk_*_from_iter` functions, this one uses `I:
     // IntoIterator` instead of `I: Iterator`, and it doesn't have a slice
     // variant, because of the need to combine `inputs` and `output`. This
@@ -2277,6 +2236,14 @@ impl<'tcx> TyCtxt<'tcx> {
         T::collect_and_apply(iter, |xs| self.mk_place_elems(xs))
     }
 
+    pub fn mk_fields_from_iter<I, T>(self, iter: I) -> T::Output
+    where
+        I: Iterator<Item = T>,
+        T: CollectAndApply<FieldIdx, &'tcx List<FieldIdx>>,
+    {
+        T::collect_and_apply(iter, |xs| self.mk_fields(xs))
+    }
+
     pub fn mk_substs_trait(
         self,
         self_ty: Ty<'tcx>,
@@ -2285,15 +2252,6 @@ impl<'tcx> TyCtxt<'tcx> {
         self.mk_substs_from_iter(iter::once(self_ty.into()).chain(rest))
     }
 
-    pub fn mk_trait_ref(
-        self,
-        trait_def_id: DefId,
-        substs: impl IntoIterator<Item: Into<GenericArg<'tcx>>>,
-    ) -> ty::TraitRef<'tcx> {
-        let substs = self.check_and_mk_substs(trait_def_id, substs);
-        ty::TraitRef { def_id: trait_def_id, substs, _use_mk_trait_ref_instead: () }
-    }
-
     pub fn mk_alias_ty(
         self,
         def_id: DefId,
@@ -2389,7 +2347,7 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     pub fn is_late_bound(self, id: HirId) -> bool {
-        self.is_late_bound_map(id.owner).map_or(false, |set| set.contains(&id.local_id))
+        self.is_late_bound_map(id.owner).is_some_and(|set| set.contains(&id.local_id))
     }
 
     pub fn late_bound_vars(self, id: HirId) -> &'tcx List<ty::BoundVariableKind> {
@@ -2461,26 +2419,17 @@ impl<'tcx> TyCtxt<'tcx> {
         }
     }
 
-    /// Named module children from all items except `use` and `extern crate` imports.
-    ///
-    /// In addition to regular items this list also includes struct or variant constructors, and
+    /// Named module children from all kinds of items, including imports.
+    /// In addition to regular items this list also includes struct and variant constructors, and
     /// items inside `extern {}` blocks because all of them introduce names into parent module.
-    /// For non-reexported children every such name is associated with a separate `DefId`.
     ///
     /// Module here is understood in name resolution sense - it can be a `mod` item,
     /// or a crate root, or an enum, or a trait.
-    pub fn module_children_non_reexports(self, def_id: LocalDefId) -> &'tcx [LocalDefId] {
-        self.resolutions(()).module_children_non_reexports.get(&def_id).map_or(&[], |v| &v[..])
-    }
-
-    /// Named module children from `use` and `extern crate` imports.
     ///
-    /// Reexported names are not associated with individual `DefId`s,
-    /// e.g. a glob import can introduce a lot of names, all with the same `DefId`.
-    /// That's why the list needs to contain `ModChild` structures describing all the names
-    /// individually instead of `DefId`s.
-    pub fn module_children_reexports(self, def_id: LocalDefId) -> &'tcx [ModChild] {
-        self.resolutions(()).module_children_reexports.get(&def_id).map_or(&[], |v| &v[..])
+    /// This is not a query, making it a query causes perf regressions
+    /// (probably due to hashing spans in `ModChild`ren).
+    pub fn module_children_local(self, def_id: LocalDefId) -> &'tcx [ModChild] {
+        self.resolutions(()).module_children.get(&def_id).map_or(&[], |v| &v[..])
     }
 }
 
@@ -2494,18 +2443,9 @@ impl<'tcx> TyCtxtAt<'tcx> {
     /// Constructs a `TyKind::Error` type and registers a `delay_span_bug` with the given `msg` to
     /// ensure it gets used.
     #[track_caller]
-    pub fn ty_error_with_message(self, msg: &str) -> Ty<'tcx> {
+    pub fn ty_error_with_message(self, msg: impl Into<DiagnosticMessage>) -> Ty<'tcx> {
         self.tcx.ty_error_with_message(self.span, msg)
     }
-
-    pub fn mk_trait_ref(
-        self,
-        trait_lang_item: LangItem,
-        substs: impl IntoIterator<Item: Into<ty::GenericArg<'tcx>>>,
-    ) -> ty::TraitRef<'tcx> {
-        let trait_def_id = self.require_lang_item(trait_lang_item, Some(self.span));
-        self.tcx.mk_trait_ref(trait_def_id, substs)
-    }
 }
 
 /// Parameter attributes that can only be determined by examining the body of a function instead
@@ -2523,7 +2463,7 @@ pub struct DeducedParamAttrs {
     pub read_only: bool,
 }
 
-pub fn provide(providers: &mut ty::query::Providers) {
+pub fn provide(providers: &mut Providers) {
     providers.maybe_unused_trait_imports =
         |tcx, ()| &tcx.resolutions(()).maybe_unused_trait_imports;
     providers.names_imported_by_glob_use = |tcx, id| {
@@ -2540,7 +2480,7 @@ pub fn provide(providers: &mut ty::query::Providers) {
         |tcx, LocalCrate| attr::contains_name(tcx.hir().krate_attrs(), sym::compiler_builtins);
     providers.has_panic_handler = |tcx, LocalCrate| {
         // We want to check if the panic handler was defined in this crate
-        tcx.lang_items().panic_impl().map_or(false, |did| did.is_local())
+        tcx.lang_items().panic_impl().is_some_and(|did| did.is_local())
     };
     providers.source_span = |tcx, def_id| tcx.untracked.source_span.get(def_id).unwrap_or(DUMMY_SP);
 }
diff --git a/compiler/rustc_middle/src/ty/context/tls.rs b/compiler/rustc_middle/src/ty/context/tls.rs
index fb0d90930..9de77b9fd 100644
--- a/compiler/rustc_middle/src/ty/context/tls.rs
+++ b/compiler/rustc_middle/src/ty/context/tls.rs
@@ -1,7 +1,7 @@
 use super::{GlobalCtxt, TyCtxt};
 
 use crate::dep_graph::TaskDepsRef;
-use crate::ty::query;
+use crate::query::plumbing::QueryJobId;
 use rustc_data_structures::sync::{self, Lock};
 use rustc_errors::Diagnostic;
 #[cfg(not(parallel_compiler))]
@@ -22,7 +22,7 @@ pub struct ImplicitCtxt<'a, 'tcx> {
 
     /// The current query job, if any. This is updated by `JobOwner::start` in
     /// `ty::query::plumbing` when executing a query.
-    pub query: Option<query::QueryJobId>,
+    pub query: Option<QueryJobId>,
 
     /// Where to store diagnostics for the current query job, if any.
     /// This is updated by `JobOwner::start` in `ty::query::plumbing` when executing a query.
@@ -78,7 +78,7 @@ where
 {
     TLV.with(|tlv| {
         let old = tlv.replace(erase(context));
-        let _reset = rustc_data_structures::OnDrop(move || tlv.set(old));
+        let _reset = rustc_data_structures::defer(move || tlv.set(old));
         f()
     })
 }
@@ -94,8 +94,8 @@ where
         f(None)
     } else {
         // We could get an `ImplicitCtxt` pointer from another thread.
-        // Ensure that `ImplicitCtxt` is `Sync`.
-        sync::assert_sync::<ImplicitCtxt<'_, '_>>();
+        // Ensure that `ImplicitCtxt` is `DynSync`.
+        sync::assert_dyn_sync::<ImplicitCtxt<'_, '_>>();
 
         unsafe { f(Some(downcast(context))) }
     }
diff --git a/compiler/rustc_middle/src/ty/diagnostics.rs b/compiler/rustc_middle/src/ty/diagnostics.rs
index ae0bb4949..6a29063b8 100644
--- a/compiler/rustc_middle/src/ty/diagnostics.rs
+++ b/compiler/rustc_middle/src/ty/diagnostics.rs
@@ -139,7 +139,7 @@ pub fn suggest_arbitrary_trait_bound<'tcx>(
     // Suggest a where clause bound for a non-type parameter.
     err.span_suggestion_verbose(
         generics.tail_span_for_predicate_suggestion(),
-        &format!(
+        format!(
             "consider {} `where` clause, but there might be an alternative better way to express \
              this requirement",
             if generics.where_clause_span.is_empty() { "introducing a" } else { "extending the" },
@@ -242,7 +242,7 @@ pub fn suggest_constraining_type_params<'a>(
 
                 err.span_label(
                     param.span,
-                    &format!("this type parameter needs to be `{}`", constraint),
+                    format!("this type parameter needs to be `{}`", constraint),
                 );
                 suggest_removing_unsized_bound(generics, &mut suggestions, param, def_id);
             }
diff --git a/compiler/rustc_middle/src/ty/erase_regions.rs b/compiler/rustc_middle/src/ty/erase_regions.rs
index 383773248..7895993cc 100644
--- a/compiler/rustc_middle/src/ty/erase_regions.rs
+++ b/compiler/rustc_middle/src/ty/erase_regions.rs
@@ -1,8 +1,9 @@
+use crate::query::Providers;
 use crate::ty::fold::{TypeFoldable, TypeFolder, TypeSuperFoldable};
 use crate::ty::{self, Ty, TyCtxt, TypeFlags, TypeVisitableExt};
 
-pub(super) fn provide(providers: &mut ty::query::Providers) {
-    *providers = ty::query::Providers { erase_regions_ty, ..*providers };
+pub(super) fn provide(providers: &mut Providers) {
+    *providers = Providers { erase_regions_ty, ..*providers };
 }
 
 fn erase_regions_ty<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Ty<'tcx> {
@@ -40,7 +41,7 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for RegionEraserVisitor<'tcx> {
     }
 
     fn fold_ty(&mut self, ty: Ty<'tcx>) -> Ty<'tcx> {
-        if ty.needs_infer() { ty.super_fold_with(self) } else { self.tcx.erase_regions_ty(ty) }
+        if ty.has_infer() { ty.super_fold_with(self) } else { self.tcx.erase_regions_ty(ty) }
     }
 
     fn fold_binder<T>(&mut self, t: ty::Binder<'tcx, T>) -> ty::Binder<'tcx, T>
diff --git a/compiler/rustc_middle/src/ty/error.rs b/compiler/rustc_middle/src/ty/error.rs
index aff6c77e0..49ab9b79e 100644
--- a/compiler/rustc_middle/src/ty/error.rs
+++ b/compiler/rustc_middle/src/ty/error.rs
@@ -28,7 +28,7 @@ impl<T> ExpectedFound<T> {
 }
 
 // Data structures used in type unification
-#[derive(Copy, Clone, Debug, TypeFoldable, TypeVisitable, Lift, PartialEq, Eq)]
+#[derive(Copy, Clone, Debug, TypeVisitable, Lift, PartialEq, Eq)]
 #[rustc_pass_by_value]
 pub enum TypeError<'tcx> {
     Mismatch,
@@ -265,7 +265,7 @@ impl<'tcx> Ty<'tcx> {
             ty::Infer(ty::FreshTy(_)) => "fresh type".into(),
             ty::Infer(ty::FreshIntTy(_)) => "fresh integral type".into(),
             ty::Infer(ty::FreshFloatTy(_)) => "fresh floating-point type".into(),
-            ty::Alias(ty::Projection, _) => "associated type".into(),
+            ty::Alias(ty::Projection | ty::Inherent, _) => "associated type".into(),
             ty::Param(p) => format!("type parameter `{p}`").into(),
             ty::Alias(ty::Opaque, ..) => if tcx.ty_is_opaque_future(self) { "future".into() } else { "opaque type".into() },
             ty::Error(_) => "type error".into(),
@@ -312,7 +312,7 @@ impl<'tcx> Ty<'tcx> {
             ty::Tuple(..) => "tuple".into(),
             ty::Placeholder(..) => "higher-ranked type".into(),
             ty::Bound(..) => "bound type variable".into(),
-            ty::Alias(ty::Projection, _) => "associated type".into(),
+            ty::Alias(ty::Projection | ty::Inherent, _) => "associated type".into(),
             ty::Param(_) => "type parameter".into(),
             ty::Alias(ty::Opaque, ..) => "opaque type".into(),
         }
diff --git a/compiler/rustc_middle/src/ty/fast_reject.rs b/compiler/rustc_middle/src/ty/fast_reject.rs
index 31d00b65e..76f61d9ac 100644
--- a/compiler/rustc_middle/src/ty/fast_reject.rs
+++ b/compiler/rustc_middle/src/ty/fast_reject.rs
@@ -68,7 +68,7 @@ pub enum TreatParams {
 }
 
 /// During fast-rejection, we have the choice of treating projection types
-/// as either simplifyable or not, depending on whether we expect the projection
+/// as either simplifiable or not, depending on whether we expect the projection
 /// to be normalized/rigid.
 #[derive(PartialEq, Eq, Debug, Clone, Copy)]
 pub enum TreatProjections {
diff --git a/compiler/rustc_middle/src/ty/flags.rs b/compiler/rustc_middle/src/ty/flags.rs
index 5a6ee1238..d64875a9f 100644
--- a/compiler/rustc_middle/src/ty/flags.rs
+++ b/compiler/rustc_middle/src/ty/flags.rs
@@ -176,14 +176,14 @@ impl FlagComputation {
                 self.add_substs(substs);
             }
 
-            &ty::Alias(ty::Projection, data) => {
-                self.add_flags(TypeFlags::HAS_TY_PROJECTION);
-                self.add_projection_ty(data);
-            }
+            &ty::Alias(kind, data) => {
+                self.add_flags(match kind {
+                    ty::Projection => TypeFlags::HAS_TY_PROJECTION,
+                    ty::Inherent => TypeFlags::HAS_TY_INHERENT,
+                    ty::Opaque => TypeFlags::HAS_TY_OPAQUE,
+                });
 
-            &ty::Alias(ty::Opaque, ty::AliasTy { substs, .. }) => {
-                self.add_flags(TypeFlags::HAS_TY_OPAQUE);
-                self.add_substs(substs);
+                self.add_alias_ty(data);
             }
 
             &ty::Dynamic(obj, r, _) => {
@@ -267,7 +267,7 @@ impl FlagComputation {
                 projection_ty,
                 term,
             })) => {
-                self.add_projection_ty(projection_ty);
+                self.add_alias_ty(projection_ty);
                 self.add_term(term);
             }
             ty::PredicateKind::WellFormed(arg) => {
@@ -372,8 +372,8 @@ impl FlagComputation {
         }
     }
 
-    fn add_projection_ty(&mut self, projection_ty: ty::AliasTy<'_>) {
-        self.add_substs(projection_ty.substs);
+    fn add_alias_ty(&mut self, alias_ty: ty::AliasTy<'_>) {
+        self.add_substs(alias_ty.substs);
     }
 
     fn add_substs(&mut self, substs: &[GenericArg<'_>]) {
diff --git a/compiler/rustc_middle/src/ty/fold.rs b/compiler/rustc_middle/src/ty/fold.rs
index 203e16bea..25890eb15 100644
--- a/compiler/rustc_middle/src/ty/fold.rs
+++ b/compiler/rustc_middle/src/ty/fold.rs
@@ -37,7 +37,8 @@ where
     }
 
     fn fold_region(&mut self, r: ty::Region<'tcx>) -> ty::Region<'tcx> {
-        let r = r.super_fold_with(self);
+        // This one is a little different, because `super_fold_with` is not
+        // implemented on non-recursive `Region`.
         (self.lt_op)(r)
     }
 
diff --git a/compiler/rustc_middle/src/ty/generics.rs b/compiler/rustc_middle/src/ty/generics.rs
index baef4ffed..b0ffe7829 100644
--- a/compiler/rustc_middle/src/ty/generics.rs
+++ b/compiler/rustc_middle/src/ty/generics.rs
@@ -103,7 +103,7 @@ impl GenericParamDef {
             ty::GenericParamDefKind::Lifetime => tcx.mk_re_error_misc().into(),
             ty::GenericParamDefKind::Type { .. } => tcx.ty_error_misc().into(),
             ty::GenericParamDefKind::Const { .. } => {
-                tcx.const_error(tcx.type_of(self.def_id).subst(tcx, preceding_substs)).into()
+                tcx.const_error_misc(tcx.type_of(self.def_id).subst(tcx, preceding_substs)).into()
             }
         }
     }
@@ -298,7 +298,7 @@ impl<'tcx> Generics {
             .iter()
             .rev()
             .take_while(|param| {
-                param.default_value(tcx).map_or(false, |default| {
+                param.default_value(tcx).is_some_and(|default| {
                     default.subst(tcx, substs) == substs[param.index as usize]
                 })
             })
diff --git a/compiler/rustc_middle/src/ty/impls_ty.rs b/compiler/rustc_middle/src/ty/impls_ty.rs
index 4c7822acd..02baa395c 100644
--- a/compiler/rustc_middle/src/ty/impls_ty.rs
+++ b/compiler/rustc_middle/src/ty/impls_ty.rs
@@ -73,34 +73,6 @@ impl<'a, 'tcx> HashStable<StableHashingContext<'a>> for ty::subst::GenericArg<'t
     }
 }
 
-impl<'a, 'tcx> HashStable<StableHashingContext<'a>> for ty::subst::GenericArgKind<'tcx> {
-    fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
-        match self {
-            // WARNING: We dedup cache the `HashStable` results for `List`
-            // while ignoring types and freely transmute
-            // between `List<Ty<'tcx>>` and `List<GenericArg<'tcx>>`.
-            // See `fn mk_type_list` for more details.
-            //
-            // We therefore hash types without adding a hash for their discriminant.
-            //
-            // In order to make it very unlikely for the sequence of bytes being hashed for
-            // a `GenericArgKind::Type` to be the same as the sequence of bytes being
-            // hashed for one of the other variants, we hash some very high number instead
-            // of their actual discriminant since `TyKind` should never start with anything
-            // that high.
-            ty::subst::GenericArgKind::Type(ty) => ty.hash_stable(hcx, hasher),
-            ty::subst::GenericArgKind::Const(ct) => {
-                0xF3u8.hash_stable(hcx, hasher);
-                ct.hash_stable(hcx, hasher);
-            }
-            ty::subst::GenericArgKind::Lifetime(lt) => {
-                0xF5u8.hash_stable(hcx, hasher);
-                lt.hash_stable(hcx, hasher);
-            }
-        }
-    }
-}
-
 // AllocIds get resolved to whatever they point to (to be stable)
 impl<'a> HashStable<StableHashingContext<'a>> for mir::interpret::AllocId {
     fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
diff --git a/compiler/rustc_middle/src/ty/inhabitedness/mod.rs b/compiler/rustc_middle/src/ty/inhabitedness/mod.rs
index 92a040068..422350284 100644
--- a/compiler/rustc_middle/src/ty/inhabitedness/mod.rs
+++ b/compiler/rustc_middle/src/ty/inhabitedness/mod.rs
@@ -43,6 +43,7 @@
 //! This code should only compile in modules where the uninhabitedness of `Foo`
 //! is visible.
 
+use crate::query::Providers;
 use crate::ty::context::TyCtxt;
 use crate::ty::{self, DefId, Ty, VariantDef, Visibility};
 
@@ -52,9 +53,8 @@ pub mod inhabited_predicate;
 
 pub use inhabited_predicate::InhabitedPredicate;
 
-pub(crate) fn provide(providers: &mut ty::query::Providers) {
-    *providers =
-        ty::query::Providers { inhabited_predicate_adt, inhabited_predicate_type, ..*providers };
+pub(crate) fn provide(providers: &mut Providers) {
+    *providers = Providers { inhabited_predicate_adt, inhabited_predicate_type, ..*providers };
 }
 
 /// Returns an `InhabitedPredicate` that is generic over type parameters and
@@ -113,6 +113,12 @@ impl<'tcx> Ty<'tcx> {
             }
             Never => InhabitedPredicate::False,
             Param(_) | Alias(ty::Projection, _) => InhabitedPredicate::GenericType(self),
+            // FIXME(inherent_associated_types): Most likely we can just map to `GenericType` like above.
+            // However it's unclear if the substs passed to `InhabitedPredicate::subst` are of the correct
+            // format, i.e. don't contain parent substs. If you hit this case, please verify this beforehand.
+            Alias(ty::Inherent, _) => {
+                bug!("unimplemented: inhabitedness checking for inherent projections")
+            }
             Tuple(tys) if tys.is_empty() => InhabitedPredicate::True,
             // use a query for more complex cases
             Adt(..) | Array(..) | Tuple(_) => tcx.inhabited_predicate_type(self),
diff --git a/compiler/rustc_middle/src/ty/instance.rs b/compiler/rustc_middle/src/ty/instance.rs
index e73225f70..e641d1ef1 100644
--- a/compiler/rustc_middle/src/ty/instance.rs
+++ b/compiler/rustc_middle/src/ty/instance.rs
@@ -34,7 +34,7 @@ pub enum InstanceDef<'tcx> {
     /// - `fn` items
     /// - closures
     /// - generators
-    Item(ty::WithOptConstParam<DefId>),
+    Item(DefId),
 
     /// An intrinsic `fn` item (with `"rust-intrinsic"` or `"platform-intrinsic"` ABI).
     ///
@@ -115,7 +115,7 @@ impl<'tcx> Instance<'tcx> {
     /// lifetimes erased, allowing a `ParamEnv` to be specified for use during normalization.
     pub fn ty(&self, tcx: TyCtxt<'tcx>, param_env: ty::ParamEnv<'tcx>) -> Ty<'tcx> {
         let ty = tcx.type_of(self.def.def_id());
-        tcx.subst_and_normalize_erasing_regions(self.substs, param_env, ty.skip_binder())
+        tcx.subst_and_normalize_erasing_regions(self.substs, param_env, ty)
     }
 
     /// Finds a crate that contains a monomorphization of this instance that
@@ -143,7 +143,7 @@ impl<'tcx> Instance<'tcx> {
 
         match self.def {
             InstanceDef::Item(def) => tcx
-                .upstream_monomorphizations_for(def.did)
+                .upstream_monomorphizations_for(def)
                 .and_then(|monos| monos.get(&self.substs).cloned()),
             InstanceDef::DropGlue(_, Some(_)) => tcx.upstream_drop_glue_for(self.substs),
             _ => None,
@@ -155,8 +155,8 @@ impl<'tcx> InstanceDef<'tcx> {
     #[inline]
     pub fn def_id(self) -> DefId {
         match self {
-            InstanceDef::Item(def) => def.did,
-            InstanceDef::VTableShim(def_id)
+            InstanceDef::Item(def_id)
+            | InstanceDef::VTableShim(def_id)
             | InstanceDef::ReifyShim(def_id)
             | InstanceDef::FnPtrShim(def_id, _)
             | InstanceDef::Virtual(def_id, _)
@@ -172,7 +172,7 @@ impl<'tcx> InstanceDef<'tcx> {
     /// Returns the `DefId` of instances which might not require codegen locally.
     pub fn def_id_if_not_guaranteed_local_codegen(self) -> Option<DefId> {
         match self {
-            ty::InstanceDef::Item(def) => Some(def.did),
+            ty::InstanceDef::Item(def) => Some(def),
             ty::InstanceDef::DropGlue(def_id, Some(_)) | InstanceDef::ThreadLocalShim(def_id) => {
                 Some(def_id)
             }
@@ -189,23 +189,6 @@ impl<'tcx> InstanceDef<'tcx> {
     }
 
     #[inline]
-    pub fn with_opt_param(self) -> ty::WithOptConstParam<DefId> {
-        match self {
-            InstanceDef::Item(def) => def,
-            InstanceDef::VTableShim(def_id)
-            | InstanceDef::ReifyShim(def_id)
-            | InstanceDef::FnPtrShim(def_id, _)
-            | InstanceDef::Virtual(def_id, _)
-            | InstanceDef::Intrinsic(def_id)
-            | InstanceDef::ClosureOnceShim { call_once: def_id, track_caller: _ }
-            | InstanceDef::DropGlue(def_id, _)
-            | InstanceDef::CloneShim(def_id, _)
-            | InstanceDef::ThreadLocalShim(def_id)
-            | InstanceDef::FnPtrAddrShim(def_id, _) => ty::WithOptConstParam::unknown(def_id),
-        }
-    }
-
-    #[inline]
     pub fn get_attrs(
         &self,
         tcx: TyCtxt<'tcx>,
@@ -222,7 +205,7 @@ impl<'tcx> InstanceDef<'tcx> {
     pub fn requires_inline(&self, tcx: TyCtxt<'tcx>) -> bool {
         use rustc_hir::definitions::DefPathData;
         let def_id = match *self {
-            ty::InstanceDef::Item(def) => def.did,
+            ty::InstanceDef::Item(def) => def,
             ty::InstanceDef::DropGlue(_, Some(_)) => return false,
             ty::InstanceDef::ThreadLocalShim(_) => return false,
             _ => return true,
@@ -273,8 +256,7 @@ impl<'tcx> InstanceDef<'tcx> {
 
     pub fn requires_caller_location(&self, tcx: TyCtxt<'_>) -> bool {
         match *self {
-            InstanceDef::Item(ty::WithOptConstParam { did: def_id, .. })
-            | InstanceDef::Virtual(def_id, _) => {
+            InstanceDef::Item(def_id) | InstanceDef::Virtual(def_id, _) => {
                 tcx.body_codegen_attrs(def_id).flags.contains(CodegenFnAttrFlags::TRACK_CALLER)
             }
             InstanceDef::ClosureOnceShim { call_once: _, track_caller } => track_caller,
@@ -358,7 +340,7 @@ impl<'tcx> Instance<'tcx> {
             def_id,
             substs
         );
-        Instance { def: InstanceDef::Item(ty::WithOptConstParam::unknown(def_id)), substs }
+        Instance { def: InstanceDef::Item(def_id), substs }
     }
 
     pub fn mono(tcx: TyCtxt<'tcx>, def_id: DefId) -> Instance<'tcx> {
@@ -403,18 +385,21 @@ impl<'tcx> Instance<'tcx> {
     /// couldn't complete due to errors elsewhere - this is distinct
     /// from `Ok(None)` to avoid misleading diagnostics when an error
     /// has already been/will be emitted, for the original cause
+    #[instrument(level = "debug", skip(tcx), ret)]
     pub fn resolve(
         tcx: TyCtxt<'tcx>,
         param_env: ty::ParamEnv<'tcx>,
         def_id: DefId,
         substs: SubstsRef<'tcx>,
     ) -> Result<Option<Instance<'tcx>>, ErrorGuaranteed> {
-        Instance::resolve_opt_const_arg(
-            tcx,
-            param_env,
-            ty::WithOptConstParam::unknown(def_id),
-            substs,
-        )
+        // All regions in the result of this query are erased, so it's
+        // fine to erase all of the input regions.
+
+        // HACK(eddyb) erase regions in `substs` first, so that `param_env.and(...)`
+        // below is more likely to ignore the bounds in scope (e.g. if the only
+        // generic parameters mentioned by `substs` were lifetime ones).
+        let substs = tcx.erase_regions(substs);
+        tcx.resolve_instance(tcx.erase_regions(param_env.and((def_id, substs))))
     }
 
     pub fn expect_resolve(
@@ -432,31 +417,6 @@ impl<'tcx> Instance<'tcx> {
         }
     }
 
-    // This should be kept up to date with `resolve`.
-    pub fn resolve_opt_const_arg(
-        tcx: TyCtxt<'tcx>,
-        param_env: ty::ParamEnv<'tcx>,
-        def: ty::WithOptConstParam<DefId>,
-        substs: SubstsRef<'tcx>,
-    ) -> Result<Option<Instance<'tcx>>, ErrorGuaranteed> {
-        // All regions in the result of this query are erased, so it's
-        // fine to erase all of the input regions.
-
-        // HACK(eddyb) erase regions in `substs` first, so that `param_env.and(...)`
-        // below is more likely to ignore the bounds in scope (e.g. if the only
-        // generic parameters mentioned by `substs` were lifetime ones).
-        let substs = tcx.erase_regions(substs);
-
-        // FIXME(eddyb) should this always use `param_env.with_reveal_all()`?
-        if let Some((did, param_did)) = def.as_const_arg() {
-            tcx.resolve_instance_of_const_arg(
-                tcx.erase_regions(param_env.and((did, param_did, substs))),
-            )
-        } else {
-            tcx.resolve_instance(tcx.erase_regions(param_env.and((def.did, substs))))
-        }
-    }
-
     pub fn resolve_for_fn_ptr(
         tcx: TyCtxt<'tcx>,
         param_env: ty::ParamEnv<'tcx>,
@@ -470,7 +430,7 @@ impl<'tcx> Instance<'tcx> {
             match resolved.def {
                 InstanceDef::Item(def) if resolved.def.requires_caller_location(tcx) => {
                     debug!(" => fn pointer created for function with #[track_caller]");
-                    resolved.def = InstanceDef::ReifyShim(def.did);
+                    resolved.def = InstanceDef::ReifyShim(def);
                 }
                 InstanceDef::Virtual(def_id, _) => {
                     debug!(" => fn pointer created for virtual call");
@@ -513,23 +473,23 @@ impl<'tcx> Instance<'tcx> {
                         if resolved.def.requires_caller_location(tcx)
                             // 2) The caller location parameter comes from having `#[track_caller]`
                             // on the implementation, and *not* on the trait method.
-                            && !tcx.should_inherit_track_caller(def.did)
+                            && !tcx.should_inherit_track_caller(def)
                             // If the method implementation comes from the trait definition itself
                             // (e.g. `trait Foo { #[track_caller] my_fn() { /* impl */ } }`),
                             // then we don't need to generate a shim. This check is needed because
                             // `should_inherit_track_caller` returns `false` if our method
                             // implementation comes from the trait block, and not an impl block
                             && !matches!(
-                                tcx.opt_associated_item(def.did),
+                                tcx.opt_associated_item(def),
                                 Some(ty::AssocItem {
                                     container: ty::AssocItemContainer::TraitContainer,
                                     ..
                                 })
                             )
                         {
-                            if tcx.is_closure(def.did) {
+                            if tcx.is_closure(def) {
                                 debug!(" => vtable fn pointer created for closure with #[track_caller]: {:?} for method {:?} {:?}",
-                                       def.did, def_id, substs);
+                                       def, def_id, substs);
 
                                 // Create a shim for the `FnOnce/FnMut/Fn` method we are calling
                                 // - unlike functions, invoking a closure always goes through a
@@ -537,9 +497,9 @@ impl<'tcx> Instance<'tcx> {
                                 resolved = Instance { def: InstanceDef::ReifyShim(def_id), substs };
                             } else {
                                 debug!(
-                                    " => vtable fn pointer created for function with #[track_caller]: {:?}", def.did
+                                    " => vtable fn pointer created for function with #[track_caller]: {:?}", def
                                 );
-                                resolved.def = InstanceDef::ReifyShim(def.did);
+                                resolved.def = InstanceDef::ReifyShim(def);
                             }
                         }
                     }
@@ -618,14 +578,15 @@ impl<'tcx> Instance<'tcx> {
         self.def.has_polymorphic_mir_body().then_some(self.substs)
     }
 
-    pub fn subst_mir<T>(&self, tcx: TyCtxt<'tcx>, v: &T) -> T
+    pub fn subst_mir<T>(&self, tcx: TyCtxt<'tcx>, v: EarlyBinder<&T>) -> T
     where
         T: TypeFoldable<TyCtxt<'tcx>> + Copy,
     {
+        let v = v.map_bound(|v| *v);
         if let Some(substs) = self.substs_for_mir_body() {
-            EarlyBinder(*v).subst(tcx, substs)
+            v.subst(tcx, substs)
         } else {
-            *v
+            v.skip_binder()
         }
     }
 
@@ -634,7 +595,7 @@ impl<'tcx> Instance<'tcx> {
         &self,
         tcx: TyCtxt<'tcx>,
         param_env: ty::ParamEnv<'tcx>,
-        v: T,
+        v: EarlyBinder<T>,
     ) -> T
     where
         T: TypeFoldable<TyCtxt<'tcx>> + Clone,
@@ -642,7 +603,7 @@ impl<'tcx> Instance<'tcx> {
         if let Some(substs) = self.substs_for_mir_body() {
             tcx.subst_and_normalize_erasing_regions(substs, param_env, v)
         } else {
-            tcx.normalize_erasing_regions(param_env, v)
+            tcx.normalize_erasing_regions(param_env, v.skip_binder())
         }
     }
 
@@ -651,7 +612,7 @@ impl<'tcx> Instance<'tcx> {
         &self,
         tcx: TyCtxt<'tcx>,
         param_env: ty::ParamEnv<'tcx>,
-        v: T,
+        v: EarlyBinder<T>,
     ) -> Result<T, NormalizationError<'tcx>>
     where
         T: TypeFoldable<TyCtxt<'tcx>> + Clone,
@@ -659,7 +620,7 @@ impl<'tcx> Instance<'tcx> {
         if let Some(substs) = self.substs_for_mir_body() {
             tcx.try_subst_and_normalize_erasing_regions(substs, param_env, v)
         } else {
-            tcx.try_normalize_erasing_regions(param_env, v)
+            tcx.try_normalize_erasing_regions(param_env, v.skip_binder())
         }
     }
 
@@ -698,7 +659,7 @@ fn polymorphize<'tcx>(
     } else {
         None
     };
-    let has_upvars = upvars_ty.map_or(false, |ty| !ty.tuple_fields().is_empty());
+    let has_upvars = upvars_ty.is_some_and(|ty| !ty.tuple_fields().is_empty());
     debug!("polymorphize: upvars_ty={:?} has_upvars={:?}", upvars_ty, has_upvars);
 
     struct PolymorphizationFolder<'tcx> {
@@ -714,11 +675,8 @@ fn polymorphize<'tcx>(
             debug!("fold_ty: ty={:?}", ty);
             match *ty.kind() {
                 ty::Closure(def_id, substs) => {
-                    let polymorphized_substs = polymorphize(
-                        self.tcx,
-                        ty::InstanceDef::Item(ty::WithOptConstParam::unknown(def_id)),
-                        substs,
-                    );
+                    let polymorphized_substs =
+                        polymorphize(self.tcx, ty::InstanceDef::Item(def_id), substs);
                     if substs == polymorphized_substs {
                         ty
                     } else {
@@ -726,11 +684,8 @@ fn polymorphize<'tcx>(
                     }
                 }
                 ty::Generator(def_id, substs, movability) => {
-                    let polymorphized_substs = polymorphize(
-                        self.tcx,
-                        ty::InstanceDef::Item(ty::WithOptConstParam::unknown(def_id)),
-                        substs,
-                    );
+                    let polymorphized_substs =
+                        polymorphize(self.tcx, ty::InstanceDef::Item(def_id), substs);
                     if substs == polymorphized_substs {
                         ty
                     } else {
diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs
index 195d951f9..b5a743cfe 100644
--- a/compiler/rustc_middle/src/ty/layout.rs
+++ b/compiler/rustc_middle/src/ty/layout.rs
@@ -1,11 +1,12 @@
 use crate::fluent_generated as fluent;
 use crate::middle::codegen_fn_attrs::CodegenFnAttrFlags;
+use crate::query::TyCtxtAt;
 use crate::ty::normalize_erasing_regions::NormalizationError;
 use crate::ty::{self, ReprOptions, Ty, TyCtxt, TypeVisitableExt};
 use rustc_errors::{DiagnosticBuilder, Handler, IntoDiagnostic};
 use rustc_hir as hir;
 use rustc_hir::def_id::DefId;
-use rustc_index::vec::IndexVec;
+use rustc_index::IndexVec;
 use rustc_session::config::OptLevel;
 use rustc_span::symbol::{sym, Symbol};
 use rustc_span::{Span, DUMMY_SP};
@@ -210,6 +211,7 @@ pub enum LayoutError<'tcx> {
     Unknown(Ty<'tcx>),
     SizeOverflow(Ty<'tcx>),
     NormalizationFailure(Ty<'tcx>, NormalizationError<'tcx>),
+    Cycle,
 }
 
 impl IntoDiagnostic<'_, !> for LayoutError<'_> {
@@ -230,12 +232,15 @@ impl IntoDiagnostic<'_, !> for LayoutError<'_> {
                 diag.set_arg("failure_ty", e.get_type_for_failure());
                 diag.set_primary_message(fluent::middle_cannot_be_normalized);
             }
+            LayoutError::Cycle => {
+                diag.set_primary_message(fluent::middle_cycle);
+            }
         }
         diag
     }
 }
 
-// FIXME: Once the other errors that embed this error have been converted to translateable
+// FIXME: Once the other errors that embed this error have been converted to translatable
 // diagnostics, this Display impl should be removed.
 impl<'tcx> fmt::Display for LayoutError<'tcx> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
@@ -250,6 +255,7 @@ impl<'tcx> fmt::Display for LayoutError<'tcx> {
                 t,
                 e.get_type_for_failure()
             ),
+            LayoutError::Cycle => write!(f, "a cycle occurred during layout computation"),
         }
     }
 }
@@ -263,7 +269,7 @@ pub struct LayoutCx<'tcx, C> {
 impl<'tcx> LayoutCalculator for LayoutCx<'tcx, TyCtxt<'tcx>> {
     type TargetDataLayoutRef = &'tcx TargetDataLayout;
 
-    fn delay_bug(&self, txt: &str) {
+    fn delay_bug(&self, txt: String) {
         self.tcx.sess.delay_span_bug(DUMMY_SP, txt);
     }
 
@@ -319,7 +325,7 @@ impl<'tcx> SizeSkeleton<'tcx> {
                 let non_zero = !ty.is_unsafe_ptr();
                 let tail = tcx.struct_tail_erasing_lifetimes(pointee, param_env);
                 match tail.kind() {
-                    ty::Param(_) | ty::Alias(ty::Projection, _) => {
+                    ty::Param(_) | ty::Alias(ty::Projection | ty::Inherent, _) => {
                         debug_assert!(tail.has_non_region_param());
                         Ok(SizeSkeleton::Pointer { non_zero, tail: tcx.erase_regions(tail) })
                     }
@@ -458,10 +464,10 @@ impl<'tcx> SizeSkeleton<'tcx> {
     }
 }
 
-/// When creating the layout for types with abstract conts in their size (i.e. [usize; 4 * N]),
+/// When creating the layout for types with abstract consts in their size (i.e. [usize; 4 * N]),
 /// to ensure that they have a canonical order and can be compared directly we combine all
 /// constants, and sort the other terms. This allows comparison of expressions of sizes,
-/// allowing for things like transmutating between types that depend on generic consts.
+/// allowing for things like transmuting between types that depend on generic consts.
 /// This returns `None` if multiplication of constants overflows.
 fn mul_sorted_consts<'tcx>(
     tcx: TyCtxt<'tcx>,
@@ -538,20 +544,20 @@ impl<'tcx> HasTyCtxt<'tcx> for TyCtxt<'tcx> {
     }
 }
 
-impl<'tcx> HasDataLayout for ty::query::TyCtxtAt<'tcx> {
+impl<'tcx> HasDataLayout for TyCtxtAt<'tcx> {
     #[inline]
     fn data_layout(&self) -> &TargetDataLayout {
         &self.data_layout
     }
 }
 
-impl<'tcx> HasTargetSpec for ty::query::TyCtxtAt<'tcx> {
+impl<'tcx> HasTargetSpec for TyCtxtAt<'tcx> {
     fn target_spec(&self) -> &Target {
         &self.sess.target
     }
 }
 
-impl<'tcx> HasTyCtxt<'tcx> for ty::query::TyCtxtAt<'tcx> {
+impl<'tcx> HasTyCtxt<'tcx> for TyCtxtAt<'tcx> {
     #[inline]
     fn tcx(&self) -> TyCtxt<'tcx> {
         **self
@@ -678,7 +684,7 @@ impl<'tcx> LayoutOfHelpers<'tcx> for LayoutCx<'tcx, TyCtxt<'tcx>> {
     }
 }
 
-impl<'tcx> LayoutOfHelpers<'tcx> for LayoutCx<'tcx, ty::query::TyCtxtAt<'tcx>> {
+impl<'tcx> LayoutOfHelpers<'tcx> for LayoutCx<'tcx, TyCtxtAt<'tcx>> {
     type LayoutOfResult = Result<TyAndLayout<'tcx>, LayoutError<'tcx>>;
 
     #[inline]
diff --git a/compiler/rustc_middle/src/ty/list.rs b/compiler/rustc_middle/src/ty/list.rs
index 79365ef28..71911a5a6 100644
--- a/compiler/rustc_middle/src/ty/list.rs
+++ b/compiler/rustc_middle/src/ty/list.rs
@@ -1,4 +1,5 @@
 use crate::arena::Arena;
+use rustc_data_structures::aligned::{align_of, Aligned};
 use rustc_serialize::{Encodable, Encoder};
 use std::alloc::Layout;
 use std::cmp::Ordering;
@@ -198,22 +199,23 @@ impl<'a, T: Copy> IntoIterator for &'a List<T> {
 
 unsafe impl<T: Sync> Sync for List<T> {}
 
-unsafe impl<'a, T: 'a> rustc_data_structures::tagged_ptr::Pointer for &'a List<T> {
-    const BITS: usize = std::mem::align_of::<usize>().trailing_zeros() as usize;
-
-    #[inline]
-    fn into_usize(self) -> usize {
-        self as *const List<T> as usize
-    }
-
-    #[inline]
-    unsafe fn from_usize(ptr: usize) -> &'a List<T> {
-        &*(ptr as *const List<T>)
-    }
+// We need this since `List` uses extern type `OpaqueListContents`.
+#[cfg(parallel_compiler)]
+use rustc_data_structures::sync::DynSync;
+#[cfg(parallel_compiler)]
+unsafe impl<T: DynSync> DynSync for List<T> {}
+
+// Safety:
+// Layouts of `Equivalent<T>` and `List<T>` are the same, modulo opaque tail,
+// thus aligns of `Equivalent<T>` and `List<T>` must be the same.
+unsafe impl<T> Aligned for List<T> {
+    const ALIGN: ptr::Alignment = {
+        #[repr(C)]
+        struct Equivalent<T> {
+            _len: usize,
+            _data: [T; 0],
+        }
 
-    unsafe fn with_ref<R, F: FnOnce(&Self) -> R>(ptr: usize, f: F) -> R {
-        // `Self` is `&'a List<T>` which impls `Copy`, so this is fine.
-        let ptr = Self::from_usize(ptr);
-        f(&ptr)
-    }
+        align_of::<Equivalent<T>>()
+    };
 }
diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs
index 2e516f291..a8d0dca37 100644
--- a/compiler/rustc_middle/src/ty/mod.rs
+++ b/compiler/rustc_middle/src/ty/mod.rs
@@ -21,6 +21,7 @@ use crate::error::{OpaqueHiddenTypeMismatch, TypeMismatchReason};
 use crate::metadata::ModChild;
 use crate::middle::privacy::EffectiveVisibilities;
 use crate::mir::{Body, GeneratorLayout};
+use crate::query::Providers;
 use crate::traits::{self, Reveal};
 use crate::ty;
 use crate::ty::fast_reject::SimplifiedType;
@@ -36,12 +37,12 @@ use rustc_data_structures::intern::Interned;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::steal::Steal;
 use rustc_data_structures::tagged_ptr::CopyTaggedPtr;
-use rustc_errors::ErrorGuaranteed;
+use rustc_errors::{DiagnosticBuilder, ErrorGuaranteed, StashKey};
 use rustc_hir as hir;
 use rustc_hir::def::{CtorKind, CtorOf, DefKind, DocLinkResMap, LifetimeRes, Res};
 use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LocalDefId, LocalDefIdMap};
 use rustc_hir::Node;
-use rustc_index::vec::IndexVec;
+use rustc_index::IndexVec;
 use rustc_macros::HashStable;
 use rustc_query_system::ich::StableHashingContext;
 use rustc_serialize::{Decodable, Encodable};
@@ -84,8 +85,7 @@ pub use self::consts::{
     Const, ConstData, ConstInt, ConstKind, Expr, InferConst, ScalarInt, UnevaluatedConst, ValTree,
 };
 pub use self::context::{
-    tls, CtxtInterners, DeducedParamAttrs, FreeRegionInfo, GlobalCtxt, Lift, OnDiskCache, TyCtxt,
-    TyCtxtFeed,
+    tls, CtxtInterners, DeducedParamAttrs, FreeRegionInfo, GlobalCtxt, Lift, TyCtxt, TyCtxtFeed,
 };
 pub use self::instance::{Instance, InstanceDef, ShortInstance, UnusedGenericParams};
 pub use self::list::List;
@@ -122,7 +122,6 @@ pub mod inhabitedness;
 pub mod layout;
 pub mod normalize_erasing_regions;
 pub mod print;
-pub mod query;
 pub mod relate;
 pub mod subst;
 pub mod trait_def;
@@ -166,8 +165,7 @@ pub struct ResolverGlobalCtxt {
     pub effective_visibilities: EffectiveVisibilities,
     pub extern_crate_map: FxHashMap<LocalDefId, CrateNum>,
     pub maybe_unused_trait_imports: FxIndexSet<LocalDefId>,
-    pub module_children_non_reexports: LocalDefIdMap<Vec<LocalDefId>>,
-    pub module_children_reexports: LocalDefIdMap<Vec<ModChild>>,
+    pub module_children: LocalDefIdMap<Vec<ModChild>>,
     pub glob_map: FxHashMap<LocalDefId, FxHashSet<Symbol>>,
     pub main_def: Option<MainDefinition>,
     pub trait_impls: FxIndexMap<DefId, Vec<LocalDefId>>,
@@ -861,6 +859,11 @@ impl<'tcx> PolyTraitPredicate<'tcx> {
     pub fn is_const_if_const(self) -> bool {
         self.skip_binder().is_const_if_const()
     }
+
+    #[inline]
+    pub fn polarity(self) -> ImplPolarity {
+        self.skip_binder().polarity
+    }
 }
 
 /// `A: B`
@@ -993,21 +996,15 @@ impl<'tcx> Term<'tcx> {
         }
     }
 
-    /// This function returns the inner `AliasTy` if this term is a projection.
-    ///
-    /// FIXME: rename `AliasTy` to `AliasTerm` and make sure we correctly
-    /// deal with constants.
-    pub fn to_projection_term(&self, tcx: TyCtxt<'tcx>) -> Option<AliasTy<'tcx>> {
+    /// This function returns the inner `AliasTy` for a `ty::Alias` or `ConstKind::Unevaluated`.
+    pub fn to_alias_ty(&self, tcx: TyCtxt<'tcx>) -> Option<AliasTy<'tcx>> {
         match self.unpack() {
-            TermKind::Ty(ty) => match ty.kind() {
-                ty::Alias(kind, alias_ty) => match kind {
-                    AliasKind::Projection => Some(*alias_ty),
-                    AliasKind::Opaque => None,
-                },
+            TermKind::Ty(ty) => match *ty.kind() {
+                ty::Alias(_kind, alias_ty) => Some(alias_ty),
                 _ => None,
             },
             TermKind::Const(ct) => match ct.kind() {
-                ConstKind::Unevaluated(uv) => Some(tcx.mk_alias_ty(uv.def.did, uv.substs)),
+                ConstKind::Unevaluated(uv) => Some(tcx.mk_alias_ty(uv.def, uv.substs)),
                 _ => None,
             },
         }
@@ -1067,6 +1064,24 @@ impl ParamTerm {
     }
 }
 
+#[derive(Copy, Clone, Eq, PartialEq, Debug)]
+pub enum TermVid<'tcx> {
+    Ty(ty::TyVid),
+    Const(ty::ConstVid<'tcx>),
+}
+
+impl From<ty::TyVid> for TermVid<'_> {
+    fn from(value: ty::TyVid) -> Self {
+        TermVid::Ty(value)
+    }
+}
+
+impl<'tcx> From<ty::ConstVid<'tcx>> for TermVid<'tcx> {
+    fn from(value: ty::ConstVid<'tcx>) -> Self {
+        TermVid::Const(value)
+    }
+}
+
 /// This kind of predicate has no *direct* correspondent in the
 /// syntax, but it roughly corresponds to the syntactic forms:
 ///
@@ -1207,6 +1222,18 @@ impl<'tcx> ToPredicate<'tcx, PolyTraitPredicate<'tcx>> for Binder<'tcx, TraitRef
     }
 }
 
+impl<'tcx> ToPredicate<'tcx, PolyTraitPredicate<'tcx>> for TraitRef<'tcx> {
+    fn to_predicate(self, tcx: TyCtxt<'tcx>) -> PolyTraitPredicate<'tcx> {
+        ty::Binder::dummy(self).to_predicate(tcx)
+    }
+}
+
+impl<'tcx> ToPredicate<'tcx, PolyTraitPredicate<'tcx>> for TraitPredicate<'tcx> {
+    fn to_predicate(self, _tcx: TyCtxt<'tcx>) -> PolyTraitPredicate<'tcx> {
+        ty::Binder::dummy(self)
+    }
+}
+
 impl<'tcx> ToPredicate<'tcx> for PolyTraitPredicate<'tcx> {
     fn to_predicate(self, tcx: TyCtxt<'tcx>) -> Predicate<'tcx> {
         self.map_bound(|p| PredicateKind::Clause(Clause::Trait(p))).to_predicate(tcx)
@@ -1231,6 +1258,12 @@ impl<'tcx> ToPredicate<'tcx> for PolyProjectionPredicate<'tcx> {
     }
 }
 
+impl<'tcx> ToPredicate<'tcx> for TraitPredicate<'tcx> {
+    fn to_predicate(self, tcx: TyCtxt<'tcx>) -> Predicate<'tcx> {
+        PredicateKind::Clause(Clause::Trait(self)).to_predicate(tcx)
+    }
+}
+
 impl<'tcx> Predicate<'tcx> {
     pub fn to_opt_poly_trait_pred(self) -> Option<PolyTraitPredicate<'tcx>> {
         let predicate = self.kind();
@@ -1400,14 +1433,26 @@ pub struct OpaqueHiddenType<'tcx> {
 }
 
 impl<'tcx> OpaqueHiddenType<'tcx> {
-    pub fn report_mismatch(&self, other: &Self, tcx: TyCtxt<'tcx>) -> ErrorGuaranteed {
+    pub fn report_mismatch(
+        &self,
+        other: &Self,
+        opaque_def_id: LocalDefId,
+        tcx: TyCtxt<'tcx>,
+    ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
+        if let Some(diag) = tcx
+            .sess
+            .diagnostic()
+            .steal_diagnostic(tcx.def_span(opaque_def_id), StashKey::OpaqueHiddenTypeMismatch)
+        {
+            diag.cancel();
+        }
         // Found different concrete types for the opaque type.
         let sub_diag = if self.span == other.span {
             TypeMismatchReason::ConflictType { span: self.span }
         } else {
             TypeMismatchReason::PreviousUse { span: self.span }
         };
-        tcx.sess.emit_err(OpaqueHiddenTypeMismatch {
+        tcx.sess.create_err(OpaqueHiddenTypeMismatch {
             self_ty: self.ty,
             other_ty: other.ty,
             other_span: other.span,
@@ -1471,135 +1516,6 @@ pub struct BoundConst<'tcx> {
 
 pub type PlaceholderConst<'tcx> = Placeholder<BoundVar>;
 
-/// A `DefId` which, in case it is a const argument, is potentially bundled with
-/// the `DefId` of the generic parameter it instantiates.
-///
-/// This is used to avoid calls to `type_of` for const arguments during typeck
-/// which cause cycle errors.
-///
-/// ```rust
-/// struct A;
-/// impl A {
-///     fn foo<const N: usize>(&self) -> [u8; N] { [0; N] }
-///     //           ^ const parameter
-/// }
-/// struct B;
-/// impl B {
-///     fn foo<const M: u8>(&self) -> usize { 42 }
-///     //           ^ const parameter
-/// }
-///
-/// fn main() {
-///     let a = A;
-///     let _b = a.foo::<{ 3 + 7 }>();
-///     //               ^^^^^^^^^ const argument
-/// }
-/// ```
-///
-/// Let's look at the call `a.foo::<{ 3 + 7 }>()` here. We do not know
-/// which `foo` is used until we know the type of `a`.
-///
-/// We only know the type of `a` once we are inside of `typeck(main)`.
-/// We also end up normalizing the type of `_b` during `typeck(main)` which
-/// requires us to evaluate the const argument.
-///
-/// To evaluate that const argument we need to know its type,
-/// which we would get using `type_of(const_arg)`. This requires us to
-/// resolve `foo` as it can be either `usize` or `u8` in this example.
-/// However, resolving `foo` once again requires `typeck(main)` to get the type of `a`,
-/// which results in a cycle.
-///
-/// In short we must not call `type_of(const_arg)` during `typeck(main)`.
-///
-/// When first creating the `ty::Const` of the const argument inside of `typeck` we have
-/// already resolved `foo` so we know which const parameter this argument instantiates.
-/// This means that we also know the expected result of `type_of(const_arg)` even if we
-/// aren't allowed to call that query: it is equal to `type_of(const_param)` which is
-/// trivial to compute.
-///
-/// If we now want to use that constant in a place which potentially needs its type
-/// we also pass the type of its `const_param`. This is the point of `WithOptConstParam`,
-/// except that instead of a `Ty` we bundle the `DefId` of the const parameter.
-/// Meaning that we need to use `type_of(const_param_did)` if `const_param_did` is `Some`
-/// to get the type of `did`.
-#[derive(Copy, Clone, Debug, TypeFoldable, TypeVisitable, Lift, TyEncodable, TyDecodable)]
-#[derive(PartialEq, Eq, PartialOrd, Ord)]
-#[derive(Hash, HashStable)]
-pub struct WithOptConstParam<T> {
-    pub did: T,
-    /// The `DefId` of the corresponding generic parameter in case `did` is
-    /// a const argument.
-    ///
-    /// Note that even if `did` is a const argument, this may still be `None`.
-    /// All queries taking `WithOptConstParam` start by calling `tcx.opt_const_param_of(def.did)`
-    /// to potentially update `param_did` in the case it is `None`.
-    pub const_param_did: Option<DefId>,
-}
-
-impl<T> WithOptConstParam<T> {
-    /// Creates a new `WithOptConstParam` setting `const_param_did` to `None`.
-    #[inline(always)]
-    pub fn unknown(did: T) -> WithOptConstParam<T> {
-        WithOptConstParam { did, const_param_did: None }
-    }
-}
-
-impl WithOptConstParam<LocalDefId> {
-    /// Returns `Some((did, param_did))` if `def_id` is a const argument,
-    /// `None` otherwise.
-    #[inline(always)]
-    pub fn try_lookup(did: LocalDefId, tcx: TyCtxt<'_>) -> Option<(LocalDefId, DefId)> {
-        tcx.opt_const_param_of(did).map(|param_did| (did, param_did))
-    }
-
-    /// In case `self` is unknown but `self.did` is a const argument, this returns
-    /// a `WithOptConstParam` with the correct `const_param_did`.
-    #[inline(always)]
-    pub fn try_upgrade(self, tcx: TyCtxt<'_>) -> Option<WithOptConstParam<LocalDefId>> {
-        if self.const_param_did.is_none() {
-            if let const_param_did @ Some(_) = tcx.opt_const_param_of(self.did) {
-                return Some(WithOptConstParam { did: self.did, const_param_did });
-            }
-        }
-
-        None
-    }
-
-    pub fn to_global(self) -> WithOptConstParam<DefId> {
-        WithOptConstParam { did: self.did.to_def_id(), const_param_did: self.const_param_did }
-    }
-
-    pub fn def_id_for_type_of(self) -> DefId {
-        if let Some(did) = self.const_param_did { did } else { self.did.to_def_id() }
-    }
-}
-
-impl WithOptConstParam<DefId> {
-    pub fn as_local(self) -> Option<WithOptConstParam<LocalDefId>> {
-        self.did
-            .as_local()
-            .map(|did| WithOptConstParam { did, const_param_did: self.const_param_did })
-    }
-
-    pub fn as_const_arg(self) -> Option<(LocalDefId, DefId)> {
-        if let Some(param_did) = self.const_param_did {
-            if let Some(did) = self.did.as_local() {
-                return Some((did, param_did));
-            }
-        }
-
-        None
-    }
-
-    pub fn is_local(self) -> bool {
-        self.did.is_local()
-    }
-
-    pub fn def_id_for_type_of(self) -> DefId {
-        self.const_param_did.unwrap_or(self.did)
-    }
-}
-
 /// When type checking, we use the `ParamEnv` to track
 /// details about the set of where-clauses that are in scope at this
 /// particular point.
@@ -1626,27 +1542,12 @@ struct ParamTag {
     constness: hir::Constness,
 }
 
-unsafe impl rustc_data_structures::tagged_ptr::Tag for ParamTag {
-    const BITS: usize = 2;
-    #[inline]
-    fn into_usize(self) -> usize {
-        match self {
-            Self { reveal: traits::Reveal::UserFacing, constness: hir::Constness::NotConst } => 0,
-            Self { reveal: traits::Reveal::All, constness: hir::Constness::NotConst } => 1,
-            Self { reveal: traits::Reveal::UserFacing, constness: hir::Constness::Const } => 2,
-            Self { reveal: traits::Reveal::All, constness: hir::Constness::Const } => 3,
-        }
-    }
-    #[inline]
-    unsafe fn from_usize(ptr: usize) -> Self {
-        match ptr {
-            0 => Self { reveal: traits::Reveal::UserFacing, constness: hir::Constness::NotConst },
-            1 => Self { reveal: traits::Reveal::All, constness: hir::Constness::NotConst },
-            2 => Self { reveal: traits::Reveal::UserFacing, constness: hir::Constness::Const },
-            3 => Self { reveal: traits::Reveal::All, constness: hir::Constness::Const },
-            _ => std::hint::unreachable_unchecked(),
-        }
-    }
+impl_tag! {
+    impl Tag for ParamTag;
+    ParamTag { reveal: traits::Reveal::UserFacing, constness: hir::Constness::NotConst },
+    ParamTag { reveal: traits::Reveal::All,        constness: hir::Constness::NotConst },
+    ParamTag { reveal: traits::Reveal::UserFacing, constness: hir::Constness::Const    },
+    ParamTag { reveal: traits::Reveal::All,        constness: hir::Constness::Const    },
 }
 
 impl<'tcx> fmt::Debug for ParamEnv<'tcx> {
@@ -1850,12 +1751,6 @@ impl<'tcx, T> ParamEnvAnd<'tcx, T> {
     pub fn into_parts(self) -> (ParamEnv<'tcx>, T) {
         (self.param_env, self.value)
     }
-
-    #[inline]
-    pub fn without_const(mut self) -> Self {
-        self.param_env = self.param_env.without_const();
-        self
-    }
 }
 
 #[derive(Copy, Clone, Debug, HashStable, Encodable, Decodable)]
@@ -1868,7 +1763,7 @@ pub struct Destructor {
 
 bitflags! {
     #[derive(HashStable, TyEncodable, TyDecodable)]
-    pub struct VariantFlags: u32 {
+    pub struct VariantFlags: u8 {
         const NO_VARIANT_FLAGS        = 0;
         /// Indicates whether the field list of this variant is `#[non_exhaustive]`.
         const IS_FIELD_LIST_NON_EXHAUSTIVE = 1 << 0;
@@ -1969,6 +1864,16 @@ impl VariantDef {
     pub fn ctor_def_id(&self) -> Option<DefId> {
         self.ctor.map(|(_, def_id)| def_id)
     }
+
+    /// Returns the one field in this variant.
+    ///
+    /// `panic!`s if there are no fields or multiple fields.
+    #[inline]
+    pub fn single_field(&self) -> &FieldDef {
+        assert!(self.fields.len() == 1);
+
+        &self.fields[FieldIdx::from_u32(0)]
+    }
 }
 
 impl PartialEq for VariantDef {
@@ -1983,7 +1888,20 @@ impl PartialEq for VariantDef {
 
         let Self { def_id: lhs_def_id, ctor: _, name: _, discr: _, fields: _, flags: _ } = &self;
         let Self { def_id: rhs_def_id, ctor: _, name: _, discr: _, fields: _, flags: _ } = other;
-        lhs_def_id == rhs_def_id
+
+        let res = lhs_def_id == rhs_def_id;
+
+        // Double check that implicit assumption detailed above.
+        if cfg!(debug_assertions) && res {
+            let deep = self.ctor == other.ctor
+                && self.name == other.name
+                && self.discr == other.discr
+                && self.fields == other.fields
+                && self.flags == other.flags;
+            assert!(deep, "VariantDef for the same def-id has differing data");
+        }
+
+        res
     }
 }
 
@@ -2038,7 +1956,15 @@ impl PartialEq for FieldDef {
 
         let Self { did: rhs_did, name: _, vis: _ } = other;
 
-        lhs_did == rhs_did
+        let res = lhs_did == rhs_did;
+
+        // Double check that implicit assumption detailed above.
+        if cfg!(debug_assertions) && res {
+            let deep = self.name == other.name && self.vis == other.vis;
+            assert!(deep, "FieldDef for the same def-id has differing data");
+        }
+
+        res
     }
 }
 
@@ -2245,10 +2171,9 @@ impl<'tcx> TyCtxt<'tcx> {
     /// See [`item_name`][Self::item_name] for more information.
     pub fn opt_item_ident(self, def_id: DefId) -> Option<Ident> {
         let def = self.opt_item_name(def_id)?;
-        let span = def_id
-            .as_local()
-            .and_then(|id| self.def_ident_span(id))
-            .unwrap_or(rustc_span::DUMMY_SP);
+        let span = self
+            .def_ident_span(def_id)
+            .unwrap_or_else(|| bug!("missing ident span for {def_id:?}"));
         Some(Ident::new(def, span))
     }
 
@@ -2289,8 +2214,8 @@ impl<'tcx> TyCtxt<'tcx> {
         let impl_trait_ref2 = self.impl_trait_ref(def_id2);
         // If either trait impl references an error, they're allowed to overlap,
         // as one of them essentially doesn't exist.
-        if impl_trait_ref1.map_or(false, |tr| tr.subst_identity().references_error())
-            || impl_trait_ref2.map_or(false, |tr| tr.subst_identity().references_error())
+        if impl_trait_ref1.is_some_and(|tr| tr.subst_identity().references_error())
+            || impl_trait_ref2.is_some_and(|tr| tr.subst_identity().references_error())
         {
             return Some(ImplOverlapKind::Permitted { marker: false });
         }
@@ -2311,7 +2236,7 @@ impl<'tcx> TyCtxt<'tcx> {
 
         let is_marker_overlap = {
             let is_marker_impl = |trait_ref: Option<EarlyBinder<TraitRef<'_>>>| -> bool {
-                trait_ref.map_or(false, |tr| self.trait_def(tr.skip_binder().def_id).is_marker)
+                trait_ref.is_some_and(|tr| self.trait_def(tr.skip_binder().def_id).is_marker)
             };
             is_marker_impl(impl_trait_ref1) && is_marker_impl(impl_trait_ref2)
         };
@@ -2361,7 +2286,7 @@ impl<'tcx> TyCtxt<'tcx> {
         match instance {
             ty::InstanceDef::Item(def) => {
                 debug!("calling def_kind on def: {:?}", def);
-                let def_kind = self.def_kind(def.did);
+                let def_kind = self.def_kind(def);
                 debug!("returned from def_kind: {:?}", def_kind);
                 match def_kind {
                     DefKind::Const
@@ -2369,13 +2294,10 @@ impl<'tcx> TyCtxt<'tcx> {
                     | DefKind::AssocConst
                     | DefKind::Ctor(..)
                     | DefKind::AnonConst
-                    | DefKind::InlineConst => self.mir_for_ctfe_opt_const_arg(def),
+                    | DefKind::InlineConst => self.mir_for_ctfe(def),
                     // If the caller wants `mir_for_ctfe` of a function they should not be using
                     // `instance_mir`, so we'll assume const fn also wants the optimized version.
-                    _ => {
-                        assert_eq!(def.const_param_did, None);
-                        self.optimized_mir(def.did)
-                    }
+                    _ => self.optimized_mir(def),
                 }
             }
             ty::InstanceDef::VTableShim(..)
@@ -2566,9 +2488,7 @@ impl<'tcx> TyCtxt<'tcx> {
             && if self.features().collapse_debuginfo {
                 span.in_macro_expansion_with_collapse_debuginfo()
             } else {
-                // Inlined spans should not be collapsed as that leads to all of the
-                // inlined code being attributed to the inline callsite.
-                span.from_expansion() && !span.is_inlined()
+                span.from_expansion()
             }
     }
 
@@ -2599,6 +2519,18 @@ impl<'tcx> TyCtxt<'tcx> {
         }
     }
 
+    /// Returns the `DefId` of the item within which the `impl Trait` is declared.
+    /// For type-alias-impl-trait this is the `type` alias.
+    /// For impl-trait-in-assoc-type this is the assoc type.
+    /// For return-position-impl-trait this is the function.
+    pub fn impl_trait_parent(self, mut def_id: LocalDefId) -> LocalDefId {
+        // Find the surrounding item (type alias or assoc type)
+        while let DefKind::OpaqueTy = self.def_kind(def_id) {
+            def_id = self.local_parent(def_id);
+        }
+        def_id
+    }
+
     pub fn impl_method_has_trait_impl_trait_tys(self, def_id: DefId) -> bool {
         if self.def_kind(def_id) != DefKind::AssocFn {
             return false;
@@ -2643,7 +2575,7 @@ pub fn is_impl_trait_defn(tcx: TyCtxt<'_>, def_id: DefId) -> Option<LocalDefId>
                 hir::OpaqueTyOrigin::FnReturn(parent) | hir::OpaqueTyOrigin::AsyncFn(parent) => {
                     Some(parent)
                 }
-                hir::OpaqueTyOrigin::TyAlias => None,
+                hir::OpaqueTyOrigin::TyAlias { .. } => None,
             };
         }
     }
@@ -2701,7 +2633,7 @@ pub fn ast_uint_ty(uty: UintTy) -> ast::UintTy {
     }
 }
 
-pub fn provide(providers: &mut ty::query::Providers) {
+pub fn provide(providers: &mut Providers) {
     closure::provide(providers);
     context::provide(providers);
     erase_regions::provide(providers);
@@ -2710,7 +2642,7 @@ pub fn provide(providers: &mut ty::query::Providers) {
     print::provide(providers);
     super::util::bug::provide(providers);
     super::middle::provide(providers);
-    *providers = ty::query::Providers {
+    *providers = Providers {
         trait_impls_of: trait_def::trait_impls_of_provider,
         incoherent_impls: trait_def::incoherent_impls_provider,
         const_param_default: consts::const_param_default,
diff --git a/compiler/rustc_middle/src/ty/normalize_erasing_regions.rs b/compiler/rustc_middle/src/ty/normalize_erasing_regions.rs
index 7c59879a1..a0c8d299f 100644
--- a/compiler/rustc_middle/src/ty/normalize_erasing_regions.rs
+++ b/compiler/rustc_middle/src/ty/normalize_erasing_regions.rs
@@ -32,7 +32,7 @@ impl<'tcx> TyCtxt<'tcx> {
     ///
     /// This should only be used outside of type inference. For example,
     /// it assumes that normalization will succeed.
-    #[tracing::instrument(level = "debug", skip(self, param_env))]
+    #[tracing::instrument(level = "debug", skip(self, param_env), ret)]
     pub fn normalize_erasing_regions<T>(self, param_env: ty::ParamEnv<'tcx>, value: T) -> T
     where
         T: TypeFoldable<TyCtxt<'tcx>>,
@@ -139,7 +139,7 @@ impl<'tcx> TyCtxt<'tcx> {
         self,
         param_substs: SubstsRef<'tcx>,
         param_env: ty::ParamEnv<'tcx>,
-        value: T,
+        value: EarlyBinder<T>,
     ) -> T
     where
         T: TypeFoldable<TyCtxt<'tcx>>,
@@ -151,7 +151,7 @@ impl<'tcx> TyCtxt<'tcx> {
              param_env={:?})",
             param_substs, value, param_env,
         );
-        let substituted = EarlyBinder(value).subst(self, param_substs);
+        let substituted = value.subst(self, param_substs);
         self.normalize_erasing_regions(param_env, substituted)
     }
 
@@ -163,7 +163,7 @@ impl<'tcx> TyCtxt<'tcx> {
         self,
         param_substs: SubstsRef<'tcx>,
         param_env: ty::ParamEnv<'tcx>,
-        value: T,
+        value: EarlyBinder<T>,
     ) -> Result<T, NormalizationError<'tcx>>
     where
         T: TypeFoldable<TyCtxt<'tcx>>,
@@ -175,7 +175,7 @@ impl<'tcx> TyCtxt<'tcx> {
              param_env={:?})",
             param_substs, value, param_env,
         );
-        let substituted = EarlyBinder(value).subst(self, param_substs);
+        let substituted = value.subst(self, param_substs);
         self.try_normalize_erasing_regions(param_env, substituted)
     }
 }
diff --git a/compiler/rustc_middle/src/ty/opaque_types.rs b/compiler/rustc_middle/src/ty/opaque_types.rs
index 751f3066c..1b336b7bf 100644
--- a/compiler/rustc_middle/src/ty/opaque_types.rs
+++ b/compiler/rustc_middle/src/ty/opaque_types.rs
@@ -177,7 +177,7 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for ReverseMapper<'tcx> {
                                 .sess
                                 .struct_span_err(
                                     self.span,
-                                    &format!(
+                                    format!(
                                         "type parameter `{}` is part of concrete type but not \
                                           used in parameter list for the `impl Trait` type alias",
                                         ty
@@ -207,14 +207,16 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for ReverseMapper<'tcx> {
                     Some(GenericArgKind::Const(c1)) => c1,
                     Some(u) => panic!("const mapped to unexpected kind: {:?}", u),
                     None => {
-                        if !self.ignore_errors {
-                            self.tcx.sess.emit_err(ConstNotUsedTraitAlias {
+                        let guar = self
+                            .tcx
+                            .sess
+                            .create_err(ConstNotUsedTraitAlias {
                                 ct: ct.to_string(),
                                 span: self.span,
-                            });
-                        }
+                            })
+                            .emit_unless(self.ignore_errors);
 
-                        self.interner().const_error(ct.ty())
+                        self.interner().const_error(ct.ty(), guar)
                     }
                 }
             }
diff --git a/compiler/rustc_middle/src/ty/parameterized.rs b/compiler/rustc_middle/src/ty/parameterized.rs
index 7534d06ae..a2e77d9cd 100644
--- a/compiler/rustc_middle/src/ty/parameterized.rs
+++ b/compiler/rustc_middle/src/ty/parameterized.rs
@@ -1,6 +1,6 @@
 use rustc_data_structures::fx::FxHashMap;
 use rustc_hir::def_id::DefIndex;
-use rustc_index::vec::{Idx, IndexVec};
+use rustc_index::{Idx, IndexVec};
 
 use crate::ty;
 
@@ -56,6 +56,7 @@ trivially_parameterized_over_tcx! {
     std::string::String,
     crate::metadata::ModChild,
     crate::middle::codegen_fn_attrs::CodegenFnAttrs,
+    crate::middle::debugger_visualizer::DebuggerVisualizerFile,
     crate::middle::exported_symbols::SymbolExportInfo,
     crate::middle::resolve_bound_vars::ObjectLifetimeDefault,
     crate::mir::ConstQualifs,
@@ -91,7 +92,6 @@ trivially_parameterized_over_tcx! {
     rustc_session::cstore::ForeignModule,
     rustc_session::cstore::LinkagePreference,
     rustc_session::cstore::NativeLib,
-    rustc_span::DebuggerVisualizerFile,
     rustc_span::ExpnData,
     rustc_span::ExpnHash,
     rustc_span::ExpnId,
diff --git a/compiler/rustc_middle/src/ty/print/mod.rs b/compiler/rustc_middle/src/ty/print/mod.rs
index d947d9604..64e7480e6 100644
--- a/compiler/rustc_middle/src/ty/print/mod.rs
+++ b/compiler/rustc_middle/src/ty/print/mod.rs
@@ -169,8 +169,11 @@ pub trait Printer<'tcx>: Sized {
                 self.path_append(
                     |cx: Self| {
                         if trait_qualify_parent {
-                            let trait_ref =
-                                cx.tcx().mk_trait_ref(parent_def_id, parent_substs.iter().copied());
+                            let trait_ref = ty::TraitRef::new(
+                                cx.tcx(),
+                                parent_def_id,
+                                parent_substs.iter().copied(),
+                            );
                             cx.path_qualified(trait_ref.self_ty(), Some(trait_ref))
                         } else {
                             cx.print_def_path(parent_def_id, parent_substs)
@@ -327,6 +330,6 @@ pub fn describe_as_module(def_id: LocalDefId, tcx: TyCtxt<'_>) -> String {
     if def_id.is_top_level_module() {
         "top-level module".to_string()
     } else {
-        format!("module `{}`", tcx.def_path_str(def_id.to_def_id()))
+        format!("module `{}`", tcx.def_path_str(def_id))
     }
 }
diff --git a/compiler/rustc_middle/src/ty/print/pretty.rs b/compiler/rustc_middle/src/ty/print/pretty.rs
index 72caadaf6..d6c88ea96 100644
--- a/compiler/rustc_middle/src/ty/print/pretty.rs
+++ b/compiler/rustc_middle/src/ty/print/pretty.rs
@@ -1,4 +1,6 @@
 use crate::mir::interpret::{AllocRange, GlobalAlloc, Pointer, Provenance, Scalar};
+use crate::query::IntoQueryParam;
+use crate::query::Providers;
 use crate::ty::{
     self, ConstInt, ParamConst, ScalarInt, Term, TermKind, Ty, TyCtxt, TypeFoldable,
     TypeSuperFoldable, TypeSuperVisitable, TypeVisitable, TypeVisitableExt,
@@ -698,10 +700,10 @@ pub trait PrettyPrinter<'tcx>:
                     if verbose { p!(write("{:?}", infer_ty)) } else { p!(write("{}", infer_ty)) }
                 }
             }
-            ty::Error(_) => p!("[type error]"),
+            ty::Error(_) => p!("{{type error}}"),
             ty::Param(ref param_ty) => p!(print(param_ty)),
             ty::Bound(debruijn, bound_ty) => match bound_ty.kind {
-                ty::BoundTyKind::Anon => self.pretty_print_bound_var(debruijn, bound_ty.var)?,
+                ty::BoundTyKind::Anon => debug_bound_var(&mut self, debruijn, bound_ty.var)?,
                 ty::BoundTyKind::Param(_, s) => match self.should_print_verbose() {
                     true if debruijn == ty::INNERMOST => p!(write("^{}", s)),
                     true => p!(write("^{}_{}", debruijn.index(), s)),
@@ -728,7 +730,7 @@ pub trait PrettyPrinter<'tcx>:
             ty::Foreign(def_id) => {
                 p!(print_def_path(def_id, &[]));
             }
-            ty::Alias(ty::Projection, ref data) => {
+            ty::Alias(ty::Projection | ty::Inherent, ref data) => {
                 if !(self.should_print_verbose() || NO_QUERIES.with(|q| q.get()))
                     && self.tcx().is_impl_trait_in_trait(data.def_id)
                 {
@@ -738,7 +740,9 @@ pub trait PrettyPrinter<'tcx>:
                 }
             }
             ty::Placeholder(placeholder) => match placeholder.bound.kind {
-                ty::BoundTyKind::Anon => p!(write("Placeholder({:?})", placeholder)),
+                ty::BoundTyKind::Anon => {
+                    debug_placeholder_var(&mut self, placeholder.universe, placeholder.bound.var)?;
+                }
                 ty::BoundTyKind::Param(_, name) => p!(write("{}", name)),
             },
             ty::Alias(ty::Opaque, ty::AliasTy { def_id, substs, .. }) => {
@@ -911,7 +915,7 @@ pub trait PrettyPrinter<'tcx>:
 
         // Grab the "TraitA + TraitB" from `impl TraitA + TraitB`,
         // by looking up the projections associated with the def_id.
-        let bounds = tcx.bound_explicit_item_bounds(def_id);
+        let bounds = tcx.explicit_item_bounds(def_id);
 
         let mut traits = FxIndexMap::default();
         let mut fn_traits = FxIndexMap::default();
@@ -1160,16 +1164,20 @@ pub trait PrettyPrinter<'tcx>:
         traits.entry(trait_ref).or_default().extend(proj_ty);
     }
 
-    fn pretty_print_bound_var(
-        &mut self,
-        debruijn: ty::DebruijnIndex,
-        var: ty::BoundVar,
-    ) -> Result<(), Self::Error> {
-        if debruijn == ty::INNERMOST {
-            write!(self, "^{}", var.index())
-        } else {
-            write!(self, "^{}_{}", debruijn.index(), var.index())
-        }
+    fn pretty_print_inherent_projection(
+        self,
+        alias_ty: &ty::AliasTy<'tcx>,
+    ) -> Result<Self::Path, Self::Error> {
+        let def_key = self.tcx().def_key(alias_ty.def_id);
+        self.path_generic_args(
+            |cx| {
+                cx.path_append(
+                    |cx| cx.path_qualified(alias_ty.self_ty(), None),
+                    &def_key.disambiguated_data,
+                )
+            },
+            &alias_ty.substs[1..],
+        )
     }
 
     fn ty_infer_name(&self, _: ty::TyVid) -> Option<Symbol> {
@@ -1305,7 +1313,7 @@ pub trait PrettyPrinter<'tcx>:
         define_scoped_cx!(self);
 
         if self.should_print_verbose() {
-            p!(write("Const({:?}: {:?})", ct.kind(), ct.ty()));
+            p!(write("{:?}", ct));
             return Ok(self);
         }
 
@@ -1328,13 +1336,13 @@ pub trait PrettyPrinter<'tcx>:
 
         match ct.kind() {
             ty::ConstKind::Unevaluated(ty::UnevaluatedConst { def, substs }) => {
-                match self.tcx().def_kind(def.did) {
+                match self.tcx().def_kind(def) {
                     DefKind::Const | DefKind::AssocConst => {
-                        p!(print_value_path(def.did, substs))
+                        p!(print_value_path(def, substs))
                     }
                     DefKind::AnonConst => {
                         if def.is_local()
-                            && let span = self.tcx().def_span(def.did)
+                            && let span = self.tcx().def_span(def)
                             && let Ok(snip) = self.tcx().sess.source_map().span_to_snippet(span)
                         {
                             p!(write("{}", snip))
@@ -1344,7 +1352,7 @@ pub trait PrettyPrinter<'tcx>:
                             // cause printing to enter an infinite recursion if the anon const is in the self type i.e.
                             // `impl<T: Default> Default for [T; 32 - 1 - 1 - 1] {`
                             // where we would try to print `<[T; /* print `constant#0` again */] as Default>::{constant#0}`
-                            p!(write("{}::{}", self.tcx().crate_name(def.did.krate), self.tcx().def_path(def.did).to_string_no_crate_verbose()))
+                            p!(write("{}::{}", self.tcx().crate_name(def.krate), self.tcx().def_path(def).to_string_no_crate_verbose()))
                         }
                     }
                     defkind => bug!("`{:?}` has unexpected defkind {:?}", ct, defkind),
@@ -1364,13 +1372,15 @@ pub trait PrettyPrinter<'tcx>:
             }
 
             ty::ConstKind::Bound(debruijn, bound_var) => {
-                self.pretty_print_bound_var(debruijn, bound_var)?
+                debug_bound_var(&mut self, debruijn, bound_var)?
             }
-            ty::ConstKind::Placeholder(placeholder) => p!(write("Placeholder({:?})", placeholder)),
+            ty::ConstKind::Placeholder(placeholder) => {
+                debug_placeholder_var(&mut self, placeholder.universe, placeholder.bound)?;
+            },
             // FIXME(generic_const_exprs):
             // write out some legible representation of an abstract const?
-            ty::ConstKind::Expr(_) => p!("[const expr]"),
-            ty::ConstKind::Error(_) => p!("[const error]"),
+            ty::ConstKind::Expr(_) => p!("{{const expr}}"),
+            ty::ConstKind::Error(_) => p!("{{const error}}"),
         };
         Ok(self)
     }
@@ -1787,17 +1797,27 @@ fn guess_def_namespace(tcx: TyCtxt<'_>, def_id: DefId) -> Namespace {
 impl<'t> TyCtxt<'t> {
     /// Returns a string identifying this `DefId`. This string is
     /// suitable for user output.
-    pub fn def_path_str(self, def_id: DefId) -> String {
+    pub fn def_path_str(self, def_id: impl IntoQueryParam<DefId>) -> String {
         self.def_path_str_with_substs(def_id, &[])
     }
 
-    pub fn def_path_str_with_substs(self, def_id: DefId, substs: &'t [GenericArg<'t>]) -> String {
+    pub fn def_path_str_with_substs(
+        self,
+        def_id: impl IntoQueryParam<DefId>,
+        substs: &'t [GenericArg<'t>],
+    ) -> String {
+        let def_id = def_id.into_query_param();
         let ns = guess_def_namespace(self, def_id);
         debug!("def_path_str: def_id={:?}, ns={:?}", def_id, ns);
         FmtPrinter::new(self, ns).print_def_path(def_id, substs).unwrap().into_buffer()
     }
 
-    pub fn value_path_str_with_substs(self, def_id: DefId, substs: &'t [GenericArg<'t>]) -> String {
+    pub fn value_path_str_with_substs(
+        self,
+        def_id: impl IntoQueryParam<DefId>,
+        substs: &'t [GenericArg<'t>],
+    ) -> String {
+        let def_id = def_id.into_query_param();
         let ns = guess_def_namespace(self, def_id);
         debug!("value_path_str: def_id={:?}, ns={:?}", def_id, ns);
         FmtPrinter::new(self, ns).print_value_path(def_id, substs).unwrap().into_buffer()
@@ -2518,7 +2538,7 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
                     self.used_region_names.insert(name);
                 }
 
-                r.super_visit_with(self)
+                ControlFlow::Continue(())
             }
 
             // We collect types in order to prevent really large types from compiling for
@@ -2608,6 +2628,12 @@ macro_rules! define_print_and_forward_display {
 #[derive(Copy, Clone, TypeFoldable, TypeVisitable, Lift)]
 pub struct TraitRefPrintOnlyTraitPath<'tcx>(ty::TraitRef<'tcx>);
 
+impl<'tcx> rustc_errors::IntoDiagnosticArg for TraitRefPrintOnlyTraitPath<'tcx> {
+    fn into_diagnostic_arg(self) -> rustc_errors::DiagnosticArgValue<'static> {
+        self.to_string().into_diagnostic_arg()
+    }
+}
+
 impl<'tcx> fmt::Debug for TraitRefPrintOnlyTraitPath<'tcx> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         fmt::Display::fmt(self, f)
@@ -2665,7 +2691,7 @@ impl<'tcx> ty::PolyTraitPredicate<'tcx> {
     }
 }
 
-#[derive(Debug, Copy, Clone, TypeFoldable, TypeVisitable, Lift)]
+#[derive(Debug, Copy, Clone, Lift)]
 pub struct PrintClosureAsImpl<'tcx> {
     pub closure: ty::ClosureSubsts<'tcx>,
 }
@@ -2791,6 +2817,9 @@ define_print_and_forward_display! {
         if let ty::BoundConstness::ConstIfConst = self.constness && cx.tcx().features().const_trait_impl {
             p!("~const ");
         }
+        if let ty::ImplPolarity::Negative = self.polarity {
+            p!("!");
+        }
         p!(print(self.trait_ref.print_only_trait_path()))
     }
 
@@ -2808,7 +2837,11 @@ define_print_and_forward_display! {
     }
 
     ty::AliasTy<'tcx> {
-        p!(print_def_path(self.def_id, self.substs));
+        if let DefKind::Impl { of_trait: false } = cx.tcx().def_kind(cx.tcx().parent(self.def_id)) {
+            p!(pretty_print_inherent_projection(self))
+        } else {
+            p!(print_def_path(self.def_id, self.substs));
+        }
     }
 
     ty::ClosureKind {
@@ -3020,8 +3053,8 @@ fn trimmed_def_paths(tcx: TyCtxt<'_>, (): ()) -> FxHashMap<DefId, Symbol> {
     map
 }
 
-pub fn provide(providers: &mut ty::query::Providers) {
-    *providers = ty::query::Providers { trimmed_def_paths, ..*providers };
+pub fn provide(providers: &mut Providers) {
+    *providers = Providers { trimmed_def_paths, ..*providers };
 }
 
 #[derive(Default)]
@@ -3032,3 +3065,27 @@ pub struct OpaqueFnEntry<'tcx> {
     fn_trait_ref: Option<ty::PolyTraitRef<'tcx>>,
     return_ty: Option<ty::Binder<'tcx, Term<'tcx>>>,
 }
+
+pub fn debug_bound_var<T: std::fmt::Write>(
+    fmt: &mut T,
+    debruijn: ty::DebruijnIndex,
+    var: ty::BoundVar,
+) -> Result<(), std::fmt::Error> {
+    if debruijn == ty::INNERMOST {
+        write!(fmt, "^{}", var.index())
+    } else {
+        write!(fmt, "^{}_{}", debruijn.index(), var.index())
+    }
+}
+
+pub fn debug_placeholder_var<T: std::fmt::Write>(
+    fmt: &mut T,
+    universe: ty::UniverseIndex,
+    bound: ty::BoundVar,
+) -> Result<(), std::fmt::Error> {
+    if universe == ty::UniverseIndex::ROOT {
+        write!(fmt, "!{}", bound.index())
+    } else {
+        write!(fmt, "!{}_{}", universe.index(), bound.index())
+    }
+}
diff --git a/compiler/rustc_middle/src/ty/relate.rs b/compiler/rustc_middle/src/ty/relate.rs
index 46c931d61..3bbe6a23b 100644
--- a/compiler/rustc_middle/src/ty/relate.rs
+++ b/compiler/rustc_middle/src/ty/relate.rs
@@ -7,7 +7,7 @@
 use crate::ty::error::{ExpectedFound, TypeError};
 use crate::ty::{self, Expr, ImplSubject, Term, TermKind, Ty, TyCtxt, TypeFoldable};
 use crate::ty::{GenericArg, GenericArgKind, SubstsRef};
-use rustc_hir as ast;
+use rustc_hir as hir;
 use rustc_hir::def_id::DefId;
 use rustc_target::spec::abi;
 use std::iter;
@@ -123,8 +123,8 @@ pub fn relate_type_and_mut<'tcx, R: TypeRelation<'tcx>>(
     } else {
         let mutbl = a.mutbl;
         let (variance, info) = match mutbl {
-            ast::Mutability::Not => (ty::Covariant, ty::VarianceDiagInfo::None),
-            ast::Mutability::Mut => {
+            hir::Mutability::Not => (ty::Covariant, ty::VarianceDiagInfo::None),
+            hir::Mutability::Mut => {
                 (ty::Invariant, ty::VarianceDiagInfo::Invariant { ty: base_ty, param_index: 0 })
             }
         };
@@ -239,12 +239,12 @@ impl<'tcx> Relate<'tcx> for ty::BoundConstness {
     }
 }
 
-impl<'tcx> Relate<'tcx> for ast::Unsafety {
+impl<'tcx> Relate<'tcx> for hir::Unsafety {
     fn relate<R: TypeRelation<'tcx>>(
         relation: &mut R,
-        a: ast::Unsafety,
-        b: ast::Unsafety,
-    ) -> RelateResult<'tcx, ast::Unsafety> {
+        a: hir::Unsafety,
+        b: hir::Unsafety,
+    ) -> RelateResult<'tcx, hir::Unsafety> {
         if a != b {
             Err(TypeError::UnsafetyMismatch(expected_found(relation, a, b)))
         } else {
@@ -315,7 +315,7 @@ impl<'tcx> Relate<'tcx> for ty::TraitRef<'tcx> {
             Err(TypeError::Traits(expected_found(relation, a.def_id, b.def_id)))
         } else {
             let substs = relate_substs(relation, a.substs, b.substs)?;
-            Ok(relation.tcx().mk_trait_ref(a.def_id, substs))
+            Ok(ty::TraitRef::new(relation.tcx(), a.def_id, substs))
         }
     }
 }
@@ -388,24 +388,24 @@ impl<'tcx> Relate<'tcx> for Ty<'tcx> {
     }
 }
 
-/// The main "type relation" routine. Note that this does not handle
-/// inference artifacts, so you should filter those out before calling
-/// it.
-pub fn super_relate_tys<'tcx, R: TypeRelation<'tcx>>(
+/// Relates `a` and `b` structurally, calling the relation for all nested values.
+/// Any semantic equality, e.g. of projections, and inference variables have to be
+/// handled by the caller.
+pub fn structurally_relate_tys<'tcx, R: TypeRelation<'tcx>>(
     relation: &mut R,
     a: Ty<'tcx>,
     b: Ty<'tcx>,
 ) -> RelateResult<'tcx, Ty<'tcx>> {
     let tcx = relation.tcx();
-    debug!("super_relate_tys: a={:?} b={:?}", a, b);
+    debug!("structurally_relate_tys: a={:?} b={:?}", a, b);
     match (a.kind(), b.kind()) {
         (&ty::Infer(_), _) | (_, &ty::Infer(_)) => {
             // The caller should handle these cases!
-            bug!("var types encountered in super_relate_tys")
+            bug!("var types encountered in structurally_relate_tys")
         }
 
         (ty::Bound(..), _) | (_, ty::Bound(..)) => {
-            bug!("bound types encountered in super_relate_tys")
+            bug!("bound types encountered in structurally_relate_tys")
         }
 
         (&ty::Error(guar), _) | (_, &ty::Error(guar)) => Ok(tcx.ty_error(guar)),
@@ -550,6 +550,11 @@ pub fn super_relate_tys<'tcx, R: TypeRelation<'tcx>>(
             Ok(tcx.mk_projection(projection_ty.def_id, projection_ty.substs))
         }
 
+        (&ty::Alias(ty::Inherent, a_data), &ty::Alias(ty::Inherent, b_data)) => {
+            let alias_ty = relation.relate(a_data, b_data)?;
+            Ok(tcx.mk_alias(ty::Inherent, tcx.mk_alias_ty(alias_ty.def_id, alias_ty.substs)))
+        }
+
         (
             &ty::Alias(ty::Opaque, ty::AliasTy { def_id: a_def_id, substs: a_substs, .. }),
             &ty::Alias(ty::Opaque, ty::AliasTy { def_id: b_def_id, substs: b_substs, .. }),
@@ -570,15 +575,18 @@ pub fn super_relate_tys<'tcx, R: TypeRelation<'tcx>>(
     }
 }
 
-/// The main "const relation" routine. Note that this does not handle
-/// inference artifacts, so you should filter those out before calling
-/// it.
-pub fn super_relate_consts<'tcx, R: TypeRelation<'tcx>>(
+/// Relates `a` and `b` structurally, calling the relation for all nested values.
+/// Any semantic equality, e.g. of unevaluated consts, and inference variables have
+/// to be handled by the caller.
+///
+/// FIXME: This is not totally structual, which probably should be fixed.
+/// See the HACKs below.
+pub fn structurally_relate_consts<'tcx, R: TypeRelation<'tcx>>(
     relation: &mut R,
     mut a: ty::Const<'tcx>,
     mut b: ty::Const<'tcx>,
 ) -> RelateResult<'tcx, ty::Const<'tcx>> {
-    debug!("{}.super_relate_consts(a = {:?}, b = {:?})", relation.tag(), a, b);
+    debug!("{}.structurally_relate_consts(a = {:?}, b = {:?})", relation.tag(), a, b);
     let tcx = relation.tcx();
 
     // HACK(const_generics): We still need to eagerly evaluate consts when
@@ -597,7 +605,7 @@ pub fn super_relate_consts<'tcx, R: TypeRelation<'tcx>>(
         b = tcx.expand_abstract_consts(b);
     }
 
-    debug!("{}.super_relate_consts(normed_a = {:?}, normed_b = {:?})", relation.tag(), a, b);
+    debug!("{}.structurally_relate_consts(normed_a = {:?}, normed_b = {:?})", relation.tag(), a, b);
 
     // Currently, the values that can be unified are primitive types,
     // and those that derive both `PartialEq` and `Eq`, corresponding
@@ -605,7 +613,7 @@ pub fn super_relate_consts<'tcx, R: TypeRelation<'tcx>>(
     let is_match = match (a.kind(), b.kind()) {
         (ty::ConstKind::Infer(_), _) | (_, ty::ConstKind::Infer(_)) => {
             // The caller should handle these cases!
-            bug!("var types encountered in super_relate_consts: {:?} {:?}", a, b)
+            bug!("var types encountered in structurally_relate_consts: {:?} {:?}", a, b)
         }
 
         (ty::ConstKind::Error(_), _) => return Ok(a),
diff --git a/compiler/rustc_middle/src/ty/structural_impls.rs b/compiler/rustc_middle/src/ty/structural_impls.rs
index 5c604bb6d..16cb6c910 100644
--- a/compiler/rustc_middle/src/ty/structural_impls.rs
+++ b/compiler/rustc_middle/src/ty/structural_impls.rs
@@ -4,13 +4,12 @@
 //! to help with the tedium.
 
 use crate::mir::interpret;
-use crate::mir::ProjectionKind;
 use crate::ty::fold::{FallibleTypeFolder, TypeFoldable, TypeSuperFoldable};
 use crate::ty::print::{with_no_trimmed_paths, FmtPrinter, Printer};
 use crate::ty::visit::{TypeSuperVisitable, TypeVisitable, TypeVisitor};
 use crate::ty::{self, AliasTy, InferConst, Lift, Term, TermKind, Ty, TyCtxt};
 use rustc_hir::def::Namespace;
-use rustc_index::vec::{Idx, IndexVec};
+use rustc_index::{Idx, IndexVec};
 use rustc_target::abi::TyAndLayout;
 
 use std::fmt;
@@ -95,7 +94,7 @@ impl<'tcx> fmt::Debug for ty::FnSig<'tcx> {
 
 impl<'tcx> fmt::Debug for ty::ConstVid<'tcx> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "_#{}c", self.index)
+        write!(f, "?{}c", self.index)
     }
 }
 
@@ -193,11 +192,49 @@ impl<'tcx> fmt::Debug for AliasTy<'tcx> {
     }
 }
 
+impl<'tcx> fmt::Debug for ty::InferConst<'tcx> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            InferConst::Var(var) => write!(f, "{var:?}"),
+            InferConst::Fresh(var) => write!(f, "Fresh({var:?})"),
+        }
+    }
+}
+
+impl<'tcx> fmt::Debug for ty::Const<'tcx> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        // This reflects what `Const` looked liked before `Interned` was
+        // introduced. We print it like this to avoid having to update expected
+        // output in a lot of tests.
+        write!(f, "Const {{ ty: {:?}, kind: {:?} }}", self.ty(), self.kind())
+    }
+}
+
+impl<'tcx> fmt::Debug for ty::ConstKind<'tcx> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        use ty::ConstKind::*;
+        match self {
+            Param(param) => write!(f, "{param:?}"),
+            Infer(var) => write!(f, "{var:?}"),
+            Bound(debruijn, var) => ty::print::debug_bound_var(f, *debruijn, *var),
+            Placeholder(placeholder) => {
+                ty::print::debug_placeholder_var(f, placeholder.universe, placeholder.bound)
+            }
+            Unevaluated(uv) => {
+                f.debug_tuple("Unevaluated").field(&uv.substs).field(&uv.def).finish()
+            }
+            Value(valtree) => write!(f, "{valtree:?}"),
+            Error(_) => write!(f, "[const error]"),
+            Expr(expr) => write!(f, "{expr:?}"),
+        }
+    }
+}
+
 ///////////////////////////////////////////////////////////////////////////
 // Atomic structs
 //
 // For things that don't carry any arena-allocated data (and are
-// copy...), just add them to one of these lists as appropriat.
+// copy...), just add them to one of these lists as appropriate.
 
 // For things for which the type library provides traversal implementations
 // for all Interners, we only need to provide a Lift implementation:
@@ -205,6 +242,7 @@ CloneLiftImpls! {
     (),
     bool,
     usize,
+    u8,
     u16,
     u32,
     u64,
@@ -276,9 +314,7 @@ TrivialTypeTraversalAndLiftImpls! {
 }
 
 TrivialTypeTraversalAndLiftImpls! {
-    for<'tcx> {
-        ty::ValTree<'tcx>,
-    }
+    ty::ValTree<'tcx>,
 }
 
 ///////////////////////////////////////////////////////////////////////////
@@ -375,16 +411,6 @@ impl<'a, 'tcx> Lift<'tcx> for ty::ParamEnv<'a> {
 ///////////////////////////////////////////////////////////////////////////
 // Traversal implementations.
 
-/// AdtDefs are basically the same as a DefId.
-impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for ty::AdtDef<'tcx> {
-    fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
-        self,
-        _folder: &mut F,
-    ) -> Result<Self, F::Error> {
-        Ok(self)
-    }
-}
-
 impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for ty::AdtDef<'tcx> {
     fn visit_with<V: TypeVisitor<TyCtxt<'tcx>>>(
         &self,
@@ -447,15 +473,6 @@ impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for &'tcx ty::List<ty::Const<'tcx>> {
     }
 }
 
-impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for &'tcx ty::List<ProjectionKind> {
-    fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
-        self,
-        folder: &mut F,
-    ) -> Result<Self, F::Error> {
-        ty::util::fold_list(self, folder, |tcx, v| tcx.mk_projs(v))
-    }
-}
-
 impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for Ty<'tcx> {
     fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
         self,
@@ -583,24 +600,6 @@ impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for ty::Region<'tcx> {
     }
 }
 
-impl<'tcx> TypeSuperFoldable<TyCtxt<'tcx>> for ty::Region<'tcx> {
-    fn try_super_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
-        self,
-        _folder: &mut F,
-    ) -> Result<Self, F::Error> {
-        Ok(self)
-    }
-}
-
-impl<'tcx> TypeSuperVisitable<TyCtxt<'tcx>> for ty::Region<'tcx> {
-    fn super_visit_with<V: TypeVisitor<TyCtxt<'tcx>>>(
-        &self,
-        _visitor: &mut V,
-    ) -> ControlFlow<V::BreakTy> {
-        ControlFlow::Continue(())
-    }
-}
-
 impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for ty::Predicate<'tcx> {
     fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
         self,
diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs
index 96c1577d5..e6d51c4ec 100644
--- a/compiler/rustc_middle/src/ty/sty.rs
+++ b/compiler/rustc_middle/src/ty/sty.rs
@@ -19,7 +19,7 @@ use rustc_errors::{DiagnosticArgValue, IntoDiagnosticArg};
 use rustc_hir as hir;
 use rustc_hir::def_id::DefId;
 use rustc_hir::LangItem;
-use rustc_index::vec::Idx;
+use rustc_index::Idx;
 use rustc_macros::HashStable;
 use rustc_span::symbol::{kw, sym, Symbol};
 use rustc_span::Span;
@@ -631,7 +631,7 @@ impl<'tcx> UpvarSubsts<'tcx> {
 /// type of the constant. The reason that `R` is represented as an extra type parameter
 /// is the same reason that [`ClosureSubsts`] have `CS` and `U` as type parameters:
 /// inline const can reference lifetimes that are internal to the creating function.
-#[derive(Copy, Clone, Debug, TypeFoldable, TypeVisitable)]
+#[derive(Copy, Clone, Debug)]
 pub struct InlineConstSubsts<'tcx> {
     /// Generic parameters from the enclosing item,
     /// concatenated with the inferred type of the constant.
@@ -727,13 +727,13 @@ impl<'tcx> PolyExistentialPredicate<'tcx> {
             ExistentialPredicate::AutoTrait(did) => {
                 let generics = tcx.generics_of(did);
                 let trait_ref = if generics.params.len() == 1 {
-                    tcx.mk_trait_ref(did, [self_ty])
+                    ty::TraitRef::new(tcx, did, [self_ty])
                 } else {
                     // If this is an ill-formed auto trait, then synthesize
                     // new error substs for the missing generics.
                     let err_substs =
                         ty::InternalSubsts::extend_with_error(tcx, did, &[self_ty.into()]);
-                    tcx.mk_trait_ref(did, err_substs)
+                    ty::TraitRef::new(tcx, did, err_substs)
                 };
                 self.rebind(trait_ref).without_const().to_predicate(tcx)
             }
@@ -820,36 +820,68 @@ pub struct TraitRef<'tcx> {
     pub def_id: DefId,
     pub substs: SubstsRef<'tcx>,
     /// This field exists to prevent the creation of `TraitRef` without
-    /// calling [TyCtxt::mk_trait_ref].
-    pub(super) _use_mk_trait_ref_instead: (),
+    /// calling [`TraitRef::new`].
+    pub(super) _use_trait_ref_new_instead: (),
 }
 
 impl<'tcx> TraitRef<'tcx> {
+    pub fn new(
+        tcx: TyCtxt<'tcx>,
+        trait_def_id: DefId,
+        substs: impl IntoIterator<Item: Into<GenericArg<'tcx>>>,
+    ) -> Self {
+        let substs = tcx.check_and_mk_substs(trait_def_id, substs);
+        Self { def_id: trait_def_id, substs, _use_trait_ref_new_instead: () }
+    }
+
+    pub fn from_lang_item(
+        tcx: TyCtxt<'tcx>,
+        trait_lang_item: LangItem,
+        span: Span,
+        substs: impl IntoIterator<Item: Into<ty::GenericArg<'tcx>>>,
+    ) -> Self {
+        let trait_def_id = tcx.require_lang_item(trait_lang_item, Some(span));
+        Self::new(tcx, trait_def_id, substs)
+    }
+
+    pub fn from_method(
+        tcx: TyCtxt<'tcx>,
+        trait_id: DefId,
+        substs: SubstsRef<'tcx>,
+    ) -> ty::TraitRef<'tcx> {
+        let defs = tcx.generics_of(trait_id);
+        ty::TraitRef::new(tcx, trait_id, tcx.mk_substs(&substs[..defs.params.len()]))
+    }
+
+    /// Returns a `TraitRef` of the form `P0: Foo<P1..Pn>` where `Pi`
+    /// are the parameters defined on trait.
+    pub fn identity(tcx: TyCtxt<'tcx>, def_id: DefId) -> TraitRef<'tcx> {
+        ty::TraitRef::new(tcx, def_id, InternalSubsts::identity_for_item(tcx, def_id))
+    }
+
     pub fn with_self_ty(self, tcx: TyCtxt<'tcx>, self_ty: Ty<'tcx>) -> Self {
-        tcx.mk_trait_ref(
+        ty::TraitRef::new(
+            tcx,
             self.def_id,
             [self_ty.into()].into_iter().chain(self.substs.iter().skip(1)),
         )
     }
 
-    /// Returns a `TraitRef` of the form `P0: Foo<P1..Pn>` where `Pi`
-    /// are the parameters defined on trait.
-    pub fn identity(tcx: TyCtxt<'tcx>, def_id: DefId) -> Binder<'tcx, TraitRef<'tcx>> {
-        ty::Binder::dummy(tcx.mk_trait_ref(def_id, InternalSubsts::identity_for_item(tcx, def_id)))
+    /// Converts this trait ref to a trait predicate with a given `constness` and a positive polarity.
+    #[inline]
+    pub fn with_constness(self, constness: ty::BoundConstness) -> ty::TraitPredicate<'tcx> {
+        ty::TraitPredicate { trait_ref: self, constness, polarity: ty::ImplPolarity::Positive }
     }
 
+    /// Converts this trait ref to a trait predicate without `const` and a positive polarity.
     #[inline]
-    pub fn self_ty(&self) -> Ty<'tcx> {
-        self.substs.type_at(0)
+    pub fn without_const(self) -> ty::TraitPredicate<'tcx> {
+        self.with_constness(ty::BoundConstness::NotConst)
     }
 
-    pub fn from_method(
-        tcx: TyCtxt<'tcx>,
-        trait_id: DefId,
-        substs: SubstsRef<'tcx>,
-    ) -> ty::TraitRef<'tcx> {
-        let defs = tcx.generics_of(trait_id);
-        tcx.mk_trait_ref(trait_id, tcx.mk_substs(&substs[..defs.params.len()]))
+    #[inline]
+    pub fn self_ty(&self) -> Ty<'tcx> {
+        self.substs.type_at(0)
     }
 }
 
@@ -907,7 +939,7 @@ impl<'tcx> ExistentialTraitRef<'tcx> {
         // otherwise the escaping vars would be captured by the binder
         // debug_assert!(!self_ty.has_escaping_bound_vars());
 
-        tcx.mk_trait_ref(self.def_id, [self_ty.into()].into_iter().chain(self.substs.iter()))
+        ty::TraitRef::new(tcx, self.def_id, [self_ty.into()].into_iter().chain(self.substs.iter()))
     }
 }
 
@@ -1158,9 +1190,9 @@ where
 
 /// Represents the projection of an associated type.
 ///
-/// For a projection, this would be `<Ty as Trait<...>>::N`.
-///
-/// For an opaque type, there is no explicit syntax.
+/// * For a projection, this would be `<Ty as Trait<...>>::N<...>`.
+/// * For an inherent projection, this would be `Ty::N<...>`.
+/// * For an opaque type, there is no explicit syntax.
 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, TyEncodable, TyDecodable)]
 #[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub struct AliasTy<'tcx> {
@@ -1169,12 +1201,16 @@ pub struct AliasTy<'tcx> {
     /// For a projection, these are the substitutions for the trait and the
     /// GAT substitutions, if there are any.
     ///
+    /// For an inherent projection, they consist of the self type and the GAT substitutions,
+    /// if there are any.
+    ///
     /// For RPIT the substitutions are for the generics of the function,
     /// while for TAIT it is used for the generic parameters of the alias.
     pub substs: SubstsRef<'tcx>,
 
-    /// The `DefId` of the `TraitItem` for the associated type `N` if this is a projection,
-    /// or the `OpaqueType` item if this is an opaque.
+    /// The `DefId` of the `TraitItem` or `ImplItem` for the associated type `N` depending on whether
+    /// this is a projection or an inherent projection or the `DefId` of the `OpaqueType` item if
+    /// this is an opaque.
     ///
     /// During codegen, `tcx.type_of(def_id)` can be used to get the type of the
     /// underlying type if the type is an opaque.
@@ -1192,6 +1228,7 @@ pub struct AliasTy<'tcx> {
 impl<'tcx> AliasTy<'tcx> {
     pub fn kind(self, tcx: TyCtxt<'tcx>) -> ty::AliasKind {
         match tcx.def_kind(self.def_id) {
+            DefKind::AssocTy if let DefKind::Impl { of_trait: false } = tcx.def_kind(tcx.parent(self.def_id)) => ty::Inherent,
             DefKind::AssocTy | DefKind::ImplTraitPlaceholder => ty::Projection,
             DefKind::OpaqueTy => ty::Opaque,
             kind => bug!("unexpected DefKind in AliasTy: {kind:?}"),
@@ -1205,6 +1242,17 @@ impl<'tcx> AliasTy<'tcx> {
 
 /// The following methods work only with associated type projections.
 impl<'tcx> AliasTy<'tcx> {
+    pub fn self_ty(self) -> Ty<'tcx> {
+        self.substs.type_at(0)
+    }
+
+    pub fn with_self_ty(self, tcx: TyCtxt<'tcx>, self_ty: Ty<'tcx>) -> Self {
+        tcx.mk_alias_ty(self.def_id, [self_ty.into()].into_iter().chain(self.substs.iter().skip(1)))
+    }
+}
+
+/// The following methods work only with trait associated type projections.
+impl<'tcx> AliasTy<'tcx> {
     pub fn trait_def_id(self, tcx: TyCtxt<'tcx>) -> DefId {
         match tcx.def_kind(self.def_id) {
             DefKind::AssocTy | DefKind::AssocConst => tcx.parent(self.def_id),
@@ -1217,7 +1265,7 @@ impl<'tcx> AliasTy<'tcx> {
 
     /// Extracts the underlying trait reference and own substs from this projection.
     /// For example, if this is a projection of `<T as StreamingIterator>::Item<'a>`,
-    /// then this function would return a `T: Iterator` trait reference and `['a]` as the own substs
+    /// then this function would return a `T: StreamingIterator` trait reference and `['a]` as the own substs
     pub fn trait_ref_and_own_substs(
         self,
         tcx: TyCtxt<'tcx>,
@@ -1226,7 +1274,7 @@ impl<'tcx> AliasTy<'tcx> {
         let trait_def_id = self.trait_def_id(tcx);
         let trait_generics = tcx.generics_of(trait_def_id);
         (
-            tcx.mk_trait_ref(trait_def_id, self.substs.truncate_to(tcx, trait_generics)),
+            ty::TraitRef::new(tcx, trait_def_id, self.substs.truncate_to(tcx, trait_generics)),
             &self.substs[trait_generics.count()..],
         )
     }
@@ -1240,15 +1288,30 @@ impl<'tcx> AliasTy<'tcx> {
     /// as well.
     pub fn trait_ref(self, tcx: TyCtxt<'tcx>) -> ty::TraitRef<'tcx> {
         let def_id = self.trait_def_id(tcx);
-        tcx.mk_trait_ref(def_id, self.substs.truncate_to(tcx, tcx.generics_of(def_id)))
+        ty::TraitRef::new(tcx, def_id, self.substs.truncate_to(tcx, tcx.generics_of(def_id)))
     }
+}
 
-    pub fn self_ty(self) -> Ty<'tcx> {
-        self.substs.type_at(0)
-    }
+/// The following methods work only with inherent associated type projections.
+impl<'tcx> AliasTy<'tcx> {
+    /// Transform the substitutions to have the given `impl` substs as the base and the GAT substs on top of that.
+    ///
+    /// Does the following transformation:
+    ///
+    /// ```text
+    /// [Self, P_0...P_m] -> [I_0...I_n, P_0...P_m]
+    ///
+    ///     I_i impl subst
+    ///     P_j GAT subst
+    /// ```
+    pub fn rebase_substs_onto_impl(
+        self,
+        impl_substs: ty::SubstsRef<'tcx>,
+        tcx: TyCtxt<'tcx>,
+    ) -> ty::SubstsRef<'tcx> {
+        debug_assert_eq!(self.kind(tcx), ty::Inherent);
 
-    pub fn with_self_ty(self, tcx: TyCtxt<'tcx>, self_ty: Ty<'tcx>) -> Self {
-        tcx.mk_alias_ty(self.def_id, [self_ty.into()].into_iter().chain(self.substs.iter().skip(1)))
+        tcx.mk_substs_from_iter(impl_substs.into_iter().chain(self.substs.into_iter().skip(1)))
     }
 }
 
@@ -1436,7 +1499,7 @@ pub struct ConstVid<'tcx> {
 rustc_index::newtype_index! {
     /// A **region** (lifetime) **v**ariable **ID**.
     #[derive(HashStable)]
-    #[debug_format = "'_#{}r"]
+    #[debug_format = "'?{}"]
     pub struct RegionVid {}
 }
 
@@ -1645,7 +1708,9 @@ impl<'tcx> Region<'tcx> {
             ty::ReErased => {
                 flags = flags | TypeFlags::HAS_RE_ERASED;
             }
-            ty::ReError(_) => {}
+            ty::ReError(_) => {
+                flags = flags | TypeFlags::HAS_FREE_REGIONS;
+            }
         }
 
         debug!("type_flags({:?}) = {:?}", self, flags);
@@ -2303,13 +2368,11 @@ impl<'tcx> Ty<'tcx> {
 
             ty::Adt(def, _substs) => def.sized_constraint(tcx).0.is_empty(),
 
-            ty::Alias(..) | ty::Param(_) => false,
+            ty::Alias(..) | ty::Param(_) | ty::Placeholder(..) => false,
 
             ty::Infer(ty::TyVar(_)) => false,
 
-            ty::Bound(..)
-            | ty::Placeholder(..)
-            | ty::Infer(ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_)) => {
+            ty::Bound(..) | ty::Infer(ty::FreshTy(_) | ty::FreshIntTy(_) | ty::FreshFloatTy(_)) => {
                 bug!("`is_trivially_sized` applied to unexpected type: {:?}", self)
             }
         }
@@ -2400,6 +2463,13 @@ impl<'tcx> Ty<'tcx> {
             _ => None,
         }
     }
+
+    pub fn is_c_void(self, tcx: TyCtxt<'_>) -> bool {
+        match self.kind() {
+            ty::Adt(adt, _) => tcx.lang_items().get(LangItem::CVoid) == Some(adt.did()),
+            _ => false,
+        }
+    }
 }
 
 /// Extra information about why we ended up with a particular variance.
diff --git a/compiler/rustc_middle/src/ty/subst.rs b/compiler/rustc_middle/src/ty/subst.rs
index f05b87343..43f95635a 100644
--- a/compiler/rustc_middle/src/ty/subst.rs
+++ b/compiler/rustc_middle/src/ty/subst.rs
@@ -21,7 +21,6 @@ use std::marker::PhantomData;
 use std::mem;
 use std::num::NonZeroUsize;
 use std::ops::{ControlFlow, Deref};
-use std::slice;
 
 /// An entity in the Rust type system, which can be one of
 /// several kinds (types, lifetimes, and consts).
@@ -48,38 +47,13 @@ const TYPE_TAG: usize = 0b00;
 const REGION_TAG: usize = 0b01;
 const CONST_TAG: usize = 0b10;
 
-#[derive(Debug, TyEncodable, TyDecodable, PartialEq, Eq, PartialOrd, Ord)]
+#[derive(Debug, TyEncodable, TyDecodable, PartialEq, Eq, PartialOrd, Ord, HashStable)]
 pub enum GenericArgKind<'tcx> {
     Lifetime(ty::Region<'tcx>),
     Type(Ty<'tcx>),
     Const(ty::Const<'tcx>),
 }
 
-/// This function goes from `&'a [Ty<'tcx>]` to `&'a [GenericArg<'tcx>]`
-///
-/// This is sound as, for types, `GenericArg` is just
-/// `NonZeroUsize::new_unchecked(ty as *const _ as usize)` as
-/// long as we use `0` for the `TYPE_TAG`.
-pub fn ty_slice_as_generic_args<'a, 'tcx>(ts: &'a [Ty<'tcx>]) -> &'a [GenericArg<'tcx>] {
-    assert_eq!(TYPE_TAG, 0);
-    // SAFETY: the whole slice is valid and immutable.
-    // `Ty` and `GenericArg` is explained above.
-    unsafe { slice::from_raw_parts(ts.as_ptr().cast(), ts.len()) }
-}
-
-impl<'tcx> List<Ty<'tcx>> {
-    /// Allows to freely switch between `List<Ty<'tcx>>` and `List<GenericArg<'tcx>>`.
-    ///
-    /// As lists are interned, `List<Ty<'tcx>>` and `List<GenericArg<'tcx>>` have
-    /// be interned together, see `mk_type_list` for more details.
-    #[inline]
-    pub fn as_substs(&'tcx self) -> SubstsRef<'tcx> {
-        assert_eq!(TYPE_TAG, 0);
-        // SAFETY: `List<T>` is `#[repr(C)]`. `Ty` and `GenericArg` is explained above.
-        unsafe { &*(self as *const List<Ty<'tcx>> as *const List<GenericArg<'tcx>>) }
-    }
-}
-
 impl<'tcx> GenericArgKind<'tcx> {
     #[inline]
     fn pack(self) -> GenericArg<'tcx> {
@@ -180,30 +154,45 @@ impl<'tcx> GenericArg<'tcx> {
         }
     }
 
-    /// Unpack the `GenericArg` as a region when it is known certainly to be a region.
-    pub fn expect_region(self) -> ty::Region<'tcx> {
+    #[inline]
+    pub fn as_type(self) -> Option<Ty<'tcx>> {
         match self.unpack() {
-            GenericArgKind::Lifetime(lt) => lt,
-            _ => bug!("expected a region, but found another kind"),
+            GenericArgKind::Type(ty) => Some(ty),
+            _ => None,
         }
     }
 
+    #[inline]
+    pub fn as_region(self) -> Option<ty::Region<'tcx>> {
+        match self.unpack() {
+            GenericArgKind::Lifetime(re) => Some(re),
+            _ => None,
+        }
+    }
+
+    #[inline]
+    pub fn as_const(self) -> Option<ty::Const<'tcx>> {
+        match self.unpack() {
+            GenericArgKind::Const(ct) => Some(ct),
+            _ => None,
+        }
+    }
+
+    /// Unpack the `GenericArg` as a region when it is known certainly to be a region.
+    pub fn expect_region(self) -> ty::Region<'tcx> {
+        self.as_region().unwrap_or_else(|| bug!("expected a region, but found another kind"))
+    }
+
     /// Unpack the `GenericArg` as a type when it is known certainly to be a type.
     /// This is true in cases where `Substs` is used in places where the kinds are known
     /// to be limited (e.g. in tuples, where the only parameters are type parameters).
     pub fn expect_ty(self) -> Ty<'tcx> {
-        match self.unpack() {
-            GenericArgKind::Type(ty) => ty,
-            _ => bug!("expected a type, but found another kind"),
-        }
+        self.as_type().unwrap_or_else(|| bug!("expected a type, but found another kind"))
     }
 
     /// Unpack the `GenericArg` as a const when it is known certainly to be a const.
     pub fn expect_const(self) -> ty::Const<'tcx> {
-        match self.unpack() {
-            GenericArgKind::Const(c) => c,
-            _ => bug!("expected a const, but found another kind"),
-        }
+        self.as_const().unwrap_or_else(|| bug!("expected a const, but found another kind"))
     }
 
     pub fn is_non_region_infer(self) -> bool {
@@ -268,13 +257,16 @@ pub type InternalSubsts<'tcx> = List<GenericArg<'tcx>>;
 pub type SubstsRef<'tcx> = &'tcx InternalSubsts<'tcx>;
 
 impl<'tcx> InternalSubsts<'tcx> {
-    /// Checks whether all elements of this list are types, if so, transmute.
-    pub fn try_as_type_list(&'tcx self) -> Option<&'tcx List<Ty<'tcx>>> {
-        self.iter().all(|arg| matches!(arg.unpack(), GenericArgKind::Type(_))).then(|| {
-            assert_eq!(TYPE_TAG, 0);
-            // SAFETY: All elements are types, see `List<Ty<'tcx>>::as_substs`.
-            unsafe { &*(self as *const List<GenericArg<'tcx>> as *const List<Ty<'tcx>>) }
-        })
+    /// Converts substs to a type list.
+    ///
+    /// # Panics
+    ///
+    /// If any of the generic arguments are not types.
+    pub fn into_type_list(&self, tcx: TyCtxt<'tcx>) -> &'tcx List<Ty<'tcx>> {
+        tcx.mk_type_list_from_iter(self.iter().map(|arg| match arg.unpack() {
+            GenericArgKind::Type(ty) => ty,
+            _ => bug!("`into_type_list` called on substs with non-types"),
+        }))
     }
 
     /// Interpret these substitutions as the substitutions of a closure type.
@@ -379,22 +371,17 @@ impl<'tcx> InternalSubsts<'tcx> {
 
     #[inline]
     pub fn types(&'tcx self) -> impl DoubleEndedIterator<Item = Ty<'tcx>> + 'tcx {
-        self.iter()
-            .filter_map(|k| if let GenericArgKind::Type(ty) = k.unpack() { Some(ty) } else { None })
+        self.iter().filter_map(|k| k.as_type())
     }
 
     #[inline]
     pub fn regions(&'tcx self) -> impl DoubleEndedIterator<Item = ty::Region<'tcx>> + 'tcx {
-        self.iter().filter_map(|k| {
-            if let GenericArgKind::Lifetime(lt) = k.unpack() { Some(lt) } else { None }
-        })
+        self.iter().filter_map(|k| k.as_region())
     }
 
     #[inline]
     pub fn consts(&'tcx self) -> impl DoubleEndedIterator<Item = ty::Const<'tcx>> + 'tcx {
-        self.iter().filter_map(|k| {
-            if let GenericArgKind::Const(ct) = k.unpack() { Some(ct) } else { None }
-        })
+        self.iter().filter_map(|k| k.as_const())
     }
 
     #[inline]
@@ -410,31 +397,21 @@ impl<'tcx> InternalSubsts<'tcx> {
     #[inline]
     #[track_caller]
     pub fn type_at(&self, i: usize) -> Ty<'tcx> {
-        if let GenericArgKind::Type(ty) = self[i].unpack() {
-            ty
-        } else {
-            bug!("expected type for param #{} in {:?}", i, self);
-        }
+        self[i].as_type().unwrap_or_else(|| bug!("expected type for param #{} in {:?}", i, self))
     }
 
     #[inline]
     #[track_caller]
     pub fn region_at(&self, i: usize) -> ty::Region<'tcx> {
-        if let GenericArgKind::Lifetime(lt) = self[i].unpack() {
-            lt
-        } else {
-            bug!("expected region for param #{} in {:?}", i, self);
-        }
+        self[i]
+            .as_region()
+            .unwrap_or_else(|| bug!("expected region for param #{} in {:?}", i, self))
     }
 
     #[inline]
     #[track_caller]
     pub fn const_at(&self, i: usize) -> ty::Const<'tcx> {
-        if let GenericArgKind::Const(ct) = self[i].unpack() {
-            ct
-        } else {
-            bug!("expected const for param #{} in {:?}", i, self);
-        }
+        self[i].as_const().unwrap_or_else(|| bug!("expected const for param #{} in {:?}", i, self))
     }
 
     #[inline]
@@ -635,6 +612,12 @@ where
     ) -> SubstIter<'s, 'tcx, I> {
         SubstIter { it: self.0.into_iter(), tcx, substs }
     }
+
+    /// Similar to [`subst_identity`](EarlyBinder::subst_identity),
+    /// but on an iterator of `TypeFoldable` values.
+    pub fn subst_identity_iter(self) -> I::IntoIter {
+        self.0.into_iter()
+    }
 }
 
 pub struct SubstIter<'s, 'tcx, I: IntoIterator> {
@@ -687,6 +670,12 @@ where
     ) -> SubstIterCopied<'s, 'tcx, I> {
         SubstIterCopied { it: self.0.into_iter(), tcx, substs }
     }
+
+    /// Similar to [`subst_identity`](EarlyBinder::subst_identity),
+    /// but on an iterator of values that deref to a `TypeFoldable`.
+    pub fn subst_identity_iter_copied(self) -> impl Iterator<Item = <I::Item as Deref>::Target> {
+        self.0.into_iter().map(|v| *v)
+    }
 }
 
 pub struct SubstIterCopied<'a, 'tcx, I: IntoIterator> {
@@ -772,7 +761,7 @@ impl<'tcx, T: TypeFoldable<TyCtxt<'tcx>>> ty::EarlyBinder<T> {
 
     /// Returns the inner value, but only if it contains no bound vars.
     pub fn no_bound_vars(self) -> Option<T> {
-        if !self.0.needs_subst() { Some(self.0) } else { None }
+        if !self.0.has_param() { Some(self.0) } else { None }
     }
 }
 
@@ -840,12 +829,18 @@ impl<'a, 'tcx> TypeFolder<TyCtxt<'tcx>> for SubstFolder<'a, 'tcx> {
                     None => region_param_out_of_range(data, self.substs),
                 }
             }
-            _ => r,
+            ty::ReLateBound(..)
+            | ty::ReFree(_)
+            | ty::ReStatic
+            | ty::RePlaceholder(_)
+            | ty::ReErased
+            | ty::ReError(_) => r,
+            ty::ReVar(_) => bug!("unexpected region: {r:?}"),
         }
     }
 
     fn fold_ty(&mut self, t: Ty<'tcx>) -> Ty<'tcx> {
-        if !t.needs_subst() {
+        if !t.has_param() {
             return t;
         }
 
diff --git a/compiler/rustc_middle/src/ty/trait_def.rs b/compiler/rustc_middle/src/ty/trait_def.rs
index 6747da7ab..e61037e5e 100644
--- a/compiler/rustc_middle/src/ty/trait_def.rs
+++ b/compiler/rustc_middle/src/ty/trait_def.rs
@@ -139,40 +139,6 @@ impl<'tcx> TyCtxt<'tcx> {
         treat_projections: TreatProjections,
         mut f: impl FnMut(DefId),
     ) {
-        let _: Option<()> =
-            self.find_map_relevant_impl(trait_def_id, self_ty, treat_projections, |did| {
-                f(did);
-                None
-            });
-    }
-
-    /// `trait_def_id` MUST BE the `DefId` of a trait.
-    pub fn non_blanket_impls_for_ty(
-        self,
-        trait_def_id: DefId,
-        self_ty: Ty<'tcx>,
-    ) -> impl Iterator<Item = DefId> + 'tcx {
-        let impls = self.trait_impls_of(trait_def_id);
-        if let Some(simp) = fast_reject::simplify_type(self, self_ty, TreatParams::AsCandidateKey) {
-            if let Some(impls) = impls.non_blanket_impls.get(&simp) {
-                return impls.iter().copied();
-            }
-        }
-
-        [].iter().copied()
-    }
-
-    /// Applies function to every impl that could possibly match the self type `self_ty` and returns
-    /// the first non-none value.
-    ///
-    /// `trait_def_id` MUST BE the `DefId` of a trait.
-    pub fn find_map_relevant_impl<T>(
-        self,
-        trait_def_id: DefId,
-        self_ty: Ty<'tcx>,
-        treat_projections: TreatProjections,
-        mut f: impl FnMut(DefId) -> Option<T>,
-    ) -> Option<T> {
         // FIXME: This depends on the set of all impls for the trait. That is
         // unfortunate wrt. incremental compilation.
         //
@@ -181,9 +147,7 @@ impl<'tcx> TyCtxt<'tcx> {
         let impls = self.trait_impls_of(trait_def_id);
 
         for &impl_def_id in impls.blanket_impls.iter() {
-            if let result @ Some(_) = f(impl_def_id) {
-                return result;
-            }
+            f(impl_def_id);
         }
 
         // Note that we're using `TreatParams::ForLookup` to query `non_blanket_impls` while using
@@ -199,20 +163,30 @@ impl<'tcx> TyCtxt<'tcx> {
         if let Some(simp) = fast_reject::simplify_type(self, self_ty, treat_params) {
             if let Some(impls) = impls.non_blanket_impls.get(&simp) {
                 for &impl_def_id in impls {
-                    if let result @ Some(_) = f(impl_def_id) {
-                        return result;
-                    }
+                    f(impl_def_id);
                 }
             }
         } else {
             for &impl_def_id in impls.non_blanket_impls.values().flatten() {
-                if let result @ Some(_) = f(impl_def_id) {
-                    return result;
-                }
+                f(impl_def_id);
             }
         }
+    }
 
-        None
+    /// `trait_def_id` MUST BE the `DefId` of a trait.
+    pub fn non_blanket_impls_for_ty(
+        self,
+        trait_def_id: DefId,
+        self_ty: Ty<'tcx>,
+    ) -> impl Iterator<Item = DefId> + 'tcx {
+        let impls = self.trait_impls_of(trait_def_id);
+        if let Some(simp) = fast_reject::simplify_type(self, self_ty, TreatParams::AsCandidateKey) {
+            if let Some(impls) = impls.non_blanket_impls.get(&simp) {
+                return impls.iter().copied();
+            }
+        }
+
+        [].iter().copied()
     }
 
     /// Returns an iterator containing all impls for `trait_def_id`.
diff --git a/compiler/rustc_middle/src/ty/typeck_results.rs b/compiler/rustc_middle/src/ty/typeck_results.rs
index 47943b94c..e04dbbff9 100644
--- a/compiler/rustc_middle/src/ty/typeck_results.rs
+++ b/compiler/rustc_middle/src/ty/typeck_results.rs
@@ -20,7 +20,7 @@ use rustc_hir::{
     hir_id::OwnerId,
     HirId, ItemLocalId, ItemLocalMap, ItemLocalSet,
 };
-use rustc_index::vec::{Idx, IndexVec};
+use rustc_index::{Idx, IndexVec};
 use rustc_macros::HashStable;
 use rustc_middle::mir::FakeReadCause;
 use rustc_session::Session;
@@ -151,10 +151,14 @@ pub struct TypeckResults<'tcx> {
     /// this field will be set to `Some(ErrorGuaranteed)`.
     pub tainted_by_errors: Option<ErrorGuaranteed>,
 
-    /// All the opaque types that have hidden types set
-    /// by this function. We also store the
-    /// type here, so that mir-borrowck can use it as a hint for figuring out hidden types,
-    /// even if they are only set in dead code (which doesn't show up in MIR).
+    /// All the opaque types that have hidden types set by this function.
+    /// We also store the type here, so that the compiler can use it as a hint
+    /// for figuring out hidden types, even if they are only set in dead code
+    /// (which doesn't show up in MIR).
+    ///
+    /// These types are mapped back to the opaque's identity substitutions
+    /// (with erased regions), which is why we don't associated substs with any
+    /// of these usages.
     pub concrete_opaque_types: FxIndexMap<LocalDefId, ty::OpaqueHiddenType<'tcx>>,
 
     /// Tracks the minimum captures required for a closure;
@@ -208,6 +212,9 @@ pub struct TypeckResults<'tcx> {
     /// Contains the data for evaluating the effect of feature `capture_disjoint_fields`
     /// on closure size.
     pub closure_size_eval: FxHashMap<LocalDefId, ClosureSizeProfileData<'tcx>>,
+
+    /// Container types and field indices of `offset_of!` expressions
+    offset_of_data: ItemLocalMap<(Ty<'tcx>, Vec<FieldIdx>)>,
 }
 
 /// Whenever a value may be live across a generator yield, the type of that value winds up in the
@@ -280,6 +287,7 @@ impl<'tcx> TypeckResults<'tcx> {
             generator_interior_predicates: Default::default(),
             treat_byte_string_as_slice: Default::default(),
             closure_size_eval: Default::default(),
+            offset_of_data: Default::default(),
         }
     }
 
@@ -530,6 +538,14 @@ impl<'tcx> TypeckResults<'tcx> {
     pub fn coercion_casts(&self) -> &ItemLocalSet {
         &self.coercion_casts
     }
+
+    pub fn offset_of_data(&self) -> LocalTableInContext<'_, (Ty<'tcx>, Vec<FieldIdx>)> {
+        LocalTableInContext { hir_owner: self.hir_owner, data: &self.offset_of_data }
+    }
+
+    pub fn offset_of_data_mut(&mut self) -> LocalTableInContextMut<'_, (Ty<'tcx>, Vec<FieldIdx>)> {
+        LocalTableInContextMut { hir_owner: self.hir_owner, data: &mut self.offset_of_data }
+    }
 }
 
 /// Validate that the given HirId (respectively its `local_id` part) can be
diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs
index c8a78ec03..ba0513563 100644
--- a/compiler/rustc_middle/src/ty/util.rs
+++ b/compiler/rustc_middle/src/ty/util.rs
@@ -2,7 +2,7 @@
 
 use crate::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use crate::mir;
-use crate::ty::fast_reject::TreatProjections;
+use crate::query::Providers;
 use crate::ty::layout::IntegerExt;
 use crate::ty::{
     self, FallibleTypeFolder, ToPredicate, Ty, TyCtxt, TypeFoldable, TypeFolder, TypeSuperFoldable,
@@ -11,13 +11,13 @@ use crate::ty::{
 use crate::ty::{GenericArgKind, SubstsRef};
 use rustc_apfloat::Float as _;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
-use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
+use rustc_data_structures::stable_hasher::{Hash64, HashStable, StableHasher};
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir as hir;
 use rustc_hir::def::{CtorOf, DefKind, Res};
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_index::bit_set::GrowableBitSet;
-use rustc_index::vec::{Idx, IndexVec};
+use rustc_index::{Idx, IndexVec};
 use rustc_macros::HashStable;
 use rustc_session::Limit;
 use rustc_span::sym;
@@ -35,9 +35,14 @@ pub struct Discr<'tcx> {
 
 /// Used as an input to [`TyCtxt::uses_unique_generic_params`].
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
-pub enum IgnoreRegions {
-    Yes,
+pub enum CheckRegions {
     No,
+    /// Only permit early bound regions. This is useful for Adts which
+    /// can never have late bound regions.
+    OnlyEarlyBound,
+    /// Permit both late bound and early bound regions. Use this for functions,
+    /// which frequently have late bound regions.
+    Bound,
 }
 
 #[derive(Copy, Clone, Debug)]
@@ -124,7 +129,7 @@ impl IntTypeExt for IntegerType {
 impl<'tcx> TyCtxt<'tcx> {
     /// Creates a hash of the type `Ty` which will be the same no matter what crate
     /// context it's calculated within. This is used by the `type_id` intrinsic.
-    pub fn type_id_hash(self, ty: Ty<'tcx>) -> u64 {
+    pub fn type_id_hash(self, ty: Ty<'tcx>) -> Hash64 {
         // We want the type_id be independent of the types free regions, so we
         // erase them. The erase_regions() call will also anonymize bound
         // regions, which is desirable too.
@@ -359,21 +364,29 @@ impl<'tcx> TyCtxt<'tcx> {
         self.ensure().coherent_trait(drop_trait);
 
         let ty = self.type_of(adt_did).subst_identity();
-        let (did, constness) = self.find_map_relevant_impl(
-            drop_trait,
-            ty,
-            // FIXME: This could also be some other mode, like "unexpected"
-            TreatProjections::ForLookup,
-            |impl_did| {
-                if let Some(item_id) = self.associated_item_def_ids(impl_did).first() {
-                    if validate(self, impl_did).is_ok() {
-                        return Some((*item_id, self.constness(impl_did)));
-                    }
-                }
-                None
-            },
-        )?;
+        let mut dtor_candidate = None;
+        self.for_each_relevant_impl(drop_trait, ty, |impl_did| {
+            if validate(self, impl_did).is_err() {
+                // Already `ErrorGuaranteed`, no need to delay a span bug here.
+                return;
+            }
 
+            let Some(item_id) = self.associated_item_def_ids(impl_did).first() else {
+                self.sess.delay_span_bug(self.def_span(impl_did), "Drop impl without drop function");
+                return;
+            };
+
+            if let Some((old_item_id, _)) = dtor_candidate {
+                self.sess
+                    .struct_span_err(self.def_span(item_id), "multiple drop impls found")
+                    .span_note(self.def_span(old_item_id), "other impl here")
+                    .delay_as_bug();
+            }
+
+            dtor_candidate = Some((*item_id, self.constness(impl_did)));
+        });
+
+        let (did, constness) = dtor_candidate?;
         Some(ty::Destructor { did, constness })
     }
 
@@ -461,21 +474,28 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn uses_unique_generic_params(
         self,
         substs: SubstsRef<'tcx>,
-        ignore_regions: IgnoreRegions,
+        ignore_regions: CheckRegions,
     ) -> Result<(), NotUniqueParam<'tcx>> {
         let mut seen = GrowableBitSet::default();
+        let mut seen_late = FxHashSet::default();
         for arg in substs {
             match arg.unpack() {
-                GenericArgKind::Lifetime(lt) => {
-                    if ignore_regions == IgnoreRegions::No {
-                        let ty::ReEarlyBound(p) = lt.kind() else {
-                            return Err(NotUniqueParam::NotParam(lt.into()))
-                        };
+                GenericArgKind::Lifetime(lt) => match (ignore_regions, lt.kind()) {
+                    (CheckRegions::Bound, ty::ReLateBound(di, reg)) => {
+                        if !seen_late.insert((di, reg)) {
+                            return Err(NotUniqueParam::DuplicateParam(lt.into()));
+                        }
+                    }
+                    (CheckRegions::OnlyEarlyBound | CheckRegions::Bound, ty::ReEarlyBound(p)) => {
                         if !seen.insert(p.index) {
                             return Err(NotUniqueParam::DuplicateParam(lt.into()));
                         }
                     }
-                }
+                    (CheckRegions::OnlyEarlyBound | CheckRegions::Bound, _) => {
+                        return Err(NotUniqueParam::NotParam(lt.into()));
+                    }
+                    (CheckRegions::No, _) => {}
+                },
                 GenericArgKind::Type(t) => match t.kind() {
                     ty::Param(p) => {
                         if !seen.insert(p.index) {
@@ -498,6 +518,42 @@ impl<'tcx> TyCtxt<'tcx> {
         Ok(())
     }
 
+    /// Checks whether each generic argument is simply a unique generic placeholder.
+    ///
+    /// This is used in the new solver, which canonicalizes params to placeholders
+    /// for better caching.
+    pub fn uses_unique_placeholders_ignoring_regions(
+        self,
+        substs: SubstsRef<'tcx>,
+    ) -> Result<(), NotUniqueParam<'tcx>> {
+        let mut seen = GrowableBitSet::default();
+        for arg in substs {
+            match arg.unpack() {
+                // Ignore regions, since we can't resolve those in a canonicalized
+                // query in the trait solver.
+                GenericArgKind::Lifetime(_) => {}
+                GenericArgKind::Type(t) => match t.kind() {
+                    ty::Placeholder(p) => {
+                        if !seen.insert(p.bound.var) {
+                            return Err(NotUniqueParam::DuplicateParam(t.into()));
+                        }
+                    }
+                    _ => return Err(NotUniqueParam::NotParam(t.into())),
+                },
+                GenericArgKind::Const(c) => match c.kind() {
+                    ty::ConstKind::Placeholder(p) => {
+                        if !seen.insert(p.bound) {
+                            return Err(NotUniqueParam::DuplicateParam(c.into()));
+                        }
+                    }
+                    _ => return Err(NotUniqueParam::NotParam(c.into())),
+                },
+            }
+        }
+
+        Ok(())
+    }
+
     /// Returns `true` if `def_id` refers to a closure (e.g., `|x| x * 2`). Note
     /// that closures have a `DefId`, but the closure *expression* also
     /// has a `HirId` that is located within the context where the
@@ -642,16 +698,16 @@ impl<'tcx> TyCtxt<'tcx> {
         }
     }
 
-    /// Return the set of types that should be taken into accound when checking
+    /// Return the set of types that should be taken into account when checking
     /// trait bounds on a generator's internal state.
     pub fn generator_hidden_types(
         self,
         def_id: DefId,
     ) -> impl Iterator<Item = ty::EarlyBinder<Ty<'tcx>>> {
-        let generator_layout = &self.mir_generator_witnesses(def_id);
+        let generator_layout = self.mir_generator_witnesses(def_id);
         generator_layout
-            .field_tys
-            .iter()
+            .as_ref()
+            .map_or_else(|| [].iter(), |l| l.field_tys.iter())
             .filter(|decl| !decl.ignore_for_traits)
             .map(|decl| ty::EarlyBinder(decl.ty))
     }
@@ -694,20 +750,6 @@ impl<'tcx> TyCtxt<'tcx> {
         if visitor.found_recursion { Err(expanded_type) } else { Ok(expanded_type) }
     }
 
-    pub fn bound_return_position_impl_trait_in_trait_tys(
-        self,
-        def_id: DefId,
-    ) -> ty::EarlyBinder<Result<&'tcx FxHashMap<DefId, Ty<'tcx>>, ErrorGuaranteed>> {
-        ty::EarlyBinder(self.collect_return_position_impl_trait_in_trait_tys(def_id))
-    }
-
-    pub fn bound_explicit_item_bounds(
-        self,
-        def_id: DefId,
-    ) -> ty::EarlyBinder<&'tcx [(ty::Predicate<'tcx>, rustc_span::Span)]> {
-        ty::EarlyBinder(self.explicit_item_bounds(def_id))
-    }
-
     /// Returns names of captured upvars for closures and generators.
     ///
     /// Here are some examples:
@@ -1158,7 +1200,7 @@ impl<'tcx> Ty<'tcx> {
                 // context, or *something* like that, but for now just avoid passing inference
                 // variables to queries that can't cope with them. Instead, conservatively
                 // return "true" (may change drop order).
-                if query_ty.needs_infer() {
+                if query_ty.has_infer() {
                     return true;
                 }
 
@@ -1260,7 +1302,7 @@ pub enum ExplicitSelf<'tcx> {
 
 impl<'tcx> ExplicitSelf<'tcx> {
     /// Categorizes an explicit self declaration like `self: SomeType`
-    /// into either `self`, `&self`, `&mut self`, `Box<self>`, or
+    /// into either `self`, `&self`, `&mut self`, `Box<Self>`, or
     /// `Other`.
     /// This is mainly used to require the arbitrary_self_types feature
     /// in the case of `Other`, to improve error messages in the common cases,
@@ -1402,7 +1444,7 @@ pub fn is_trivially_const_drop(ty: Ty<'_>) -> bool {
 }
 
 /// Does the equivalent of
-/// ```ignore (ilustrative)
+/// ```ignore (illustrative)
 /// let v = self.iter().map(|p| p.fold_with(folder)).collect::<SmallVec<[_; 8]>>();
 /// folder.tcx().intern_*(&v)
 /// ```
@@ -1479,8 +1521,8 @@ pub fn is_intrinsic(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
     matches!(tcx.fn_sig(def_id).skip_binder().abi(), Abi::RustIntrinsic | Abi::PlatformIntrinsic)
 }
 
-pub fn provide(providers: &mut ty::query::Providers) {
-    *providers = ty::query::Providers {
+pub fn provide(providers: &mut Providers) {
+    *providers = Providers {
         reveal_opaque_types_in_bounds,
         is_doc_hidden,
         is_doc_notable_trait,
diff --git a/compiler/rustc_middle/src/ty/visit.rs b/compiler/rustc_middle/src/ty/visit.rs
index 08a62c900..520bb55e0 100644
--- a/compiler/rustc_middle/src/ty/visit.rs
+++ b/compiler/rustc_middle/src/ty/visit.rs
@@ -33,6 +33,14 @@ pub trait TypeVisitableExt<'tcx>: TypeVisitable<TyCtxt<'tcx>> {
     }
 
     fn has_type_flags(&self, flags: TypeFlags) -> bool {
+        // N.B. Even though this uses a visitor, the visitor does not actually
+        //      recurse through the whole `TypeVisitable` implementor type.
+        //
+        //      Instead it stops on the first "level", visiting types, regions,
+        //      consts and predicates just fetches their type flags.
+        //
+        //      Thus this is a lot faster than it might seem and should be
+        //      optimized to a simple field access.
         let res =
             self.visit_with(&mut HasTypeFlagsVisitor { flags }).break_value() == Some(FoundFlags);
         trace!(?self, ?flags, ?res, "has_type_flags");
@@ -41,6 +49,9 @@ pub trait TypeVisitableExt<'tcx>: TypeVisitable<TyCtxt<'tcx>> {
     fn has_projections(&self) -> bool {
         self.has_type_flags(TypeFlags::HAS_PROJECTION)
     }
+    fn has_inherent_projections(&self) -> bool {
+        self.has_type_flags(TypeFlags::HAS_TY_INHERENT)
+    }
     fn has_opaque_types(&self) -> bool {
         self.has_type_flags(TypeFlags::HAS_TY_OPAQUE)
     }
@@ -62,7 +73,7 @@ pub trait TypeVisitableExt<'tcx>: TypeVisitable<TyCtxt<'tcx>> {
         }
     }
     fn has_non_region_param(&self) -> bool {
-        self.has_type_flags(TypeFlags::NEEDS_SUBST - TypeFlags::HAS_RE_PARAM)
+        self.has_type_flags(TypeFlags::HAS_PARAM - TypeFlags::HAS_RE_PARAM)
     }
     fn has_infer_regions(&self) -> bool {
         self.has_type_flags(TypeFlags::HAS_RE_INFER)
@@ -71,10 +82,10 @@ pub trait TypeVisitableExt<'tcx>: TypeVisitable<TyCtxt<'tcx>> {
         self.has_type_flags(TypeFlags::HAS_TY_INFER)
     }
     fn has_non_region_infer(&self) -> bool {
-        self.has_type_flags(TypeFlags::NEEDS_INFER - TypeFlags::HAS_RE_INFER)
+        self.has_type_flags(TypeFlags::HAS_INFER - TypeFlags::HAS_RE_INFER)
     }
-    fn needs_infer(&self) -> bool {
-        self.has_type_flags(TypeFlags::NEEDS_INFER)
+    fn has_infer(&self) -> bool {
+        self.has_type_flags(TypeFlags::HAS_INFER)
     }
     fn has_placeholders(&self) -> bool {
         self.has_type_flags(
@@ -86,8 +97,8 @@ pub trait TypeVisitableExt<'tcx>: TypeVisitable<TyCtxt<'tcx>> {
     fn has_non_region_placeholders(&self) -> bool {
         self.has_type_flags(TypeFlags::HAS_TY_PLACEHOLDER | TypeFlags::HAS_CT_PLACEHOLDER)
     }
-    fn needs_subst(&self) -> bool {
-        self.has_type_flags(TypeFlags::NEEDS_SUBST)
+    fn has_param(&self) -> bool {
+        self.has_type_flags(TypeFlags::HAS_PARAM)
     }
     /// "Free" regions in this context means that it has any region
     /// that is not (a) erased or (b) late-bound.
@@ -364,7 +375,7 @@ impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for ValidateBoundVars<'tcx> {
             _ => (),
         };
 
-        r.super_visit_with(self)
+        ControlFlow::Continue(())
     }
 }
 
diff --git a/compiler/rustc_middle/src/ty/walk.rs b/compiler/rustc_middle/src/ty/walk.rs
index 182945b9c..04a635a68 100644
--- a/compiler/rustc_middle/src/ty/walk.rs
+++ b/compiler/rustc_middle/src/ty/walk.rs
@@ -194,7 +194,7 @@ fn push_inner<'tcx>(stack: &mut TypeWalkerStack<'tcx>, parent: GenericArg<'tcx>)
             | ty::FnDef(_, substs) => {
                 stack.extend(substs.iter().rev());
             }
-            ty::Tuple(ts) => stack.extend(ts.as_substs().iter().rev()),
+            ty::Tuple(ts) => stack.extend(ts.iter().rev().map(GenericArg::from)),
             ty::GeneratorWitness(ts) => {
                 stack.extend(ts.skip_binder().iter().rev().map(|ty| ty.into()));
             }
diff --git a/compiler/rustc_middle/src/util/bug.rs b/compiler/rustc_middle/src/util/bug.rs
index b73ae5939..43ee0343f 100644
--- a/compiler/rustc_middle/src/util/bug.rs
+++ b/compiler/rustc_middle/src/util/bug.rs
@@ -31,8 +31,8 @@ fn opt_span_bug_fmt<S: Into<MultiSpan>>(
     tls::with_opt(move |tcx| {
         let msg = format!("{}: {}", location, args);
         match (tcx, span) {
-            (Some(tcx), Some(span)) => tcx.sess.diagnostic().span_bug(span, &msg),
-            (Some(tcx), None) => tcx.sess.diagnostic().bug(&msg),
+            (Some(tcx), Some(span)) => tcx.sess.diagnostic().span_bug(span, msg),
+            (Some(tcx), None) => tcx.sess.diagnostic().bug(msg),
             (None, _) => panic_any(msg),
         }
     })
@@ -48,6 +48,6 @@ pub fn trigger_delay_span_bug(tcx: TyCtxt<'_>, key: rustc_hir::def_id::DefId) {
     );
 }
 
-pub fn provide(providers: &mut crate::ty::query::Providers) {
-    *providers = crate::ty::query::Providers { trigger_delay_span_bug, ..*providers };
+pub fn provide(providers: &mut crate::query::Providers) {
+    *providers = crate::query::Providers { trigger_delay_span_bug, ..*providers };
 }
diff --git a/compiler/rustc_middle/src/util/call_kind.rs b/compiler/rustc_middle/src/util/call_kind.rs
new file mode 100644
index 000000000..98d55ea6d
--- /dev/null
+++ b/compiler/rustc_middle/src/util/call_kind.rs
@@ -0,0 +1,142 @@
+//! Common logic for borrowck use-after-move errors when moved into a `fn(self)`,
+//! as well as errors when attempting to call a non-const function in a const
+//! context.
+
+use crate::ty::subst::SubstsRef;
+use crate::ty::{AssocItemContainer, Instance, ParamEnv, Ty, TyCtxt};
+use rustc_hir::def_id::DefId;
+use rustc_hir::{lang_items, LangItem};
+use rustc_span::symbol::Ident;
+use rustc_span::{sym, DesugaringKind, Span};
+
+#[derive(Clone, Copy, PartialEq, Eq, Debug)]
+pub enum CallDesugaringKind {
+    /// for _ in x {} calls x.into_iter()
+    ForLoopIntoIter,
+    /// x? calls x.branch()
+    QuestionBranch,
+    /// x? calls type_of(x)::from_residual()
+    QuestionFromResidual,
+    /// try { ..; x } calls type_of(x)::from_output(x)
+    TryBlockFromOutput,
+    /// `.await` calls `IntoFuture::into_future`
+    Await,
+}
+
+impl CallDesugaringKind {
+    pub fn trait_def_id(self, tcx: TyCtxt<'_>) -> DefId {
+        match self {
+            Self::ForLoopIntoIter => tcx.get_diagnostic_item(sym::IntoIterator).unwrap(),
+            Self::QuestionBranch | Self::TryBlockFromOutput => {
+                tcx.require_lang_item(LangItem::Try, None)
+            }
+            Self::QuestionFromResidual => tcx.get_diagnostic_item(sym::FromResidual).unwrap(),
+            Self::Await => tcx.get_diagnostic_item(sym::IntoFuture).unwrap(),
+        }
+    }
+}
+
+#[derive(Clone, Copy, PartialEq, Eq, Debug)]
+pub enum CallKind<'tcx> {
+    /// A normal method call of the form `receiver.foo(a, b, c)`
+    Normal {
+        self_arg: Option<Ident>,
+        desugaring: Option<(CallDesugaringKind, Ty<'tcx>)>,
+        method_did: DefId,
+        method_substs: SubstsRef<'tcx>,
+    },
+    /// A call to `Fn(..)::call(..)`, desugared from `my_closure(a, b, c)`
+    FnCall { fn_trait_id: DefId, self_ty: Ty<'tcx> },
+    /// A call to an operator trait, desugared from operator syntax (e.g. `a << b`)
+    Operator { self_arg: Option<Ident>, trait_id: DefId, self_ty: Ty<'tcx> },
+    DerefCoercion {
+        /// The `Span` of the `Target` associated type
+        /// in the `Deref` impl we are using.
+        deref_target: Span,
+        /// The type `T::Deref` we are dereferencing to
+        deref_target_ty: Ty<'tcx>,
+        self_ty: Ty<'tcx>,
+    },
+}
+
+pub fn call_kind<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    param_env: ParamEnv<'tcx>,
+    method_did: DefId,
+    method_substs: SubstsRef<'tcx>,
+    fn_call_span: Span,
+    from_hir_call: bool,
+    self_arg: Option<Ident>,
+) -> CallKind<'tcx> {
+    let parent = tcx.opt_associated_item(method_did).and_then(|assoc| {
+        let container_id = assoc.container_id(tcx);
+        match assoc.container {
+            AssocItemContainer::ImplContainer => tcx.trait_id_of_impl(container_id),
+            AssocItemContainer::TraitContainer => Some(container_id),
+        }
+    });
+
+    let fn_call = parent.and_then(|p| {
+        lang_items::FN_TRAITS.iter().filter_map(|&l| tcx.lang_items().get(l)).find(|&id| id == p)
+    });
+
+    let operator = if !from_hir_call && let Some(p) = parent {
+        lang_items::OPERATORS.iter().filter_map(|&l| tcx.lang_items().get(l)).find(|&id| id == p)
+    } else {
+        None
+    };
+
+    let is_deref = !from_hir_call && tcx.is_diagnostic_item(sym::deref_method, method_did);
+
+    // Check for a 'special' use of 'self' -
+    // an FnOnce call, an operator (e.g. `<<`), or a
+    // deref coercion.
+    let kind = if let Some(trait_id) = fn_call {
+        Some(CallKind::FnCall { fn_trait_id: trait_id, self_ty: method_substs.type_at(0) })
+    } else if let Some(trait_id) = operator {
+        Some(CallKind::Operator { self_arg, trait_id, self_ty: method_substs.type_at(0) })
+    } else if is_deref {
+        let deref_target = tcx.get_diagnostic_item(sym::deref_target).and_then(|deref_target| {
+            Instance::resolve(tcx, param_env, deref_target, method_substs).transpose()
+        });
+        if let Some(Ok(instance)) = deref_target {
+            let deref_target_ty = instance.ty(tcx, param_env);
+            Some(CallKind::DerefCoercion {
+                deref_target: tcx.def_span(instance.def_id()),
+                deref_target_ty,
+                self_ty: method_substs.type_at(0),
+            })
+        } else {
+            None
+        }
+    } else {
+        None
+    };
+
+    kind.unwrap_or_else(|| {
+        // This isn't a 'special' use of `self`
+        debug!(?method_did, ?fn_call_span);
+        let desugaring = if Some(method_did) == tcx.lang_items().into_iter_fn()
+            && fn_call_span.desugaring_kind() == Some(DesugaringKind::ForLoop)
+        {
+            Some((CallDesugaringKind::ForLoopIntoIter, method_substs.type_at(0)))
+        } else if fn_call_span.desugaring_kind() == Some(DesugaringKind::QuestionMark) {
+            if Some(method_did) == tcx.lang_items().branch_fn() {
+                Some((CallDesugaringKind::QuestionBranch, method_substs.type_at(0)))
+            } else if Some(method_did) == tcx.lang_items().from_residual_fn() {
+                Some((CallDesugaringKind::QuestionFromResidual, method_substs.type_at(0)))
+            } else {
+                None
+            }
+        } else if Some(method_did) == tcx.lang_items().from_output_fn()
+            && fn_call_span.desugaring_kind() == Some(DesugaringKind::TryBlock)
+        {
+            Some((CallDesugaringKind::TryBlockFromOutput, method_substs.type_at(0)))
+        } else if fn_call_span.is_desugaring(DesugaringKind::Await) {
+            Some((CallDesugaringKind::Await, method_substs.type_at(0)))
+        } else {
+            None
+        };
+        CallKind::Normal { self_arg, desugaring, method_did, method_substs }
+    })
+}
diff --git a/compiler/rustc_middle/src/util/find_self_call.rs b/compiler/rustc_middle/src/util/find_self_call.rs
new file mode 100644
index 000000000..0eab0adf0
--- /dev/null
+++ b/compiler/rustc_middle/src/util/find_self_call.rs
@@ -0,0 +1,36 @@
+use crate::mir::*;
+use crate::ty::subst::SubstsRef;
+use crate::ty::{self, TyCtxt};
+use rustc_span::def_id::DefId;
+
+/// Checks if the specified `local` is used as the `self` parameter of a method call
+/// in the provided `BasicBlock`. If it is, then the `DefId` of the called method is
+/// returned.
+pub fn find_self_call<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    body: &Body<'tcx>,
+    local: Local,
+    block: BasicBlock,
+) -> Option<(DefId, SubstsRef<'tcx>)> {
+    debug!("find_self_call(local={:?}): terminator={:?}", local, &body[block].terminator);
+    if let Some(Terminator { kind: TerminatorKind::Call { func, args, .. }, .. }) =
+        &body[block].terminator
+    {
+        debug!("find_self_call: func={:?}", func);
+        if let Operand::Constant(box Constant { literal, .. }) = func {
+            if let ty::FnDef(def_id, substs) = *literal.ty().kind() {
+                if let Some(ty::AssocItem { fn_has_self_parameter: true, .. }) =
+                    tcx.opt_associated_item(def_id)
+                {
+                    debug!("find_self_call: args={:?}", args);
+                    if let [Operand::Move(self_place) | Operand::Copy(self_place), ..] = **args {
+                        if self_place.as_local() == Some(local) {
+                            return Some((def_id, substs));
+                        }
+                    }
+                }
+            }
+        }
+    }
+    None
+}
diff --git a/compiler/rustc_middle/src/util/mod.rs b/compiler/rustc_middle/src/util/mod.rs
new file mode 100644
index 000000000..53b425789
--- /dev/null
+++ b/compiler/rustc_middle/src/util/mod.rs
@@ -0,0 +1,7 @@
+pub mod bug;
+pub mod call_kind;
+pub mod common;
+pub mod find_self_call;
+
+pub use call_kind::{call_kind, CallDesugaringKind, CallKind};
+pub use find_self_call::find_self_call;
diff --git a/compiler/rustc_middle/src/values.rs b/compiler/rustc_middle/src/values.rs
index 55aa4fcff..c62c33d4d 100644
--- a/compiler/rustc_middle/src/values.rs
+++ b/compiler/rustc_middle/src/values.rs
@@ -106,6 +106,12 @@ impl<'tcx> Value<TyCtxt<'tcx>, DepKind> for ty::EarlyBinder<ty::Binder<'_, ty::F
     }
 }
 
+impl<'tcx, T> Value<TyCtxt<'tcx>, DepKind> for Result<T, ty::layout::LayoutError<'_>> {
+    fn from_cycle_error(_tcx: TyCtxt<'tcx>, _cycle: &[QueryInfo<DepKind>]) -> Self {
+        Err(ty::layout::LayoutError::Cycle)
+    }
+}
+
 // item_and_field_ids should form a cycle where each field contains the
 // type in the next element in the list
 pub fn recursive_type_error(
@@ -158,8 +164,8 @@ pub fn recursive_type_error(
     }
     let items_list = {
         let mut s = String::new();
-        for (i, (item_id, _)) in item_and_field_ids.iter().enumerate() {
-            let path = tcx.def_path_str(item_id.to_def_id());
+        for (i, &(item_id, _)) in item_and_field_ids.iter().enumerate() {
+            let path = tcx.def_path_str(item_id);
             write!(&mut s, "`{path}`").unwrap();
             if i == (ITEM_LIMIT - 1) && cycle_len > ITEM_LIMIT {
                 write!(&mut s, " and {} more", cycle_len - 5).unwrap();