Adding debian version 1.65.0+dfsg1-2.debian/1.65.0+dfsg1-2

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

34 files changed, 1307 insertions, 1125 deletions

diff --git a/compiler/rustc_middle/src/ty/abstract_const.rs b/compiler/rustc_middle/src/ty/abstract_const.rs
index bed809930..8f79b4705 100644
--- a/compiler/rustc_middle/src/ty/abstract_const.rs
+++ b/compiler/rustc_middle/src/ty/abstract_const.rs
@@ -42,7 +42,7 @@ impl<'tcx> AbstractConst<'tcx> {
         ct: ty::Const<'tcx>,
     ) -> Result<Option<AbstractConst<'tcx>>, ErrorGuaranteed> {
         match ct.kind() {
-            ty::ConstKind::Unevaluated(uv) => AbstractConst::new(tcx, uv.shrink()),
+            ty::ConstKind::Unevaluated(uv) => AbstractConst::new(tcx, uv),
             ty::ConstKind::Error(DelaySpanBugEmitted { reported, .. }) => Err(reported),
             _ => Ok(None),
         }
diff --git a/compiler/rustc_middle/src/ty/adjustment.rs b/compiler/rustc_middle/src/ty/adjustment.rs
index d36cf2fe3..b809f1767 100644
--- a/compiler/rustc_middle/src/ty/adjustment.rs
+++ b/compiler/rustc_middle/src/ty/adjustment.rs
@@ -77,7 +77,7 @@ pub enum PointerCast {
 ///    At some point, of course, `Box` should move out of the compiler, in which
 ///    case this is analogous to transforming a struct. E.g., `Box<[i32; 4]>` ->
 ///    `Box<[i32]>` is an `Adjust::Unsize` with the target `Box<[i32]>`.
-#[derive(Clone, TyEncodable, TyDecodable, HashStable, TypeFoldable, TypeVisitable)]
+#[derive(Clone, TyEncodable, TyDecodable, HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub struct Adjustment<'tcx> {
     pub kind: Adjust<'tcx>,
     pub target: Ty<'tcx>,
@@ -89,7 +89,7 @@ impl<'tcx> Adjustment<'tcx> {
     }
 }
 
-#[derive(Clone, Debug, TyEncodable, TyDecodable, HashStable, TypeFoldable, TypeVisitable)]
+#[derive(Clone, Debug, TyEncodable, TyDecodable, HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub enum Adjust<'tcx> {
     /// Go from ! to any type.
     NeverToAny,
@@ -108,7 +108,7 @@ pub enum Adjust<'tcx> {
 /// The target type is `U` in both cases, with the region and mutability
 /// being those shared by both the receiver and the returned reference.
 #[derive(Copy, Clone, PartialEq, Debug, TyEncodable, TyDecodable, HashStable)]
-#[derive(TypeFoldable, TypeVisitable)]
+#[derive(TypeFoldable, TypeVisitable, Lift)]
 pub struct OverloadedDeref<'tcx> {
     pub region: ty::Region<'tcx>,
     pub mutbl: hir::Mutability,
@@ -167,7 +167,7 @@ impl From<AutoBorrowMutability> for hir::Mutability {
 }
 
 #[derive(Copy, Clone, PartialEq, Debug, TyEncodable, TyDecodable, HashStable)]
-#[derive(TypeFoldable, TypeVisitable)]
+#[derive(TypeFoldable, TypeVisitable, Lift)]
 pub enum AutoBorrow<'tcx> {
     /// Converts from T to &T.
     Ref(ty::Region<'tcx>, AutoBorrowMutability),
diff --git a/compiler/rustc_middle/src/ty/assoc.rs b/compiler/rustc_middle/src/ty/assoc.rs
index c97156ac1..55ee5bd2f 100644
--- a/compiler/rustc_middle/src/ty/assoc.rs
+++ b/compiler/rustc_middle/src/ty/assoc.rs
@@ -42,7 +42,7 @@ impl AssocItem {
     }
 
     #[inline]
-    pub fn visibility(&self, tcx: TyCtxt<'_>) -> Visibility {
+    pub fn visibility(&self, tcx: TyCtxt<'_>) -> Visibility<DefId> {
         tcx.visibility(self.def_id)
     }
 
diff --git a/compiler/rustc_middle/src/ty/binding.rs b/compiler/rustc_middle/src/ty/binding.rs
index 3d65429f2..a5b05a4f9 100644
--- a/compiler/rustc_middle/src/ty/binding.rs
+++ b/compiler/rustc_middle/src/ty/binding.rs
@@ -1,6 +1,4 @@
-use rustc_hir::BindingAnnotation;
-use rustc_hir::BindingAnnotation::*;
-use rustc_hir::Mutability;
+use rustc_hir::{BindingAnnotation, ByRef, Mutability};
 
 #[derive(Clone, PartialEq, TyEncodable, TyDecodable, Debug, Copy, HashStable)]
 pub enum BindingMode {
@@ -11,12 +9,10 @@ pub enum BindingMode {
 TrivialTypeTraversalAndLiftImpls! { BindingMode, }
 
 impl BindingMode {
-    pub fn convert(ba: BindingAnnotation) -> BindingMode {
-        match ba {
-            Unannotated => BindingMode::BindByValue(Mutability::Not),
-            Mutable => BindingMode::BindByValue(Mutability::Mut),
-            Ref => BindingMode::BindByReference(Mutability::Not),
-            RefMut => BindingMode::BindByReference(Mutability::Mut),
+    pub fn convert(BindingAnnotation(by_ref, mutbl): BindingAnnotation) -> BindingMode {
+        match by_ref {
+            ByRef::No => BindingMode::BindByValue(mutbl),
+            ByRef::Yes => BindingMode::BindByReference(mutbl),
         }
     }
 }
diff --git a/compiler/rustc_middle/src/ty/cast.rs b/compiler/rustc_middle/src/ty/cast.rs
index c4b743dd4..981e2d3b6 100644
--- a/compiler/rustc_middle/src/ty/cast.rs
+++ b/compiler/rustc_middle/src/ty/cast.rs
@@ -33,6 +33,8 @@ pub enum CastTy<'tcx> {
     FnPtr,
     /// Raw pointers.
     Ptr(ty::TypeAndMut<'tcx>),
+    /// Casting into a `dyn*` value.
+    DynStar,
 }
 
 /// Cast Kind. See [RFC 401](https://rust-lang.github.io/rfcs/0401-coercions.html)
@@ -50,6 +52,7 @@ pub enum CastKind {
     ArrayPtrCast,
     FnPtrPtrCast,
     FnPtrAddrCast,
+    DynStarCast,
 }
 
 impl<'tcx> CastTy<'tcx> {
@@ -67,6 +70,7 @@ impl<'tcx> CastTy<'tcx> {
             ty::Adt(d, _) if d.is_enum() && d.is_payloadfree() => Some(CastTy::Int(IntTy::CEnum)),
             ty::RawPtr(mt) => Some(CastTy::Ptr(mt)),
             ty::FnPtr(..) => Some(CastTy::FnPtr),
+            ty::Dynamic(_, _, ty::DynStar) => Some(CastTy::DynStar),
             _ => None,
         }
     }
diff --git a/compiler/rustc_middle/src/ty/consts.rs b/compiler/rustc_middle/src/ty/consts.rs
index f8792edc0..339ff4d35 100644
--- a/compiler/rustc_middle/src/ty/consts.rs
+++ b/compiler/rustc_middle/src/ty/consts.rs
@@ -41,7 +41,7 @@ pub struct ConstS<'tcx> {
 }
 
 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
-static_assert_size!(ConstS<'_>, 48);
+static_assert_size!(ConstS<'_>, 40);
 
 impl<'tcx> Const<'tcx> {
     #[inline]
@@ -65,8 +65,6 @@ impl<'tcx> Const<'tcx> {
         tcx: TyCtxt<'tcx>,
         def: ty::WithOptConstParam<LocalDefId>,
     ) -> Self {
-        debug!("Const::from_anon_const(def={:?})", def);
-
         let body_id = match tcx.hir().get_by_def_id(def.did) {
             hir::Node::AnonConst(ac) => ac.body,
             _ => span_bug!(
@@ -86,7 +84,7 @@ impl<'tcx> Const<'tcx> {
                 kind: ty::ConstKind::Unevaluated(ty::Unevaluated {
                     def: def.to_global(),
                     substs: InternalSubsts::identity_for_item(tcx, def.did.to_def_id()),
-                    promoted: None,
+                    promoted: (),
                 }),
                 ty,
             }),
@@ -183,7 +181,7 @@ impl<'tcx> Const<'tcx> {
                     kind: ty::ConstKind::Unevaluated(ty::Unevaluated {
                         def: ty::WithOptConstParam::unknown(def_id).to_global(),
                         substs,
-                        promoted: None,
+                        promoted: (),
                     }),
                     ty,
                 })
diff --git a/compiler/rustc_middle/src/ty/consts/kind.rs b/compiler/rustc_middle/src/ty/consts/kind.rs
index cb0137d2e..455015280 100644
--- a/compiler/rustc_middle/src/ty/consts/kind.rs
+++ b/compiler/rustc_middle/src/ty/consts/kind.rs
@@ -11,6 +11,7 @@ use rustc_macros::HashStable;
 use rustc_target::abi::Size;
 
 use super::ScalarInt;
+
 /// An unevaluated, potentially generic, constant.
 #[derive(Copy, Clone, Debug, Eq, PartialEq, PartialOrd, Ord, TyEncodable, TyDecodable, Lift)]
 #[derive(Hash, HashStable)]
@@ -20,6 +21,12 @@ pub struct Unevaluated<'tcx, P = Option<Promoted>> {
     pub promoted: P,
 }
 
+impl rustc_errors::IntoDiagnosticArg for Unevaluated<'_> {
+    fn into_diagnostic_arg(self) -> rustc_errors::DiagnosticArgValue<'static> {
+        format!("{:?}", self).into_diagnostic_arg()
+    }
+}
+
 impl<'tcx> Unevaluated<'tcx> {
     #[inline]
     pub fn shrink(self) -> Unevaluated<'tcx, ()> {
@@ -44,7 +51,7 @@ impl<'tcx, P: Default> Unevaluated<'tcx, P> {
 
 /// Represents a constant in Rust.
 #[derive(Copy, Clone, Debug, Eq, PartialEq, PartialOrd, Ord, TyEncodable, TyDecodable)]
-#[derive(Hash, HashStable)]
+#[derive(Hash, HashStable, TypeFoldable, TypeVisitable)]
 pub enum ConstKind<'tcx> {
     /// A const generic parameter.
     Param(ty::ParamConst),
@@ -60,7 +67,7 @@ pub enum ConstKind<'tcx> {
 
     /// Used in the HIR by using `Unevaluated` everywhere and later normalizing to one of the other
     /// variants when the code is monomorphic enough for that.
-    Unevaluated(Unevaluated<'tcx>),
+    Unevaluated(Unevaluated<'tcx, ()>),
 
     /// Used to hold computed value.
     Value(ty::ValTree<'tcx>),
@@ -71,7 +78,7 @@ pub enum ConstKind<'tcx> {
 }
 
 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
-static_assert_size!(ConstKind<'_>, 40);
+static_assert_size!(ConstKind<'_>, 32);
 
 impl<'tcx> ConstKind<'tcx> {
     #[inline]
@@ -174,9 +181,12 @@ impl<'tcx> ConstKind<'tcx> {
         param_env: ParamEnv<'tcx>,
         eval_mode: EvalMode,
     ) -> Option<Result<EvalResult<'tcx>, ErrorGuaranteed>> {
+        assert!(!self.has_escaping_bound_vars(), "escaping vars in {self:?}");
         if let ConstKind::Unevaluated(unevaluated) = self {
             use crate::mir::interpret::ErrorHandled;
 
+            assert_eq!(unevaluated.promoted, ());
+
             // HACK(eddyb) this erases lifetimes even though `const_eval_resolve`
             // also does later, but we want to do it before checking for
             // inference variables.
@@ -197,7 +207,7 @@ impl<'tcx> ConstKind<'tcx> {
                 tcx.param_env(unevaluated.def.did).and(ty::Unevaluated {
                     def: unevaluated.def,
                     substs: InternalSubsts::identity_for_item(tcx, unevaluated.def.did),
-                    promoted: unevaluated.promoted,
+                    promoted: (),
                 })
             } else {
                 param_env_and
@@ -221,7 +231,7 @@ impl<'tcx> ConstKind<'tcx> {
                     }
                 }
                 EvalMode::Mir => {
-                    match tcx.const_eval_resolve(param_env, unevaluated, None) {
+                    match tcx.const_eval_resolve(param_env, unevaluated.expand(), None) {
                         // NOTE(eddyb) `val` contains no lifetimes/types/consts,
                         // and we use the original type, so nothing from `substs`
                         // (which may be identity substs, see above),
diff --git a/compiler/rustc_middle/src/ty/consts/valtree.rs b/compiler/rustc_middle/src/ty/consts/valtree.rs
index 93707bb18..a803fca0d 100644
--- a/compiler/rustc_middle/src/ty/consts/valtree.rs
+++ b/compiler/rustc_middle/src/ty/consts/valtree.rs
@@ -18,7 +18,7 @@ use rustc_macros::{HashStable, TyDecodable, TyEncodable};
 /// `ValTree` does not have this problem with representation, as it only contains integers or
 /// lists of (nested) `ValTree`.
 pub enum ValTree<'tcx> {
-    /// ZSTs, integers, `bool`, `char` are represented as scalars.
+    /// integers, `bool`, `char` are represented as scalars.
     /// See the `ScalarInt` documentation for how `ScalarInt` guarantees that equal values
     /// of these types have the same representation.
     Leaf(ScalarInt),
@@ -27,8 +27,11 @@ pub enum ValTree<'tcx> {
     // dont use SliceOrStr for now
     /// The fields of any kind of aggregate. Structs, tuples and arrays are represented by
     /// listing their fields' values in order.
+    ///
     /// Enums are represented by storing their discriminant as a field, followed by all
     /// the fields of the variant.
+    ///
+    /// ZST types are represented as an empty slice.
     Branch(&'tcx [ValTree<'tcx>]),
 }
 
diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index 0a0f45ce1..0b497fa4a 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -22,6 +22,7 @@ use crate::ty::{
     FloatVar, FloatVid, GenericParamDefKind, InferConst, InferTy, IntTy, IntVar, IntVid, List,
     ParamConst, ParamTy, PolyFnSig, Predicate, PredicateKind, PredicateS, ProjectionTy, Region,
     RegionKind, ReprOptions, TraitObjectVisitor, Ty, TyKind, TyS, TyVar, TyVid, TypeAndMut, UintTy,
+    Visibility,
 };
 use rustc_ast as ast;
 use rustc_data_structures::fingerprint::Fingerprint;
@@ -62,7 +63,7 @@ use rustc_span::{Span, DUMMY_SP};
 use rustc_target::abi::{Layout, LayoutS, TargetDataLayout, VariantIdx};
 use rustc_target::spec::abi;
 use rustc_type_ir::sty::TyKind::*;
-use rustc_type_ir::{InternAs, InternIteratorElement, Interner, TypeFlags};
+use rustc_type_ir::{DynKind, InternAs, InternIteratorElement, Interner, TypeFlags};
 
 use std::any::Any;
 use std::borrow::Borrow;
@@ -275,9 +276,6 @@ pub struct CommonTypes<'tcx> {
 }
 
 pub struct CommonLifetimes<'tcx> {
-    /// `ReEmpty` in the root universe.
-    pub re_root_empty: Region<'tcx>,
-
     /// `ReStatic`
     pub re_static: Region<'tcx>,
 
@@ -874,7 +872,7 @@ pub type CanonicalUserTypeAnnotations<'tcx> =
 
 #[derive(Clone, Debug, TyEncodable, TyDecodable, HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub struct CanonicalUserTypeAnnotation<'tcx> {
-    pub user_ty: CanonicalUserType<'tcx>,
+    pub user_ty: Box<CanonicalUserType<'tcx>>,
     pub span: Span,
     pub inferred_ty: Ty<'tcx>,
 }
@@ -986,11 +984,7 @@ impl<'tcx> CommonLifetimes<'tcx> {
             ))
         };
 
-        CommonLifetimes {
-            re_root_empty: mk(ty::ReEmpty(ty::UniverseIndex::ROOT)),
-            re_static: mk(ty::ReStatic),
-            re_erased: mk(ty::ReErased),
-        }
+        CommonLifetimes { re_static: mk(ty::ReStatic), re_erased: mk(ty::ReErased) }
     }
 }
 
@@ -1089,7 +1083,7 @@ pub struct GlobalCtxt<'tcx> {
 
     pub queries: &'tcx dyn query::QueryEngine<'tcx>,
     pub query_caches: query::QueryCaches<'tcx>,
-    query_kinds: &'tcx [DepKindStruct],
+    query_kinds: &'tcx [DepKindStruct<'tcx>],
 
     // Internal caches for metadata decoding. No need to track deps on this.
     pub ty_rcache: Lock<FxHashMap<ty::CReaderCacheKey, Ty<'tcx>>>,
@@ -1246,12 +1240,12 @@ impl<'tcx> TyCtxt<'tcx> {
         dep_graph: DepGraph,
         on_disk_cache: Option<&'tcx dyn OnDiskCache<'tcx>>,
         queries: &'tcx dyn query::QueryEngine<'tcx>,
-        query_kinds: &'tcx [DepKindStruct],
+        query_kinds: &'tcx [DepKindStruct<'tcx>],
         crate_name: &str,
         output_filenames: OutputFilenames,
     ) -> GlobalCtxt<'tcx> {
         let data_layout = TargetDataLayout::parse(&s.target).unwrap_or_else(|err| {
-            s.fatal(&err);
+            s.emit_fatal(err);
         });
         let interners = CtxtInterners::new(arena);
         let common_types = CommonTypes::new(
@@ -1296,7 +1290,7 @@ impl<'tcx> TyCtxt<'tcx> {
         }
     }
 
-    pub(crate) fn query_kind(self, k: DepKind) -> &'tcx DepKindStruct {
+    pub(crate) fn query_kind(self, k: DepKind) -> &'tcx DepKindStruct<'tcx> {
         &self.query_kinds[k as usize]
     }
 
@@ -1498,17 +1492,17 @@ impl<'tcx> TyCtxt<'tcx> {
         // Create a dependency to the crate to be sure we re-execute this when the amount of
         // definitions change.
         self.ensure().hir_crate(());
-        // Leak a read lock once we start iterating on definitions, to prevent adding new onces
+        // Leak a read lock once we start iterating on definitions, to prevent adding new ones
         // while iterating.  If some query needs to add definitions, it should be `ensure`d above.
         let definitions = self.definitions.leak();
         definitions.iter_local_def_id()
     }
 
     pub fn def_path_table(self) -> &'tcx rustc_hir::definitions::DefPathTable {
-        // Create a dependency to the crate to be sure we reexcute this when the amount of
+        // Create a dependency to the crate to be sure we re-execute this when the amount of
         // definitions change.
         self.ensure().hir_crate(());
-        // Leak a read lock once we start iterating on definitions, to prevent adding new onces
+        // Leak a read lock once we start iterating on definitions, to prevent adding new ones
         // while iterating.  If some query needs to add definitions, it should be `ensure`d above.
         let definitions = self.definitions.leak();
         definitions.def_path_table()
@@ -1517,10 +1511,10 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn def_path_hash_to_def_index_map(
         self,
     ) -> &'tcx rustc_hir::def_path_hash_map::DefPathHashMap {
-        // Create a dependency to the crate to be sure we reexcute this when the amount of
+        // Create a dependency to the crate to be sure we re-execute this when the amount of
         // definitions change.
         self.ensure().hir_crate(());
-        // Leak a read lock once we start iterating on definitions, to prevent adding new onces
+        // Leak a read lock once we start iterating on definitions, to prevent adding new ones
         // while iterating.  If some query needs to add definitions, it should be `ensure`d above.
         let definitions = self.definitions.leak();
         definitions.def_path_hash_to_def_index_map()
@@ -1596,7 +1590,7 @@ impl<'tcx> TyCtxt<'tcx> {
         })
     }
 
-    // Returns the `DefId` and the `BoundRegionKind` corresponding to the given region.
+    /// Returns the `DefId` and the `BoundRegionKind` corresponding to the given region.
     pub fn is_suitable_region(self, region: Region<'tcx>) -> Option<FreeRegionInfo> {
         let (suitable_region_binding_scope, bound_region) = match *region {
             ty::ReFree(ref free_region) => {
@@ -1728,6 +1722,11 @@ impl<'tcx> TyCtxt<'tcx> {
             .chain(self.crates(()).iter().copied())
             .flat_map(move |cnum| self.traits_in_crate(cnum).iter().copied())
     }
+
+    #[inline]
+    pub fn local_visibility(self, def_id: LocalDefId) -> Visibility {
+        self.visibility(def_id.to_def_id()).expect_local()
+    }
 }
 
 /// A trait implemented for all `X<'a>` types that can be safely and
@@ -1821,7 +1820,9 @@ nop_list_lift! {bound_variable_kinds; ty::BoundVariableKind => ty::BoundVariable
 // This is the impl for `&'a InternalSubsts<'a>`.
 nop_list_lift! {substs; GenericArg<'a> => GenericArg<'tcx>}
 
-CloneLiftImpls! { for<'tcx> { Constness, traits::WellFormedLoc, } }
+CloneLiftImpls! { for<'tcx> {
+    Constness, traits::WellFormedLoc, ImplPolarity, crate::mir::ReturnConstraint,
+} }
 
 pub mod tls {
     use super::{ptr_eq, GlobalCtxt, TyCtxt};
@@ -1829,9 +1830,9 @@ pub mod tls {
     use crate::dep_graph::TaskDepsRef;
     use crate::ty::query;
     use rustc_data_structures::sync::{self, Lock};
-    use rustc_data_structures::thin_vec::ThinVec;
     use rustc_errors::Diagnostic;
     use std::mem;
+    use thin_vec::ThinVec;
 
     #[cfg(not(parallel_compiler))]
     use std::cell::Cell;
@@ -1857,8 +1858,8 @@ pub mod tls {
         /// This is updated by `JobOwner::start` in `ty::query::plumbing` when executing a query.
         pub diagnostics: Option<&'a Lock<ThinVec<Diagnostic>>>,
 
-        /// Used to prevent layout from recursing too deeply.
-        pub layout_depth: usize,
+        /// Used to prevent queries from calling too deeply.
+        pub query_depth: usize,
 
         /// The current dep graph task. This is used to add dependencies to queries
         /// when executing them.
@@ -1872,7 +1873,7 @@ pub mod tls {
                 tcx,
                 query: None,
                 diagnostics: None,
-                layout_depth: 0,
+                query_depth: 0,
                 task_deps: TaskDepsRef::Ignore,
             }
         }
@@ -2546,8 +2547,9 @@ impl<'tcx> TyCtxt<'tcx> {
         self,
         obj: &'tcx List<ty::Binder<'tcx, ExistentialPredicate<'tcx>>>,
         reg: ty::Region<'tcx>,
+        repr: DynKind,
     ) -> Ty<'tcx> {
-        self.mk_ty(Dynamic(obj, reg))
+        self.mk_ty(Dynamic(obj, reg, repr))
     }
 
     #[inline]
diff --git a/compiler/rustc_middle/src/ty/diagnostics.rs b/compiler/rustc_middle/src/ty/diagnostics.rs
index dd2f43210..855917fb8 100644
--- a/compiler/rustc_middle/src/ty/diagnostics.rs
+++ b/compiler/rustc_middle/src/ty/diagnostics.rs
@@ -102,13 +102,25 @@ pub fn suggest_arbitrary_trait_bound<'tcx>(
     generics: &hir::Generics<'_>,
     err: &mut Diagnostic,
     trait_pred: PolyTraitPredicate<'tcx>,
+    associated_ty: Option<(&'static str, Ty<'tcx>)>,
 ) -> bool {
     if !trait_pred.is_suggestable(tcx, false) {
         return false;
     }
 
     let param_name = trait_pred.skip_binder().self_ty().to_string();
-    let constraint = trait_pred.print_modifiers_and_trait_path().to_string();
+    let mut constraint = trait_pred.print_modifiers_and_trait_path().to_string();
+
+    if let Some((name, term)) = associated_ty {
+        // FIXME: this case overlaps with code in TyCtxt::note_and_explain_type_err.
+        // That should be extracted into a helper function.
+        if constraint.ends_with('>') {
+            constraint = format!("{}, {} = {}>", &constraint[..constraint.len() - 1], name, term);
+        } else {
+            constraint.push_str(&format!("<{} = {}>", name, term));
+        }
+    }
+
     let param = generics.params.iter().find(|p| p.name.ident().as_str() == param_name);
 
     // Skip, there is a param named Self
@@ -396,7 +408,7 @@ impl<'v> hir::intravisit::Visitor<'v> for TraitObjectVisitor<'v> {
             ) => {
                 self.0.push(ty);
             }
-            hir::TyKind::OpaqueDef(item_id, _) => {
+            hir::TyKind::OpaqueDef(item_id, _, _) => {
                 self.0.push(ty);
                 let item = self.1.item(item_id);
                 hir::intravisit::walk_item(self, item);
@@ -455,7 +467,7 @@ impl<'tcx> TypeVisitor<'tcx> for IsSuggestableVisitor<'tcx> {
                 }
             }
 
-            Dynamic(dty, _) => {
+            Dynamic(dty, _, _) => {
                 for pred in *dty {
                     match pred.skip_binder() {
                         ExistentialPredicate::Trait(_) | ExistentialPredicate::Projection(_) => {
diff --git a/compiler/rustc_middle/src/ty/error.rs b/compiler/rustc_middle/src/ty/error.rs
index 4b0bc3c11..01e1e97b2 100644
--- a/compiler/rustc_middle/src/ty/error.rs
+++ b/compiler/rustc_middle/src/ty/error.rs
@@ -2,6 +2,7 @@ use crate::traits::{ObligationCause, ObligationCauseCode};
 use crate::ty::diagnostics::suggest_constraining_type_param;
 use crate::ty::print::{FmtPrinter, Printer};
 use crate::ty::{self, BoundRegionKind, Region, Ty, TyCtxt};
+use hir::def::DefKind;
 use rustc_errors::Applicability::{MachineApplicable, MaybeIncorrect};
 use rustc_errors::{pluralize, Diagnostic, MultiSpan};
 use rustc_hir as hir;
@@ -13,7 +14,7 @@ use rustc_target::spec::abi;
 use std::borrow::Cow;
 use std::fmt;
 
-#[derive(Clone, Copy, Debug, PartialEq, Eq, TypeFoldable, TypeVisitable)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, TypeFoldable, TypeVisitable, Lift)]
 pub struct ExpectedFound<T> {
     pub expected: T,
     pub found: T,
@@ -30,7 +31,8 @@ impl<T> ExpectedFound<T> {
 }
 
 // Data structures used in type unification
-#[derive(Clone, Debug, TypeFoldable, TypeVisitable)]
+#[derive(Copy, Clone, Debug, TypeFoldable, TypeVisitable, Lift)]
+#[rustc_pass_by_value]
 pub enum TypeError<'tcx> {
     Mismatch,
     ConstnessMismatch(ExpectedFound<ty::BoundConstness>),
@@ -73,6 +75,18 @@ pub enum TypeError<'tcx> {
     TargetFeatureCast(DefId),
 }
 
+impl TypeError<'_> {
+    pub fn involves_regions(self) -> bool {
+        match self {
+            TypeError::RegionsDoesNotOutlive(_, _)
+            | TypeError::RegionsInsufficientlyPolymorphic(_, _)
+            | TypeError::RegionsOverlyPolymorphic(_, _)
+            | TypeError::RegionsPlaceholderMismatch => true,
+            _ => false,
+        }
+    }
+}
+
 /// Explains the source of a type err in a short, human readable way. This is meant to be placed
 /// in parentheses after some larger message. You should also invoke `note_and_explain_type_err()`
 /// afterwards to present additional details, particularly when it comes to lifetime-related
@@ -211,7 +225,7 @@ impl<'tcx> fmt::Display for TypeError<'tcx> {
 }
 
 impl<'tcx> TypeError<'tcx> {
-    pub fn must_include_note(&self) -> bool {
+    pub fn must_include_note(self) -> bool {
         use self::TypeError::*;
         match self {
             CyclicTy(_) | CyclicConst(_) | UnsafetyMismatch(_) | ConstnessMismatch(_)
@@ -263,10 +277,23 @@ impl<'tcx> Ty<'tcx> {
             }
             ty::Slice(ty) if ty.is_simple_ty() => format!("slice `{}`", self).into(),
             ty::Slice(_) => "slice".into(),
-            ty::RawPtr(_) => "*-ptr".into(),
+            ty::RawPtr(tymut) => {
+                let tymut_string = match tymut.mutbl {
+                    hir::Mutability::Mut => tymut.to_string(),
+                    hir::Mutability::Not => format!("const {}", tymut.ty),
+                };
+
+                if tymut_string != "_" && (tymut.ty.is_simple_text() || tymut_string.len() < "const raw pointer".len()) {
+                    format!("`*{}`", tymut_string).into()
+                } else {
+                    // Unknown type name, it's long or has type arguments
+                    "raw pointer".into()
+                }
+            },
             ty::Ref(_, ty, mutbl) => {
                 let tymut = ty::TypeAndMut { ty, mutbl };
                 let tymut_string = tymut.to_string();
+
                 if tymut_string != "_"
                     && (ty.is_simple_text() || tymut_string.len() < "mutable reference".len())
                 {
@@ -347,7 +374,7 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn note_and_explain_type_err(
         self,
         diag: &mut Diagnostic,
-        err: &TypeError<'tcx>,
+        err: TypeError<'tcx>,
         cause: &ObligationCause<'tcx>,
         sp: Span,
         body_owner_def_id: DefId,
@@ -512,7 +539,7 @@ impl<T> Trait<T> for X {
                             diag.span_label(p_span, "this type parameter");
                         }
                     }
-                    (ty::Projection(proj_ty), _) => {
+                    (ty::Projection(proj_ty), _) if self.def_kind(proj_ty.item_def_id) != DefKind::ImplTraitPlaceholder => {
                         self.expected_projection(
                             diag,
                             proj_ty,
@@ -521,7 +548,7 @@ impl<T> Trait<T> for X {
                             cause.code(),
                         );
                     }
-                    (_, ty::Projection(proj_ty)) => {
+                    (_, ty::Projection(proj_ty)) if self.def_kind(proj_ty.item_def_id) != DefKind::ImplTraitPlaceholder => {
                         let msg = format!(
                             "consider constraining the associated type `{}` to `{}`",
                             values.found, values.expected,
@@ -568,7 +595,7 @@ impl<T> Trait<T> for X {
             }
             TargetFeatureCast(def_id) => {
                 let target_spans =
-                    self.get_attrs(*def_id, sym::target_feature).map(|attr| attr.span);
+                    self.get_attrs(def_id, sym::target_feature).map(|attr| attr.span);
                 diag.note(
                     "functions with `#[target_feature]` can only be coerced to `unsafe` function pointers"
                 );
@@ -640,7 +667,7 @@ impl<T> Trait<T> for X {
         self,
         diag: &mut Diagnostic,
         proj_ty: &ty::ProjectionTy<'tcx>,
-        values: &ExpectedFound<Ty<'tcx>>,
+        values: ExpectedFound<Ty<'tcx>>,
         body_owner_def_id: DefId,
         cause_code: &ObligationCauseCode<'_>,
     ) {
diff --git a/compiler/rustc_middle/src/ty/flags.rs b/compiler/rustc_middle/src/ty/flags.rs
index ea6bb8a7a..98b8a7386 100644
--- a/compiler/rustc_middle/src/ty/flags.rs
+++ b/compiler/rustc_middle/src/ty/flags.rs
@@ -1,5 +1,5 @@
 use crate::ty::subst::{GenericArg, GenericArgKind};
-use crate::ty::{self, InferConst, Term, Ty, TypeFlags};
+use crate::ty::{self, InferConst, Ty, TypeFlags};
 use std::slice;
 
 #[derive(Debug)]
@@ -171,7 +171,7 @@ impl FlagComputation {
                 self.add_substs(substs);
             }
 
-            &ty::Dynamic(obj, r) => {
+            &ty::Dynamic(obj, r, _) => {
                 for predicate in obj.iter() {
                     self.bound_computation(predicate, |computation, predicate| match predicate {
                         ty::ExistentialPredicate::Trait(tr) => computation.add_substs(tr.substs),
@@ -243,9 +243,9 @@ impl FlagComputation {
             }
             ty::PredicateKind::Projection(ty::ProjectionPredicate { projection_ty, term }) => {
                 self.add_projection_ty(projection_ty);
-                match term {
-                    Term::Ty(ty) => self.add_ty(ty),
-                    Term::Const(c) => self.add_const(c),
+                match term.unpack() {
+                    ty::TermKind::Ty(ty) => self.add_ty(ty),
+                    ty::TermKind::Const(c) => self.add_const(c),
                 }
             }
             ty::PredicateKind::WellFormed(arg) => {
@@ -320,9 +320,9 @@ impl FlagComputation {
 
     fn add_existential_projection(&mut self, projection: &ty::ExistentialProjection<'_>) {
         self.add_substs(projection.substs);
-        match projection.term {
-            ty::Term::Ty(ty) => self.add_ty(ty),
-            ty::Term::Const(ct) => self.add_const(ct),
+        match projection.term.unpack() {
+            ty::TermKind::Ty(ty) => self.add_ty(ty),
+            ty::TermKind::Const(ct) => self.add_const(ct),
         }
     }
 
diff --git a/compiler/rustc_middle/src/ty/fold.rs b/compiler/rustc_middle/src/ty/fold.rs
index 5e96e278b..cac95e14a 100644
--- a/compiler/rustc_middle/src/ty/fold.rs
+++ b/compiler/rustc_middle/src/ty/fold.rs
@@ -302,6 +302,17 @@ impl<'tcx> TyCtxt<'tcx> {
     {
         value.fold_with(&mut RegionFolder::new(self, &mut f))
     }
+
+    pub fn super_fold_regions<T>(
+        self,
+        value: T,
+        mut f: impl FnMut(ty::Region<'tcx>, ty::DebruijnIndex) -> ty::Region<'tcx>,
+    ) -> T
+    where
+        T: TypeSuperFoldable<'tcx>,
+    {
+        value.super_fold_with(&mut RegionFolder::new(self, &mut f))
+    }
 }
 
 /// Folds over the substructure of a type, visiting its component
@@ -353,7 +364,7 @@ impl<'a, 'tcx> TypeFolder<'tcx> for RegionFolder<'a, 'tcx> {
         t
     }
 
-    #[instrument(skip(self), level = "debug")]
+    #[instrument(skip(self), level = "debug", ret)]
     fn fold_region(&mut self, r: ty::Region<'tcx>) -> ty::Region<'tcx> {
         match *r {
             ty::ReLateBound(debruijn, _) if debruijn < self.current_index => {
@@ -377,17 +388,13 @@ pub trait BoundVarReplacerDelegate<'tcx> {
     fn replace_const(&mut self, bv: ty::BoundVar, ty: Ty<'tcx>) -> ty::Const<'tcx>;
 }
 
-pub struct FnMutDelegate<R, T, C> {
-    pub regions: R,
-    pub types: T,
-    pub consts: C,
+pub struct FnMutDelegate<'a, 'tcx> {
+    pub regions: &'a mut (dyn FnMut(ty::BoundRegion) -> ty::Region<'tcx> + 'a),
+    pub types: &'a mut (dyn FnMut(ty::BoundTy) -> Ty<'tcx> + 'a),
+    pub consts: &'a mut (dyn FnMut(ty::BoundVar, Ty<'tcx>) -> ty::Const<'tcx> + 'a),
 }
-impl<'tcx, R, T, C> BoundVarReplacerDelegate<'tcx> for FnMutDelegate<R, T, C>
-where
-    R: FnMut(ty::BoundRegion) -> ty::Region<'tcx>,
-    T: FnMut(ty::BoundTy) -> Ty<'tcx>,
-    C: FnMut(ty::BoundVar, Ty<'tcx>) -> ty::Const<'tcx>,
-{
+
+impl<'a, 'tcx> BoundVarReplacerDelegate<'tcx> for FnMutDelegate<'a, 'tcx> {
     fn replace_region(&mut self, br: ty::BoundRegion) -> ty::Region<'tcx> {
         (self.regions)(br)
     }
@@ -511,7 +518,7 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn replace_late_bound_regions_uncached<T, F>(
         self,
         value: Binder<'tcx, T>,
-        replace_regions: F,
+        mut replace_regions: F,
     ) -> T
     where
         F: FnMut(ty::BoundRegion) -> ty::Region<'tcx>,
@@ -522,9 +529,9 @@ impl<'tcx> TyCtxt<'tcx> {
             value
         } else {
             let delegate = FnMutDelegate {
-                regions: replace_regions,
-                types: |b| bug!("unexpected bound ty in binder: {b:?}"),
-                consts: |b, ty| bug!("unexpected bound ct in binder: {b:?} {ty}"),
+                regions: &mut replace_regions,
+                types: &mut |b| bug!("unexpected bound ty in binder: {b:?}"),
+                consts: &mut |b, ty| bug!("unexpected bound ct in binder: {b:?} {ty}"),
             };
             let mut replacer = BoundVarReplacer::new(self, delegate);
             value.fold_with(&mut replacer)
@@ -584,19 +591,19 @@ impl<'tcx> TyCtxt<'tcx> {
         self.replace_escaping_bound_vars_uncached(
             value,
             FnMutDelegate {
-                regions: |r: ty::BoundRegion| {
+                regions: &mut |r: ty::BoundRegion| {
                     self.mk_region(ty::ReLateBound(
                         ty::INNERMOST,
                         ty::BoundRegion { var: shift_bv(r.var), kind: r.kind },
                     ))
                 },
-                types: |t: ty::BoundTy| {
+                types: &mut |t: ty::BoundTy| {
                     self.mk_ty(ty::Bound(
                         ty::INNERMOST,
                         ty::BoundTy { var: shift_bv(t.var), kind: t.kind },
                     ))
                 },
-                consts: |c, ty: Ty<'tcx>| {
+                consts: &mut |c, ty: Ty<'tcx>| {
                     self.mk_const(ty::ConstS {
                         kind: ty::ConstKind::Bound(ty::INNERMOST, shift_bv(c)),
                         ty,
diff --git a/compiler/rustc_middle/src/ty/generics.rs b/compiler/rustc_middle/src/ty/generics.rs
index add2df258..0c8bdde9c 100644
--- a/compiler/rustc_middle/src/ty/generics.rs
+++ b/compiler/rustc_middle/src/ty/generics.rs
@@ -1,4 +1,3 @@
-use crate::middle::resolve_lifetime::ObjectLifetimeDefault;
 use crate::ty;
 use crate::ty::subst::{Subst, SubstsRef};
 use crate::ty::EarlyBinder;
@@ -13,7 +12,7 @@ use super::{EarlyBoundRegion, InstantiatedPredicates, ParamConst, ParamTy, Predi
 #[derive(Clone, Debug, TyEncodable, TyDecodable, HashStable)]
 pub enum GenericParamDefKind {
     Lifetime,
-    Type { has_default: bool, object_lifetime_default: ObjectLifetimeDefault, synthetic: bool },
+    Type { has_default: bool, synthetic: bool },
     Const { has_default: bool },
 }
 
@@ -28,8 +27,9 @@ impl GenericParamDefKind {
     pub fn to_ord(&self) -> ast::ParamKindOrd {
         match self {
             GenericParamDefKind::Lifetime => ast::ParamKindOrd::Lifetime,
-            GenericParamDefKind::Type { .. } => ast::ParamKindOrd::Type,
-            GenericParamDefKind::Const { .. } => ast::ParamKindOrd::Const,
+            GenericParamDefKind::Type { .. } | GenericParamDefKind::Const { .. } => {
+                ast::ParamKindOrd::TypeOrConst
+            }
         }
     }
 
@@ -122,6 +122,21 @@ pub struct Generics {
 }
 
 impl<'tcx> Generics {
+    /// Looks through the generics and all parents to find the index of the
+    /// given param def-id. This is in comparison to the `param_def_id_to_index`
+    /// struct member, which only stores information about this item's own
+    /// generics.
+    pub fn param_def_id_to_index(&self, tcx: TyCtxt<'tcx>, def_id: DefId) -> Option<u32> {
+        if let Some(idx) = self.param_def_id_to_index.get(&def_id) {
+            Some(*idx)
+        } else if let Some(parent) = self.parent {
+            let parent = tcx.generics_of(parent);
+            parent.param_def_id_to_index(tcx, def_id)
+        } else {
+            None
+        }
+    }
+
     #[inline]
     pub fn count(&self) -> usize {
         self.parent_count + self.params.len()
@@ -252,7 +267,7 @@ impl<'tcx> Generics {
         // Filter the default arguments.
         //
         // This currently uses structural equality instead
-        // of semantic equivalance. While not ideal, that's
+        // of semantic equivalence. While not ideal, that's
         // good enough for now as this should only be used
         // for diagnostics anyways.
         own_params.end -= self
@@ -314,6 +329,7 @@ impl<'tcx> GenericPredicates<'tcx> {
         }
     }
 
+    #[instrument(level = "debug", skip(self, tcx))]
     fn instantiate_into(
         &self,
         tcx: TyCtxt<'tcx>,
diff --git a/compiler/rustc_middle/src/ty/impls_ty.rs b/compiler/rustc_middle/src/ty/impls_ty.rs
index cd00b26b8..d1c0d62ac 100644
--- a/compiler/rustc_middle/src/ty/impls_ty.rs
+++ b/compiler/rustc_middle/src/ty/impls_ty.rs
@@ -113,7 +113,7 @@ impl<'a> HashStable<StableHashingContext<'a>> for mir::interpret::AllocId {
 }
 
 // `Relocations` with default type parameters is a sorted map.
-impl<'a, Prov> HashStable<StableHashingContext<'a>> for mir::interpret::Relocations<Prov>
+impl<'a, Prov> HashStable<StableHashingContext<'a>> for mir::interpret::ProvenanceMap<Prov>
 where
     Prov: HashStable<StableHashingContext<'a>>,
 {
diff --git a/compiler/rustc_middle/src/ty/inhabitedness/mod.rs b/compiler/rustc_middle/src/ty/inhabitedness/mod.rs
index 3d22f5a04..aaa66deb2 100644
--- a/compiler/rustc_middle/src/ty/inhabitedness/mod.rs
+++ b/compiler/rustc_middle/src/ty/inhabitedness/mod.rs
@@ -169,14 +169,10 @@ impl<'tcx> FieldDef {
         param_env: ty::ParamEnv<'tcx>,
     ) -> DefIdForest<'tcx> {
         let data_uninhabitedness = move || self.ty(tcx, substs).uninhabited_from(tcx, param_env);
-        // FIXME(canndrew): Currently enum fields are (incorrectly) stored with
-        // `Visibility::Invisible` so we need to override `self.vis` if we're
-        // dealing with an enum.
         if is_enum {
             data_uninhabitedness()
         } else {
             match self.vis {
-                Visibility::Invisible => DefIdForest::empty(),
                 Visibility::Restricted(from) => {
                     let forest = DefIdForest::from_id(from);
                     let iter = Some(forest).into_iter().chain(Some(data_uninhabitedness()));
diff --git a/compiler/rustc_middle/src/ty/instance.rs b/compiler/rustc_middle/src/ty/instance.rs
index 53218225d..9afd66207 100644
--- a/compiler/rustc_middle/src/ty/instance.rs
+++ b/compiler/rustc_middle/src/ty/instance.rs
@@ -20,14 +20,14 @@ use std::fmt;
 /// simply couples a potentially generic `InstanceDef` with some substs, and codegen and const eval
 /// will do all required substitution as they run.
 #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, TyEncodable, TyDecodable)]
-#[derive(HashStable, Lift)]
+#[derive(HashStable, Lift, TypeFoldable, TypeVisitable)]
 pub struct Instance<'tcx> {
     pub def: InstanceDef<'tcx>,
     pub substs: SubstsRef<'tcx>,
 }
 
 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
-#[derive(TyEncodable, TyDecodable, HashStable, TypeFoldable, TypeVisitable)]
+#[derive(TyEncodable, TyDecodable, HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub enum InstanceDef<'tcx> {
     /// A user-defined callable item.
     ///
diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs
index ad78d24e9..042eeec3f 100644
--- a/compiler/rustc_middle/src/ty/layout.rs
+++ b/compiler/rustc_middle/src/ty/layout.rs
@@ -2,7 +2,10 @@ use crate::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use crate::mir::{GeneratorLayout, GeneratorSavedLocal};
 use crate::ty::normalize_erasing_regions::NormalizationError;
 use crate::ty::subst::Subst;
-use crate::ty::{self, subst::SubstsRef, EarlyBinder, ReprOptions, Ty, TyCtxt, TypeVisitable};
+use crate::ty::{
+    self, layout_sanity_check::sanity_check_layout, subst::SubstsRef, EarlyBinder, ReprOptions, Ty,
+    TyCtxt, TypeVisitable,
+};
 use rustc_ast as ast;
 use rustc_attr as attr;
 use rustc_hir as hir;
@@ -19,7 +22,7 @@ use rustc_target::abi::call::{
 use rustc_target::abi::*;
 use rustc_target::spec::{abi::Abi as SpecAbi, HasTargetSpec, PanicStrategy, Target};
 
-use std::cmp;
+use std::cmp::{self, Ordering};
 use std::fmt;
 use std::iter;
 use std::num::NonZeroUsize;
@@ -221,164 +224,46 @@ impl<'tcx> fmt::Display for LayoutError<'tcx> {
     }
 }
 
-/// Enforce some basic invariants on layouts.
-fn sanity_check_layout<'tcx>(
-    tcx: TyCtxt<'tcx>,
-    param_env: ty::ParamEnv<'tcx>,
-    layout: &TyAndLayout<'tcx>,
-) {
-    // Type-level uninhabitedness should always imply ABI uninhabitedness.
-    if tcx.conservative_is_privately_uninhabited(param_env.and(layout.ty)) {
-        assert!(layout.abi.is_uninhabited());
-    }
-
-    if layout.size.bytes() % layout.align.abi.bytes() != 0 {
-        bug!("size is not a multiple of align, in the following layout:\n{layout:#?}");
-    }
-
-    if cfg!(debug_assertions) {
-        fn check_layout_abi<'tcx>(tcx: TyCtxt<'tcx>, layout: Layout<'tcx>) {
-            match layout.abi() {
-                Abi::Scalar(scalar) => {
-                    // No padding in scalars.
-                    assert_eq!(
-                        layout.align().abi,
-                        scalar.align(&tcx).abi,
-                        "alignment mismatch between ABI and layout in {layout:#?}"
-                    );
-                    assert_eq!(
-                        layout.size(),
-                        scalar.size(&tcx),
-                        "size mismatch between ABI and layout in {layout:#?}"
-                    );
-                }
-                Abi::Vector { count, element } => {
-                    // No padding in vectors. Alignment can be strengthened, though.
-                    assert!(
-                        layout.align().abi >= element.align(&tcx).abi,
-                        "alignment mismatch between ABI and layout in {layout:#?}"
-                    );
-                    let size = element.size(&tcx) * count;
-                    assert_eq!(
-                        layout.size(),
-                        size.align_to(tcx.data_layout().vector_align(size).abi),
-                        "size mismatch between ABI and layout in {layout:#?}"
-                    );
-                }
-                Abi::ScalarPair(scalar1, scalar2) => {
-                    // Sanity-check scalar pairs. These are a bit more flexible and support
-                    // padding, but we can at least ensure both fields actually fit into the layout
-                    // and the alignment requirement has not been weakened.
-                    let align1 = scalar1.align(&tcx).abi;
-                    let align2 = scalar2.align(&tcx).abi;
-                    assert!(
-                        layout.align().abi >= cmp::max(align1, align2),
-                        "alignment mismatch between ABI and layout in {layout:#?}",
-                    );
-                    let field2_offset = scalar1.size(&tcx).align_to(align2);
-                    assert!(
-                        layout.size() >= field2_offset + scalar2.size(&tcx),
-                        "size mismatch between ABI and layout in {layout:#?}"
-                    );
-                }
-                Abi::Uninhabited | Abi::Aggregate { .. } => {} // Nothing to check.
-            }
-        }
-
-        check_layout_abi(tcx, layout.layout);
-
-        if let Variants::Multiple { variants, .. } = &layout.variants {
-            for variant in variants {
-                check_layout_abi(tcx, *variant);
-                // No nested "multiple".
-                assert!(matches!(variant.variants(), Variants::Single { .. }));
-                // Skip empty variants.
-                if variant.size() == Size::ZERO
-                    || variant.fields().count() == 0
-                    || variant.abi().is_uninhabited()
-                {
-                    // These are never actually accessed anyway, so we can skip them. (Note that
-                    // sometimes, variants with fields have size 0, and sometimes, variants without
-                    // fields have non-0 size.)
-                    continue;
-                }
-                // Variants should have the same or a smaller size as the full thing.
-                if variant.size() > layout.size {
-                    bug!(
-                        "Type with size {} bytes has variant with size {} bytes: {layout:#?}",
-                        layout.size.bytes(),
-                        variant.size().bytes(),
-                    )
-                }
-                // The top-level ABI and the ABI of the variants should be coherent.
-                let abi_coherent = match (layout.abi, variant.abi()) {
-                    (Abi::Scalar(..), Abi::Scalar(..)) => true,
-                    (Abi::ScalarPair(..), Abi::ScalarPair(..)) => true,
-                    (Abi::Uninhabited, _) => true,
-                    (Abi::Aggregate { .. }, _) => true,
-                    _ => false,
-                };
-                if !abi_coherent {
-                    bug!(
-                        "Variant ABI is incompatible with top-level ABI:\nvariant={:#?}\nTop-level: {layout:#?}",
-                        variant
-                    );
-                }
-            }
-        }
-    }
-}
-
 #[instrument(skip(tcx, query), level = "debug")]
 fn layout_of<'tcx>(
     tcx: TyCtxt<'tcx>,
     query: ty::ParamEnvAnd<'tcx, Ty<'tcx>>,
 ) -> Result<TyAndLayout<'tcx>, LayoutError<'tcx>> {
-    ty::tls::with_related_context(tcx, move |icx| {
-        let (param_env, ty) = query.into_parts();
-        debug!(?ty);
-
-        if !tcx.recursion_limit().value_within_limit(icx.layout_depth) {
-            tcx.sess.fatal(&format!("overflow representing the type `{}`", ty));
+    let (param_env, ty) = query.into_parts();
+    debug!(?ty);
+
+    let param_env = param_env.with_reveal_all_normalized(tcx);
+    let unnormalized_ty = ty;
+
+    // FIXME: We might want to have two different versions of `layout_of`:
+    // One that can be called after typecheck has completed and can use
+    // `normalize_erasing_regions` here and another one that can be called
+    // before typecheck has completed and uses `try_normalize_erasing_regions`.
+    let ty = match tcx.try_normalize_erasing_regions(param_env, ty) {
+        Ok(t) => t,
+        Err(normalization_error) => {
+            return Err(LayoutError::NormalizationFailure(ty, normalization_error));
         }
+    };
 
-        // Update the ImplicitCtxt to increase the layout_depth
-        let icx = ty::tls::ImplicitCtxt { layout_depth: icx.layout_depth + 1, ..icx.clone() };
-
-        ty::tls::enter_context(&icx, |_| {
-            let param_env = param_env.with_reveal_all_normalized(tcx);
-            let unnormalized_ty = ty;
-
-            // FIXME: We might want to have two different versions of `layout_of`:
-            // One that can be called after typecheck has completed and can use
-            // `normalize_erasing_regions` here and another one that can be called
-            // before typecheck has completed and uses `try_normalize_erasing_regions`.
-            let ty = match tcx.try_normalize_erasing_regions(param_env, ty) {
-                Ok(t) => t,
-                Err(normalization_error) => {
-                    return Err(LayoutError::NormalizationFailure(ty, normalization_error));
-                }
-            };
-
-            if ty != unnormalized_ty {
-                // Ensure this layout is also cached for the normalized type.
-                return tcx.layout_of(param_env.and(ty));
-            }
+    if ty != unnormalized_ty {
+        // Ensure this layout is also cached for the normalized type.
+        return tcx.layout_of(param_env.and(ty));
+    }
 
-            let cx = LayoutCx { tcx, param_env };
+    let cx = LayoutCx { tcx, param_env };
 
-            let layout = cx.layout_of_uncached(ty)?;
-            let layout = TyAndLayout { ty, layout };
+    let layout = cx.layout_of_uncached(ty)?;
+    let layout = TyAndLayout { ty, layout };
 
-            cx.record_layout_for_printing(layout);
+    cx.record_layout_for_printing(layout);
 
-            sanity_check_layout(tcx, param_env, &layout);
+    sanity_check_layout(&cx, &layout);
 
-            Ok(layout)
-        })
-    })
+    Ok(layout)
 }
 
+#[derive(Clone, Copy)]
 pub struct LayoutCx<'tcx, C> {
     pub tcx: C,
     pub param_env: ty::ParamEnv<'tcx>,
@@ -740,6 +625,14 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
                 tcx.intern_layout(self.scalar_pair(data_ptr, metadata))
             }
 
+            ty::Dynamic(_, _, ty::DynStar) => {
+                let mut data = scalar_unit(Int(dl.ptr_sized_integer(), false));
+                data.valid_range_mut().start = 0;
+                let mut vtable = scalar_unit(Pointer);
+                vtable.valid_range_mut().start = 1;
+                tcx.intern_layout(self.scalar_pair(data, vtable))
+            }
+
             // Arrays and slices.
             ty::Array(element, mut count) => {
                 if count.has_projections() {
@@ -794,7 +687,7 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
 
             // Odd unit types.
             ty::FnDef(..) => univariant(&[], &ReprOptions::default(), StructKind::AlwaysSized)?,
-            ty::Dynamic(..) | ty::Foreign(..) => {
+            ty::Dynamic(_, _, ty::Dyn) | ty::Foreign(..) => {
                 let mut unit = self.univariant_uninterned(
                     ty,
                     &[],
@@ -872,7 +765,7 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
                 // * the element type and length of the single array field, if
                 // the first field is of array type, or
                 //
-                // * the homogenous field type and the number of fields.
+                // * the homogeneous field type and the number of fields.
                 let (e_ty, e_len, is_array) = if let ty::Array(e_ty, _) = f0_ty.kind() {
                     // First ADT field is an array:
 
@@ -1161,131 +1054,191 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
                 // that allow representation optimization.)
                 assert!(def.is_enum());
 
-                // The current code for niche-filling relies on variant indices
-                // instead of actual discriminants, so dataful enums with
-                // explicit discriminants (RFC #2363) would misbehave.
-                let no_explicit_discriminants = def
-                    .variants()
-                    .iter_enumerated()
-                    .all(|(i, v)| v.discr == ty::VariantDiscr::Relative(i.as_u32()));
-
-                let mut niche_filling_layout = None;
-
-                // Niche-filling enum optimization.
-                if !def.repr().inhibit_enum_layout_opt() && no_explicit_discriminants {
-                    let mut dataful_variant = None;
-                    let mut niche_variants = VariantIdx::MAX..=VariantIdx::new(0);
+                // Until we've decided whether to use the tagged or
+                // niche filling LayoutS, we don't want to intern the
+                // variant layouts, so we can't store them in the
+                // overall LayoutS. Store the overall LayoutS
+                // and the variant LayoutSs here until then.
+                struct TmpLayout<'tcx> {
+                    layout: LayoutS<'tcx>,
+                    variants: IndexVec<VariantIdx, LayoutS<'tcx>>,
+                }
 
-                    // Find one non-ZST variant.
-                    'variants: for (v, fields) in variants.iter_enumerated() {
-                        if absent(fields) {
-                            continue 'variants;
+                let calculate_niche_filling_layout =
+                    || -> Result<Option<TmpLayout<'tcx>>, LayoutError<'tcx>> {
+                        // The current code for niche-filling relies on variant indices
+                        // instead of actual discriminants, so enums with
+                        // explicit discriminants (RFC #2363) would misbehave.
+                        if def.repr().inhibit_enum_layout_opt()
+                            || def
+                                .variants()
+                                .iter_enumerated()
+                                .any(|(i, v)| v.discr != ty::VariantDiscr::Relative(i.as_u32()))
+                        {
+                            return Ok(None);
                         }
-                        for f in fields {
-                            if !f.is_zst() {
-                                if dataful_variant.is_none() {
-                                    dataful_variant = Some(v);
-                                    continue 'variants;
-                                } else {
-                                    dataful_variant = None;
-                                    break 'variants;
-                                }
-                            }
+
+                        if variants.len() < 2 {
+                            return Ok(None);
                         }
-                        niche_variants = *niche_variants.start().min(&v)..=v;
-                    }
 
-                    if niche_variants.start() > niche_variants.end() {
-                        dataful_variant = None;
-                    }
+                        let mut align = dl.aggregate_align;
+                        let mut variant_layouts = variants
+                            .iter_enumerated()
+                            .map(|(j, v)| {
+                                let mut st = self.univariant_uninterned(
+                                    ty,
+                                    v,
+                                    &def.repr(),
+                                    StructKind::AlwaysSized,
+                                )?;
+                                st.variants = Variants::Single { index: j };
+
+                                align = align.max(st.align);
+
+                                Ok(st)
+                            })
+                            .collect::<Result<IndexVec<VariantIdx, _>, _>>()?;
+
+                        let largest_variant_index = match variant_layouts
+                            .iter_enumerated()
+                            .max_by_key(|(_i, layout)| layout.size.bytes())
+                            .map(|(i, _layout)| i)
+                        {
+                            None => return Ok(None),
+                            Some(i) => i,
+                        };
+
+                        let all_indices = VariantIdx::new(0)..=VariantIdx::new(variants.len() - 1);
+                        let needs_disc = |index: VariantIdx| {
+                            index != largest_variant_index && !absent(&variants[index])
+                        };
+                        let niche_variants = all_indices.clone().find(|v| needs_disc(*v)).unwrap()
+                            ..=all_indices.rev().find(|v| needs_disc(*v)).unwrap();
 
-                    if let Some(i) = dataful_variant {
-                        let count = (niche_variants.end().as_u32()
-                            - niche_variants.start().as_u32()
-                            + 1) as u128;
+                        let count = niche_variants.size_hint().1.unwrap() as u128;
 
                         // Find the field with the largest niche
-                        let niche_candidate = variants[i]
+                        let (field_index, niche, (niche_start, niche_scalar)) = match variants
+                            [largest_variant_index]
                             .iter()
                             .enumerate()
                             .filter_map(|(j, field)| Some((j, field.largest_niche?)))
-                            .max_by_key(|(_, niche)| niche.available(dl));
-
-                        if let Some((field_index, niche, (niche_start, niche_scalar))) =
-                            niche_candidate.and_then(|(field_index, niche)| {
-                                Some((field_index, niche, niche.reserve(self, count)?))
-                            })
+                            .max_by_key(|(_, niche)| niche.available(dl))
+                            .and_then(|(j, niche)| Some((j, niche, niche.reserve(self, count)?)))
                         {
-                            let mut align = dl.aggregate_align;
-                            let st = variants
-                                .iter_enumerated()
-                                .map(|(j, v)| {
-                                    let mut st = self.univariant_uninterned(
-                                        ty,
-                                        v,
-                                        &def.repr(),
-                                        StructKind::AlwaysSized,
-                                    )?;
-                                    st.variants = Variants::Single { index: j };
+                            None => return Ok(None),
+                            Some(x) => x,
+                        };
+
+                        let niche_offset = niche.offset
+                            + variant_layouts[largest_variant_index].fields.offset(field_index);
+                        let niche_size = niche.value.size(dl);
+                        let size = variant_layouts[largest_variant_index].size.align_to(align.abi);
 
-                                    align = align.max(st.align);
+                        let all_variants_fit =
+                            variant_layouts.iter_enumerated_mut().all(|(i, layout)| {
+                                if i == largest_variant_index {
+                                    return true;
+                                }
 
-                                    Ok(tcx.intern_layout(st))
-                                })
-                                .collect::<Result<IndexVec<VariantIdx, _>, _>>()?;
+                                layout.largest_niche = None;
 
-                            let offset = st[i].fields().offset(field_index) + niche.offset;
+                                if layout.size <= niche_offset {
+                                    // This variant will fit before the niche.
+                                    return true;
+                                }
 
-                            // Align the total size to the largest alignment.
-                            let size = st[i].size().align_to(align.abi);
+                                // Determine if it'll fit after the niche.
+                                let this_align = layout.align.abi;
+                                let this_offset = (niche_offset + niche_size).align_to(this_align);
 
-                            let abi = if st.iter().all(|v| v.abi().is_uninhabited()) {
-                                Abi::Uninhabited
-                            } else if align == st[i].align() && size == st[i].size() {
-                                // When the total alignment and size match, we can use the
-                                // same ABI as the scalar variant with the reserved niche.
-                                match st[i].abi() {
-                                    Abi::Scalar(_) => Abi::Scalar(niche_scalar),
-                                    Abi::ScalarPair(first, second) => {
-                                        // Only the niche is guaranteed to be initialised,
-                                        // so use union layout for the other primitive.
-                                        if offset.bytes() == 0 {
-                                            Abi::ScalarPair(niche_scalar, second.to_union())
-                                        } else {
-                                            Abi::ScalarPair(first.to_union(), niche_scalar)
+                                if this_offset + layout.size > size {
+                                    return false;
+                                }
+
+                                // It'll fit, but we need to make some adjustments.
+                                match layout.fields {
+                                    FieldsShape::Arbitrary { ref mut offsets, .. } => {
+                                        for (j, offset) in offsets.iter_mut().enumerate() {
+                                            if !variants[i][j].is_zst() {
+                                                *offset += this_offset;
+                                            }
                                         }
                                     }
-                                    _ => Abi::Aggregate { sized: true },
+                                    _ => {
+                                        panic!("Layout of fields should be Arbitrary for variants")
+                                    }
                                 }
-                            } else {
-                                Abi::Aggregate { sized: true }
-                            };
 
-                            let largest_niche = Niche::from_scalar(dl, offset, niche_scalar);
-
-                            niche_filling_layout = Some(LayoutS {
-                                variants: Variants::Multiple {
-                                    tag: niche_scalar,
-                                    tag_encoding: TagEncoding::Niche {
-                                        dataful_variant: i,
-                                        niche_variants,
-                                        niche_start,
-                                    },
-                                    tag_field: 0,
-                                    variants: st,
-                                },
-                                fields: FieldsShape::Arbitrary {
-                                    offsets: vec![offset],
-                                    memory_index: vec![0],
-                                },
-                                abi,
-                                largest_niche,
-                                size,
-                                align,
+                                // It can't be a Scalar or ScalarPair because the offset isn't 0.
+                                if !layout.abi.is_uninhabited() {
+                                    layout.abi = Abi::Aggregate { sized: true };
+                                }
+                                layout.size += this_offset;
+
+                                true
                             });
+
+                        if !all_variants_fit {
+                            return Ok(None);
                         }
-                    }
-                }
+
+                        let largest_niche = Niche::from_scalar(dl, niche_offset, niche_scalar);
+
+                        let others_zst = variant_layouts.iter_enumerated().all(|(i, layout)| {
+                            i == largest_variant_index || layout.size == Size::ZERO
+                        });
+                        let same_size = size == variant_layouts[largest_variant_index].size;
+                        let same_align = align == variant_layouts[largest_variant_index].align;
+
+                        let abi = if variant_layouts.iter().all(|v| v.abi.is_uninhabited()) {
+                            Abi::Uninhabited
+                        } else if same_size && same_align && others_zst {
+                            match variant_layouts[largest_variant_index].abi {
+                                // When the total alignment and size match, we can use the
+                                // same ABI as the scalar variant with the reserved niche.
+                                Abi::Scalar(_) => Abi::Scalar(niche_scalar),
+                                Abi::ScalarPair(first, second) => {
+                                    // Only the niche is guaranteed to be initialised,
+                                    // so use union layouts for the other primitive.
+                                    if niche_offset == Size::ZERO {
+                                        Abi::ScalarPair(niche_scalar, second.to_union())
+                                    } else {
+                                        Abi::ScalarPair(first.to_union(), niche_scalar)
+                                    }
+                                }
+                                _ => Abi::Aggregate { sized: true },
+                            }
+                        } else {
+                            Abi::Aggregate { sized: true }
+                        };
+
+                        let layout = LayoutS {
+                            variants: Variants::Multiple {
+                                tag: niche_scalar,
+                                tag_encoding: TagEncoding::Niche {
+                                    untagged_variant: largest_variant_index,
+                                    niche_variants,
+                                    niche_start,
+                                },
+                                tag_field: 0,
+                                variants: IndexVec::new(),
+                            },
+                            fields: FieldsShape::Arbitrary {
+                                offsets: vec![niche_offset],
+                                memory_index: vec![0],
+                            },
+                            abi,
+                            largest_niche,
+                            size,
+                            align,
+                        };
+
+                        Ok(Some(TmpLayout { layout, variants: variant_layouts }))
+                    };
+
+                let niche_filling_layout = calculate_niche_filling_layout()?;
 
                 let (mut min, mut max) = (i128::MAX, i128::MIN);
                 let discr_type = def.repr().discr_type();
@@ -1540,15 +1493,12 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
 
                 let largest_niche = Niche::from_scalar(dl, Size::ZERO, tag);
 
-                let layout_variants =
-                    layout_variants.into_iter().map(|v| tcx.intern_layout(v)).collect();
-
                 let tagged_layout = LayoutS {
                     variants: Variants::Multiple {
                         tag,
                         tag_encoding: TagEncoding::Direct,
                         tag_field: 0,
-                        variants: layout_variants,
+                        variants: IndexVec::new(),
                     },
                     fields: FieldsShape::Arbitrary {
                         offsets: vec![Size::ZERO],
@@ -1560,20 +1510,45 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
                     size,
                 };
 
-                let best_layout = match (tagged_layout, niche_filling_layout) {
-                    (tagged_layout, Some(niche_filling_layout)) => {
+                let tagged_layout = TmpLayout { layout: tagged_layout, variants: layout_variants };
+
+                let mut best_layout = match (tagged_layout, niche_filling_layout) {
+                    (tl, Some(nl)) => {
                         // Pick the smaller layout; otherwise,
                         // pick the layout with the larger niche; otherwise,
                         // pick tagged as it has simpler codegen.
-                        cmp::min_by_key(tagged_layout, niche_filling_layout, |layout| {
-                            let niche_size = layout.largest_niche.map_or(0, |n| n.available(dl));
-                            (layout.size, cmp::Reverse(niche_size))
-                        })
+                        use Ordering::*;
+                        let niche_size = |tmp_l: &TmpLayout<'_>| {
+                            tmp_l.layout.largest_niche.map_or(0, |n| n.available(dl))
+                        };
+                        match (
+                            tl.layout.size.cmp(&nl.layout.size),
+                            niche_size(&tl).cmp(&niche_size(&nl)),
+                        ) {
+                            (Greater, _) => nl,
+                            (Equal, Less) => nl,
+                            _ => tl,
+                        }
                     }
-                    (tagged_layout, None) => tagged_layout,
+                    (tl, None) => tl,
                 };
 
-                tcx.intern_layout(best_layout)
+                // Now we can intern the variant layouts and store them in the enum layout.
+                best_layout.layout.variants = match best_layout.layout.variants {
+                    Variants::Multiple { tag, tag_encoding, tag_field, .. } => Variants::Multiple {
+                        tag,
+                        tag_encoding,
+                        tag_field,
+                        variants: best_layout
+                            .variants
+                            .into_iter()
+                            .map(|layout| tcx.intern_layout(layout))
+                            .collect(),
+                    },
+                    _ => bug!(),
+                };
+
+                tcx.intern_layout(best_layout.layout)
             }
 
             // Types with no meaningful known layout.
@@ -2468,7 +2443,9 @@ where
                 | ty::FnDef(..)
                 | ty::GeneratorWitness(..)
                 | ty::Foreign(..)
-                | ty::Dynamic(..) => bug!("TyAndLayout::field({:?}): not applicable", this),
+                | ty::Dynamic(_, _, ty::Dyn) => {
+                    bug!("TyAndLayout::field({:?}): not applicable", this)
+                }
 
                 // Potentially-fat pointers.
                 ty::Ref(_, pointee, _) | ty::RawPtr(ty::TypeAndMut { ty: pointee, .. }) => {
@@ -2497,7 +2474,7 @@ where
 
                     match tcx.struct_tail_erasing_lifetimes(pointee, cx.param_env()).kind() {
                         ty::Slice(_) | ty::Str => TyMaybeWithLayout::Ty(tcx.types.usize),
-                        ty::Dynamic(_, _) => {
+                        ty::Dynamic(_, _, ty::Dyn) => {
                             TyMaybeWithLayout::Ty(tcx.mk_imm_ref(
                                 tcx.lifetimes.re_static,
                                 tcx.mk_array(tcx.types.usize, 3),
@@ -2566,6 +2543,22 @@ where
                     }
                 }
 
+                ty::Dynamic(_, _, ty::DynStar) => {
+                    if i == 0 {
+                        TyMaybeWithLayout::Ty(tcx.types.usize)
+                    } else if i == 1 {
+                        // FIXME(dyn-star) same FIXME as above applies here too
+                        TyMaybeWithLayout::Ty(
+                            tcx.mk_imm_ref(
+                                tcx.lifetimes.re_static,
+                                tcx.mk_array(tcx.types.usize, 3),
+                            ),
+                        )
+                    } else {
+                        bug!("no field {i} on dyn*")
+                    }
+                }
+
                 ty::Projection(_)
                 | ty::Bound(..)
                 | ty::Placeholder(..)
@@ -2674,11 +2667,11 @@ where
                     // using more niches than just null (e.g., the first page of
                     // the address space, or unaligned pointers).
                     Variants::Multiple {
-                        tag_encoding: TagEncoding::Niche { dataful_variant, .. },
+                        tag_encoding: TagEncoding::Niche { untagged_variant, .. },
                         tag_field,
                         ..
                     } if this.fields.offset(tag_field) == offset => {
-                        Some(this.for_variant(cx, dataful_variant))
+                        Some(this.for_variant(cx, untagged_variant))
                     }
                     _ => Some(this),
                 };
@@ -2761,6 +2754,7 @@ impl<'tcx> ty::Instance<'tcx> {
     // for `Instance` (e.g. typeck would use `Ty::fn_sig` instead),
     // or should go through `FnAbi` instead, to avoid losing any
     // adjustments `fn_abi_of_instance` might be performing.
+    #[tracing::instrument(level = "debug", skip(tcx, param_env))]
     fn fn_sig_for_fn_abi(
         &self,
         tcx: TyCtxt<'tcx>,
@@ -2907,6 +2901,7 @@ impl<'tcx> ty::Instance<'tcx> {
 /// with `-Cpanic=abort` will look like they can't unwind when in fact they
 /// might (from a foreign exception or similar).
 #[inline]
+#[tracing::instrument(level = "debug", skip(tcx))]
 pub fn fn_can_unwind<'tcx>(tcx: TyCtxt<'tcx>, fn_def_id: Option<DefId>, abi: SpecAbi) -> bool {
     if let Some(did) = fn_def_id {
         // Special attribute for functions which can't unwind.
@@ -3123,6 +3118,7 @@ pub trait FnAbiOf<'tcx>: FnAbiOfHelpers<'tcx> {
     /// NB: that includes virtual calls, which are represented by "direct calls"
     /// to an `InstanceDef::Virtual` instance (of `<dyn Trait as Trait>::fn`).
     #[inline]
+    #[tracing::instrument(level = "debug", skip(self))]
     fn fn_abi_of_instance(
         &self,
         instance: ty::Instance<'tcx>,
@@ -3179,9 +3175,100 @@ fn fn_abi_of_instance<'tcx>(
     )
 }
 
+// Handle safe Rust thin and fat pointers.
+pub fn adjust_for_rust_scalar<'tcx>(
+    cx: LayoutCx<'tcx, TyCtxt<'tcx>>,
+    attrs: &mut ArgAttributes,
+    scalar: Scalar,
+    layout: TyAndLayout<'tcx>,
+    offset: Size,
+    is_return: bool,
+) {
+    // Booleans are always a noundef i1 that needs to be zero-extended.
+    if scalar.is_bool() {
+        attrs.ext(ArgExtension::Zext);
+        attrs.set(ArgAttribute::NoUndef);
+        return;
+    }
+
+    // Scalars which have invalid values cannot be undef.
+    if !scalar.is_always_valid(&cx) {
+        attrs.set(ArgAttribute::NoUndef);
+    }
+
+    // Only pointer types handled below.
+    let Scalar::Initialized { value: Pointer, valid_range} = scalar else { return };
+
+    if !valid_range.contains(0) {
+        attrs.set(ArgAttribute::NonNull);
+    }
+
+    if let Some(pointee) = layout.pointee_info_at(&cx, offset) {
+        if let Some(kind) = pointee.safe {
+            attrs.pointee_align = Some(pointee.align);
+
+            // `Box` (`UniqueBorrowed`) are not necessarily dereferenceable
+            // for the entire duration of the function as they can be deallocated
+            // at any time. Same for shared mutable references. If LLVM had a
+            // way to say "dereferenceable on entry" we could use it here.
+            attrs.pointee_size = match kind {
+                PointerKind::UniqueBorrowed
+                | PointerKind::UniqueBorrowedPinned
+                | PointerKind::Frozen => pointee.size,
+                PointerKind::SharedMutable | PointerKind::UniqueOwned => Size::ZERO,
+            };
+
+            // `Box`, `&T`, and `&mut T` cannot be undef.
+            // Note that this only applies to the value of the pointer itself;
+            // this attribute doesn't make it UB for the pointed-to data to be undef.
+            attrs.set(ArgAttribute::NoUndef);
+
+            // The aliasing rules for `Box<T>` are still not decided, but currently we emit
+            // `noalias` for it. This can be turned off using an unstable flag.
+            // See https://github.com/rust-lang/unsafe-code-guidelines/issues/326
+            let noalias_for_box = cx.tcx.sess.opts.unstable_opts.box_noalias.unwrap_or(true);
+
+            // `&mut` pointer parameters never alias other parameters,
+            // or mutable global data
+            //
+            // `&T` where `T` contains no `UnsafeCell<U>` is immutable,
+            // and can be marked as both `readonly` and `noalias`, as
+            // LLVM's definition of `noalias` is based solely on memory
+            // dependencies rather than pointer equality
+            //
+            // Due to past miscompiles in LLVM, we apply a separate NoAliasMutRef attribute
+            // for UniqueBorrowed arguments, so that the codegen backend can decide whether
+            // or not to actually emit the attribute. It can also be controlled with the
+            // `-Zmutable-noalias` debugging option.
+            let no_alias = match kind {
+                PointerKind::SharedMutable
+                | PointerKind::UniqueBorrowed
+                | PointerKind::UniqueBorrowedPinned => false,
+                PointerKind::UniqueOwned => noalias_for_box,
+                PointerKind::Frozen => !is_return,
+            };
+            if no_alias {
+                attrs.set(ArgAttribute::NoAlias);
+            }
+
+            if kind == PointerKind::Frozen && !is_return {
+                attrs.set(ArgAttribute::ReadOnly);
+            }
+
+            if kind == PointerKind::UniqueBorrowed && !is_return {
+                attrs.set(ArgAttribute::NoAliasMutRef);
+            }
+        }
+    }
+}
+
 impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
     // FIXME(eddyb) perhaps group the signature/type-containing (or all of them?)
     // arguments of this method, into a separate `struct`.
+    #[tracing::instrument(
+        level = "debug",
+        skip(self, caller_location, fn_def_id, force_thin_self_ptr)
+    )]
     fn fn_abi_new_uncached(
         &self,
         sig: ty::PolyFnSig<'tcx>,
@@ -3191,8 +3278,6 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
         // FIXME(eddyb) replace this with something typed, like an `enum`.
         force_thin_self_ptr: bool,
     ) -> Result<&'tcx FnAbi<'tcx, Ty<'tcx>>, FnAbiError<'tcx>> {
-        debug!("fn_abi_new_uncached({:?}, {:?})", sig, extra_args);
-
         let sig = self.tcx.normalize_erasing_late_bound_regions(self.param_env, sig);
 
         let conv = conv_from_spec_abi(self.tcx(), sig.abi);
@@ -3234,92 +3319,9 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
         use SpecAbi::*;
         let rust_abi = matches!(sig.abi, RustIntrinsic | PlatformIntrinsic | Rust | RustCall);
 
-        // Handle safe Rust thin and fat pointers.
-        let adjust_for_rust_scalar = |attrs: &mut ArgAttributes,
-                                      scalar: Scalar,
-                                      layout: TyAndLayout<'tcx>,
-                                      offset: Size,
-                                      is_return: bool| {
-            // Booleans are always a noundef i1 that needs to be zero-extended.
-            if scalar.is_bool() {
-                attrs.ext(ArgExtension::Zext);
-                attrs.set(ArgAttribute::NoUndef);
-                return;
-            }
-
-            // Scalars which have invalid values cannot be undef.
-            if !scalar.is_always_valid(self) {
-                attrs.set(ArgAttribute::NoUndef);
-            }
-
-            // Only pointer types handled below.
-            let Scalar::Initialized { value: Pointer, valid_range} = scalar else { return };
-
-            if !valid_range.contains(0) {
-                attrs.set(ArgAttribute::NonNull);
-            }
-
-            if let Some(pointee) = layout.pointee_info_at(self, offset) {
-                if let Some(kind) = pointee.safe {
-                    attrs.pointee_align = Some(pointee.align);
-
-                    // `Box` (`UniqueBorrowed`) are not necessarily dereferenceable
-                    // for the entire duration of the function as they can be deallocated
-                    // at any time. Same for shared mutable references. If LLVM had a
-                    // way to say "dereferenceable on entry" we could use it here.
-                    attrs.pointee_size = match kind {
-                        PointerKind::UniqueBorrowed
-                        | PointerKind::UniqueBorrowedPinned
-                        | PointerKind::Frozen => pointee.size,
-                        PointerKind::SharedMutable | PointerKind::UniqueOwned => Size::ZERO,
-                    };
-
-                    // `Box`, `&T`, and `&mut T` cannot be undef.
-                    // Note that this only applies to the value of the pointer itself;
-                    // this attribute doesn't make it UB for the pointed-to data to be undef.
-                    attrs.set(ArgAttribute::NoUndef);
-
-                    // The aliasing rules for `Box<T>` are still not decided, but currently we emit
-                    // `noalias` for it. This can be turned off using an unstable flag.
-                    // See https://github.com/rust-lang/unsafe-code-guidelines/issues/326
-                    let noalias_for_box =
-                        self.tcx().sess.opts.unstable_opts.box_noalias.unwrap_or(true);
-
-                    // `&mut` pointer parameters never alias other parameters,
-                    // or mutable global data
-                    //
-                    // `&T` where `T` contains no `UnsafeCell<U>` is immutable,
-                    // and can be marked as both `readonly` and `noalias`, as
-                    // LLVM's definition of `noalias` is based solely on memory
-                    // dependencies rather than pointer equality
-                    //
-                    // Due to past miscompiles in LLVM, we apply a separate NoAliasMutRef attribute
-                    // for UniqueBorrowed arguments, so that the codegen backend can decide whether
-                    // or not to actually emit the attribute. It can also be controlled with the
-                    // `-Zmutable-noalias` debugging option.
-                    let no_alias = match kind {
-                        PointerKind::SharedMutable
-                        | PointerKind::UniqueBorrowed
-                        | PointerKind::UniqueBorrowedPinned => false,
-                        PointerKind::UniqueOwned => noalias_for_box,
-                        PointerKind::Frozen => !is_return,
-                    };
-                    if no_alias {
-                        attrs.set(ArgAttribute::NoAlias);
-                    }
-
-                    if kind == PointerKind::Frozen && !is_return {
-                        attrs.set(ArgAttribute::ReadOnly);
-                    }
-
-                    if kind == PointerKind::UniqueBorrowed && !is_return {
-                        attrs.set(ArgAttribute::NoAliasMutRef);
-                    }
-                }
-            }
-        };
-
         let arg_of = |ty: Ty<'tcx>, arg_idx: Option<usize>| -> Result<_, FnAbiError<'tcx>> {
+            let span = tracing::debug_span!("arg_of");
+            let _entered = span.enter();
             let is_return = arg_idx.is_none();
 
             let layout = self.layout_of(ty)?;
@@ -3334,7 +3336,7 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
 
             let mut arg = ArgAbi::new(self, layout, |layout, scalar, offset| {
                 let mut attrs = ArgAttributes::new();
-                adjust_for_rust_scalar(&mut attrs, scalar, *layout, offset, is_return);
+                adjust_for_rust_scalar(*self, &mut attrs, scalar, *layout, offset, is_return);
                 attrs
             });
 
@@ -3367,7 +3369,7 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
                 .map(|(i, ty)| arg_of(ty, Some(i)))
                 .collect::<Result<_, _>>()?,
             c_variadic: sig.c_variadic,
-            fixed_count: inputs.len(),
+            fixed_count: inputs.len() as u32,
             conv,
             can_unwind: fn_can_unwind(self.tcx(), fn_def_id, sig.abi),
         };
@@ -3376,6 +3378,7 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
         Ok(self.tcx.arena.alloc(fn_abi))
     }
 
+    #[tracing::instrument(level = "trace", skip(self))]
     fn fn_abi_adjust_for_abi(
         &self,
         fn_abi: &mut FnAbi<'tcx, Ty<'tcx>>,
@@ -3439,7 +3442,7 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
                 }
             };
             fixup(&mut fn_abi.ret);
-            for arg in &mut fn_abi.args {
+            for arg in fn_abi.args.iter_mut() {
                 fixup(arg);
             }
         } else {
@@ -3450,6 +3453,7 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
     }
 }
 
+#[tracing::instrument(level = "debug", skip(cx))]
 fn make_thin_self_ptr<'tcx>(
     cx: &(impl HasTyCtxt<'tcx> + HasParamEnv<'tcx>),
     layout: TyAndLayout<'tcx>,
@@ -3461,7 +3465,7 @@ fn make_thin_self_ptr<'tcx>(
         tcx.mk_mut_ptr(layout.ty)
     } else {
         match layout.abi {
-            Abi::ScalarPair(..) => (),
+            Abi::ScalarPair(..) | Abi::Scalar(..) => (),
             _ => bug!("receiver type has unsupported layout: {:?}", layout),
         }
 
diff --git a/compiler/rustc_middle/src/ty/layout_sanity_check.rs b/compiler/rustc_middle/src/ty/layout_sanity_check.rs
new file mode 100644
index 000000000..87c85dcff
--- /dev/null
+++ b/compiler/rustc_middle/src/ty/layout_sanity_check.rs
@@ -0,0 +1,303 @@
+use crate::ty::{
+    layout::{LayoutCx, TyAndLayout},
+    TyCtxt,
+};
+use rustc_target::abi::*;
+
+use std::cmp;
+
+/// Enforce some basic invariants on layouts.
+pub(super) fn sanity_check_layout<'tcx>(
+    cx: &LayoutCx<'tcx, TyCtxt<'tcx>>,
+    layout: &TyAndLayout<'tcx>,
+) {
+    // Type-level uninhabitedness should always imply ABI uninhabitedness.
+    if cx.tcx.conservative_is_privately_uninhabited(cx.param_env.and(layout.ty)) {
+        assert!(layout.abi.is_uninhabited());
+    }
+
+    if layout.size.bytes() % layout.align.abi.bytes() != 0 {
+        bug!("size is not a multiple of align, in the following layout:\n{layout:#?}");
+    }
+
+    if cfg!(debug_assertions) {
+        /// Yields non-ZST fields of the type
+        fn non_zst_fields<'tcx, 'a>(
+            cx: &'a LayoutCx<'tcx, TyCtxt<'tcx>>,
+            layout: &'a TyAndLayout<'tcx>,
+        ) -> impl Iterator<Item = (Size, TyAndLayout<'tcx>)> + 'a {
+            (0..layout.layout.fields().count()).filter_map(|i| {
+                let field = layout.field(cx, i);
+                // Also checking `align == 1` here leads to test failures in
+                // `layout/zero-sized-array-union.rs`, where a type has a zero-size field with
+                // alignment 4 that still gets ignored during layout computation (which is okay
+                // since other fields already force alignment 4).
+                let zst = field.is_zst();
+                (!zst).then(|| (layout.fields.offset(i), field))
+            })
+        }
+
+        fn skip_newtypes<'tcx>(
+            cx: &LayoutCx<'tcx, TyCtxt<'tcx>>,
+            layout: &TyAndLayout<'tcx>,
+        ) -> TyAndLayout<'tcx> {
+            if matches!(layout.layout.variants(), Variants::Multiple { .. }) {
+                // Definitely not a newtype of anything.
+                return *layout;
+            }
+            let mut fields = non_zst_fields(cx, layout);
+            let Some(first) = fields.next() else {
+                // No fields here, so this could be a primitive or enum -- either way it's not a newtype around a thing
+                return *layout
+            };
+            if fields.next().is_none() {
+                let (offset, first) = first;
+                if offset == Size::ZERO && first.layout.size() == layout.size {
+                    // This is a newtype, so keep recursing.
+                    // FIXME(RalfJung): I don't think it would be correct to do any checks for
+                    // alignment here, so we don't. Is that correct?
+                    return skip_newtypes(cx, &first);
+                }
+            }
+            // No more newtypes here.
+            *layout
+        }
+
+        fn check_layout_abi<'tcx>(cx: &LayoutCx<'tcx, TyCtxt<'tcx>>, layout: &TyAndLayout<'tcx>) {
+            match layout.layout.abi() {
+                Abi::Scalar(scalar) => {
+                    // No padding in scalars.
+                    let size = scalar.size(cx);
+                    let align = scalar.align(cx).abi;
+                    assert_eq!(
+                        layout.layout.size(),
+                        size,
+                        "size mismatch between ABI and layout in {layout:#?}"
+                    );
+                    assert_eq!(
+                        layout.layout.align().abi,
+                        align,
+                        "alignment mismatch between ABI and layout in {layout:#?}"
+                    );
+                    // Check that this matches the underlying field.
+                    let inner = skip_newtypes(cx, layout);
+                    assert!(
+                        matches!(inner.layout.abi(), Abi::Scalar(_)),
+                        "`Scalar` type {} is newtype around non-`Scalar` type {}",
+                        layout.ty,
+                        inner.ty
+                    );
+                    match inner.layout.fields() {
+                        FieldsShape::Primitive => {
+                            // Fine.
+                        }
+                        FieldsShape::Union(..) => {
+                            // FIXME: I guess we could also check something here? Like, look at all fields?
+                            return;
+                        }
+                        FieldsShape::Arbitrary { .. } => {
+                            // Should be an enum, the only field is the discriminant.
+                            assert!(
+                                inner.ty.is_enum(),
+                                "`Scalar` layout for non-primitive non-enum type {}",
+                                inner.ty
+                            );
+                            assert_eq!(
+                                inner.layout.fields().count(),
+                                1,
+                                "`Scalar` layout for multiple-field type in {inner:#?}",
+                            );
+                            let offset = inner.layout.fields().offset(0);
+                            let field = inner.field(cx, 0);
+                            // The field should be at the right offset, and match the `scalar` layout.
+                            assert_eq!(
+                                offset,
+                                Size::ZERO,
+                                "`Scalar` field at non-0 offset in {inner:#?}",
+                            );
+                            assert_eq!(
+                                field.size, size,
+                                "`Scalar` field with bad size in {inner:#?}",
+                            );
+                            assert_eq!(
+                                field.align.abi, align,
+                                "`Scalar` field with bad align in {inner:#?}",
+                            );
+                            assert!(
+                                matches!(field.abi, Abi::Scalar(_)),
+                                "`Scalar` field with bad ABI in {inner:#?}",
+                            );
+                        }
+                        _ => {
+                            panic!("`Scalar` layout for non-primitive non-enum type {}", inner.ty);
+                        }
+                    }
+                }
+                Abi::ScalarPair(scalar1, scalar2) => {
+                    // Sanity-check scalar pairs. These are a bit more flexible and support
+                    // padding, but we can at least ensure both fields actually fit into the layout
+                    // and the alignment requirement has not been weakened.
+                    let size1 = scalar1.size(cx);
+                    let align1 = scalar1.align(cx).abi;
+                    let size2 = scalar2.size(cx);
+                    let align2 = scalar2.align(cx).abi;
+                    assert!(
+                        layout.layout.align().abi >= cmp::max(align1, align2),
+                        "alignment mismatch between ABI and layout in {layout:#?}",
+                    );
+                    let field2_offset = size1.align_to(align2);
+                    assert!(
+                        layout.layout.size() >= field2_offset + size2,
+                        "size mismatch between ABI and layout in {layout:#?}"
+                    );
+                    // Check that the underlying pair of fields matches.
+                    let inner = skip_newtypes(cx, layout);
+                    assert!(
+                        matches!(inner.layout.abi(), Abi::ScalarPair(..)),
+                        "`ScalarPair` type {} is newtype around non-`ScalarPair` type {}",
+                        layout.ty,
+                        inner.ty
+                    );
+                    if matches!(inner.layout.variants(), Variants::Multiple { .. }) {
+                        // FIXME: ScalarPair for enums is enormously complicated and it is very hard
+                        // to check anything about them.
+                        return;
+                    }
+                    match inner.layout.fields() {
+                        FieldsShape::Arbitrary { .. } => {
+                            // Checked below.
+                        }
+                        FieldsShape::Union(..) => {
+                            // FIXME: I guess we could also check something here? Like, look at all fields?
+                            return;
+                        }
+                        _ => {
+                            panic!("`ScalarPair` layout with unexpected field shape in {inner:#?}");
+                        }
+                    }
+                    let mut fields = non_zst_fields(cx, &inner);
+                    let (offset1, field1) = fields.next().unwrap_or_else(|| {
+                        panic!("`ScalarPair` layout for type with not even one non-ZST field: {inner:#?}")
+                    });
+                    let (offset2, field2) = fields.next().unwrap_or_else(|| {
+                        panic!("`ScalarPair` layout for type with less than two non-ZST fields: {inner:#?}")
+                    });
+                    assert!(
+                        fields.next().is_none(),
+                        "`ScalarPair` layout for type with at least three non-ZST fields: {inner:#?}"
+                    );
+                    // The fields might be in opposite order.
+                    let (offset1, field1, offset2, field2) = if offset1 <= offset2 {
+                        (offset1, field1, offset2, field2)
+                    } else {
+                        (offset2, field2, offset1, field1)
+                    };
+                    // The fields should be at the right offset, and match the `scalar` layout.
+                    assert_eq!(
+                        offset1,
+                        Size::ZERO,
+                        "`ScalarPair` first field at non-0 offset in {inner:#?}",
+                    );
+                    assert_eq!(
+                        field1.size, size1,
+                        "`ScalarPair` first field with bad size in {inner:#?}",
+                    );
+                    assert_eq!(
+                        field1.align.abi, align1,
+                        "`ScalarPair` first field with bad align in {inner:#?}",
+                    );
+                    assert!(
+                        matches!(field1.abi, Abi::Scalar(_)),
+                        "`ScalarPair` first field with bad ABI in {inner:#?}",
+                    );
+                    assert_eq!(
+                        offset2, field2_offset,
+                        "`ScalarPair` second field at bad offset in {inner:#?}",
+                    );
+                    assert_eq!(
+                        field2.size, size2,
+                        "`ScalarPair` second field with bad size in {inner:#?}",
+                    );
+                    assert_eq!(
+                        field2.align.abi, align2,
+                        "`ScalarPair` second field with bad align in {inner:#?}",
+                    );
+                    assert!(
+                        matches!(field2.abi, Abi::Scalar(_)),
+                        "`ScalarPair` second field with bad ABI in {inner:#?}",
+                    );
+                }
+                Abi::Vector { count, element } => {
+                    // No padding in vectors. Alignment can be strengthened, though.
+                    assert!(
+                        layout.layout.align().abi >= element.align(cx).abi,
+                        "alignment mismatch between ABI and layout in {layout:#?}"
+                    );
+                    let size = element.size(cx) * count;
+                    assert_eq!(
+                        layout.layout.size(),
+                        size.align_to(cx.data_layout().vector_align(size).abi),
+                        "size mismatch between ABI and layout in {layout:#?}"
+                    );
+                }
+                Abi::Uninhabited | Abi::Aggregate { .. } => {} // Nothing to check.
+            }
+        }
+
+        check_layout_abi(cx, layout);
+
+        if let Variants::Multiple { variants, .. } = &layout.variants {
+            for variant in variants.iter() {
+                // No nested "multiple".
+                assert!(matches!(variant.variants(), Variants::Single { .. }));
+                // Variants should have the same or a smaller size as the full thing,
+                // and same for alignment.
+                if variant.size() > layout.size {
+                    bug!(
+                        "Type with size {} bytes has variant with size {} bytes: {layout:#?}",
+                        layout.size.bytes(),
+                        variant.size().bytes(),
+                    )
+                }
+                if variant.align().abi > layout.align.abi {
+                    bug!(
+                        "Type with alignment {} bytes has variant with alignment {} bytes: {layout:#?}",
+                        layout.align.abi.bytes(),
+                        variant.align().abi.bytes(),
+                    )
+                }
+                // Skip empty variants.
+                if variant.size() == Size::ZERO
+                    || variant.fields().count() == 0
+                    || variant.abi().is_uninhabited()
+                {
+                    // These are never actually accessed anyway, so we can skip the coherence check
+                    // for them. They also fail that check, since they have
+                    // `Aggregate`/`Uninhbaited` ABI even when the main type is
+                    // `Scalar`/`ScalarPair`. (Note that sometimes, variants with fields have size
+                    // 0, and sometimes, variants without fields have non-0 size.)
+                    continue;
+                }
+                // The top-level ABI and the ABI of the variants should be coherent.
+                let scalar_coherent = |s1: Scalar, s2: Scalar| {
+                    s1.size(cx) == s2.size(cx) && s1.align(cx) == s2.align(cx)
+                };
+                let abi_coherent = match (layout.abi, variant.abi()) {
+                    (Abi::Scalar(s1), Abi::Scalar(s2)) => scalar_coherent(s1, s2),
+                    (Abi::ScalarPair(a1, b1), Abi::ScalarPair(a2, b2)) => {
+                        scalar_coherent(a1, a2) && scalar_coherent(b1, b2)
+                    }
+                    (Abi::Uninhabited, _) => true,
+                    (Abi::Aggregate { .. }, _) => true,
+                    _ => false,
+                };
+                if !abi_coherent {
+                    bug!(
+                        "Variant ABI is incompatible with top-level ABI:\nvariant={:#?}\nTop-level: {layout:#?}",
+                        variant
+                    );
+                }
+            }
+        }
+    }
+}
diff --git a/compiler/rustc_middle/src/ty/list.rs b/compiler/rustc_middle/src/ty/list.rs
index db3b5cfd1..79365ef28 100644
--- a/compiler/rustc_middle/src/ty/list.rs
+++ b/compiler/rustc_middle/src/ty/list.rs
@@ -65,6 +65,10 @@ impl<T> List<T> {
     pub fn len(&self) -> usize {
         self.len
     }
+
+    pub fn as_slice(&self) -> &[T] {
+        self
+    }
 }
 
 impl<T: Copy> List<T> {
diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs
index 02da02568..3f9871190 100644
--- a/compiler/rustc_middle/src/ty/mod.rs
+++ b/compiler/rustc_middle/src/ty/mod.rs
@@ -15,6 +15,7 @@ pub use self::AssocItemContainer::*;
 pub use self::BorrowKind::*;
 pub use self::IntVarValue::*;
 pub use self::Variance::*;
+use crate::error::{OpaqueHiddenTypeMismatch, TypeMismatchReason};
 use crate::metadata::ModChild;
 use crate::middle::privacy::AccessLevels;
 use crate::mir::{Body, GeneratorLayout};
@@ -40,6 +41,7 @@ use rustc_hir::Node;
 use rustc_index::vec::IndexVec;
 use rustc_macros::HashStable;
 use rustc_query_system::ich::StableHashingContext;
+use rustc_serialize::{Decodable, Encodable};
 use rustc_span::hygiene::MacroKind;
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::{ExpnId, Span};
@@ -49,10 +51,14 @@ pub use vtable::*;
 
 use std::fmt::Debug;
 use std::hash::{Hash, Hasher};
+use std::marker::PhantomData;
+use std::mem;
+use std::num::NonZeroUsize;
 use std::ops::ControlFlow;
 use std::{fmt, str};
 
 pub use crate::ty::diagnostics::*;
+pub use rustc_type_ir::DynKind::*;
 pub use rustc_type_ir::InferTy::*;
 pub use rustc_type_ir::RegionKind::*;
 pub use rustc_type_ir::TyKind::*;
@@ -124,6 +130,7 @@ mod erase_regions;
 mod generics;
 mod impls_ty;
 mod instance;
+mod layout_sanity_check;
 mod list;
 mod parameterized;
 mod rvalue_scopes;
@@ -177,11 +184,6 @@ pub struct ResolverAstLowering {
     pub label_res_map: NodeMap<ast::NodeId>,
     /// Resolutions for lifetimes.
     pub lifetimes_res_map: NodeMap<LifetimeRes>,
-    /// Mapping from generics `def_id`s to TAIT generics `def_id`s.
-    /// For each captured lifetime (e.g., 'a), we create a new lifetime parameter that is a generic
-    /// defined on the TAIT, so we have type Foo<'a1> = ... and we establish a mapping in this
-    /// field from the original parameter 'a to the new parameter 'a1.
-    pub generics_def_id_map: Vec<FxHashMap<LocalDefId, LocalDefId>>,
     /// Lifetime parameters that lowering will have to introduce.
     pub extra_lifetime_params_map: NodeMap<Vec<(Ident, ast::NodeId, LifetimeRes)>>,
 
@@ -262,13 +264,11 @@ impl fmt::Display for ImplPolarity {
 }
 
 #[derive(Clone, Debug, PartialEq, Eq, Copy, Hash, Encodable, Decodable, HashStable)]
-pub enum Visibility {
+pub enum Visibility<Id = LocalDefId> {
     /// Visible everywhere (including in other crates).
     Public,
     /// Visible only in the given crate-local module.
-    Restricted(DefId),
-    /// Not visible anywhere in the local crate. This is the visibility of private external items.
-    Invisible,
+    Restricted(Id),
 }
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, HashStable, TyEncodable, TyDecodable)]
@@ -359,31 +359,45 @@ impl<'tcx> DefIdTree for TyCtxt<'tcx> {
     }
 }
 
-impl Visibility {
-    /// Returns `true` if an item with this visibility is accessible from the given block.
-    pub fn is_accessible_from<T: DefIdTree>(self, module: DefId, tree: T) -> bool {
-        let restriction = match self {
-            // Public items are visible everywhere.
-            Visibility::Public => return true,
-            // Private items from other crates are visible nowhere.
-            Visibility::Invisible => return false,
-            // Restricted items are visible in an arbitrary local module.
-            Visibility::Restricted(other) if other.krate != module.krate => return false,
-            Visibility::Restricted(module) => module,
-        };
+impl<Id> Visibility<Id> {
+    pub fn is_public(self) -> bool {
+        matches!(self, Visibility::Public)
+    }
+
+    pub fn map_id<OutId>(self, f: impl FnOnce(Id) -> OutId) -> Visibility<OutId> {
+        match self {
+            Visibility::Public => Visibility::Public,
+            Visibility::Restricted(id) => Visibility::Restricted(f(id)),
+        }
+    }
+}
 
-        tree.is_descendant_of(module, restriction)
+impl<Id: Into<DefId>> Visibility<Id> {
+    pub fn to_def_id(self) -> Visibility<DefId> {
+        self.map_id(Into::into)
+    }
+
+    /// Returns `true` if an item with this visibility is accessible from the given module.
+    pub fn is_accessible_from(self, module: impl Into<DefId>, tree: impl DefIdTree) -> bool {
+        match self {
+            // Public items are visible everywhere.
+            Visibility::Public => true,
+            Visibility::Restricted(id) => tree.is_descendant_of(module.into(), id.into()),
+        }
     }
 
     /// Returns `true` if this visibility is at least as accessible as the given visibility
-    pub fn is_at_least<T: DefIdTree>(self, vis: Visibility, tree: T) -> bool {
-        let vis_restriction = match vis {
-            Visibility::Public => return self == Visibility::Public,
-            Visibility::Invisible => return true,
-            Visibility::Restricted(module) => module,
-        };
+    pub fn is_at_least(self, vis: Visibility<impl Into<DefId>>, tree: impl DefIdTree) -> bool {
+        match vis {
+            Visibility::Public => self.is_public(),
+            Visibility::Restricted(id) => self.is_accessible_from(id, tree),
+        }
+    }
+}
 
-        self.is_accessible_from(vis_restriction, tree)
+impl Visibility<DefId> {
+    pub fn expect_local(self) -> Visibility {
+        self.map_id(|id| id.expect_local())
     }
 
     // Returns `true` if this item is visible anywhere in the local crate.
@@ -391,13 +405,8 @@ impl Visibility {
         match self {
             Visibility::Public => true,
             Visibility::Restricted(def_id) => def_id.is_local(),
-            Visibility::Invisible => false,
         }
     }
-
-    pub fn is_public(self) -> bool {
-        matches!(self, Visibility::Public)
-    }
 }
 
 /// The crate variances map is computed during typeck and contains the
@@ -468,15 +477,6 @@ pub(crate) struct TyS<'tcx> {
     outer_exclusive_binder: ty::DebruijnIndex,
 }
 
-// `TyS` is used a lot. Make sure it doesn't unintentionally get bigger.
-#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
-static_assert_size!(TyS<'_>, 40);
-
-// We are actually storing a stable hash cache next to the type, so let's
-// also check the full size
-#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
-static_assert_size!(WithStableHash<TyS<'_>>, 56);
-
 /// Use this rather than `TyS`, whenever possible.
 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, HashStable)]
 #[rustc_diagnostic_item = "Ty"]
@@ -533,10 +533,6 @@ pub(crate) struct PredicateS<'tcx> {
     outer_exclusive_binder: ty::DebruijnIndex,
 }
 
-// This type is used a lot. Make sure it doesn't unintentionally get bigger.
-#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
-static_assert_size!(PredicateS<'_>, 56);
-
 /// Use this rather than `PredicateS`, whenever possible.
 #[derive(Clone, Copy, PartialEq, Eq, Hash)]
 #[rustc_pass_by_value]
@@ -593,6 +589,29 @@ impl<'tcx> Predicate<'tcx> {
         }
         self
     }
+
+    /// Whether this projection can be soundly normalized.
+    ///
+    /// Wf predicates must not be normalized, as normalization
+    /// can remove required bounds which would cause us to
+    /// unsoundly accept some programs. See #91068.
+    #[inline]
+    pub fn allow_normalization(self) -> bool {
+        match self.kind().skip_binder() {
+            PredicateKind::WellFormed(_) => false,
+            PredicateKind::Trait(_)
+            | PredicateKind::RegionOutlives(_)
+            | PredicateKind::TypeOutlives(_)
+            | PredicateKind::Projection(_)
+            | PredicateKind::ObjectSafe(_)
+            | PredicateKind::ClosureKind(_, _, _)
+            | PredicateKind::Subtype(_)
+            | PredicateKind::Coerce(_)
+            | PredicateKind::ConstEvaluatable(_)
+            | PredicateKind::ConstEquate(_, _)
+            | PredicateKind::TypeWellFormedFromEnv(_) => true,
+        }
+    }
 }
 
 impl<'a, 'tcx> HashStable<StableHashingContext<'a>> for Predicate<'tcx> {
@@ -617,7 +636,7 @@ impl rustc_errors::IntoDiagnosticArg for Predicate<'_> {
 }
 
 #[derive(Clone, Copy, PartialEq, Eq, Hash, TyEncodable, TyDecodable)]
-#[derive(HashStable, TypeFoldable, TypeVisitable)]
+#[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub enum PredicateKind<'tcx> {
     /// Corresponds to `where Foo: Bar<A, B, C>`. `Foo` here would be
     /// the `Self` type of the trait reference and `A`, `B`, and `C`
@@ -789,7 +808,7 @@ impl<'tcx> Predicate<'tcx> {
 }
 
 #[derive(Clone, Copy, PartialEq, Eq, Hash, TyEncodable, TyDecodable)]
-#[derive(HashStable, TypeFoldable, TypeVisitable)]
+#[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub struct TraitPredicate<'tcx> {
     pub trait_ref: TraitRef<'tcx>,
 
@@ -869,7 +888,7 @@ impl<'tcx> PolyTraitPredicate<'tcx> {
 }
 
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, TyEncodable, TyDecodable)]
-#[derive(HashStable, TypeFoldable, TypeVisitable)]
+#[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub struct OutlivesPredicate<A, B>(pub A, pub B); // `A: B`
 pub type RegionOutlivesPredicate<'tcx> = OutlivesPredicate<ty::Region<'tcx>, ty::Region<'tcx>>;
 pub type TypeOutlivesPredicate<'tcx> = OutlivesPredicate<Ty<'tcx>, ty::Region<'tcx>>;
@@ -880,7 +899,7 @@ pub type PolyTypeOutlivesPredicate<'tcx> = ty::Binder<'tcx, TypeOutlivesPredicat
 /// whether the `a` type is the type that we should label as "expected" when
 /// presenting user diagnostics.
 #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, TyEncodable, TyDecodable)]
-#[derive(HashStable, TypeFoldable, TypeVisitable)]
+#[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub struct SubtypePredicate<'tcx> {
     pub a_is_expected: bool,
     pub a: Ty<'tcx>,
@@ -890,49 +909,142 @@ pub type PolySubtypePredicate<'tcx> = ty::Binder<'tcx, SubtypePredicate<'tcx>>;
 
 /// Encodes that we have to coerce *from* the `a` type to the `b` type.
 #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, TyEncodable, TyDecodable)]
-#[derive(HashStable, TypeFoldable, TypeVisitable)]
+#[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub struct CoercePredicate<'tcx> {
     pub a: Ty<'tcx>,
     pub b: Ty<'tcx>,
 }
 pub type PolyCoercePredicate<'tcx> = ty::Binder<'tcx, CoercePredicate<'tcx>>;
 
-#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, TyEncodable, TyDecodable)]
-#[derive(HashStable, TypeFoldable, TypeVisitable)]
-pub enum Term<'tcx> {
-    Ty(Ty<'tcx>),
-    Const(Const<'tcx>),
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct Term<'tcx> {
+    ptr: NonZeroUsize,
+    marker: PhantomData<(Ty<'tcx>, Const<'tcx>)>,
+}
+
+impl Debug for Term<'_> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let data = if let Some(ty) = self.ty() {
+            format!("Term::Ty({:?})", ty)
+        } else if let Some(ct) = self.ct() {
+            format!("Term::Ct({:?})", ct)
+        } else {
+            unreachable!()
+        };
+        f.write_str(&data)
+    }
 }
 
 impl<'tcx> From<Ty<'tcx>> for Term<'tcx> {
     fn from(ty: Ty<'tcx>) -> Self {
-        Term::Ty(ty)
+        TermKind::Ty(ty).pack()
     }
 }
 
 impl<'tcx> From<Const<'tcx>> for Term<'tcx> {
     fn from(c: Const<'tcx>) -> Self {
-        Term::Const(c)
+        TermKind::Const(c).pack()
+    }
+}
+
+impl<'a, 'tcx> HashStable<StableHashingContext<'a>> for Term<'tcx> {
+    fn hash_stable(&self, hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher) {
+        self.unpack().hash_stable(hcx, hasher);
+    }
+}
+
+impl<'tcx> TypeFoldable<'tcx> for Term<'tcx> {
+    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
+        Ok(self.unpack().try_fold_with(folder)?.pack())
+    }
+}
+
+impl<'tcx> TypeVisitable<'tcx> for Term<'tcx> {
+    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
+        self.unpack().visit_with(visitor)
+    }
+}
+
+impl<'tcx, E: TyEncoder<I = TyCtxt<'tcx>>> Encodable<E> for Term<'tcx> {
+    fn encode(&self, e: &mut E) {
+        self.unpack().encode(e)
+    }
+}
+
+impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> Decodable<D> for Term<'tcx> {
+    fn decode(d: &mut D) -> Self {
+        let res: TermKind<'tcx> = Decodable::decode(d);
+        res.pack()
     }
 }
 
 impl<'tcx> Term<'tcx> {
+    #[inline]
+    pub fn unpack(self) -> TermKind<'tcx> {
+        let ptr = self.ptr.get();
+        // SAFETY: use of `Interned::new_unchecked` here is ok because these
+        // pointers were originally created from `Interned` types in `pack()`,
+        // and this is just going in the other direction.
+        unsafe {
+            match ptr & TAG_MASK {
+                TYPE_TAG => TermKind::Ty(Ty(Interned::new_unchecked(
+                    &*((ptr & !TAG_MASK) as *const WithStableHash<ty::TyS<'tcx>>),
+                ))),
+                CONST_TAG => TermKind::Const(ty::Const(Interned::new_unchecked(
+                    &*((ptr & !TAG_MASK) as *const ty::ConstS<'tcx>),
+                ))),
+                _ => core::intrinsics::unreachable(),
+            }
+        }
+    }
+
     pub fn ty(&self) -> Option<Ty<'tcx>> {
-        if let Term::Ty(ty) = self { Some(*ty) } else { None }
+        if let TermKind::Ty(ty) = self.unpack() { Some(ty) } else { None }
     }
 
     pub fn ct(&self) -> Option<Const<'tcx>> {
-        if let Term::Const(c) = self { Some(*c) } else { None }
+        if let TermKind::Const(c) = self.unpack() { Some(c) } else { None }
     }
 
     pub fn into_arg(self) -> GenericArg<'tcx> {
-        match self {
-            Term::Ty(ty) => ty.into(),
-            Term::Const(c) => c.into(),
+        match self.unpack() {
+            TermKind::Ty(ty) => ty.into(),
+            TermKind::Const(c) => c.into(),
         }
     }
 }
 
+const TAG_MASK: usize = 0b11;
+const TYPE_TAG: usize = 0b00;
+const CONST_TAG: usize = 0b01;
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, TyEncodable, TyDecodable)]
+#[derive(HashStable, TypeFoldable, TypeVisitable)]
+pub enum TermKind<'tcx> {
+    Ty(Ty<'tcx>),
+    Const(Const<'tcx>),
+}
+
+impl<'tcx> TermKind<'tcx> {
+    #[inline]
+    fn pack(self) -> Term<'tcx> {
+        let (tag, ptr) = match self {
+            TermKind::Ty(ty) => {
+                // Ensure we can use the tag bits.
+                assert_eq!(mem::align_of_val(&*ty.0.0) & TAG_MASK, 0);
+                (TYPE_TAG, ty.0.0 as *const WithStableHash<ty::TyS<'tcx>> as usize)
+            }
+            TermKind::Const(ct) => {
+                // Ensure we can use the tag bits.
+                assert_eq!(mem::align_of_val(&*ct.0.0) & TAG_MASK, 0);
+                (CONST_TAG, ct.0.0 as *const ty::ConstS<'tcx> as usize)
+            }
+        };
+
+        Term { ptr: unsafe { NonZeroUsize::new_unchecked(ptr | tag) }, marker: PhantomData }
+    }
+}
+
 /// This kind of predicate has no *direct* correspondent in the
 /// syntax, but it roughly corresponds to the syntactic forms:
 ///
@@ -946,7 +1058,7 @@ impl<'tcx> Term<'tcx> {
 /// Form #2 eventually yields one of these `ProjectionPredicate`
 /// instances to normalize the LHS.
 #[derive(Copy, Clone, PartialEq, Eq, Hash, TyEncodable, TyDecodable)]
-#[derive(HashStable, TypeFoldable, TypeVisitable)]
+#[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub struct ProjectionPredicate<'tcx> {
     pub projection_ty: ProjectionTy<'tcx>,
     pub term: Term<'tcx>,
@@ -1002,6 +1114,12 @@ pub trait ToPredicate<'tcx> {
     fn to_predicate(self, tcx: TyCtxt<'tcx>) -> Predicate<'tcx>;
 }
 
+impl<'tcx> ToPredicate<'tcx> for Predicate<'tcx> {
+    fn to_predicate(self, _tcx: TyCtxt<'tcx>) -> Predicate<'tcx> {
+        self
+    }
+}
+
 impl<'tcx> ToPredicate<'tcx> for Binder<'tcx, PredicateKind<'tcx>> {
     #[inline(always)]
     fn to_predicate(self, tcx: TyCtxt<'tcx>) -> Predicate<'tcx> {
@@ -1166,20 +1284,17 @@ pub struct OpaqueHiddenType<'tcx> {
 impl<'tcx> OpaqueHiddenType<'tcx> {
     pub fn report_mismatch(&self, other: &Self, tcx: TyCtxt<'tcx>) {
         // Found different concrete types for the opaque type.
-        let mut err = tcx.sess.struct_span_err(
-            other.span,
-            "concrete type differs from previous defining opaque type use",
-        );
-        err.span_label(other.span, format!("expected `{}`, got `{}`", self.ty, other.ty));
-        if self.span == other.span {
-            err.span_label(
-                self.span,
-                "this expression supplies two conflicting concrete types for the same opaque type",
-            );
+        let sub_diag = if self.span == other.span {
+            TypeMismatchReason::ConflictType { span: self.span }
         } else {
-            err.span_note(self.span, "previous use here");
-        }
-        err.emit();
+            TypeMismatchReason::PreviousUse { span: self.span }
+        };
+        tcx.sess.emit_err(OpaqueHiddenTypeMismatch {
+            self_ty: self.ty,
+            other_ty: other.ty,
+            other_span: other.span,
+            sub: sub_diag,
+        });
     }
 }
 
@@ -1411,7 +1526,7 @@ impl<'tcx> TypeFoldable<'tcx> for ParamEnv<'tcx> {
         Ok(ParamEnv::new(
             self.caller_bounds().try_fold_with(folder)?,
             self.reveal().try_fold_with(folder)?,
-            self.constness().try_fold_with(folder)?,
+            self.constness(),
         ))
     }
 }
@@ -1419,8 +1534,7 @@ impl<'tcx> TypeFoldable<'tcx> for ParamEnv<'tcx> {
 impl<'tcx> TypeVisitable<'tcx> for ParamEnv<'tcx> {
     fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
         self.caller_bounds().visit_with(visitor)?;
-        self.reveal().visit_with(visitor)?;
-        self.constness().visit_with(visitor)
+        self.reveal().visit_with(visitor)
     }
 }
 
@@ -1577,7 +1691,7 @@ impl<'tcx> PolyTraitRef<'tcx> {
 }
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, TypeFoldable, TypeVisitable)]
-#[derive(HashStable)]
+#[derive(HashStable, Lift)]
 pub struct ParamEnvAnd<'tcx, T> {
     pub param_env: ParamEnv<'tcx>,
     pub value: T,
@@ -1779,7 +1893,7 @@ pub enum VariantDiscr {
 pub struct FieldDef {
     pub did: DefId,
     pub name: Symbol,
-    pub vis: Visibility,
+    pub vis: Visibility<DefId>,
 }
 
 impl PartialEq for FieldDef {
@@ -2256,7 +2370,11 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     pub fn get_attr(self, did: DefId, attr: Symbol) -> Option<&'tcx ast::Attribute> {
-        self.get_attrs(did, attr).next()
+        if cfg!(debug_assertions) && !rustc_feature::is_valid_for_get_attr(attr) {
+            bug!("get_attr: unexpected called with DefId `{:?}`, attr `{:?}`", did, attr);
+        } else {
+            self.get_attrs(did, attr).next()
+        }
     }
 
     /// Determines whether an item is annotated with an attribute.
@@ -2358,6 +2476,25 @@ impl<'tcx> TyCtxt<'tcx> {
         (ident, scope)
     }
 
+    /// Returns `true` if the debuginfo for `span` should be collapsed to the outermost expansion
+    /// site. Only applies when `Span` is the result of macro expansion.
+    ///
+    /// - If the `collapse_debuginfo` feature is enabled then debuginfo is not collapsed by default
+    ///   and only when a macro definition is annotated with `#[collapse_debuginfo]`.
+    /// - If `collapse_debuginfo` is not enabled, then debuginfo is collapsed by default.
+    ///
+    /// When `-Zdebug-macros` is provided then debuginfo will never be collapsed.
+    pub fn should_collapse_debuginfo(self, span: Span) -> bool {
+        !self.sess.opts.unstable_opts.debug_macros
+            && if self.features().collapse_debuginfo {
+                span.in_macro_expansion_with_collapse_debuginfo()
+            } else {
+                // Inlined spans should not be collapsed as that leads to all of the
+                // inlined code being attributed to the inline callsite.
+                span.from_expansion() && !span.is_inlined()
+            }
+    }
+
     pub fn is_object_safe(self, key: DefId) -> bool {
         self.object_safety_violations(key).is_empty()
     }
@@ -2372,6 +2509,14 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn is_const_default_method(self, def_id: DefId) -> bool {
         matches!(self.trait_of_item(def_id), Some(trait_id) if self.has_attr(trait_id, sym::const_trait))
     }
+
+    pub fn impl_trait_in_trait_parent(self, mut def_id: DefId) -> DefId {
+        while let def_kind = self.def_kind(def_id) && def_kind != DefKind::AssocFn {
+            debug_assert_eq!(def_kind, DefKind::ImplTraitPlaceholder);
+            def_id = self.parent(def_id);
+        }
+        def_id
+    }
 }
 
 /// Yields the parent function's `LocalDefId` if `def_id` is an `impl Trait` definition.
@@ -2516,3 +2661,14 @@ pub struct DestructuredConst<'tcx> {
     pub variant: Option<VariantIdx>,
     pub fields: &'tcx [ty::Const<'tcx>],
 }
+
+// Some types are used a lot. Make sure they don't unintentionally get bigger.
+#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
+mod size_asserts {
+    use super::*;
+    use rustc_data_structures::static_assert_size;
+    // These are in alphabetical order, which is easy to maintain.
+    static_assert_size!(PredicateS<'_>, 48);
+    static_assert_size!(TyS<'_>, 40);
+    static_assert_size!(WithStableHash<TyS<'_>>, 56);
+}
diff --git a/compiler/rustc_middle/src/ty/normalize_erasing_regions.rs b/compiler/rustc_middle/src/ty/normalize_erasing_regions.rs
index 9d8a81165..9db5a2894 100644
--- a/compiler/rustc_middle/src/ty/normalize_erasing_regions.rs
+++ b/compiler/rustc_middle/src/ty/normalize_erasing_regions.rs
@@ -36,6 +36,7 @@ impl<'tcx> TyCtxt<'tcx> {
     ///
     /// This should only be used outside of type inference. For example,
     /// it assumes that normalization will succeed.
+    #[tracing::instrument(level = "debug", skip(self, param_env))]
     pub fn normalize_erasing_regions<T>(self, param_env: ty::ParamEnv<'tcx>, value: T) -> T
     where
         T: TypeFoldable<'tcx>,
@@ -100,6 +101,7 @@ impl<'tcx> TyCtxt<'tcx> {
     /// N.B., currently, higher-ranked type bounds inhibit
     /// normalization. Therefore, each time we erase them in
     /// codegen, we need to normalize the contents.
+    #[tracing::instrument(level = "debug", skip(self, param_env))]
     pub fn normalize_erasing_late_bound_regions<T>(
         self,
         param_env: ty::ParamEnv<'tcx>,
@@ -188,13 +190,11 @@ struct NormalizeAfterErasingRegionsFolder<'tcx> {
 }
 
 impl<'tcx> NormalizeAfterErasingRegionsFolder<'tcx> {
-    #[instrument(skip(self), level = "debug")]
     fn normalize_generic_arg_after_erasing_regions(
         &self,
         arg: ty::GenericArg<'tcx>,
     ) -> ty::GenericArg<'tcx> {
         let arg = self.param_env.and(arg);
-        debug!(?arg);
 
         self.tcx.try_normalize_generic_arg_after_erasing_regions(arg).unwrap_or_else(|_| bug!(
                 "Failed to normalize {:?}, maybe try to call `try_normalize_erasing_regions` instead",
diff --git a/compiler/rustc_middle/src/ty/parameterized.rs b/compiler/rustc_middle/src/ty/parameterized.rs
index e189ee2fc..9c8dc30e2 100644
--- a/compiler/rustc_middle/src/ty/parameterized.rs
+++ b/compiler/rustc_middle/src/ty/parameterized.rs
@@ -1,4 +1,4 @@
-use rustc_hir::def_id::DefId;
+use rustc_hir::def_id::{DefId, DefIndex};
 use rustc_index::vec::{Idx, IndexVec};
 
 use crate::middle::exported_symbols::ExportedSymbol;
@@ -53,17 +53,20 @@ trivially_parameterized_over_tcx! {
     crate::metadata::ModChild,
     crate::middle::codegen_fn_attrs::CodegenFnAttrs,
     crate::middle::exported_symbols::SymbolExportInfo,
+    crate::middle::resolve_lifetime::ObjectLifetimeDefault,
     crate::mir::ConstQualifs,
+    ty::AssocItemContainer,
     ty::Generics,
     ty::ImplPolarity,
     ty::ReprOptions,
     ty::TraitDef,
-    ty::Visibility,
+    ty::Visibility<DefIndex>,
     ty::adjustment::CoerceUnsizedInfo,
     ty::fast_reject::SimplifiedTypeGen<DefId>,
     rustc_ast::Attribute,
     rustc_ast::MacArgs,
     rustc_attr::ConstStability,
+    rustc_attr::DefaultBodyStability,
     rustc_attr::Deprecation,
     rustc_attr::Stability,
     rustc_hir::Constness,
diff --git a/compiler/rustc_middle/src/ty/print/pretty.rs b/compiler/rustc_middle/src/ty/print/pretty.rs
index 7f2e81a71..97bddb93e 100644
--- a/compiler/rustc_middle/src/ty/print/pretty.rs
+++ b/compiler/rustc_middle/src/ty/print/pretty.rs
@@ -1,7 +1,7 @@
 use crate::mir::interpret::{AllocRange, GlobalAlloc, Pointer, Provenance, Scalar};
 use crate::ty::subst::{GenericArg, GenericArgKind, Subst};
 use crate::ty::{
-    self, ConstInt, DefIdTree, ParamConst, ScalarInt, Term, Ty, TyCtxt, TypeFoldable,
+    self, ConstInt, DefIdTree, ParamConst, ScalarInt, Term, TermKind, Ty, TyCtxt, TypeFoldable,
     TypeSuperFoldable, TypeSuperVisitable, TypeVisitable,
 };
 use rustc_apfloat::ieee::{Double, Single};
@@ -619,12 +619,16 @@ pub trait PrettyPrinter<'tcx>:
             ty::Adt(def, substs) => {
                 p!(print_def_path(def.did(), substs));
             }
-            ty::Dynamic(data, r) => {
+            ty::Dynamic(data, r, repr) => {
                 let print_r = self.should_print_region(r);
                 if print_r {
                     p!("(");
                 }
-                p!("dyn ", print(data));
+                match repr {
+                    ty::Dyn => p!("dyn "),
+                    ty::DynStar => p!("dyn* "),
+                }
+                p!(print(data));
                 if print_r {
                     p!(" + ", print(r), ")");
                 }
@@ -632,7 +636,13 @@ pub trait PrettyPrinter<'tcx>:
             ty::Foreign(def_id) => {
                 p!(print_def_path(def_id, &[]));
             }
-            ty::Projection(ref data) => p!(print(data)),
+            ty::Projection(ref data) => {
+                if self.tcx().def_kind(data.item_def_id) == DefKind::ImplTraitPlaceholder {
+                    return self.pretty_print_opaque_impl_type(data.item_def_id, data.substs);
+                } else {
+                    p!(print(data))
+                }
+            }
             ty::Placeholder(placeholder) => p!(write("Placeholder({:?})", placeholder)),
             ty::Opaque(def_id, substs) => {
                 // FIXME(eddyb) print this with `print_def_path`.
@@ -855,7 +865,7 @@ pub trait PrettyPrinter<'tcx>:
                         }
 
                         p!(")");
-                        if let Term::Ty(ty) = return_ty.skip_binder() {
+                        if let Some(ty) = return_ty.skip_binder().ty() {
                             if !ty.is_unit() {
                                 p!(" -> ", print(return_ty));
                             }
@@ -916,12 +926,14 @@ pub trait PrettyPrinter<'tcx>:
                         // Skip printing `<[generator@] as Generator<_>>::Return` from async blocks,
                         // unless we can find out what generator return type it comes from.
                         let term = if let Some(ty) = term.skip_binder().ty()
-                            && let ty::Projection(ty::ProjectionTy { item_def_id, substs }) = ty.kind()
-                            && Some(*item_def_id) == tcx.lang_items().generator_return()
+                            && let ty::Projection(proj) = ty.kind()
+                            && let assoc = tcx.associated_item(proj.item_def_id)
+                            && assoc.trait_container(tcx) == tcx.lang_items().gen_trait()
+                            && assoc.name == rustc_span::sym::Return
                         {
                             if let ty::Generator(_, substs, _) = substs.type_at(0).kind() {
                                 let return_ty = substs.as_generator().return_ty();
-                                if !return_ty.is_ty_infer() {
+                                if !return_ty.is_ty_var() {
                                     return_ty.into()
                                 } else {
                                     continue;
@@ -942,13 +954,9 @@ pub trait PrettyPrinter<'tcx>:
 
                         p!(write("{} = ", tcx.associated_item(assoc_item_def_id).name));
 
-                        match term {
-                            Term::Ty(ty) => {
-                                p!(print(ty))
-                            }
-                            Term::Const(c) => {
-                                p!(print(c));
-                            }
+                        match term.unpack() {
+                            TermKind::Ty(ty) => p!(print(ty)),
+                            TermKind::Const(c) => p!(print(c)),
                         };
                     }
 
@@ -1193,15 +1201,9 @@ pub trait PrettyPrinter<'tcx>:
         }
 
         match ct.kind() {
-            ty::ConstKind::Unevaluated(ty::Unevaluated {
-                def,
-                substs,
-                promoted: Some(promoted),
-            }) => {
-                p!(print_value_path(def.did, substs));
-                p!(write("::{:?}", promoted));
-            }
-            ty::ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted: None }) => {
+            ty::ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted }) => {
+                assert_eq!(promoted, ());
+
                 match self.tcx().def_kind(def.did) {
                     DefKind::Static(..) | DefKind::Const | DefKind::AssocConst => {
                         p!(print_value_path(def.did, substs))
@@ -1275,7 +1277,7 @@ pub trait PrettyPrinter<'tcx>:
                                     let range =
                                         AllocRange { start: offset, size: Size::from_bytes(len) };
                                     if let Ok(byte_str) =
-                                        alloc.inner().get_bytes(&self.tcx(), range)
+                                        alloc.inner().get_bytes_strip_provenance(&self.tcx(), range)
                                     {
                                         p!(pretty_print_byte_str(byte_str))
                                     } else {
@@ -1401,14 +1403,7 @@ pub trait PrettyPrinter<'tcx>:
     }
 
     fn pretty_print_byte_str(mut self, byte_str: &'tcx [u8]) -> Result<Self::Const, Self::Error> {
-        define_scoped_cx!(self);
-        p!("b\"");
-        for &c in byte_str {
-            for e in std::ascii::escape_default(c) {
-                self.write_char(e as char)?;
-            }
-        }
-        p!("\"");
+        write!(self, "b\"{}\"", byte_str.escape_ascii())?;
         Ok(self)
     }
 
@@ -1513,6 +1508,10 @@ pub trait PrettyPrinter<'tcx>:
                 }
                 return Ok(self);
             }
+            (ty::ValTree::Leaf(leaf), ty::Ref(_, inner_ty, _)) => {
+                p!(write("&"));
+                return self.pretty_print_const_scalar_int(leaf, *inner_ty, print_ty);
+            }
             (ty::ValTree::Leaf(leaf), _) => {
                 return self.pretty_print_const_scalar_int(leaf, ty, print_ty);
             }
@@ -1532,6 +1531,34 @@ pub trait PrettyPrinter<'tcx>:
         }
         Ok(self)
     }
+
+    fn pretty_closure_as_impl(
+        mut self,
+        closure: ty::ClosureSubsts<'tcx>,
+    ) -> Result<Self::Const, Self::Error> {
+        let sig = closure.sig();
+        let kind = closure.kind_ty().to_opt_closure_kind().unwrap_or(ty::ClosureKind::Fn);
+
+        write!(self, "impl ")?;
+        self.wrap_binder(&sig, |sig, mut cx| {
+            define_scoped_cx!(cx);
+
+            p!(print(kind), "(");
+            for (i, arg) in sig.inputs()[0].tuple_fields().iter().enumerate() {
+                if i > 0 {
+                    p!(", ");
+                }
+                p!(print(arg));
+            }
+            p!(")");
+
+            if !sig.output().is_unit() {
+                p!(" -> ", print(sig.output()));
+            }
+
+            Ok(cx)
+        })
+    }
 }
 
 // HACK(eddyb) boxed to avoid moving around a large struct by-value.
@@ -1950,7 +1977,7 @@ impl<'tcx> PrettyPrinter<'tcx> for FmtPrinter<'_, 'tcx> {
 
             ty::ReVar(_) | ty::ReErased => false,
 
-            ty::ReStatic | ty::ReEmpty(_) => true,
+            ty::ReStatic => true,
         }
     }
 
@@ -2034,14 +2061,6 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
                 p!("'static");
                 return Ok(self);
             }
-            ty::ReEmpty(ty::UniverseIndex::ROOT) => {
-                p!("'<empty>");
-                return Ok(self);
-            }
-            ty::ReEmpty(ui) => {
-                p!(write("'<empty:{:?}>", ui));
-                return Ok(self);
-            }
         }
 
         p!("'_");
@@ -2446,6 +2465,11 @@ impl<'tcx> ty::PolyTraitPredicate<'tcx> {
     }
 }
 
+#[derive(Debug, Copy, Clone, TypeFoldable, TypeVisitable, Lift)]
+pub struct PrintClosureAsImpl<'tcx> {
+    pub closure: ty::ClosureSubsts<'tcx>,
+}
+
 forward_display_to_print! {
     ty::Region<'tcx>,
     Ty<'tcx>,
@@ -2538,6 +2562,10 @@ define_print_and_forward_display! {
         p!(print(self.0.trait_ref.print_only_trait_path()));
     }
 
+    PrintClosureAsImpl<'tcx> {
+        p!(pretty_closure_as_impl(self.closure))
+    }
+
     ty::ParamTy {
         p!(write("{}", self.name))
     }
@@ -2567,9 +2595,9 @@ define_print_and_forward_display! {
     }
 
     ty::Term<'tcx> {
-      match self {
-        ty::Term::Ty(ty) => p!(print(ty)),
-        ty::Term::Const(c) => p!(print(c)),
+      match self.unpack() {
+        ty::TermKind::Ty(ty) => p!(print(ty)),
+        ty::TermKind::Const(c) => p!(print(c)),
       }
     }
 
diff --git a/compiler/rustc_middle/src/ty/query.rs b/compiler/rustc_middle/src/ty/query.rs
index 2452bcf6a..a300a8df2 100644
--- a/compiler/rustc_middle/src/ty/query.rs
+++ b/compiler/rustc_middle/src/ty/query.rs
@@ -121,8 +121,8 @@ macro_rules! query_storage {
     ([][$K:ty, $V:ty]) => {
         <DefaultCacheSelector as CacheSelector<$K, $V>>::Cache
     };
-    ([(storage $ty:ty) $($rest:tt)*][$K:ty, $V:ty]) => {
-        <$ty as CacheSelector<$K, $V>>::Cache
+    ([(arena_cache) $($rest:tt)*][$K:ty, $V:ty]) => {
+        <ArenaCacheSelector<'tcx> as CacheSelector<$K, $V>>::Cache
     };
     ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
         query_storage!([$($modifiers)*][$($args)*])
@@ -173,7 +173,7 @@ macro_rules! opt_remap_env_constness {
 }
 
 macro_rules! define_callbacks {
-    (<$tcx:tt>
+    (
      $($(#[$attr:meta])*
         [$($modifiers:tt)*] fn $name:ident($($K:tt)*) -> $V:ty,)*) => {
 
@@ -187,33 +187,33 @@ macro_rules! define_callbacks {
         pub mod query_keys {
             use super::*;
 
-            $(pub type $name<$tcx> = $($K)*;)*
+            $(pub type $name<'tcx> = $($K)*;)*
         }
         #[allow(nonstandard_style, unused_lifetimes)]
         pub mod query_values {
             use super::*;
 
-            $(pub type $name<$tcx> = $V;)*
+            $(pub type $name<'tcx> = $V;)*
         }
         #[allow(nonstandard_style, unused_lifetimes)]
         pub mod query_storage {
             use super::*;
 
-            $(pub type $name<$tcx> = query_storage!([$($modifiers)*][$($K)*, $V]);)*
+            $(pub type $name<'tcx> = query_storage!([$($modifiers)*][$($K)*, $V]);)*
         }
         #[allow(nonstandard_style, unused_lifetimes)]
         pub mod query_stored {
             use super::*;
 
-            $(pub type $name<$tcx> = <query_storage::$name<$tcx> as QueryStorage>::Stored;)*
+            $(pub type $name<'tcx> = <query_storage::$name<'tcx> as QueryStorage>::Stored;)*
         }
 
         #[derive(Default)]
-        pub struct QueryCaches<$tcx> {
-            $($(#[$attr])* pub $name: query_storage::$name<$tcx>,)*
+        pub struct QueryCaches<'tcx> {
+            $($(#[$attr])* pub $name: query_storage::$name<'tcx>,)*
         }
 
-        impl<$tcx> TyCtxtEnsure<$tcx> {
+        impl<'tcx> TyCtxtEnsure<'tcx> {
             $($(#[$attr])*
             #[inline(always)]
             pub fn $name(self, key: query_helper_param_ty!($($K)*)) {
@@ -231,20 +231,20 @@ macro_rules! define_callbacks {
             })*
         }
 
-        impl<$tcx> TyCtxt<$tcx> {
+        impl<'tcx> TyCtxt<'tcx> {
             $($(#[$attr])*
             #[inline(always)]
             #[must_use]
-            pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> query_stored::$name<$tcx>
+            pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> query_stored::$name<'tcx>
             {
                 self.at(DUMMY_SP).$name(key)
             })*
         }
 
-        impl<$tcx> TyCtxtAt<$tcx> {
+        impl<'tcx> TyCtxtAt<'tcx> {
             $($(#[$attr])*
             #[inline(always)]
-            pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> query_stored::$name<$tcx>
+            pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> query_stored::$name<'tcx>
             {
                 let key = key.into_query_param();
                 opt_remap_env_constness!([$($modifiers)*][key]);
@@ -311,11 +311,11 @@ macro_rules! define_callbacks {
             $($(#[$attr])*
             fn $name(
                 &'tcx self,
-                tcx: TyCtxt<$tcx>,
+                tcx: TyCtxt<'tcx>,
                 span: Span,
-                key: query_keys::$name<$tcx>,
+                key: query_keys::$name<'tcx>,
                 mode: QueryMode,
-            ) -> Option<query_stored::$name<$tcx>>;)*
+            ) -> Option<query_stored::$name<'tcx>>;)*
         }
     };
 }
@@ -332,7 +332,7 @@ macro_rules! define_callbacks {
 // Queries marked with `fatal_cycle` do not need the latter implementation,
 // as they will raise an fatal error on query cycles instead.
 
-rustc_query_append! { [define_callbacks!][<'tcx>] }
+rustc_query_append! { define_callbacks! }
 
 mod sealed {
     use super::{DefId, LocalDefId};
diff --git a/compiler/rustc_middle/src/ty/relate.rs b/compiler/rustc_middle/src/ty/relate.rs
index 818affa71..61c34730d 100644
--- a/compiler/rustc_middle/src/ty/relate.rs
+++ b/compiler/rustc_middle/src/ty/relate.rs
@@ -6,7 +6,7 @@
 
 use crate::ty::error::{ExpectedFound, TypeError};
 use crate::ty::subst::{GenericArg, GenericArgKind, Subst, SubstsRef};
-use crate::ty::{self, ImplSubject, Term, Ty, TyCtxt, TypeFoldable};
+use crate::ty::{self, ImplSubject, Term, TermKind, Ty, TyCtxt, TypeFoldable};
 use rustc_hir as ast;
 use rustc_hir::def_id::DefId;
 use rustc_span::DUMMY_SP;
@@ -441,7 +441,9 @@ pub fn super_relate_tys<'tcx, R: TypeRelation<'tcx>>(
 
         (&ty::Foreign(a_id), &ty::Foreign(b_id)) if a_id == b_id => Ok(tcx.mk_foreign(a_id)),
 
-        (&ty::Dynamic(a_obj, a_region), &ty::Dynamic(b_obj, b_region)) => {
+        (&ty::Dynamic(a_obj, a_region, a_repr), &ty::Dynamic(b_obj, b_region, b_repr))
+            if a_repr == b_repr =>
+        {
             let region_bound = relation.with_cause(Cause::ExistentialRegionBound, |relation| {
                 relation.relate_with_variance(
                     ty::Contravariant,
@@ -450,7 +452,7 @@ pub fn super_relate_tys<'tcx, R: TypeRelation<'tcx>>(
                     b_region,
                 )
             })?;
-            Ok(tcx.mk_dynamic(relation.relate(a_obj, b_obj)?, region_bound))
+            Ok(tcx.mk_dynamic(relation.relate(a_obj, b_obj)?, region_bound, a_repr))
         }
 
         (&ty::Generator(a_id, a_substs, movability), &ty::Generator(b_id, b_substs, _))
@@ -572,8 +574,8 @@ pub fn super_relate_tys<'tcx, R: TypeRelation<'tcx>>(
 /// it.
 pub fn super_relate_consts<'tcx, R: TypeRelation<'tcx>>(
     relation: &mut R,
-    a: ty::Const<'tcx>,
-    b: ty::Const<'tcx>,
+    mut a: ty::Const<'tcx>,
+    mut b: ty::Const<'tcx>,
 ) -> RelateResult<'tcx, ty::Const<'tcx>> {
     debug!("{}.super_relate_consts(a = {:?}, b = {:?})", relation.tag(), a, b);
     let tcx = relation.tcx();
@@ -594,9 +596,16 @@ pub fn super_relate_consts<'tcx, R: TypeRelation<'tcx>>(
         );
     }
 
-    let eagerly_eval = |x: ty::Const<'tcx>| x.eval(tcx, relation.param_env());
-    let a = eagerly_eval(a);
-    let b = eagerly_eval(b);
+    // HACK(const_generics): We still need to eagerly evaluate consts when
+    // relating them because during `normalize_param_env_or_error`,
+    // we may relate an evaluated constant in a obligation against
+    // an unnormalized (i.e. unevaluated) const in the param-env.
+    // FIXME(generic_const_exprs): Once we always lazily unify unevaluated constants
+    // these `eval` calls can be removed.
+    if !relation.tcx().features().generic_const_exprs {
+        a = a.eval(tcx, relation.param_env());
+        b = b.eval(tcx, relation.param_env());
+    }
 
     // Currently, the values that can be unified are primitive types,
     // and those that derive both `PartialEq` and `Eq`, corresponding
@@ -617,7 +626,7 @@ pub fn super_relate_consts<'tcx, R: TypeRelation<'tcx>>(
         (ty::ConstKind::Unevaluated(au), ty::ConstKind::Unevaluated(bu))
             if tcx.features().generic_const_exprs =>
         {
-            tcx.try_unify_abstract_consts(relation.param_env().and((au.shrink(), bu.shrink())))
+            tcx.try_unify_abstract_consts(relation.param_env().and((au, bu)))
         }
 
         // While this is slightly incorrect, it shouldn't matter for `min_const_generics`
@@ -626,6 +635,8 @@ pub fn super_relate_consts<'tcx, R: TypeRelation<'tcx>>(
         (ty::ConstKind::Unevaluated(au), ty::ConstKind::Unevaluated(bu))
             if au.def == bu.def && au.promoted == bu.promoted =>
         {
+            assert_eq!(au.promoted, ());
+
             let substs = relation.relate_with_variance(
                 ty::Variance::Invariant,
                 ty::VarianceDiagInfo::default(),
@@ -636,7 +647,7 @@ pub fn super_relate_consts<'tcx, R: TypeRelation<'tcx>>(
                 kind: ty::ConstKind::Unevaluated(ty::Unevaluated {
                     def: au.def,
                     substs,
-                    promoted: au.promoted,
+                    promoted: (),
                 }),
                 ty: a.ty(),
             }));
@@ -803,15 +814,15 @@ impl<'tcx> Relate<'tcx> for ty::TraitPredicate<'tcx> {
     }
 }
 
-impl<'tcx> Relate<'tcx> for ty::Term<'tcx> {
+impl<'tcx> Relate<'tcx> for Term<'tcx> {
     fn relate<R: TypeRelation<'tcx>>(
         relation: &mut R,
         a: Self,
         b: Self,
     ) -> RelateResult<'tcx, Self> {
-        Ok(match (a, b) {
-            (Term::Ty(a), Term::Ty(b)) => relation.relate(a, b)?.into(),
-            (Term::Const(a), Term::Const(b)) => relation.relate(a, b)?.into(),
+        Ok(match (a.unpack(), b.unpack()) {
+            (TermKind::Ty(a), TermKind::Ty(b)) => relation.relate(a, b)?.into(),
+            (TermKind::Const(a), TermKind::Const(b)) => relation.relate(a, b)?.into(),
             _ => return Err(TypeError::Mismatch),
         })
     }
diff --git a/compiler/rustc_middle/src/ty/structural_impls.rs b/compiler/rustc_middle/src/ty/structural_impls.rs
index 7660a2f3a..84d6a8b97 100644
--- a/compiler/rustc_middle/src/ty/structural_impls.rs
+++ b/compiler/rustc_middle/src/ty/structural_impls.rs
@@ -3,13 +3,12 @@
 //! hand, though we've recently added some macros and proc-macros to help with the tedium.
 
 use crate::mir::interpret;
-use crate::mir::ProjectionKind;
+use crate::mir::{Field, ProjectionKind};
 use crate::ty::fold::{FallibleTypeFolder, TypeFoldable, TypeSuperFoldable};
 use crate::ty::print::{with_no_trimmed_paths, FmtPrinter, Printer};
 use crate::ty::visit::{TypeSuperVisitable, TypeVisitable, TypeVisitor};
-use crate::ty::{self, InferConst, Lift, Term, Ty, TyCtxt};
+use crate::ty::{self, InferConst, Lift, Term, TermKind, Ty, TyCtxt};
 use rustc_data_structures::functor::IdFunctor;
-use rustc_hir as hir;
 use rustc_hir::def::Namespace;
 use rustc_index::vec::{Idx, IndexVec};
 
@@ -238,12 +237,24 @@ TrivialTypeTraversalAndLiftImpls! {
     crate::ty::Variance,
     ::rustc_span::Span,
     ::rustc_errors::ErrorGuaranteed,
+    Field,
+    interpret::Scalar,
+    rustc_target::abi::Size,
+    ty::DelaySpanBugEmitted,
+    rustc_type_ir::DebruijnIndex,
+    ty::BoundVar,
+    ty::Placeholder<ty::BoundVar>,
+}
+
+TrivialTypeTraversalAndLiftImpls! {
+    for<'tcx> {
+        ty::ValTree<'tcx>,
+    }
 }
 
 ///////////////////////////////////////////////////////////////////////////
 // Lift implementations
 
-// FIXME(eddyb) replace all the uses of `Option::map` with `?`.
 impl<'tcx, A: Lift<'tcx>, B: Lift<'tcx>> Lift<'tcx> for (A, B) {
     type Lifted = (A::Lifted, B::Lifted);
     fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
@@ -261,10 +272,10 @@ impl<'tcx, A: Lift<'tcx>, B: Lift<'tcx>, C: Lift<'tcx>> Lift<'tcx> for (A, B, C)
 impl<'tcx, T: Lift<'tcx>> Lift<'tcx> for Option<T> {
     type Lifted = Option<T::Lifted>;
     fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        match self {
-            Some(x) => tcx.lift(x).map(Some),
-            None => Some(None),
-        }
+        Some(match self {
+            Some(x) => Some(tcx.lift(x)?),
+            None => None,
+        })
     }
 }
 
@@ -281,21 +292,21 @@ impl<'tcx, T: Lift<'tcx>, E: Lift<'tcx>> Lift<'tcx> for Result<T, E> {
 impl<'tcx, T: Lift<'tcx>> Lift<'tcx> for Box<T> {
     type Lifted = Box<T::Lifted>;
     fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        tcx.lift(*self).map(Box::new)
+        Some(Box::new(tcx.lift(*self)?))
     }
 }
 
 impl<'tcx, T: Lift<'tcx> + Clone> Lift<'tcx> for Rc<T> {
     type Lifted = Rc<T::Lifted>;
     fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        tcx.lift(self.as_ref().clone()).map(Rc::new)
+        Some(Rc::new(tcx.lift(self.as_ref().clone())?))
     }
 }
 
 impl<'tcx, T: Lift<'tcx> + Clone> Lift<'tcx> for Arc<T> {
     type Lifted = Arc<T::Lifted>;
     fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        tcx.lift(self.as_ref().clone()).map(Arc::new)
+        Some(Arc::new(tcx.lift(self.as_ref().clone())?))
     }
 }
 impl<'tcx, T: Lift<'tcx>> Lift<'tcx> for Vec<T> {
@@ -312,159 +323,18 @@ impl<'tcx, I: Idx, T: Lift<'tcx>> Lift<'tcx> for IndexVec<I, T> {
     }
 }
 
-impl<'a, 'tcx> Lift<'tcx> for ty::TraitRef<'a> {
-    type Lifted = ty::TraitRef<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        tcx.lift(self.substs).map(|substs| ty::TraitRef { def_id: self.def_id, substs })
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::ExistentialTraitRef<'a> {
-    type Lifted = ty::ExistentialTraitRef<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        tcx.lift(self.substs).map(|substs| ty::ExistentialTraitRef { def_id: self.def_id, substs })
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::ExistentialPredicate<'a> {
-    type Lifted = ty::ExistentialPredicate<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        match self {
-            ty::ExistentialPredicate::Trait(x) => tcx.lift(x).map(ty::ExistentialPredicate::Trait),
-            ty::ExistentialPredicate::Projection(x) => {
-                tcx.lift(x).map(ty::ExistentialPredicate::Projection)
-            }
-            ty::ExistentialPredicate::AutoTrait(def_id) => {
-                Some(ty::ExistentialPredicate::AutoTrait(def_id))
-            }
-        }
-    }
-}
-
 impl<'a, 'tcx> Lift<'tcx> for Term<'a> {
     type Lifted = ty::Term<'tcx>;
     fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        Some(match self {
-            Term::Ty(ty) => Term::Ty(tcx.lift(ty)?),
-            Term::Const(c) => Term::Const(tcx.lift(c)?),
-        })
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::TraitPredicate<'a> {
-    type Lifted = ty::TraitPredicate<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<ty::TraitPredicate<'tcx>> {
-        tcx.lift(self.trait_ref).map(|trait_ref| ty::TraitPredicate {
-            trait_ref,
-            constness: self.constness,
-            polarity: self.polarity,
-        })
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::SubtypePredicate<'a> {
-    type Lifted = ty::SubtypePredicate<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<ty::SubtypePredicate<'tcx>> {
-        tcx.lift((self.a, self.b)).map(|(a, b)| ty::SubtypePredicate {
-            a_is_expected: self.a_is_expected,
-            a,
-            b,
-        })
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::CoercePredicate<'a> {
-    type Lifted = ty::CoercePredicate<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<ty::CoercePredicate<'tcx>> {
-        tcx.lift((self.a, self.b)).map(|(a, b)| ty::CoercePredicate { a, b })
-    }
-}
-
-impl<'tcx, A: Copy + Lift<'tcx>, B: Copy + Lift<'tcx>> Lift<'tcx> for ty::OutlivesPredicate<A, B> {
-    type Lifted = ty::OutlivesPredicate<A::Lifted, B::Lifted>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        tcx.lift((self.0, self.1)).map(|(a, b)| ty::OutlivesPredicate(a, b))
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::ProjectionTy<'a> {
-    type Lifted = ty::ProjectionTy<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<ty::ProjectionTy<'tcx>> {
-        tcx.lift(self.substs)
-            .map(|substs| ty::ProjectionTy { item_def_id: self.item_def_id, substs })
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::ProjectionPredicate<'a> {
-    type Lifted = ty::ProjectionPredicate<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<ty::ProjectionPredicate<'tcx>> {
-        tcx.lift((self.projection_ty, self.term))
-            .map(|(projection_ty, term)| ty::ProjectionPredicate { projection_ty, term })
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::ExistentialProjection<'a> {
-    type Lifted = ty::ExistentialProjection<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        tcx.lift(self.substs).map(|substs| ty::ExistentialProjection {
-            substs,
-            term: tcx.lift(self.term).expect("type must lift when substs do"),
-            item_def_id: self.item_def_id,
-        })
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::PredicateKind<'a> {
-    type Lifted = ty::PredicateKind<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        match self {
-            ty::PredicateKind::Trait(data) => tcx.lift(data).map(ty::PredicateKind::Trait),
-            ty::PredicateKind::Subtype(data) => tcx.lift(data).map(ty::PredicateKind::Subtype),
-            ty::PredicateKind::Coerce(data) => tcx.lift(data).map(ty::PredicateKind::Coerce),
-            ty::PredicateKind::RegionOutlives(data) => {
-                tcx.lift(data).map(ty::PredicateKind::RegionOutlives)
-            }
-            ty::PredicateKind::TypeOutlives(data) => {
-                tcx.lift(data).map(ty::PredicateKind::TypeOutlives)
-            }
-            ty::PredicateKind::Projection(data) => {
-                tcx.lift(data).map(ty::PredicateKind::Projection)
-            }
-            ty::PredicateKind::WellFormed(ty) => tcx.lift(ty).map(ty::PredicateKind::WellFormed),
-            ty::PredicateKind::ClosureKind(closure_def_id, closure_substs, kind) => {
-                tcx.lift(closure_substs).map(|closure_substs| {
-                    ty::PredicateKind::ClosureKind(closure_def_id, closure_substs, kind)
-                })
-            }
-            ty::PredicateKind::ObjectSafe(trait_def_id) => {
-                Some(ty::PredicateKind::ObjectSafe(trait_def_id))
-            }
-            ty::PredicateKind::ConstEvaluatable(uv) => {
-                tcx.lift(uv).map(|uv| ty::PredicateKind::ConstEvaluatable(uv))
-            }
-            ty::PredicateKind::ConstEquate(c1, c2) => {
-                tcx.lift((c1, c2)).map(|(c1, c2)| ty::PredicateKind::ConstEquate(c1, c2))
-            }
-            ty::PredicateKind::TypeWellFormedFromEnv(ty) => {
-                tcx.lift(ty).map(ty::PredicateKind::TypeWellFormedFromEnv)
+        Some(
+            match self.unpack() {
+                TermKind::Ty(ty) => TermKind::Ty(tcx.lift(ty)?),
+                TermKind::Const(c) => TermKind::Const(tcx.lift(c)?),
             }
-        }
+            .pack(),
+        )
     }
 }
-
-impl<'a, 'tcx, T: Lift<'tcx>> Lift<'tcx> for ty::Binder<'a, T>
-where
-    <T as Lift<'tcx>>::Lifted: TypeVisitable<'tcx>,
-{
-    type Lifted = ty::Binder<'tcx, T::Lifted>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        let bound_vars = tcx.lift(self.bound_vars());
-        tcx.lift(self.skip_binder())
-            .zip(bound_vars)
-            .map(|(value, vars)| ty::Binder::bind_with_vars(value, vars))
-    }
-}
-
 impl<'a, 'tcx> Lift<'tcx> for ty::ParamEnv<'a> {
     type Lifted = ty::ParamEnv<'tcx>;
     fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
@@ -473,178 +343,6 @@ impl<'a, 'tcx> Lift<'tcx> for ty::ParamEnv<'a> {
     }
 }
 
-impl<'a, 'tcx, T: Lift<'tcx>> Lift<'tcx> for ty::ParamEnvAnd<'a, T> {
-    type Lifted = ty::ParamEnvAnd<'tcx, T::Lifted>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        tcx.lift(self.param_env).and_then(|param_env| {
-            tcx.lift(self.value).map(|value| ty::ParamEnvAnd { param_env, value })
-        })
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::ClosureSubsts<'a> {
-    type Lifted = ty::ClosureSubsts<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        tcx.lift(self.substs).map(|substs| ty::ClosureSubsts { substs })
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::GeneratorSubsts<'a> {
-    type Lifted = ty::GeneratorSubsts<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        tcx.lift(self.substs).map(|substs| ty::GeneratorSubsts { substs })
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::adjustment::Adjustment<'a> {
-    type Lifted = ty::adjustment::Adjustment<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        let ty::adjustment::Adjustment { kind, target } = self;
-        tcx.lift(kind).and_then(|kind| {
-            tcx.lift(target).map(|target| ty::adjustment::Adjustment { kind, target })
-        })
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::adjustment::Adjust<'a> {
-    type Lifted = ty::adjustment::Adjust<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        match self {
-            ty::adjustment::Adjust::NeverToAny => Some(ty::adjustment::Adjust::NeverToAny),
-            ty::adjustment::Adjust::Pointer(ptr) => Some(ty::adjustment::Adjust::Pointer(ptr)),
-            ty::adjustment::Adjust::Deref(overloaded) => {
-                tcx.lift(overloaded).map(ty::adjustment::Adjust::Deref)
-            }
-            ty::adjustment::Adjust::Borrow(autoref) => {
-                tcx.lift(autoref).map(ty::adjustment::Adjust::Borrow)
-            }
-        }
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::adjustment::OverloadedDeref<'a> {
-    type Lifted = ty::adjustment::OverloadedDeref<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        tcx.lift(self.region).map(|region| ty::adjustment::OverloadedDeref {
-            region,
-            mutbl: self.mutbl,
-            span: self.span,
-        })
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::adjustment::AutoBorrow<'a> {
-    type Lifted = ty::adjustment::AutoBorrow<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        match self {
-            ty::adjustment::AutoBorrow::Ref(r, m) => {
-                tcx.lift(r).map(|r| ty::adjustment::AutoBorrow::Ref(r, m))
-            }
-            ty::adjustment::AutoBorrow::RawPtr(m) => Some(ty::adjustment::AutoBorrow::RawPtr(m)),
-        }
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::GenSig<'a> {
-    type Lifted = ty::GenSig<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        tcx.lift((self.resume_ty, self.yield_ty, self.return_ty))
-            .map(|(resume_ty, yield_ty, return_ty)| ty::GenSig { resume_ty, yield_ty, return_ty })
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::FnSig<'a> {
-    type Lifted = ty::FnSig<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        tcx.lift(self.inputs_and_output).map(|x| ty::FnSig {
-            inputs_and_output: x,
-            c_variadic: self.c_variadic,
-            unsafety: self.unsafety,
-            abi: self.abi,
-        })
-    }
-}
-
-impl<'tcx, T: Lift<'tcx>> Lift<'tcx> for ty::error::ExpectedFound<T> {
-    type Lifted = ty::error::ExpectedFound<T::Lifted>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        let ty::error::ExpectedFound { expected, found } = self;
-        tcx.lift(expected).and_then(|expected| {
-            tcx.lift(found).map(|found| ty::error::ExpectedFound { expected, found })
-        })
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::error::TypeError<'a> {
-    type Lifted = ty::error::TypeError<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        use crate::ty::error::TypeError::*;
-
-        Some(match self {
-            Mismatch => Mismatch,
-            ConstnessMismatch(x) => ConstnessMismatch(x),
-            PolarityMismatch(x) => PolarityMismatch(x),
-            UnsafetyMismatch(x) => UnsafetyMismatch(x),
-            AbiMismatch(x) => AbiMismatch(x),
-            Mutability => Mutability,
-            ArgumentMutability(i) => ArgumentMutability(i),
-            TupleSize(x) => TupleSize(x),
-            FixedArraySize(x) => FixedArraySize(x),
-            ArgCount => ArgCount,
-            FieldMisMatch(x, y) => FieldMisMatch(x, y),
-            RegionsDoesNotOutlive(a, b) => {
-                return tcx.lift((a, b)).map(|(a, b)| RegionsDoesNotOutlive(a, b));
-            }
-            RegionsInsufficientlyPolymorphic(a, b) => {
-                return tcx.lift(b).map(|b| RegionsInsufficientlyPolymorphic(a, b));
-            }
-            RegionsOverlyPolymorphic(a, b) => {
-                return tcx.lift(b).map(|b| RegionsOverlyPolymorphic(a, b));
-            }
-            RegionsPlaceholderMismatch => RegionsPlaceholderMismatch,
-            IntMismatch(x) => IntMismatch(x),
-            FloatMismatch(x) => FloatMismatch(x),
-            Traits(x) => Traits(x),
-            VariadicMismatch(x) => VariadicMismatch(x),
-            CyclicTy(t) => return tcx.lift(t).map(|t| CyclicTy(t)),
-            CyclicConst(ct) => return tcx.lift(ct).map(|ct| CyclicConst(ct)),
-            ProjectionMismatched(x) => ProjectionMismatched(x),
-            ArgumentSorts(x, i) => return tcx.lift(x).map(|x| ArgumentSorts(x, i)),
-            Sorts(x) => return tcx.lift(x).map(Sorts),
-            ExistentialMismatch(x) => return tcx.lift(x).map(ExistentialMismatch),
-            ConstMismatch(x) => return tcx.lift(x).map(ConstMismatch),
-            IntrinsicCast => IntrinsicCast,
-            TargetFeatureCast(x) => TargetFeatureCast(x),
-            ObjectUnsafeCoercion(x) => return tcx.lift(x).map(ObjectUnsafeCoercion),
-        })
-    }
-}
-
-impl<'a, 'tcx> Lift<'tcx> for ty::InstanceDef<'a> {
-    type Lifted = ty::InstanceDef<'tcx>;
-    fn lift_to_tcx(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {
-        match self {
-            ty::InstanceDef::Item(def_id) => Some(ty::InstanceDef::Item(def_id)),
-            ty::InstanceDef::VTableShim(def_id) => Some(ty::InstanceDef::VTableShim(def_id)),
-            ty::InstanceDef::ReifyShim(def_id) => Some(ty::InstanceDef::ReifyShim(def_id)),
-            ty::InstanceDef::Intrinsic(def_id) => Some(ty::InstanceDef::Intrinsic(def_id)),
-            ty::InstanceDef::FnPtrShim(def_id, ty) => {
-                Some(ty::InstanceDef::FnPtrShim(def_id, tcx.lift(ty)?))
-            }
-            ty::InstanceDef::Virtual(def_id, n) => Some(ty::InstanceDef::Virtual(def_id, n)),
-            ty::InstanceDef::ClosureOnceShim { call_once, track_caller } => {
-                Some(ty::InstanceDef::ClosureOnceShim { call_once, track_caller })
-            }
-            ty::InstanceDef::DropGlue(def_id, ty) => {
-                Some(ty::InstanceDef::DropGlue(def_id, tcx.lift(ty)?))
-            }
-            ty::InstanceDef::CloneShim(def_id, ty) => {
-                Some(ty::InstanceDef::CloneShim(def_id, tcx.lift(ty)?))
-            }
-        }
-    }
-}
-
 ///////////////////////////////////////////////////////////////////////////
 // TypeFoldable implementations.
 
@@ -844,6 +542,12 @@ impl<'tcx, T: TypeVisitable<'tcx>> TypeVisitable<'tcx> for Vec<T> {
     }
 }
 
+impl<'tcx, T: TypeVisitable<'tcx>> TypeVisitable<'tcx> for &[T] {
+    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
+        self.iter().try_for_each(|t| t.visit_with(visitor))
+    }
+}
+
 impl<'tcx, T: TypeFoldable<'tcx>> TypeFoldable<'tcx> for Box<[T]> {
     fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
         self.try_map_id(|t| t.try_fold_with(folder))
@@ -901,88 +605,12 @@ impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::List<ty::Binder<'tcx, ty::Existentia
     }
 }
 
-impl<'tcx> TypeVisitable<'tcx>
-    for &'tcx ty::List<ty::Binder<'tcx, ty::ExistentialPredicate<'tcx>>>
-{
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
-        self.iter().try_for_each(|p| p.visit_with(visitor))
-    }
-}
-
 impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::List<ProjectionKind> {
     fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
         ty::util::fold_list(self, folder, |tcx, v| tcx.intern_projs(v))
     }
 }
 
-impl<'tcx> TypeVisitable<'tcx> for &'tcx ty::List<ProjectionKind> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
-        self.iter().try_for_each(|t| t.visit_with(visitor))
-    }
-}
-
-impl<'tcx> TypeFoldable<'tcx> for ty::instance::Instance<'tcx> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
-        use crate::ty::InstanceDef::*;
-        Ok(Self {
-            substs: self.substs.try_fold_with(folder)?,
-            def: match self.def {
-                Item(def) => Item(def.try_fold_with(folder)?),
-                VTableShim(did) => VTableShim(did.try_fold_with(folder)?),
-                ReifyShim(did) => ReifyShim(did.try_fold_with(folder)?),
-                Intrinsic(did) => Intrinsic(did.try_fold_with(folder)?),
-                FnPtrShim(did, ty) => {
-                    FnPtrShim(did.try_fold_with(folder)?, ty.try_fold_with(folder)?)
-                }
-                Virtual(did, i) => Virtual(did.try_fold_with(folder)?, i),
-                ClosureOnceShim { call_once, track_caller } => {
-                    ClosureOnceShim { call_once: call_once.try_fold_with(folder)?, track_caller }
-                }
-                DropGlue(did, ty) => {
-                    DropGlue(did.try_fold_with(folder)?, ty.try_fold_with(folder)?)
-                }
-                CloneShim(did, ty) => {
-                    CloneShim(did.try_fold_with(folder)?, ty.try_fold_with(folder)?)
-                }
-            },
-        })
-    }
-}
-
-impl<'tcx> TypeVisitable<'tcx> for ty::instance::Instance<'tcx> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
-        use crate::ty::InstanceDef::*;
-        self.substs.visit_with(visitor)?;
-        match self.def {
-            Item(def) => def.visit_with(visitor),
-            VTableShim(did) | ReifyShim(did) | Intrinsic(did) | Virtual(did, _) => {
-                did.visit_with(visitor)
-            }
-            FnPtrShim(did, ty) | CloneShim(did, ty) => {
-                did.visit_with(visitor)?;
-                ty.visit_with(visitor)
-            }
-            DropGlue(did, ty) => {
-                did.visit_with(visitor)?;
-                ty.visit_with(visitor)
-            }
-            ClosureOnceShim { call_once, track_caller: _ } => call_once.visit_with(visitor),
-        }
-    }
-}
-
-impl<'tcx> TypeFoldable<'tcx> for interpret::GlobalId<'tcx> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
-        Ok(Self { instance: self.instance.try_fold_with(folder)?, promoted: self.promoted })
-    }
-}
-
-impl<'tcx> TypeVisitable<'tcx> for interpret::GlobalId<'tcx> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
-        self.instance.visit_with(visitor)
-    }
-}
-
 impl<'tcx> TypeFoldable<'tcx> for Ty<'tcx> {
     fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
         folder.try_fold_ty(self)
@@ -1005,9 +633,11 @@ impl<'tcx> TypeSuperFoldable<'tcx> for Ty<'tcx> {
             ty::Array(typ, sz) => ty::Array(typ.try_fold_with(folder)?, sz.try_fold_with(folder)?),
             ty::Slice(typ) => ty::Slice(typ.try_fold_with(folder)?),
             ty::Adt(tid, substs) => ty::Adt(tid, substs.try_fold_with(folder)?),
-            ty::Dynamic(trait_ty, region) => {
-                ty::Dynamic(trait_ty.try_fold_with(folder)?, region.try_fold_with(folder)?)
-            }
+            ty::Dynamic(trait_ty, region, representation) => ty::Dynamic(
+                trait_ty.try_fold_with(folder)?,
+                region.try_fold_with(folder)?,
+                representation,
+            ),
             ty::Tuple(ts) => ty::Tuple(ts.try_fold_with(folder)?),
             ty::FnDef(def_id, substs) => ty::FnDef(def_id, substs.try_fold_with(folder)?),
             ty::FnPtr(f) => ty::FnPtr(f.try_fold_with(folder)?),
@@ -1051,7 +681,7 @@ impl<'tcx> TypeSuperVisitable<'tcx> for Ty<'tcx> {
             }
             ty::Slice(typ) => typ.visit_with(visitor),
             ty::Adt(_, substs) => substs.visit_with(visitor),
-            ty::Dynamic(ref trait_ty, ref reg) => {
+            ty::Dynamic(ref trait_ty, ref reg, _) => {
                 trait_ty.visit_with(visitor)?;
                 reg.visit_with(visitor)
             }
@@ -1156,12 +786,6 @@ impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::List<ty::Predicate<'tcx>> {
     }
 }
 
-impl<'tcx> TypeVisitable<'tcx> for &'tcx ty::List<ty::Predicate<'tcx>> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
-        self.iter().try_for_each(|p| p.visit_with(visitor))
-    }
-}
-
 impl<'tcx, T: TypeFoldable<'tcx>, I: Idx> TypeFoldable<'tcx> for IndexVec<I, T> {
     fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
         self.try_map_id(|x| x.try_fold_with(folder))
@@ -1208,34 +832,6 @@ impl<'tcx> TypeSuperVisitable<'tcx> for ty::Const<'tcx> {
     }
 }
 
-impl<'tcx> TypeFoldable<'tcx> for ty::ConstKind<'tcx> {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
-        Ok(match self {
-            ty::ConstKind::Infer(ic) => ty::ConstKind::Infer(ic.try_fold_with(folder)?),
-            ty::ConstKind::Param(p) => ty::ConstKind::Param(p.try_fold_with(folder)?),
-            ty::ConstKind::Unevaluated(uv) => ty::ConstKind::Unevaluated(uv.try_fold_with(folder)?),
-            ty::ConstKind::Value(_)
-            | ty::ConstKind::Bound(..)
-            | ty::ConstKind::Placeholder(..)
-            | ty::ConstKind::Error(_) => self,
-        })
-    }
-}
-
-impl<'tcx> TypeVisitable<'tcx> for ty::ConstKind<'tcx> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
-        match *self {
-            ty::ConstKind::Infer(ic) => ic.visit_with(visitor),
-            ty::ConstKind::Param(p) => p.visit_with(visitor),
-            ty::ConstKind::Unevaluated(uv) => uv.visit_with(visitor),
-            ty::ConstKind::Value(_)
-            | ty::ConstKind::Bound(..)
-            | ty::ConstKind::Placeholder(_)
-            | ty::ConstKind::Error(_) => ControlFlow::CONTINUE,
-        }
-    }
-}
-
 impl<'tcx> TypeFoldable<'tcx> for InferConst<'tcx> {
     fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, _folder: &mut F) -> Result<Self, F::Error> {
         Ok(self)
@@ -1290,15 +886,3 @@ impl<'tcx> TypeVisitable<'tcx> for ty::Unevaluated<'tcx, ()> {
         self.expand().visit_with(visitor)
     }
 }
-
-impl<'tcx> TypeFoldable<'tcx> for hir::Constness {
-    fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, _: &mut F) -> Result<Self, F::Error> {
-        Ok(self)
-    }
-}
-
-impl<'tcx> TypeVisitable<'tcx> for hir::Constness {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, _: &mut V) -> ControlFlow<V::BreakTy> {
-        ControlFlow::CONTINUE
-    }
-}
diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs
index 52c3a3886..36e560850 100644
--- a/compiler/rustc_middle/src/ty/sty.rs
+++ b/compiler/rustc_middle/src/ty/sty.rs
@@ -11,6 +11,7 @@ use crate::ty::{
     TypeVisitor,
 };
 use crate::ty::{List, ParamEnv};
+use hir::def::DefKind;
 use polonius_engine::Atom;
 use rustc_data_structures::captures::Captures;
 use rustc_data_structures::intern::Interned;
@@ -201,7 +202,7 @@ static_assert_size!(TyKind<'_>, 32);
 /// * `GR`: The "return type", which is the type of value returned upon
 ///   completion of the generator.
 /// * `GW`: The "generator witness".
-#[derive(Copy, Clone, Debug, TypeFoldable, TypeVisitable)]
+#[derive(Copy, Clone, Debug, TypeFoldable, TypeVisitable, Lift)]
 pub struct ClosureSubsts<'tcx> {
     /// Lifetime and type parameters from the enclosing function,
     /// concatenated with a tuple containing the types of the upvars.
@@ -325,10 +326,14 @@ impl<'tcx> ClosureSubsts<'tcx> {
             _ => bug!("closure_sig_as_fn_ptr_ty is not a fn-ptr: {:?}", ty.kind()),
         }
     }
+
+    pub fn print_as_impl_trait(self) -> ty::print::PrintClosureAsImpl<'tcx> {
+        ty::print::PrintClosureAsImpl { closure: self }
+    }
 }
 
 /// Similar to `ClosureSubsts`; see the above documentation for more.
-#[derive(Copy, Clone, Debug, TypeFoldable, TypeVisitable)]
+#[derive(Copy, Clone, Debug, TypeFoldable, TypeVisitable, Lift)]
 pub struct GeneratorSubsts<'tcx> {
     pub substs: SubstsRef<'tcx>,
 }
@@ -655,7 +660,7 @@ impl<'tcx> InlineConstSubsts<'tcx> {
 }
 
 #[derive(Debug, Copy, Clone, PartialEq, PartialOrd, Ord, Eq, Hash, TyEncodable, TyDecodable)]
-#[derive(HashStable, TypeFoldable, TypeVisitable)]
+#[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub enum ExistentialPredicate<'tcx> {
     /// E.g., `Iterator`.
     Trait(ExistentialTraitRef<'tcx>),
@@ -687,6 +692,9 @@ impl<'tcx> ExistentialPredicate<'tcx> {
 }
 
 impl<'tcx> Binder<'tcx, ExistentialPredicate<'tcx>> {
+    /// Given an existential predicate like `?Self: PartialEq<u32>` (e.g., derived from `dyn PartialEq<u32>`),
+    /// and a concrete type `self_ty`, returns a full predicate where the existentially quantified variable `?Self`
+    /// has been replaced with `self_ty` (e.g., `self_ty: PartialEq<u32>`, in our example).
     pub fn with_self_ty(&self, tcx: TyCtxt<'tcx>, self_ty: Ty<'tcx>) -> ty::Predicate<'tcx> {
         use crate::ty::ToPredicate;
         match self.skip_binder() {
@@ -781,7 +789,7 @@ impl<'tcx> List<ty::Binder<'tcx, ExistentialPredicate<'tcx>>> {
 /// Trait references also appear in object types like `Foo<U>`, but in
 /// that case the `Self` parameter is absent from the substitutions.
 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, TyEncodable, TyDecodable)]
-#[derive(HashStable, TypeFoldable, TypeVisitable)]
+#[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub struct TraitRef<'tcx> {
     pub def_id: DefId,
     pub substs: SubstsRef<'tcx>,
@@ -845,6 +853,12 @@ impl<'tcx> PolyTraitRef<'tcx> {
     }
 }
 
+impl rustc_errors::IntoDiagnosticArg for PolyTraitRef<'_> {
+    fn into_diagnostic_arg(self) -> rustc_errors::DiagnosticArgValue<'static> {
+        self.to_string().into_diagnostic_arg()
+    }
+}
+
 /// An existential reference to a trait, where `Self` is erased.
 /// For example, the trait object `Trait<'a, 'b, X, Y>` is:
 /// ```ignore (illustrative)
@@ -853,7 +867,7 @@ impl<'tcx> PolyTraitRef<'tcx> {
 /// The substitutions don't include the erased `Self`, only trait
 /// type and lifetime parameters (`[X, Y]` and `['a, 'b]` above).
 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, TyEncodable, TyDecodable)]
-#[derive(HashStable, TypeFoldable, TypeVisitable)]
+#[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub struct ExistentialTraitRef<'tcx> {
     pub def_id: DefId,
     pub substs: SubstsRef<'tcx>,
@@ -1009,7 +1023,7 @@ impl BoundVariableKind {
 ///
 /// `Decodable` and `Encodable` are implemented for `Binder<T>` using the `impl_binder_encode_decode!` macro.
 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
-#[derive(HashStable)]
+#[derive(HashStable, Lift)]
 pub struct Binder<'tcx, T>(T, &'tcx List<BoundVariableKind>);
 
 impl<'tcx, T> Binder<'tcx, T>
@@ -1171,7 +1185,7 @@ impl<'tcx, T> Binder<'tcx, Option<T>> {
 /// Represents the projection of an associated type. In explicit UFCS
 /// form this would be written `<T as Trait<..>>::N`.
 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, TyEncodable, TyDecodable)]
-#[derive(HashStable, TypeFoldable, TypeVisitable)]
+#[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub struct ProjectionTy<'tcx> {
     /// The parameters of the associated item.
     pub substs: SubstsRef<'tcx>,
@@ -1186,7 +1200,9 @@ pub struct ProjectionTy<'tcx> {
 
 impl<'tcx> ProjectionTy<'tcx> {
     pub fn trait_def_id(&self, tcx: TyCtxt<'tcx>) -> DefId {
-        tcx.parent(self.item_def_id)
+        let parent = tcx.parent(self.item_def_id);
+        assert_eq!(tcx.def_kind(parent), DefKind::Trait);
+        parent
     }
 
     /// Extracts the underlying trait reference and own substs from this projection.
@@ -1221,7 +1237,7 @@ impl<'tcx> ProjectionTy<'tcx> {
     }
 }
 
-#[derive(Copy, Clone, Debug, TypeFoldable, TypeVisitable)]
+#[derive(Copy, Clone, Debug, TypeFoldable, TypeVisitable, Lift)]
 pub struct GenSig<'tcx> {
     pub resume_ty: Ty<'tcx>,
     pub yield_ty: Ty<'tcx>,
@@ -1237,7 +1253,7 @@ pub type PolyGenSig<'tcx> = Binder<'tcx, GenSig<'tcx>>;
 /// - `output`: is the return type.
 /// - `c_variadic`: indicates whether this is a C-variadic function.
 #[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, TyEncodable, TyDecodable)]
-#[derive(HashStable, TypeFoldable, TypeVisitable)]
+#[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub struct FnSig<'tcx> {
     pub inputs_and_output: &'tcx List<Ty<'tcx>>,
     pub c_variadic: bool,
@@ -1419,7 +1435,7 @@ impl From<BoundVar> for BoundTy {
 
 /// A `ProjectionPredicate` for an `ExistentialTraitRef`.
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, TyEncodable, TyDecodable)]
-#[derive(HashStable, TypeFoldable, TypeVisitable)]
+#[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
 pub struct ExistentialProjection<'tcx> {
     pub item_def_id: DefId,
     pub substs: SubstsRef<'tcx>,
@@ -1501,7 +1517,6 @@ impl<'tcx> Region<'tcx> {
             ty::ReStatic => true,
             ty::ReVar(..) => false,
             ty::RePlaceholder(placeholder) => placeholder.name.is_named(),
-            ty::ReEmpty(_) => false,
             ty::ReErased => false,
         }
     }
@@ -1527,11 +1542,6 @@ impl<'tcx> Region<'tcx> {
     }
 
     #[inline]
-    pub fn is_empty(self) -> bool {
-        matches!(*self, ty::ReEmpty(..))
-    }
-
-    #[inline]
     pub fn bound_at_or_above_binder(self, index: ty::DebruijnIndex) -> bool {
         match *self {
             ty::ReLateBound(debruijn, _) => debruijn >= index,
@@ -1562,7 +1572,7 @@ impl<'tcx> Region<'tcx> {
                 flags = flags | TypeFlags::HAS_FREE_REGIONS;
                 flags = flags | TypeFlags::HAS_FREE_LOCAL_REGIONS;
             }
-            ty::ReEmpty(_) | ty::ReStatic => {
+            ty::ReStatic => {
                 flags = flags | TypeFlags::HAS_FREE_REGIONS;
             }
             ty::ReLateBound(..) => {
@@ -1617,6 +1627,10 @@ impl<'tcx> Region<'tcx> {
             _ => self.is_free(),
         }
     }
+
+    pub fn is_var(self) -> bool {
+        matches!(self.kind(), ty::ReVar(_))
+    }
 }
 
 /// Type utilities
@@ -1838,7 +1852,12 @@ impl<'tcx> Ty<'tcx> {
 
     #[inline]
     pub fn is_trait(self) -> bool {
-        matches!(self.kind(), Dynamic(..))
+        matches!(self.kind(), Dynamic(_, _, ty::Dyn))
+    }
+
+    #[inline]
+    pub fn is_dyn_star(self) -> bool {
+        matches!(self.kind(), Dynamic(_, _, ty::DynStar))
     }
 
     #[inline]
diff --git a/compiler/rustc_middle/src/ty/subst.rs b/compiler/rustc_middle/src/ty/subst.rs
index 6262aa180..8e69bf067 100644
--- a/compiler/rustc_middle/src/ty/subst.rs
+++ b/compiler/rustc_middle/src/ty/subst.rs
@@ -459,12 +459,6 @@ impl<'tcx> TypeFoldable<'tcx> for SubstsRef<'tcx> {
     }
 }
 
-impl<'tcx> TypeVisitable<'tcx> for SubstsRef<'tcx> {
-    fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
-        self.iter().try_for_each(|t| t.visit_with(visitor))
-    }
-}
-
 impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::List<Ty<'tcx>> {
     fn try_fold_with<F: FallibleTypeFolder<'tcx>>(self, folder: &mut F) -> Result<Self, F::Error> {
         // This code is fairly hot, though not as hot as `SubstsRef`.
@@ -497,7 +491,7 @@ impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::List<Ty<'tcx>> {
     }
 }
 
-impl<'tcx> TypeVisitable<'tcx> for &'tcx ty::List<Ty<'tcx>> {
+impl<'tcx, T: TypeVisitable<'tcx>> TypeVisitable<'tcx> for &'tcx ty::List<T> {
     fn visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> ControlFlow<V::BreakTy> {
         self.iter().try_for_each(|t| t.visit_with(visitor))
     }
diff --git a/compiler/rustc_middle/src/ty/trait_def.rs b/compiler/rustc_middle/src/ty/trait_def.rs
index 541dace5c..ac79949fc 100644
--- a/compiler/rustc_middle/src/ty/trait_def.rs
+++ b/compiler/rustc_middle/src/ty/trait_def.rs
@@ -256,7 +256,6 @@ pub(super) fn trait_impls_of_provider(tcx: TyCtxt<'_>, trait_id: DefId) -> Trait
 }
 
 // Query provider for `incoherent_impls`.
-#[instrument(level = "debug", skip(tcx))]
 pub(super) fn incoherent_impls_provider(tcx: TyCtxt<'_>, simp: SimplifiedType) -> &[DefId] {
     let mut impls = Vec::new();
 
diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs
index 591bb7831..23ad4d27d 100644
--- a/compiler/rustc_middle/src/ty/util.rs
+++ b/compiler/rustc_middle/src/ty/util.rs
@@ -627,7 +627,7 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     /// Expands the given impl trait type, stopping if the type is recursive.
-    #[instrument(skip(self), level = "debug")]
+    #[instrument(skip(self), level = "debug", ret)]
     pub fn try_expand_impl_trait_type(
         self,
         def_id: DefId,
@@ -644,7 +644,6 @@ impl<'tcx> TyCtxt<'tcx> {
         };
 
         let expanded_type = visitor.expand_opaque_ty(def_id, substs).unwrap();
-        trace!(?expanded_type);
         if visitor.found_recursion { Err(expanded_type) } else { Ok(expanded_type) }
     }
 
@@ -652,6 +651,13 @@ impl<'tcx> TyCtxt<'tcx> {
         ty::EarlyBinder(self.type_of(def_id))
     }
 
+    pub fn bound_trait_impl_trait_tys(
+        self,
+        def_id: DefId,
+    ) -> ty::EarlyBinder<Result<&'tcx FxHashMap<DefId, Ty<'tcx>>, ErrorGuaranteed>> {
+        ty::EarlyBinder(self.collect_trait_impl_trait_tys(def_id))
+    }
+
     pub fn bound_fn_sig(self, def_id: DefId) -> ty::EarlyBinder<ty::PolyFnSig<'tcx>> {
         ty::EarlyBinder(self.fn_sig(def_id))
     }
diff --git a/compiler/rustc_middle/src/ty/visit.rs b/compiler/rustc_middle/src/ty/visit.rs
index 536506720..5f8cb5782 100644
--- a/compiler/rustc_middle/src/ty/visit.rs
+++ b/compiler/rustc_middle/src/ty/visit.rs
@@ -84,7 +84,7 @@ pub trait TypeVisitable<'tcx>: fmt::Debug + Clone {
         self.has_vars_bound_at_or_above(ty::INNERMOST)
     }
 
-    #[instrument(level = "trace")]
+    #[instrument(level = "trace", ret)]
     fn has_type_flags(&self, flags: TypeFlags) -> bool {
         self.visit_with(&mut HasTypeFlagsVisitor { flags }).break_value() == Some(FoundFlags)
     }
@@ -560,7 +560,7 @@ impl<'tcx> TypeVisitor<'tcx> for HasTypeFlagsVisitor {
     type BreakTy = FoundFlags;
 
     #[inline]
-    #[instrument(skip(self), level = "trace")]
+    #[instrument(skip(self), level = "trace", ret)]
     fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
         let flags = t.flags();
         trace!(t.flags=?t.flags());
@@ -572,7 +572,7 @@ impl<'tcx> TypeVisitor<'tcx> for HasTypeFlagsVisitor {
     }
 
     #[inline]
-    #[instrument(skip(self), level = "trace")]
+    #[instrument(skip(self), level = "trace", ret)]
     fn visit_region(&mut self, r: ty::Region<'tcx>) -> ControlFlow<Self::BreakTy> {
         let flags = r.type_flags();
         trace!(r.flags=?flags);
@@ -584,7 +584,7 @@ impl<'tcx> TypeVisitor<'tcx> for HasTypeFlagsVisitor {
     }
 
     #[inline]
-    #[instrument(level = "trace")]
+    #[instrument(level = "trace", ret)]
     fn visit_const(&mut self, c: ty::Const<'tcx>) -> ControlFlow<Self::BreakTy> {
         let flags = FlagComputation::for_const(c);
         trace!(r.flags=?flags);
@@ -596,7 +596,7 @@ impl<'tcx> TypeVisitor<'tcx> for HasTypeFlagsVisitor {
     }
 
     #[inline]
-    #[instrument(level = "trace")]
+    #[instrument(level = "trace", ret)]
     fn visit_unevaluated(&mut self, uv: ty::Unevaluated<'tcx>) -> ControlFlow<Self::BreakTy> {
         let flags = FlagComputation::for_unevaluated_const(uv);
         trace!(r.flags=?flags);
@@ -608,7 +608,7 @@ impl<'tcx> TypeVisitor<'tcx> for HasTypeFlagsVisitor {
     }
 
     #[inline]
-    #[instrument(level = "trace")]
+    #[instrument(level = "trace", ret)]
     fn visit_predicate(&mut self, predicate: ty::Predicate<'tcx>) -> ControlFlow<Self::BreakTy> {
         debug!(
             "HasTypeFlagsVisitor: predicate={:?} predicate.flags={:?} self.flags={:?}",
@@ -666,7 +666,7 @@ impl<'tcx> TypeVisitor<'tcx> for LateBoundRegionsCollector {
         // ignore the inputs to a projection, as they may not appear
         // in the normalized form
         if self.just_constrained {
-            if let ty::Projection(..) = t.kind() {
+            if let ty::Projection(..) | ty::Opaque(..) = t.kind() {
                 return ControlFlow::CONTINUE;
             }
         }
diff --git a/compiler/rustc_middle/src/ty/vtable.rs b/compiler/rustc_middle/src/ty/vtable.rs
index 04a9fd1f7..5ca51c25a 100644
--- a/compiler/rustc_middle/src/ty/vtable.rs
+++ b/compiler/rustc_middle/src/ty/vtable.rs
@@ -1,7 +1,7 @@
 use std::convert::TryFrom;
 use std::fmt;
 
-use crate::mir::interpret::{alloc_range, AllocId, Allocation, Pointer, Scalar, ScalarMaybeUninit};
+use crate::mir::interpret::{alloc_range, AllocId, Allocation, Pointer, Scalar};
 use crate::ty::{self, Instance, PolyTraitRef, Ty, TyCtxt};
 use rustc_ast::Mutability;
 
@@ -87,7 +87,7 @@ pub(super) fn vtable_allocation_provider<'tcx>(
                 let instance = ty::Instance::resolve_drop_in_place(tcx, ty);
                 let fn_alloc_id = tcx.create_fn_alloc(instance);
                 let fn_ptr = Pointer::from(fn_alloc_id);
-                ScalarMaybeUninit::from_pointer(fn_ptr, &tcx)
+                Scalar::from_pointer(fn_ptr, &tcx)
             }
             VtblEntry::MetadataSize => Scalar::from_uint(size, ptr_size).into(),
             VtblEntry::MetadataAlign => Scalar::from_uint(align, ptr_size).into(),
@@ -97,14 +97,14 @@ pub(super) fn vtable_allocation_provider<'tcx>(
                 let instance = instance.polymorphize(tcx);
                 let fn_alloc_id = tcx.create_fn_alloc(instance);
                 let fn_ptr = Pointer::from(fn_alloc_id);
-                ScalarMaybeUninit::from_pointer(fn_ptr, &tcx)
+                Scalar::from_pointer(fn_ptr, &tcx)
             }
             VtblEntry::TraitVPtr(trait_ref) => {
                 let super_trait_ref = trait_ref
                     .map_bound(|trait_ref| ty::ExistentialTraitRef::erase_self_ty(tcx, trait_ref));
                 let supertrait_alloc_id = tcx.vtable_allocation((ty, Some(super_trait_ref)));
                 let vptr = Pointer::from(supertrait_alloc_id);
-                ScalarMaybeUninit::from_pointer(vptr, &tcx)
+                Scalar::from_pointer(vptr, &tcx)
             }
         };
         vtable
diff --git a/compiler/rustc_middle/src/ty/walk.rs b/compiler/rustc_middle/src/ty/walk.rs
index 02fe1f3a7..a3e11bbf0 100644
--- a/compiler/rustc_middle/src/ty/walk.rs
+++ b/compiler/rustc_middle/src/ty/walk.rs
@@ -152,7 +152,7 @@ fn push_inner<'tcx>(stack: &mut TypeWalkerStack<'tcx>, parent: GenericArg<'tcx>)
             ty::Projection(data) => {
                 stack.extend(data.substs.iter().rev());
             }
-            ty::Dynamic(obj, lt) => {
+            ty::Dynamic(obj, lt, _) => {
                 stack.push(lt.into());
                 stack.extend(obj.iter().rev().flat_map(|predicate| {
                     let (substs, opt_ty) = match predicate.skip_binder() {
@@ -165,9 +165,9 @@ fn push_inner<'tcx>(stack: &mut TypeWalkerStack<'tcx>, parent: GenericArg<'tcx>)
                         }
                     };
 
-                    substs.iter().rev().chain(opt_ty.map(|term| match term {
-                        ty::Term::Ty(ty) => ty.into(),
-                        ty::Term::Const(ct) => ct.into(),
+                    substs.iter().rev().chain(opt_ty.map(|term| match term.unpack() {
+                        ty::TermKind::Ty(ty) => ty.into(),
+                        ty::TermKind::Const(ct) => ct.into(),
                     }))
                 }));
             }