Merging upstream version 1.75.0+dfsg1.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

73 files changed, 2138 insertions, 1763 deletions

diff --git a/compiler/rustc_middle/Cargo.toml b/compiler/rustc_middle/Cargo.toml
index bb8e774ce..d356984c1 100644
--- a/compiler/rustc_middle/Cargo.toml
+++ b/compiler/rustc_middle/Cargo.toml
@@ -3,24 +3,24 @@ name = "rustc_middle"
 version = "0.0.0"
 edition = "2021"
 
-[lib]
-
 [dependencies]
+# tidy-alphabetical-start
 bitflags = "1.2.1"
 derive_more = "0.99.17"
 either = "1.5.0"
-gsgdt = "0.1.2"
 field-offset = "0.3.5"
+gsgdt = "0.1.2"
 measureme = "10.0.0"
 polonius-engine = "0.13.0"
+rustc-rayon = { version = "0.5.0", optional = true }
+rustc-rayon-core = { version = "0.5.0", optional = true }
 rustc_apfloat = "0.2.0"
 rustc_arena = { path = "../rustc_arena" }
 rustc_ast = { path = "../rustc_ast" }
 rustc_attr = { path = "../rustc_attr" }
 rustc_data_structures = { path = "../rustc_data_structures" }
+rustc_error_messages = { path = "../rustc_error_messages" } # Used for intra-doc links
 rustc_errors = { path = "../rustc_errors" }
-# Used for intra-doc links
-rustc_error_messages = { path = "../rustc_error_messages" }
 rustc_feature = { path = "../rustc_feature" }
 rustc_fluent_macro = { path = "../rustc_fluent_macro" }
 rustc_graphviz = { path = "../rustc_graphviz" }
@@ -28,8 +28,6 @@ rustc_hir = { path = "../rustc_hir" }
 rustc_index = { path = "../rustc_index" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_query_system = { path = "../rustc_query_system" }
-rustc-rayon-core = { version = "0.5.0", optional = true }
-rustc-rayon = { version = "0.5.0", optional = true }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
@@ -38,6 +36,9 @@ rustc_type_ir = { path = "../rustc_type_ir" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
 thin-vec = "0.2.12"
 tracing = "0.1"
+# tidy-alphabetical-end
 
 [features]
+# tidy-alphabetical-start
 rustc_use_parallel_compiler = ["rustc-rayon", "rustc-rayon-core"]
+# tidy-alphabetical-end
diff --git a/compiler/rustc_middle/messages.ftl b/compiler/rustc_middle/messages.ftl
index 82162fd85..27d555d7e 100644
--- a/compiler/rustc_middle/messages.ftl
+++ b/compiler/rustc_middle/messages.ftl
@@ -5,12 +5,14 @@ middle_assert_async_resume_after_panic = `async fn` resumed after panicking
 
 middle_assert_async_resume_after_return = `async fn` resumed after completion
 
+middle_assert_coroutine_resume_after_panic = coroutine resumed after panicking
+
+middle_assert_coroutine_resume_after_return = coroutine resumed after completion
+
 middle_assert_divide_by_zero =
     attempt to divide `{$val}` by zero
 
-middle_assert_generator_resume_after_panic = generator resumed after panicking
-
-middle_assert_generator_resume_after_return = generator resumed after completion
+middle_assert_gen_resume_after_panic = `gen` fn or block cannot be further iterated on after it panicked
 
 middle_assert_misaligned_ptr_deref =
     misaligned pointer dereference: address must be a multiple of {$required} but is {$found}
diff --git a/compiler/rustc_middle/src/arena.rs b/compiler/rustc_middle/src/arena.rs
index 952c796f5..dd761b4e3 100644
--- a/compiler/rustc_middle/src/arena.rs
+++ b/compiler/rustc_middle/src/arena.rs
@@ -8,7 +8,7 @@
 macro_rules! arena_types {
     ($macro:path) => (
         $macro!([
-            [] layout: rustc_target::abi::LayoutS,
+            [] layout: rustc_target::abi::LayoutS<rustc_target::abi::FieldIdx, rustc_target::abi::VariantIdx>,
             [] fn_abi: rustc_target::abi::call::FnAbi<'tcx, rustc_middle::ty::Ty<'tcx>>,
             // AdtDef are interned and compared by address
             [decode] adt_def: rustc_middle::ty::AdtDefData,
@@ -69,6 +69,7 @@ macro_rules! arena_types {
             [] dtorck_constraint: rustc_middle::traits::query::DropckConstraint<'tcx>,
             [] candidate_step: rustc_middle::traits::query::CandidateStep<'tcx>,
             [] autoderef_bad_ty: rustc_middle::traits::query::MethodAutoderefBadTy<'tcx>,
+            [] canonical_goal_evaluation: rustc_middle::traits::solve::inspect::GoalEvaluationStep<'tcx>,
             [] query_region_constraints: rustc_middle::infer::canonical::QueryRegionConstraints<'tcx>,
             [] type_op_subtype:
                 rustc_middle::infer::canonical::Canonical<'tcx,
diff --git a/compiler/rustc_middle/src/hir/map/mod.rs b/compiler/rustc_middle/src/hir/map/mod.rs
index 4af2d83e9..58c0c6bab 100644
--- a/compiler/rustc_middle/src/hir/map/mod.rs
+++ b/compiler/rustc_middle/src/hir/map/mod.rs
@@ -6,7 +6,7 @@ use rustc_ast as ast;
 use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::svh::Svh;
-use rustc_data_structures::sync::{par_for_each_in, DynSend, DynSync};
+use rustc_data_structures::sync::{par_for_each_in, try_par_for_each_in, DynSend, DynSync};
 use rustc_hir::def::{DefKind, Res};
 use rustc_hir::def_id::{DefId, LocalDefId, LocalModDefId, LOCAL_CRATE};
 use rustc_hir::definitions::{DefKey, DefPath, DefPathData, DefPathHash};
@@ -16,7 +16,7 @@ use rustc_index::Idx;
 use rustc_middle::hir::nested_filter;
 use rustc_span::def_id::StableCrateId;
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
-use rustc_span::Span;
+use rustc_span::{ErrorGuaranteed, Span};
 use rustc_target::spec::abi::Abi;
 
 #[inline]
@@ -240,7 +240,7 @@ impl<'hir> Map<'hir> {
             Node::Field(_) => DefKind::Field,
             Node::Expr(expr) => match expr.kind {
                 ExprKind::Closure(Closure { movability: None, .. }) => DefKind::Closure,
-                ExprKind::Closure(Closure { movability: Some(_), .. }) => DefKind::Generator,
+                ExprKind::Closure(Closure { movability: Some(_), .. }) => DefKind::Coroutine,
                 _ => bug!("def_kind: unsupported node: {}", self.node_to_string(hir_id)),
             },
             Node::GenericParam(param) => match param.kind {
@@ -445,7 +445,7 @@ impl<'hir> Map<'hir> {
             }
             DefKind::InlineConst => BodyOwnerKind::Const { inline: true },
             DefKind::Ctor(..) | DefKind::Fn | DefKind::AssocFn => BodyOwnerKind::Fn,
-            DefKind::Closure | DefKind::Generator => BodyOwnerKind::Closure,
+            DefKind::Closure | DefKind::Coroutine => BodyOwnerKind::Closure,
             DefKind::Static(mt) => BodyOwnerKind::Static(mt),
             dk => bug!("{:?} is not a body node: {:?}", def_id, dk),
         }
@@ -632,6 +632,17 @@ impl<'hir> Map<'hir> {
         })
     }
 
+    #[inline]
+    pub fn try_par_for_each_module(
+        self,
+        f: impl Fn(LocalModDefId) -> Result<(), ErrorGuaranteed> + DynSend + DynSync,
+    ) -> Result<(), ErrorGuaranteed> {
+        let crate_items = self.tcx.hir_crate_items(());
+        try_par_for_each_in(&crate_items.submodules[..], |module| {
+            f(LocalModDefId::new_unchecked(module.def_id))
+        })
+    }
+
     /// Returns an iterator for the nodes in the ancestor tree of the `current_id`
     /// until the crate root is reached. Prefer this over your own loop using `parent_id`.
     #[inline]
@@ -970,12 +981,15 @@ impl<'hir> Map<'hir> {
                 // SyntaxContext of the visibility.
                 sig.span.find_ancestor_in_same_ctxt(*outer_span).unwrap_or(*outer_span)
             }
+            // Impls, including their where clauses.
+            Node::Item(Item {
+                kind: ItemKind::Impl(Impl { generics, .. }),
+                span: outer_span,
+                ..
+            }) => until_within(*outer_span, generics.where_clause_span),
             // Constants and Statics.
             Node::Item(Item {
-                kind:
-                    ItemKind::Const(ty, ..)
-                    | ItemKind::Static(ty, ..)
-                    | ItemKind::Impl(Impl { self_ty: ty, .. }),
+                kind: ItemKind::Const(ty, ..) | ItemKind::Static(ty, ..),
                 span: outer_span,
                 ..
             })
diff --git a/compiler/rustc_middle/src/hir/mod.rs b/compiler/rustc_middle/src/hir/mod.rs
index 0da8fe9cc..f28ec7711 100644
--- a/compiler/rustc_middle/src/hir/mod.rs
+++ b/compiler/rustc_middle/src/hir/mod.rs
@@ -9,12 +9,12 @@ pub mod place;
 use crate::query::Providers;
 use crate::ty::{EarlyBinder, ImplSubject, TyCtxt};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
-use rustc_data_structures::sync::{par_for_each_in, DynSend, DynSync};
+use rustc_data_structures::sync::{try_par_for_each_in, DynSend, DynSync};
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::{DefId, LocalDefId, LocalModDefId};
 use rustc_hir::*;
 use rustc_query_system::ich::StableHashingContext;
-use rustc_span::{ExpnId, DUMMY_SP};
+use rustc_span::{ErrorGuaranteed, ExpnId, DUMMY_SP};
 
 /// Top-level HIR node for current owner. This only contains the node for which
 /// `HirId::local_id == 0`, and excludes bodies.
@@ -78,20 +78,32 @@ impl ModuleItems {
         self.owners().map(|id| id.def_id)
     }
 
-    pub fn par_items(&self, f: impl Fn(ItemId) + DynSend + DynSync) {
-        par_for_each_in(&self.items[..], |&id| f(id))
+    pub fn par_items(
+        &self,
+        f: impl Fn(ItemId) -> Result<(), ErrorGuaranteed> + DynSend + DynSync,
+    ) -> Result<(), ErrorGuaranteed> {
+        try_par_for_each_in(&self.items[..], |&id| f(id))
     }
 
-    pub fn par_trait_items(&self, f: impl Fn(TraitItemId) + DynSend + DynSync) {
-        par_for_each_in(&self.trait_items[..], |&id| f(id))
+    pub fn par_trait_items(
+        &self,
+        f: impl Fn(TraitItemId) -> Result<(), ErrorGuaranteed> + DynSend + DynSync,
+    ) -> Result<(), ErrorGuaranteed> {
+        try_par_for_each_in(&self.trait_items[..], |&id| f(id))
     }
 
-    pub fn par_impl_items(&self, f: impl Fn(ImplItemId) + DynSend + DynSync) {
-        par_for_each_in(&self.impl_items[..], |&id| f(id))
+    pub fn par_impl_items(
+        &self,
+        f: impl Fn(ImplItemId) -> Result<(), ErrorGuaranteed> + DynSend + DynSync,
+    ) -> Result<(), ErrorGuaranteed> {
+        try_par_for_each_in(&self.impl_items[..], |&id| f(id))
     }
 
-    pub fn par_foreign_items(&self, f: impl Fn(ForeignItemId) + DynSend + DynSync) {
-        par_for_each_in(&self.foreign_items[..], |&id| f(id))
+    pub fn par_foreign_items(
+        &self,
+        f: impl Fn(ForeignItemId) -> Result<(), ErrorGuaranteed> + DynSend + DynSync,
+    ) -> Result<(), ErrorGuaranteed> {
+        try_par_for_each_in(&self.foreign_items[..], |&id| f(id))
     }
 }
 
diff --git a/compiler/rustc_middle/src/hir/nested_filter.rs b/compiler/rustc_middle/src/hir/nested_filter.rs
index 6896837aa..adbe81bb2 100644
--- a/compiler/rustc_middle/src/hir/nested_filter.rs
+++ b/compiler/rustc_middle/src/hir/nested_filter.rs
@@ -4,7 +4,7 @@ use rustc_hir::intravisit::nested_filter::NestedFilter;
 /// that are inside of an item-like.
 ///
 /// Notably, possible occurrences of bodies in non-item-like things
-/// include: closures/generators, inline `const {}` blocks, and
+/// include: closures/coroutines, inline `const {}` blocks, and
 /// constant arguments of types, e.g. in `let _: [(); /* HERE */];`.
 ///
 /// **This is the most common choice.** A very common pattern is
diff --git a/compiler/rustc_middle/src/hooks/mod.rs b/compiler/rustc_middle/src/hooks/mod.rs
index 12aeae177..8588ae203 100644
--- a/compiler/rustc_middle/src/hooks/mod.rs
+++ b/compiler/rustc_middle/src/hooks/mod.rs
@@ -1,3 +1,8 @@
+//! "Hooks" provide a way for `tcx` functionality to be provided by some downstream crate without
+//! everything in rustc having to depend on that crate. This is somewhat similar to queries, but
+//! queries come with a lot of machinery for caching and incremental compilation, whereas hooks are
+//! just plain function pointers without any of the query magic.
+
 use crate::mir;
 use crate::query::TyCtxtAt;
 use crate::ty::{Ty, TyCtxt};
@@ -61,5 +66,8 @@ macro_rules! declare_hooks {
 declare_hooks! {
     /// Tries to destructure an `mir::Const` ADT or array into its variant index
     /// and its field values. This should only be used for pretty printing.
-    hook try_destructure_mir_constant_for_diagnostics(val: mir::ConstValue<'tcx>, ty: Ty<'tcx>) -> Option<mir::DestructuredConstant<'tcx>>;
+    hook try_destructure_mir_constant_for_user_output(val: mir::ConstValue<'tcx>, ty: Ty<'tcx>) -> Option<mir::DestructuredConstant<'tcx>>;
+
+    /// Getting a &core::panic::Location referring to a span.
+    hook const_caller_location(file: rustc_span::Symbol, line: u32, col: u32) -> mir::ConstValue<'tcx>;
 }
diff --git a/compiler/rustc_middle/src/infer/canonical.rs b/compiler/rustc_middle/src/infer/canonical.rs
index 41beca072..64b63f4c5 100644
--- a/compiler/rustc_middle/src/infer/canonical.rs
+++ b/compiler/rustc_middle/src/infer/canonical.rs
@@ -21,35 +21,17 @@
 //!
 //! [c]: https://rust-lang.github.io/chalk/book/canonical_queries/canonicalization.html
 
-use crate::infer::MemberConstraint;
-use crate::mir::ConstraintCategory;
-use crate::ty::GenericArg;
-use crate::ty::{self, BoundVar, List, Region, Ty, TyCtxt};
 use rustc_macros::HashStable;
+use rustc_type_ir::Canonical as IrCanonical;
 use smallvec::SmallVec;
-use std::fmt::Display;
 use std::ops::Index;
 
-/// A "canonicalized" type `V` is one where all free inference
-/// variables have been rewritten to "canonical vars". These are
-/// numbered starting from 0 in order of first appearance.
-#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, TyDecodable, TyEncodable)]
-#[derive(HashStable, TypeFoldable, TypeVisitable)]
-pub struct Canonical<'tcx, V> {
-    pub value: V,
-    pub max_universe: ty::UniverseIndex,
-    pub variables: CanonicalVarInfos<'tcx>,
-}
+use crate::infer::MemberConstraint;
+use crate::mir::ConstraintCategory;
+use crate::ty::GenericArg;
+use crate::ty::{self, BoundVar, List, Region, Ty, TyCtxt};
 
-impl<'tcx, V: Display> std::fmt::Display for Canonical<'tcx, V> {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(
-            f,
-            "Canonical {{ value: {}, max_universe: {:?}, variables: {:?} }}",
-            self.value, self.max_universe, self.variables
-        )
-    }
-}
+pub type Canonical<'tcx, V> = IrCanonical<TyCtxt<'tcx>, V>;
 
 pub type CanonicalVarInfos<'tcx> = &'tcx List<CanonicalVarInfo<'tcx>>;
 
@@ -104,7 +86,7 @@ impl CanonicalVarValues<'_> {
                     } else {
                         // It's ok if this region var isn't unique
                     }
-                },
+                }
                 ty::GenericArgKind::Type(ty) => {
                     if let ty::Bound(ty::INNERMOST, bt) = *ty.kind()
                         && var == bt.var
@@ -240,7 +222,7 @@ pub enum CanonicalVarKind<'tcx> {
     Effect,
 
     /// A "placeholder" that represents "any const".
-    PlaceholderConst(ty::PlaceholderConst<'tcx>, Ty<'tcx>),
+    PlaceholderConst(ty::PlaceholderConst, Ty<'tcx>),
 }
 
 impl<'tcx> CanonicalVarKind<'tcx> {
@@ -379,56 +361,6 @@ impl<'tcx, R> QueryResponse<'tcx, R> {
     }
 }
 
-impl<'tcx, R> Canonical<'tcx, QueryResponse<'tcx, R>> {
-    pub fn is_proven(&self) -> bool {
-        self.value.is_proven()
-    }
-
-    pub fn is_ambiguous(&self) -> bool {
-        !self.is_proven()
-    }
-}
-
-impl<'tcx, V> Canonical<'tcx, V> {
-    /// Allows you to map the `value` of a canonical while keeping the
-    /// same set of bound variables.
-    ///
-    /// **WARNING:** This function is very easy to mis-use, hence the
-    /// name!  In particular, the new value `W` must use all **the
-    /// same type/region variables** in **precisely the same order**
-    /// as the original! (The ordering is defined by the
-    /// `TypeFoldable` implementation of the type in question.)
-    ///
-    /// An example of a **correct** use of this:
-    ///
-    /// ```rust,ignore (not real code)
-    /// let a: Canonical<'_, T> = ...;
-    /// let b: Canonical<'_, (T,)> = a.unchecked_map(|v| (v, ));
-    /// ```
-    ///
-    /// An example of an **incorrect** use of this:
-    ///
-    /// ```rust,ignore (not real code)
-    /// let a: Canonical<'tcx, T> = ...;
-    /// let ty: Ty<'tcx> = ...;
-    /// let b: Canonical<'tcx, (T, Ty<'tcx>)> = a.unchecked_map(|v| (v, ty));
-    /// ```
-    pub fn unchecked_map<W>(self, map_op: impl FnOnce(V) -> W) -> Canonical<'tcx, W> {
-        let Canonical { max_universe, variables, value } = self;
-        Canonical { max_universe, variables, value: map_op(value) }
-    }
-
-    /// Allows you to map the `value` of a canonical while keeping the same set of
-    /// bound variables.
-    ///
-    /// **WARNING:** This function is very easy to mis-use, hence the name! See
-    /// the comment of [Canonical::unchecked_map] for more details.
-    pub fn unchecked_rebind<W>(self, value: W) -> Canonical<'tcx, W> {
-        let Canonical { max_universe, variables, value: _ } = self;
-        Canonical { max_universe, variables, value }
-    }
-}
-
 pub type QueryOutlivesConstraint<'tcx> =
     (ty::OutlivesPredicate<GenericArg<'tcx>, Region<'tcx>>, ConstraintCategory<'tcx>);
 
diff --git a/compiler/rustc_middle/src/infer/unify_key.rs b/compiler/rustc_middle/src/infer/unify_key.rs
index 7ca964759..041a63776 100644
--- a/compiler/rustc_middle/src/infer/unify_key.rs
+++ b/compiler/rustc_middle/src/infer/unify_key.rs
@@ -141,18 +141,30 @@ pub struct ConstVarValue<'tcx> {
     pub val: ConstVariableValue<'tcx>,
 }
 
-impl<'tcx> UnifyKey for ty::ConstVid<'tcx> {
+#[derive(PartialEq, Copy, Clone, Debug)]
+pub struct ConstVidKey<'tcx> {
+    pub vid: ty::ConstVid,
+    pub phantom: PhantomData<ty::Const<'tcx>>,
+}
+
+impl<'tcx> From<ty::ConstVid> for ConstVidKey<'tcx> {
+    fn from(vid: ty::ConstVid) -> Self {
+        ConstVidKey { vid, phantom: PhantomData }
+    }
+}
+
+impl<'tcx> UnifyKey for ConstVidKey<'tcx> {
     type Value = ConstVarValue<'tcx>;
     #[inline]
     fn index(&self) -> u32 {
-        self.index
+        self.vid.as_u32()
     }
     #[inline]
     fn from_index(i: u32) -> Self {
-        ty::ConstVid { index: i, phantom: PhantomData }
+        ConstVidKey::from(ty::ConstVid::from_u32(i))
     }
     fn tag() -> &'static str {
-        "ConstVid"
+        "ConstVidKey"
     }
 }
 
@@ -224,17 +236,29 @@ impl<'tcx> UnifyValue for EffectVarValue<'tcx> {
     }
 }
 
-impl<'tcx> UnifyKey for ty::EffectVid<'tcx> {
+#[derive(PartialEq, Copy, Clone, Debug)]
+pub struct EffectVidKey<'tcx> {
+    pub vid: ty::EffectVid,
+    pub phantom: PhantomData<EffectVarValue<'tcx>>,
+}
+
+impl<'tcx> From<ty::EffectVid> for EffectVidKey<'tcx> {
+    fn from(vid: ty::EffectVid) -> Self {
+        EffectVidKey { vid, phantom: PhantomData }
+    }
+}
+
+impl<'tcx> UnifyKey for EffectVidKey<'tcx> {
     type Value = Option<EffectVarValue<'tcx>>;
     #[inline]
     fn index(&self) -> u32 {
-        self.index
+        self.vid.as_u32()
     }
     #[inline]
     fn from_index(i: u32) -> Self {
-        ty::EffectVid { index: i, phantom: PhantomData }
+        EffectVidKey::from(ty::EffectVid::from_u32(i))
     }
     fn tag() -> &'static str {
-        "EffectVid"
+        "EffectVidKey"
     }
 }
diff --git a/compiler/rustc_middle/src/lib.rs b/compiler/rustc_middle/src/lib.rs
index fe4fc3761..448a3029a 100644
--- a/compiler/rustc_middle/src/lib.rs
+++ b/compiler/rustc_middle/src/lib.rs
@@ -23,6 +23,8 @@
 //! This API is completely unstable and subject to change.
 
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
+#![cfg_attr(not(bootstrap), doc(rust_logo))]
+#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
 #![feature(allocator_api)]
 #![feature(array_windows)]
 #![feature(assert_matches)]
@@ -30,11 +32,12 @@
 #![feature(core_intrinsics)]
 #![feature(discriminant_kind)]
 #![feature(exhaustive_patterns)]
-#![feature(generators)]
+#![cfg_attr(bootstrap, feature(generators))]
+#![cfg_attr(not(bootstrap), feature(coroutines))]
 #![feature(get_mut_unchecked)]
 #![feature(if_let_guard)]
 #![feature(inline_const)]
-#![feature(iter_from_generator)]
+#![feature(iter_from_coroutine)]
 #![feature(negative_impls)]
 #![feature(never_type)]
 #![feature(extern_types)]
diff --git a/compiler/rustc_middle/src/lint.rs b/compiler/rustc_middle/src/lint.rs
index 59849e8eb..eada116f8 100644
--- a/compiler/rustc_middle/src/lint.rs
+++ b/compiler/rustc_middle/src/lint.rs
@@ -9,17 +9,15 @@ use rustc_session::lint::{
     FutureIncompatibilityReason, Level, Lint, LintId,
 };
 use rustc_session::Session;
-use rustc_span::hygiene::MacroKind;
-use rustc_span::source_map::{DesugaringKind, ExpnKind};
-use rustc_span::{symbol, Span, Symbol, DUMMY_SP};
+use rustc_span::hygiene::{ExpnKind, MacroKind};
+use rustc_span::{symbol, DesugaringKind, Span, Symbol, DUMMY_SP};
 
 use crate::ty::TyCtxt;
 
 /// How a lint level was set.
 #[derive(Clone, Copy, PartialEq, Eq, HashStable, Debug)]
 pub enum LintLevelSource {
-    /// Lint is at the default level as declared
-    /// in rustc or a plugin.
+    /// Lint is at the default level as declared in rustc.
     Default,
 
     /// Lint level was set by an attribute.
diff --git a/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs b/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs
index 4e5725876..f758c1d5e 100644
--- a/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs
+++ b/compiler/rustc_middle/src/middle/codegen_fn_attrs.rs
@@ -126,14 +126,6 @@ impl CodegenFnAttrs {
         }
     }
 
-    /// Returns `true` if `#[inline]` or `#[inline(always)]` is present.
-    pub fn requests_inline(&self) -> bool {
-        match self.inline {
-            InlineAttr::Hint | InlineAttr::Always => true,
-            InlineAttr::None | InlineAttr::Never => false,
-        }
-    }
-
     /// Returns `true` if it looks like this symbol needs to be exported, for example:
     ///
     /// * `#[no_mangle]` is present
diff --git a/compiler/rustc_middle/src/middle/privacy.rs b/compiler/rustc_middle/src/middle/privacy.rs
index 1913421f5..500536a9e 100644
--- a/compiler/rustc_middle/src/middle/privacy.rs
+++ b/compiler/rustc_middle/src/middle/privacy.rs
@@ -244,7 +244,9 @@ impl<Id: Eq + Hash> EffectiveVisibilities<Id> {
                 if !(inherited_effective_vis_at_prev_level == inherited_effective_vis_at_level
                     && level != l)
                 {
-                    calculated_effective_vis = if let Some(max_vis) = max_vis && !max_vis.is_at_least(inherited_effective_vis_at_level, tcx) {
+                    calculated_effective_vis = if let Some(max_vis) = max_vis
+                        && !max_vis.is_at_least(inherited_effective_vis_at_level, tcx)
+                    {
                         max_vis
                     } else {
                         inherited_effective_vis_at_level
diff --git a/compiler/rustc_middle/src/middle/region.rs b/compiler/rustc_middle/src/middle/region.rs
index c50c5e6f7..56fed05c6 100644
--- a/compiler/rustc_middle/src/middle/region.rs
+++ b/compiler/rustc_middle/src/middle/region.rs
@@ -308,7 +308,7 @@ pub struct ScopeTree {
 
     /// The number of visit_expr and visit_pat calls done in the body.
     /// Used to sanity check visit_expr/visit_pat call count when
-    /// calculating generator interiors.
+    /// calculating coroutine interiors.
     pub body_expr_count: FxHashMap<hir::BodyId, usize>,
 }
 
@@ -413,7 +413,7 @@ impl ScopeTree {
 
     /// Gives the number of expressions visited in a body.
     /// Used to sanity check visit_expr call count when
-    /// calculating generator interiors.
+    /// calculating coroutine interiors.
     pub fn body_expr_count(&self, body_id: hir::BodyId) -> Option<usize> {
         self.body_expr_count.get(&body_id).copied()
     }
diff --git a/compiler/rustc_middle/src/middle/stability.rs b/compiler/rustc_middle/src/middle/stability.rs
index 908ab8b61..f7a55fa95 100644
--- a/compiler/rustc_middle/src/middle/stability.rs
+++ b/compiler/rustc_middle/src/middle/stability.rs
@@ -5,7 +5,9 @@ pub use self::StabilityLevel::*;
 
 use crate::ty::{self, TyCtxt};
 use rustc_ast::NodeId;
-use rustc_attr::{self as attr, ConstStability, DefaultBodyStability, Deprecation, Stability};
+use rustc_attr::{
+    self as attr, ConstStability, DefaultBodyStability, DeprecatedSince, Deprecation, Stability,
+};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_errors::{Applicability, Diagnostic};
 use rustc_feature::GateIssue;
@@ -123,44 +125,6 @@ pub fn report_unstable(
     }
 }
 
-/// Checks whether an item marked with `deprecated(since="X")` is currently
-/// deprecated (i.e., whether X is not greater than the current rustc version).
-pub fn deprecation_in_effect(depr: &Deprecation) -> bool {
-    let is_since_rustc_version = depr.is_since_rustc_version;
-    let since = depr.since.as_ref().map(Symbol::as_str);
-
-    fn parse_version(ver: &str) -> Vec<u32> {
-        // We ignore non-integer components of the version (e.g., "nightly").
-        ver.split(|c| c == '.' || c == '-').flat_map(|s| s.parse()).collect()
-    }
-
-    if !is_since_rustc_version {
-        // The `since` field doesn't have semantic purpose without `#![staged_api]`.
-        return true;
-    }
-
-    if let Some(since) = since {
-        if since == "TBD" {
-            return false;
-        }
-
-        if let Some(rustc) = option_env!("CFG_RELEASE") {
-            let since: Vec<u32> = parse_version(&since);
-            let rustc: Vec<u32> = parse_version(rustc);
-            // We simply treat invalid `since` attributes as relating to a previous
-            // Rust version, thus always displaying the warning.
-            if since.len() != 3 {
-                return true;
-            }
-            return since <= rustc;
-        }
-    };
-
-    // Assume deprecation is in effect if "since" field is missing
-    // or if we can't determine the current Rust version.
-    true
-}
-
 pub fn deprecation_suggestion(
     diag: &mut Diagnostic,
     kind: &str,
@@ -183,7 +147,7 @@ fn deprecation_lint(is_in_effect: bool) -> &'static Lint {
 
 fn deprecation_message(
     is_in_effect: bool,
-    since: Option<Symbol>,
+    since: DeprecatedSince,
     note: Option<Symbol>,
     kind: &str,
     path: &str,
@@ -191,17 +155,18 @@ fn deprecation_message(
     let message = if is_in_effect {
         format!("use of deprecated {kind} `{path}`")
     } else {
-        let since = since.as_ref().map(Symbol::as_str);
-
-        if since == Some("TBD") {
-            format!("use of {kind} `{path}` that will be deprecated in a future Rust version")
-        } else {
-            format!(
-                "use of {} `{}` that will be deprecated in future version {}",
-                kind,
-                path,
-                since.unwrap()
-            )
+        match since {
+            DeprecatedSince::RustcVersion(version) => format!(
+                "use of {kind} `{path}` that will be deprecated in future version {version}"
+            ),
+            DeprecatedSince::Future => {
+                format!("use of {kind} `{path}` that will be deprecated in a future Rust version")
+            }
+            DeprecatedSince::NonStandard(_)
+            | DeprecatedSince::Unspecified
+            | DeprecatedSince::Err => {
+                unreachable!("this deprecation is always in effect; {since:?}")
+            }
         }
     };
 
@@ -216,7 +181,7 @@ pub fn deprecation_message_and_lint(
     kind: &str,
     path: &str,
 ) -> (String, &'static Lint) {
-    let is_in_effect = deprecation_in_effect(depr);
+    let is_in_effect = depr.is_in_effect();
     (
         deprecation_message(is_in_effect, depr.since, depr.note, kind, path),
         deprecation_lint(is_in_effect),
@@ -384,11 +349,11 @@ impl<'tcx> TyCtxt<'tcx> {
                 // With #![staged_api], we want to emit down the whole
                 // hierarchy.
                 let depr_attr = &depr_entry.attr;
-                if !skip || depr_attr.is_since_rustc_version {
+                if !skip || depr_attr.is_since_rustc_version() {
                     // Calculating message for lint involves calling `self.def_path_str`.
                     // Which by default to calculate visible path will invoke expensive `visible_parent_map` query.
                     // So we skip message calculation altogether, if lint is allowed.
-                    let is_in_effect = deprecation_in_effect(depr_attr);
+                    let is_in_effect = depr_attr.is_in_effect();
                     let lint = deprecation_lint(is_in_effect);
                     if self.lint_level_at_node(lint, id).0 != Level::Allow {
                         let def_path = with_no_trimmed_paths!(self.def_path_str(def_id));
@@ -448,14 +413,16 @@ impl<'tcx> TyCtxt<'tcx> {
                     debug!("stability: skipping span={:?} since it is internal", span);
                     return EvalResult::Allow;
                 }
-                if self.features().active(feature) {
+                if self.features().declared(feature) {
                     return EvalResult::Allow;
                 }
 
                 // If this item was previously part of a now-stabilized feature which is still
                 // active (i.e. the user hasn't removed the attribute for the stabilized feature
                 // yet) then allow use of this item.
-                if let Some(implied_by) = implied_by && self.features().active(implied_by) {
+                if let Some(implied_by) = implied_by
+                    && self.features().declared(implied_by)
+                {
                     return EvalResult::Allow;
                 }
 
@@ -532,7 +499,7 @@ impl<'tcx> TyCtxt<'tcx> {
                     debug!("body stability: skipping span={:?} since it is internal", span);
                     return EvalResult::Allow;
                 }
-                if self.features().active(feature) {
+                if self.features().declared(feature) {
                     return EvalResult::Allow;
                 }
 
diff --git a/compiler/rustc_middle/src/mir/consts.rs b/compiler/rustc_middle/src/mir/consts.rs
index 7c8a57b84..a9d09709e 100644
--- a/compiler/rustc_middle/src/mir/consts.rs
+++ b/compiler/rustc_middle/src/mir/consts.rs
@@ -3,6 +3,7 @@ use std::fmt::{self, Debug, Display, Formatter};
 use rustc_hir;
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::{self as hir};
+use rustc_session::RemapFileNameExt;
 use rustc_span::Span;
 use rustc_target::abi::{HasDataLayout, Size};
 
@@ -172,6 +173,24 @@ impl<'tcx> ConstValue<'tcx> {
         let end = end.try_into().unwrap();
         Some(data.inner().inspect_with_uninit_and_ptr_outside_interpreter(start..end))
     }
+
+    /// Check if a constant may contain provenance information. This is used by MIR opts.
+    /// Can return `true` even if there is no provenance.
+    pub fn may_have_provenance(&self, tcx: TyCtxt<'tcx>, size: Size) -> bool {
+        match *self {
+            ConstValue::ZeroSized | ConstValue::Scalar(Scalar::Int(_)) => return false,
+            ConstValue::Scalar(Scalar::Ptr(..)) => return true,
+            // It's hard to find out the part of the allocation we point to;
+            // just conservatively check everything.
+            ConstValue::Slice { data, meta: _ } => !data.inner().provenance().ptrs().is_empty(),
+            ConstValue::Indirect { alloc_id, offset } => !tcx
+                .global_alloc(alloc_id)
+                .unwrap_memory()
+                .inner()
+                .provenance()
+                .range_empty(super::AllocRange::from(offset..offset + size), &tcx),
+        }
+    }
 }
 
 ///////////////////////////////////////////////////////////////////////////
@@ -213,10 +232,10 @@ impl<'tcx> Const<'tcx> {
     pub fn try_to_scalar(self) -> Option<Scalar> {
         match self {
             Const::Ty(c) => match c.kind() {
-                ty::ConstKind::Value(valtree) => match valtree {
-                    ty::ValTree::Leaf(scalar_int) => Some(Scalar::Int(scalar_int)),
-                    ty::ValTree::Branch(_) => None,
-                },
+                ty::ConstKind::Value(valtree) if c.ty().is_primitive() => {
+                    // A valtree of a type where leaves directly represent the scalar const value.
+                    Some(valtree.unwrap_leaf().into())
+                }
                 _ => None,
             },
             Const::Val(val, _) => val.try_to_scalar(),
@@ -279,7 +298,16 @@ impl<'tcx> Const<'tcx> {
         tcx: TyCtxt<'tcx>,
         param_env: ty::ParamEnv<'tcx>,
     ) -> Option<Scalar> {
-        self.eval(tcx, param_env, None).ok()?.try_to_scalar()
+        match self {
+            Const::Ty(c) if c.ty().is_primitive() => {
+                // Avoid the `valtree_to_const_val` query. Can only be done on primitive types that
+                // are valtree leaves, and *not* on references. (References should return the
+                // pointer here, which valtrees don't represent.)
+                let val = c.eval(tcx, param_env, None).ok()?;
+                Some(val.unwrap_leaf().into())
+            }
+            _ => self.eval(tcx, param_env, None).ok()?.try_to_scalar(),
+        }
     }
 
     #[inline]
@@ -476,6 +504,40 @@ impl<'tcx> Const<'tcx> {
             _ => Self::Ty(c),
         }
     }
+
+    /// Return true if any evaluation of this constant always returns the same value,
+    /// taking into account even pointer identity tests.
+    pub fn is_deterministic(&self) -> bool {
+        // Some constants may generate fresh allocations for pointers they contain,
+        // so using the same constant twice can yield two different results:
+        // - valtrees purposefully generate new allocations
+        // - ConstValue::Slice also generate new allocations
+        match self {
+            Const::Ty(c) => match c.kind() {
+                ty::ConstKind::Param(..) => true,
+                // A valtree may be a reference. Valtree references correspond to a
+                // different allocation each time they are evaluated. Valtrees for primitive
+                // types are fine though.
+                ty::ConstKind::Value(_) => c.ty().is_primitive(),
+                ty::ConstKind::Unevaluated(..) | ty::ConstKind::Expr(..) => false,
+                // This can happen if evaluation of a constant failed. The result does not matter
+                // much since compilation is doomed.
+                ty::ConstKind::Error(..) => false,
+                // Should not appear in runtime MIR.
+                ty::ConstKind::Infer(..)
+                | ty::ConstKind::Bound(..)
+                | ty::ConstKind::Placeholder(..) => bug!(),
+            },
+            Const::Unevaluated(..) => false,
+            // If the same slice appears twice in the MIR, we cannot guarantee that we will
+            // give the same `AllocId` to the data.
+            Const::Val(ConstValue::Slice { .. }, _) => false,
+            Const::Val(
+                ConstValue::ZeroSized | ConstValue::Scalar(_) | ConstValue::Indirect { .. },
+                _,
+            ) => true,
+        }
+    }
 }
 
 /// An unevaluated (potentially generic) constant used in MIR.
@@ -520,3 +582,20 @@ impl<'tcx> Display for Const<'tcx> {
         }
     }
 }
+
+///////////////////////////////////////////////////////////////////////////
+/// Const-related utilities
+
+impl<'tcx> TyCtxt<'tcx> {
+    pub fn span_as_caller_location(self, span: Span) -> ConstValue<'tcx> {
+        let topmost = span.ctxt().outer_expn().expansion_cause().unwrap_or(span);
+        let caller = self.sess.source_map().lookup_char_pos(topmost.lo());
+        self.const_caller_location(
+            rustc_span::symbol::Symbol::intern(
+                &caller.file.name.for_codegen(&self.sess).to_string_lossy(),
+            ),
+            caller.line as u32,
+            caller.col_display as u32 + 1,
+        )
+    }
+}
diff --git a/compiler/rustc_middle/src/mir/coverage.rs b/compiler/rustc_middle/src/mir/coverage.rs
index 9ef673922..08d377a86 100644
--- a/compiler/rustc_middle/src/mir/coverage.rs
+++ b/compiler/rustc_middle/src/mir/coverage.rs
@@ -1,5 +1,6 @@
 //! Metadata from source code coverage analysis and instrumentation.
 
+use rustc_index::IndexVec;
 use rustc_macros::HashStable;
 use rustc_span::Symbol;
 
@@ -8,6 +9,11 @@ use std::fmt::{self, Debug, Formatter};
 rustc_index::newtype_index! {
     /// ID of a coverage counter. Values ascend from 0.
     ///
+    /// Before MIR inlining, counter IDs are local to their enclosing function.
+    /// After MIR inlining, coverage statements may have been inlined into
+    /// another function, so use the statement's source-scope to find which
+    /// function/instance its IDs are meaningful for.
+    ///
     /// Note that LLVM handles counter IDs as `uint32_t`, so there is no need
     /// to use a larger representation on the Rust side.
     #[derive(HashStable)]
@@ -18,16 +24,16 @@ rustc_index::newtype_index! {
 
 impl CounterId {
     pub const START: Self = Self::from_u32(0);
-
-    #[inline(always)]
-    pub fn next_id(self) -> Self {
-        Self::from_u32(self.as_u32() + 1)
-    }
 }
 
 rustc_index::newtype_index! {
     /// ID of a coverage-counter expression. Values ascend from 0.
     ///
+    /// Before MIR inlining, expression IDs are local to their enclosing function.
+    /// After MIR inlining, coverage statements may have been inlined into
+    /// another function, so use the statement's source-scope to find which
+    /// function/instance its IDs are meaningful for.
+    ///
     /// Note that LLVM handles expression IDs as `uint32_t`, so there is no need
     /// to use a larger representation on the Rust side.
     #[derive(HashStable)]
@@ -38,26 +44,23 @@ rustc_index::newtype_index! {
 
 impl ExpressionId {
     pub const START: Self = Self::from_u32(0);
-
-    #[inline(always)]
-    pub fn next_id(self) -> Self {
-        Self::from_u32(self.as_u32() + 1)
-    }
 }
 
-/// Operand of a coverage-counter expression.
+/// Enum that can hold a constant zero value, the ID of an physical coverage
+/// counter, or the ID of a coverage-counter expression.
 ///
-/// Operands can be a constant zero value, an actual coverage counter, or another
-/// expression. Counter/expression operands are referred to by ID.
+/// This was originally only used for expression operands (and named `Operand`),
+/// but the zero/counter/expression distinction is also useful for representing
+/// the value of code/gap mappings, and the true/false arms of branch mappings.
 #[derive(Copy, Clone, PartialEq, Eq)]
 #[derive(TyEncodable, TyDecodable, Hash, HashStable, TypeFoldable, TypeVisitable)]
-pub enum Operand {
+pub enum CovTerm {
     Zero,
     Counter(CounterId),
     Expression(ExpressionId),
 }
 
-impl Debug for Operand {
+impl Debug for CovTerm {
     fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
         match self {
             Self::Zero => write!(f, "Zero"),
@@ -69,40 +72,31 @@ impl Debug for Operand {
 
 #[derive(Clone, PartialEq, TyEncodable, TyDecodable, Hash, HashStable, TypeFoldable, TypeVisitable)]
 pub enum CoverageKind {
-    Counter {
-        function_source_hash: u64,
-        /// ID of this counter within its enclosing function.
-        /// Expressions in the same function can refer to it as an operand.
-        id: CounterId,
-    },
-    Expression {
-        /// ID of this coverage-counter expression within its enclosing function.
-        /// Other expressions in the same function can refer to it as an operand.
-        id: ExpressionId,
-        lhs: Operand,
-        op: Op,
-        rhs: Operand,
-    },
-    Unreachable,
+    /// Marks the point in MIR control flow represented by a coverage counter.
+    ///
+    /// This is eventually lowered to `llvm.instrprof.increment` in LLVM IR.
+    ///
+    /// If this statement does not survive MIR optimizations, any mappings that
+    /// refer to this counter can have those references simplified to zero.
+    CounterIncrement { id: CounterId },
+
+    /// Marks the point in MIR control-flow represented by a coverage expression.
+    ///
+    /// If this statement does not survive MIR optimizations, any mappings that
+    /// refer to this expression can have those references simplified to zero.
+    ///
+    /// (This is only inserted for expression IDs that are directly used by
+    /// mappings. Intermediate expressions with no direct mappings are
+    /// retained/zeroed based on whether they are transitively used.)
+    ExpressionUsed { id: ExpressionId },
 }
 
 impl Debug for CoverageKind {
     fn fmt(&self, fmt: &mut Formatter<'_>) -> fmt::Result {
         use CoverageKind::*;
         match self {
-            Counter { id, .. } => write!(fmt, "Counter({:?})", id.index()),
-            Expression { id, lhs, op, rhs } => write!(
-                fmt,
-                "Expression({:?}) = {:?} {} {:?}",
-                id.index(),
-                lhs,
-                match op {
-                    Op::Add => "+",
-                    Op::Subtract => "-",
-                },
-                rhs,
-            ),
-            Unreachable => write!(fmt, "Unreachable"),
+            CounterIncrement { id } => write!(fmt, "CounterIncrement({:?})", id.index()),
+            ExpressionUsed { id } => write!(fmt, "ExpressionUsed({:?})", id.index()),
         }
     }
 }
@@ -143,3 +137,38 @@ impl Op {
         matches!(self, Self::Subtract)
     }
 }
+
+#[derive(Clone, Debug)]
+#[derive(TyEncodable, TyDecodable, Hash, HashStable, TypeFoldable, TypeVisitable)]
+pub struct Expression {
+    pub lhs: CovTerm,
+    pub op: Op,
+    pub rhs: CovTerm,
+}
+
+#[derive(Clone, Debug)]
+#[derive(TyEncodable, TyDecodable, Hash, HashStable, TypeFoldable, TypeVisitable)]
+pub struct Mapping {
+    pub code_region: CodeRegion,
+
+    /// Indicates whether this mapping uses a counter value, expression value,
+    /// or zero value.
+    ///
+    /// FIXME: When we add support for mapping kinds other than `Code`
+    /// (e.g. branch regions, expansion regions), replace this with a dedicated
+    /// mapping-kind enum.
+    pub term: CovTerm,
+}
+
+/// Stores per-function coverage information attached to a `mir::Body`,
+/// to be used in conjunction with the individual coverage statements injected
+/// into the function's basic blocks.
+#[derive(Clone, Debug)]
+#[derive(TyEncodable, TyDecodable, Hash, HashStable, TypeFoldable, TypeVisitable)]
+pub struct FunctionCoverageInfo {
+    pub function_source_hash: u64,
+    pub num_counters: usize,
+
+    pub expressions: IndexVec<ExpressionId, Expression>,
+    pub mappings: Vec<Mapping>,
+}
diff --git a/compiler/rustc_middle/src/mir/interpret/allocation.rs b/compiler/rustc_middle/src/mir/interpret/allocation.rs
index c787481bf..aded3e495 100644
--- a/compiler/rustc_middle/src/mir/interpret/allocation.rs
+++ b/compiler/rustc_middle/src/mir/interpret/allocation.rs
@@ -32,23 +32,16 @@ pub use init_mask::{InitChunk, InitChunkIter};
 pub trait AllocBytes:
     Clone + fmt::Debug + Eq + PartialEq + Hash + Deref<Target = [u8]> + DerefMut<Target = [u8]>
 {
-    /// Adjust the bytes to the specified alignment -- by default, this is a no-op.
-    fn adjust_to_align(self, _align: Align) -> Self;
-
     /// Create an `AllocBytes` from a slice of `u8`.
     fn from_bytes<'a>(slice: impl Into<Cow<'a, [u8]>>, _align: Align) -> Self;
 
-    /// Create a zeroed `AllocBytes` of the specified size and alignment;
-    /// call the callback error handler if there is an error in allocating the memory.
+    /// Create a zeroed `AllocBytes` of the specified size and alignment.
+    /// Returns `None` if we ran out of memory on the host.
     fn zeroed(size: Size, _align: Align) -> Option<Self>;
 }
 
 // Default `bytes` for `Allocation` is a `Box<[u8]>`.
 impl AllocBytes for Box<[u8]> {
-    fn adjust_to_align(self, _align: Align) -> Self {
-        self
-    }
-
     fn from_bytes<'a>(slice: impl Into<Cow<'a, [u8]>>, _align: Align) -> Self {
         Box::<[u8]>::from(slice.into())
     }
@@ -299,6 +292,7 @@ impl<Prov: Provenance, Bytes: AllocBytes> Allocation<Prov, (), Bytes> {
     }
 
     fn uninit_inner<R>(size: Size, align: Align, fail: impl FnOnce() -> R) -> Result<Self, R> {
+        // We raise an error if we cannot create the allocation on the host.
         // This results in an error that can happen non-deterministically, since the memory
         // available to the compiler can change between runs. Normally queries are always
         // deterministic. However, we can be non-deterministic here because all uses of const
@@ -351,10 +345,8 @@ impl<Bytes: AllocBytes> Allocation<AllocId, (), Bytes> {
         extra: Extra,
         mut adjust_ptr: impl FnMut(Pointer<AllocId>) -> Result<Pointer<Prov>, Err>,
     ) -> Result<Allocation<Prov, Extra, Bytes>, Err> {
-        // Compute new pointer provenance, which also adjusts the bytes, and realign the pointer if
-        // necessary.
-        let mut bytes = self.bytes.adjust_to_align(self.align);
-
+        let mut bytes = self.bytes;
+        // Adjust provenance of pointers stored in this allocation.
         let mut new_provenance = Vec::with_capacity(self.provenance.ptrs().len());
         let ptr_size = cx.data_layout().pointer_size.bytes_usize();
         let endian = cx.data_layout().endian;
diff --git a/compiler/rustc_middle/src/mir/interpret/allocation/provenance_map.rs b/compiler/rustc_middle/src/mir/interpret/allocation/provenance_map.rs
index 0243fc451..d504af6b7 100644
--- a/compiler/rustc_middle/src/mir/interpret/allocation/provenance_map.rs
+++ b/compiler/rustc_middle/src/mir/interpret/allocation/provenance_map.rs
@@ -315,7 +315,9 @@ impl<Prov: Provenance> ProvenanceMap<Prov> {
             self.ptrs.insert_presorted(dest_ptrs.into());
         }
         if Prov::OFFSET_IS_ADDR {
-            if let Some(dest_bytes) = copy.dest_bytes && !dest_bytes.is_empty() {
+            if let Some(dest_bytes) = copy.dest_bytes
+                && !dest_bytes.is_empty()
+            {
                 self.bytes.get_or_insert_with(Box::default).insert_presorted(dest_bytes.into());
             }
         } else {
diff --git a/compiler/rustc_middle/src/mir/interpret/error.rs b/compiler/rustc_middle/src/mir/interpret/error.rs
index bc464aca5..44b22e2d3 100644
--- a/compiler/rustc_middle/src/mir/interpret/error.rs
+++ b/compiler/rustc_middle/src/mir/interpret/error.rs
@@ -43,21 +43,6 @@ impl ErrorHandled {
         }
     }
 
-    pub fn emit_err(&self, tcx: TyCtxt<'_>) -> ErrorGuaranteed {
-        match self {
-            &ErrorHandled::Reported(err, span) => {
-                if !err.is_tainted_by_errors && !span.is_dummy() {
-                    tcx.sess.emit_err(error::ErroneousConstant { span });
-                }
-                err.error
-            }
-            &ErrorHandled::TooGeneric(span) => tcx.sess.delay_span_bug(
-                span,
-                "encountered TooGeneric error when monomorphic data was expected",
-            ),
-        }
-    }
-
     pub fn emit_note(&self, tcx: TyCtxt<'_>) {
         match self {
             &ErrorHandled::Reported(err, span) => {
@@ -231,10 +216,8 @@ pub enum InvalidProgramInfo<'tcx> {
 }
 
 /// Details of why a pointer had to be in-bounds.
-#[derive(Debug, Copy, Clone, TyEncodable, TyDecodable, HashStable)]
+#[derive(Debug, Copy, Clone)]
 pub enum CheckInAllocMsg {
-    /// We are dereferencing a pointer (i.e., creating a place).
-    DerefTest,
     /// We are access memory.
     MemoryAccessTest,
     /// We are doing pointer arithmetic.
@@ -245,7 +228,16 @@ pub enum CheckInAllocMsg {
     InboundsTest,
 }
 
-#[derive(Debug, Copy, Clone, TyEncodable, TyDecodable, HashStable)]
+/// Details of which pointer is not aligned.
+#[derive(Debug, Copy, Clone)]
+pub enum CheckAlignMsg {
+    /// The accessed pointer did not have proper alignment.
+    AccessedPtr,
+    /// The access ocurred with a place that was based on a misaligned pointer.
+    BasedOn,
+}
+
+#[derive(Debug, Copy, Clone)]
 pub enum InvalidMetaKind {
     /// Size of a `[T]` is too big
     SliceTooBig,
@@ -278,6 +270,13 @@ pub struct ScalarSizeMismatch {
     pub data_size: u64,
 }
 
+/// Information about a misaligned pointer.
+#[derive(Copy, Clone, Hash, PartialEq, Eq, Debug)]
+pub struct Misalignment {
+    pub has: Align,
+    pub required: Align,
+}
+
 macro_rules! impl_into_diagnostic_arg_through_debug {
     ($($ty:ty),*$(,)?) => {$(
         impl IntoDiagnosticArg for $ty {
@@ -339,7 +338,7 @@ pub enum UndefinedBehaviorInfo<'tcx> {
     /// Using an integer as a pointer in the wrong way.
     DanglingIntPointer(u64, CheckInAllocMsg),
     /// Used a pointer with bad alignment.
-    AlignmentCheckFailed { required: Align, has: Align },
+    AlignmentCheckFailed(Misalignment, CheckAlignMsg),
     /// Writing to read-only memory.
     WriteToReadOnly(AllocId),
     /// Trying to access the data behind a function pointer.
diff --git a/compiler/rustc_middle/src/mir/interpret/mod.rs b/compiler/rustc_middle/src/mir/interpret/mod.rs
index d21f82f04..e360fb3ea 100644
--- a/compiler/rustc_middle/src/mir/interpret/mod.rs
+++ b/compiler/rustc_middle/src/mir/interpret/mod.rs
@@ -142,11 +142,12 @@ use crate::ty::GenericArgKind;
 use crate::ty::{self, Instance, Ty, TyCtxt};
 
 pub use self::error::{
-    struct_error, BadBytesAccess, CheckInAllocMsg, ErrorHandled, EvalToAllocationRawResult,
-    EvalToConstValueResult, EvalToValTreeResult, ExpectedKind, InterpError, InterpErrorInfo,
-    InterpResult, InvalidMetaKind, InvalidProgramInfo, MachineStopType, PointerKind,
-    ReportedErrorInfo, ResourceExhaustionInfo, ScalarSizeMismatch, UndefinedBehaviorInfo,
-    UnsupportedOpInfo, ValidationErrorInfo, ValidationErrorKind,
+    struct_error, BadBytesAccess, CheckAlignMsg, CheckInAllocMsg, ErrorHandled,
+    EvalToAllocationRawResult, EvalToConstValueResult, EvalToValTreeResult, ExpectedKind,
+    InterpError, InterpErrorInfo, InterpResult, InvalidMetaKind, InvalidProgramInfo,
+    MachineStopType, Misalignment, PointerKind, ReportedErrorInfo, ResourceExhaustionInfo,
+    ScalarSizeMismatch, UndefinedBehaviorInfo, UnsupportedOpInfo, ValidationErrorInfo,
+    ValidationErrorKind,
 };
 
 pub use self::value::Scalar;
diff --git a/compiler/rustc_middle/src/mir/mod.rs b/compiler/rustc_middle/src/mir/mod.rs
index 0bb1c66da..7054cede2 100644
--- a/compiler/rustc_middle/src/mir/mod.rs
+++ b/compiler/rustc_middle/src/mir/mod.rs
@@ -2,7 +2,7 @@
 //!
 //! [rustc dev guide]: https://rustc-dev-guide.rust-lang.org/mir/index.html
 
-use crate::mir::interpret::{AllocRange, ConstAllocation, ErrorHandled, Scalar};
+use crate::mir::interpret::{AllocRange, ConstAllocation, Scalar};
 use crate::mir::visit::MirVisitable;
 use crate::ty::codec::{TyDecoder, TyEncoder};
 use crate::ty::fold::{FallibleTypeFolder, TypeFoldable};
@@ -17,7 +17,7 @@ use rustc_data_structures::captures::Captures;
 use rustc_errors::{DiagnosticArgValue, DiagnosticMessage, ErrorGuaranteed, IntoDiagnosticArg};
 use rustc_hir::def::{CtorKind, Namespace};
 use rustc_hir::def_id::{DefId, CRATE_DEF_ID};
-use rustc_hir::{self, GeneratorKind, ImplicitSelfKind};
+use rustc_hir::{self, CoroutineKind, ImplicitSelfKind};
 use rustc_hir::{self as hir, HirId};
 use rustc_session::Session;
 use rustc_target::abi::{FieldIdx, VariantIdx};
@@ -246,19 +246,19 @@ impl<'tcx> MirSource<'tcx> {
 }
 
 #[derive(Clone, TyEncodable, TyDecodable, Debug, HashStable, TypeFoldable, TypeVisitable)]
-pub struct GeneratorInfo<'tcx> {
-    /// The yield type of the function, if it is a generator.
+pub struct CoroutineInfo<'tcx> {
+    /// The yield type of the function, if it is a coroutine.
     pub yield_ty: Option<Ty<'tcx>>,
 
-    /// Generator drop glue.
-    pub generator_drop: Option<Body<'tcx>>,
+    /// Coroutine drop glue.
+    pub coroutine_drop: Option<Body<'tcx>>,
 
-    /// The layout of a generator. Produced by the state transformation.
-    pub generator_layout: Option<GeneratorLayout<'tcx>>,
+    /// The layout of a coroutine. Produced by the state transformation.
+    pub coroutine_layout: Option<CoroutineLayout<'tcx>>,
 
-    /// If this is a generator then record the type of source expression that caused this generator
+    /// If this is a coroutine then record the type of source expression that caused this coroutine
     /// to be created.
-    pub generator_kind: GeneratorKind,
+    pub coroutine_kind: CoroutineKind,
 }
 
 /// The lowered representation of a single function.
@@ -284,7 +284,7 @@ pub struct Body<'tcx> {
     /// and used for debuginfo. Indexed by a `SourceScope`.
     pub source_scopes: IndexVec<SourceScope, SourceScopeData<'tcx>>,
 
-    pub generator: Option<Box<GeneratorInfo<'tcx>>>,
+    pub coroutine: Option<Box<CoroutineInfo<'tcx>>>,
 
     /// Declarations of locals.
     ///
@@ -345,6 +345,14 @@ pub struct Body<'tcx> {
     pub injection_phase: Option<MirPhase>,
 
     pub tainted_by_errors: Option<ErrorGuaranteed>,
+
+    /// Per-function coverage information added by the `InstrumentCoverage`
+    /// pass, to be used in conjunction with the coverage statements injected
+    /// into this body's blocks.
+    ///
+    /// If `-Cinstrument-coverage` is not active, or if an individual function
+    /// is not eligible for coverage, then this should always be `None`.
+    pub function_coverage_info: Option<Box<coverage::FunctionCoverageInfo>>,
 }
 
 impl<'tcx> Body<'tcx> {
@@ -357,7 +365,7 @@ impl<'tcx> Body<'tcx> {
         arg_count: usize,
         var_debug_info: Vec<VarDebugInfo<'tcx>>,
         span: Span,
-        generator_kind: Option<GeneratorKind>,
+        coroutine_kind: Option<CoroutineKind>,
         tainted_by_errors: Option<ErrorGuaranteed>,
     ) -> Self {
         // We need `arg_count` locals, and one for the return place.
@@ -374,12 +382,12 @@ impl<'tcx> Body<'tcx> {
             source,
             basic_blocks: BasicBlocks::new(basic_blocks),
             source_scopes,
-            generator: generator_kind.map(|generator_kind| {
-                Box::new(GeneratorInfo {
+            coroutine: coroutine_kind.map(|coroutine_kind| {
+                Box::new(CoroutineInfo {
                     yield_ty: None,
-                    generator_drop: None,
-                    generator_layout: None,
-                    generator_kind,
+                    coroutine_drop: None,
+                    coroutine_layout: None,
+                    coroutine_kind,
                 })
             }),
             local_decls,
@@ -392,6 +400,7 @@ impl<'tcx> Body<'tcx> {
             is_polymorphic: false,
             injection_phase: None,
             tainted_by_errors,
+            function_coverage_info: None,
         };
         body.is_polymorphic = body.has_non_region_param();
         body
@@ -409,7 +418,7 @@ impl<'tcx> Body<'tcx> {
             source: MirSource::item(CRATE_DEF_ID.to_def_id()),
             basic_blocks: BasicBlocks::new(basic_blocks),
             source_scopes: IndexVec::new(),
-            generator: None,
+            coroutine: None,
             local_decls: IndexVec::new(),
             user_type_annotations: IndexVec::new(),
             arg_count: 0,
@@ -420,6 +429,7 @@ impl<'tcx> Body<'tcx> {
             is_polymorphic: false,
             injection_phase: None,
             tainted_by_errors: None,
+            function_coverage_info: None,
         };
         body.is_polymorphic = body.has_non_region_param();
         body
@@ -538,22 +548,22 @@ impl<'tcx> Body<'tcx> {
 
     #[inline]
     pub fn yield_ty(&self) -> Option<Ty<'tcx>> {
-        self.generator.as_ref().and_then(|generator| generator.yield_ty)
+        self.coroutine.as_ref().and_then(|coroutine| coroutine.yield_ty)
     }
 
     #[inline]
-    pub fn generator_layout(&self) -> Option<&GeneratorLayout<'tcx>> {
-        self.generator.as_ref().and_then(|generator| generator.generator_layout.as_ref())
+    pub fn coroutine_layout(&self) -> Option<&CoroutineLayout<'tcx>> {
+        self.coroutine.as_ref().and_then(|coroutine| coroutine.coroutine_layout.as_ref())
     }
 
     #[inline]
-    pub fn generator_drop(&self) -> Option<&Body<'tcx>> {
-        self.generator.as_ref().and_then(|generator| generator.generator_drop.as_ref())
+    pub fn coroutine_drop(&self) -> Option<&Body<'tcx>> {
+        self.coroutine.as_ref().and_then(|coroutine| coroutine.coroutine_drop.as_ref())
     }
 
     #[inline]
-    pub fn generator_kind(&self) -> Option<GeneratorKind> {
-        self.generator.as_ref().map(|generator| generator.generator_kind)
+    pub fn coroutine_kind(&self) -> Option<CoroutineKind> {
+        self.coroutine.as_ref().map(|coroutine| coroutine.coroutine_kind)
     }
 
     #[inline]
@@ -569,32 +579,38 @@ impl<'tcx> Body<'tcx> {
         self.injection_phase.is_some()
     }
 
-    /// *Must* be called once the full substitution for this body is known, to ensure that the body
-    /// is indeed fit for code generation or consumption more generally.
-    ///
-    /// Sadly there's no nice way to represent an "arbitrary normalizer", so we take one for
-    /// constants specifically. (`Option<GenericArgsRef>` could be used for that, but the fact
-    /// that `Instance::args_for_mir_body` is private and instead instance exposes normalization
-    /// functions makes it seem like exposing the generic args is not the intended strategy.)
-    ///
-    /// Also sadly, CTFE doesn't even know whether it runs on MIR that is already polymorphic or still monomorphic,
-    /// so we cannot just immediately ICE on TooGeneric.
-    ///
-    /// Returns Ok(()) if everything went fine, and `Err` if a problem occurred and got reported.
-    pub fn post_mono_checks(
+    /// For a `Location` in this scope, determine what the "caller location" at that point is. This
+    /// is interesting because of inlining: the `#[track_caller]` attribute of inlined functions
+    /// must be honored. Falls back to the `tracked_caller` value for `#[track_caller]` functions,
+    /// or the function's scope.
+    pub fn caller_location_span<T>(
         &self,
+        mut source_info: SourceInfo,
+        caller_location: Option<T>,
         tcx: TyCtxt<'tcx>,
-        param_env: ty::ParamEnv<'tcx>,
-        normalize_const: impl Fn(Const<'tcx>) -> Result<Const<'tcx>, ErrorHandled>,
-    ) -> Result<(), ErrorHandled> {
-        // For now, the only thing we have to check is is to ensure that all the constants used in
-        // the body successfully evaluate.
-        for &const_ in &self.required_consts {
-            let c = normalize_const(const_.const_)?;
-            c.eval(tcx, param_env, Some(const_.span))?;
+        from_span: impl FnOnce(Span) -> T,
+    ) -> T {
+        loop {
+            let scope_data = &self.source_scopes[source_info.scope];
+
+            if let Some((callee, callsite_span)) = scope_data.inlined {
+                // Stop inside the most nested non-`#[track_caller]` function,
+                // before ever reaching its caller (which is irrelevant).
+                if !callee.def.requires_caller_location(tcx) {
+                    return from_span(source_info.span);
+                }
+                source_info.span = callsite_span;
+            }
+
+            // Skip past all of the parents with `inlined: None`.
+            match scope_data.inlined_parent_scope {
+                Some(parent) => source_info.scope = parent,
+                None => break,
+            }
         }
 
-        Ok(())
+        // No inlined `SourceScope`s, or all of them were `#[track_caller]`.
+        caller_location.unwrap_or_else(|| from_span(source_info.span))
     }
 }
 
@@ -830,22 +846,6 @@ pub struct LocalDecl<'tcx> {
     // FIXME(matthewjasper) Don't store in this in `Body`
     pub local_info: ClearCrossCrate<Box<LocalInfo<'tcx>>>,
 
-    /// `true` if this is an internal local.
-    ///
-    /// These locals are not based on types in the source code and are only used
-    /// for a few desugarings at the moment.
-    ///
-    /// The generator transformation will sanity check the locals which are live
-    /// across a suspension point against the type components of the generator
-    /// which type checking knows are live across a suspension point. We need to
-    /// flag drop flags to avoid triggering this check as they are introduced
-    /// outside of type inference.
-    ///
-    /// This should be sound because the drop flags are fully algebraic, and
-    /// therefore don't affect the auto-trait or outlives properties of the
-    /// generator.
-    pub internal: bool,
-
     /// The type of this local.
     pub ty: Ty<'tcx>,
 
@@ -1058,7 +1058,7 @@ impl<'tcx> LocalDecl<'tcx> {
         self.source_info.span.desugaring_kind().is_some()
     }
 
-    /// Creates a new `LocalDecl` for a temporary: mutable, non-internal.
+    /// Creates a new `LocalDecl` for a temporary, mutable.
     #[inline]
     pub fn new(ty: Ty<'tcx>, span: Span) -> Self {
         Self::with_source_info(ty, SourceInfo::outermost(span))
@@ -1070,20 +1070,12 @@ impl<'tcx> LocalDecl<'tcx> {
         LocalDecl {
             mutability: Mutability::Mut,
             local_info: ClearCrossCrate::Set(Box::new(LocalInfo::Boring)),
-            internal: false,
             ty,
             user_ty: None,
             source_info,
         }
     }
 
-    /// Converts `self` into same `LocalDecl` except tagged as internal.
-    #[inline]
-    pub fn internal(mut self) -> Self {
-        self.internal = true;
-        self
-    }
-
     /// Converts `self` into same `LocalDecl` except tagged as immutable.
     #[inline]
     pub fn immutable(mut self) -> Self {
@@ -1614,6 +1606,23 @@ impl Location {
     }
 }
 
+/// `DefLocation` represents the location of a definition - either an argument or an assignment
+/// within MIR body.
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub enum DefLocation {
+    Argument,
+    Body(Location),
+}
+
+impl DefLocation {
+    pub fn dominates(self, location: Location, dominators: &Dominators<BasicBlock>) -> bool {
+        match self {
+            DefLocation::Argument => true,
+            DefLocation::Body(def) => def.successor_within_block().dominates(location, dominators),
+        }
+    }
+}
+
 // Some nodes are used a lot. Make sure they don't unintentionally get bigger.
 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
 mod size_asserts {
diff --git a/compiler/rustc_middle/src/mir/mono.rs b/compiler/rustc_middle/src/mir/mono.rs
index 403e80bd3..91fdf0b31 100644
--- a/compiler/rustc_middle/src/mir/mono.rs
+++ b/compiler/rustc_middle/src/mir/mono.rs
@@ -10,8 +10,8 @@ use rustc_hir::ItemId;
 use rustc_index::Idx;
 use rustc_query_system::ich::StableHashingContext;
 use rustc_session::config::OptLevel;
-use rustc_span::source_map::Span;
 use rustc_span::symbol::Symbol;
+use rustc_span::Span;
 use std::fmt;
 use std::hash::Hash;
 
diff --git a/compiler/rustc_middle/src/mir/patch.rs b/compiler/rustc_middle/src/mir/patch.rs
index da486c346..eb4aa9eb9 100644
--- a/compiler/rustc_middle/src/mir/patch.rs
+++ b/compiler/rustc_middle/src/mir/patch.rs
@@ -99,7 +99,9 @@ impl<'tcx> MirPatch<'tcx> {
     }
 
     pub fn terminate_block(&mut self, reason: UnwindTerminateReason) -> BasicBlock {
-        if let Some((cached_bb, cached_reason)) = self.terminate_block && reason == cached_reason {
+        if let Some((cached_bb, cached_reason)) = self.terminate_block
+            && reason == cached_reason
+        {
             return cached_bb;
         }
 
@@ -127,7 +129,7 @@ impl<'tcx> MirPatch<'tcx> {
         Location { block: bb, statement_index: offset }
     }
 
-    pub fn new_internal_with_info(
+    pub fn new_local_with_info(
         &mut self,
         ty: Ty<'tcx>,
         span: Span,
@@ -135,7 +137,7 @@ impl<'tcx> MirPatch<'tcx> {
     ) -> Local {
         let index = self.next_local;
         self.next_local += 1;
-        let mut new_decl = LocalDecl::new(ty, span).internal();
+        let mut new_decl = LocalDecl::new(ty, span);
         **new_decl.local_info.as_mut().assert_crate_local() = local_info;
         self.new_locals.push(new_decl);
         Local::new(index)
@@ -148,13 +150,6 @@ impl<'tcx> MirPatch<'tcx> {
         Local::new(index)
     }
 
-    pub fn new_internal(&mut self, ty: Ty<'tcx>, span: Span) -> Local {
-        let index = self.next_local;
-        self.next_local += 1;
-        self.new_locals.push(LocalDecl::new(ty, span).internal());
-        Local::new(index)
-    }
-
     pub fn new_block(&mut self, data: BasicBlockData<'tcx>) -> BasicBlock {
         let block = BasicBlock::new(self.patch_map.len());
         debug!("MirPatch: new_block: {:?}: {:?}", block, data);
diff --git a/compiler/rustc_middle/src/mir/pretty.rs b/compiler/rustc_middle/src/mir/pretty.rs
index f032fd29d..a13248584 100644
--- a/compiler/rustc_middle/src/mir/pretty.rs
+++ b/compiler/rustc_middle/src/mir/pretty.rs
@@ -16,7 +16,7 @@ use rustc_middle::mir::interpret::{
     Pointer, Provenance,
 };
 use rustc_middle::mir::visit::Visitor;
-use rustc_middle::mir::*;
+use rustc_middle::mir::{self, *};
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_target::abi::Size;
 
@@ -130,8 +130,8 @@ fn dump_matched_mir_node<'tcx, F>(
             Some(promoted) => write!(file, "::{promoted:?}`")?,
         }
         writeln!(file, " {disambiguator} {pass_name}")?;
-        if let Some(ref layout) = body.generator_layout() {
-            writeln!(file, "/* generator_layout = {layout:#?} */")?;
+        if let Some(ref layout) = body.coroutine_layout() {
+            writeln!(file, "/* coroutine_layout = {layout:#?} */")?;
         }
         writeln!(file)?;
         extra_data(PassWhere::BeforeCFG, &mut file)?;
@@ -493,6 +493,27 @@ pub fn write_mir_intro<'tcx>(
     // Add an empty line before the first block is printed.
     writeln!(w)?;
 
+    if let Some(function_coverage_info) = &body.function_coverage_info {
+        write_function_coverage_info(function_coverage_info, w)?;
+    }
+
+    Ok(())
+}
+
+fn write_function_coverage_info(
+    function_coverage_info: &coverage::FunctionCoverageInfo,
+    w: &mut dyn io::Write,
+) -> io::Result<()> {
+    let coverage::FunctionCoverageInfo { expressions, mappings, .. } = function_coverage_info;
+
+    for (id, expression) in expressions.iter_enumerated() {
+        writeln!(w, "{INDENT}coverage {id:?} => {expression:?};")?;
+    }
+    for coverage::Mapping { term, code_region } in mappings {
+        writeln!(w, "{INDENT}coverage {term:?} => {code_region:?};")?;
+    }
+    writeln!(w)?;
+
     Ok(())
 }
 
@@ -685,10 +706,7 @@ impl Debug for Statement<'_> {
             AscribeUserType(box (ref place, ref c_ty), ref variance) => {
                 write!(fmt, "AscribeUserType({place:?}, {variance:?}, {c_ty:?})")
             }
-            Coverage(box self::Coverage { ref kind, code_region: Some(ref rgn) }) => {
-                write!(fmt, "Coverage::{kind:?} for {rgn:?}")
-            }
-            Coverage(box ref coverage) => write!(fmt, "Coverage::{:?}", coverage.kind),
+            Coverage(box mir::Coverage { ref kind }) => write!(fmt, "Coverage::{kind:?}"),
             Intrinsic(box ref intrinsic) => write!(fmt, "{intrinsic}"),
             ConstEvalCounter => write!(fmt, "ConstEvalCounter"),
             Nop => write!(fmt, "nop"),
@@ -764,10 +782,10 @@ impl<'tcx> TerminatorKind<'tcx> {
             Goto { .. } => write!(fmt, "goto"),
             SwitchInt { discr, .. } => write!(fmt, "switchInt({discr:?})"),
             Return => write!(fmt, "return"),
-            GeneratorDrop => write!(fmt, "generator_drop"),
+            CoroutineDrop => write!(fmt, "coroutine_drop"),
             UnwindResume => write!(fmt, "resume"),
             UnwindTerminate(reason) => {
-                write!(fmt, "abort({})", reason.as_short_str())
+                write!(fmt, "terminate({})", reason.as_short_str())
             }
             Yield { value, resume_arg, .. } => write!(fmt, "{resume_arg:?} = yield({value:?})"),
             Unreachable => write!(fmt, "unreachable"),
@@ -847,7 +865,7 @@ impl<'tcx> TerminatorKind<'tcx> {
     pub fn fmt_successor_labels(&self) -> Vec<Cow<'static, str>> {
         use self::TerminatorKind::*;
         match *self {
-            Return | UnwindResume | UnwindTerminate(_) | Unreachable | GeneratorDrop => vec![],
+            Return | UnwindResume | UnwindTerminate(_) | Unreachable | CoroutineDrop => vec![],
             Goto { .. } => vec!["".into()],
             SwitchInt { ref targets, .. } => targets
                 .values
@@ -980,9 +998,9 @@ impl<'tcx> Debug for Rvalue<'tcx> {
                         ty::tls::with(|tcx| {
                             let variant_def = &tcx.adt_def(adt_did).variant(variant);
                             let args = tcx.lift(args).expect("could not lift for printing");
-                            let name = FmtPrinter::new(tcx, Namespace::ValueNS)
-                                .print_def_path(variant_def.def_id, args)?
-                                .into_buffer();
+                            let name = FmtPrinter::print_string(tcx, Namespace::ValueNS, |cx| {
+                                cx.print_def_path(variant_def.def_id, args)
+                            })?;
 
                             match variant_def.ctor_kind() {
                                 Some(CtorKind::Const) => fmt.write_str(&name),
@@ -1028,8 +1046,8 @@ impl<'tcx> Debug for Rvalue<'tcx> {
                         struct_fmt.finish()
                     }),
 
-                    AggregateKind::Generator(def_id, _, _) => ty::tls::with(|tcx| {
-                        let name = format!("{{generator@{:?}}}", tcx.def_span(def_id));
+                    AggregateKind::Coroutine(def_id, _, _) => ty::tls::with(|tcx| {
+                        let name = format!("{{coroutine@{:?}}}", tcx.def_span(def_id));
                         let mut struct_fmt = fmt.debug_struct(&name);
 
                         // FIXME(project-rfc-2229#48): This should be a list of capture names/places
@@ -1283,8 +1301,8 @@ impl<'tcx> Visitor<'tcx> for ExtraComments<'tcx> {
                     self.push(&format!("+ args: {args:#?}"));
                 }
 
-                AggregateKind::Generator(def_id, args, movability) => {
-                    self.push("generator");
+                AggregateKind::Coroutine(def_id, args, movability) => {
+                    self.push("coroutine");
                     self.push(&format!("+ def_id: {def_id:?}"));
                     self.push(&format!("+ args: {args:#?}"));
                     self.push(&format!("+ movability: {movability:?}"));
@@ -1695,7 +1713,7 @@ fn pretty_print_const_value_tcx<'tcx>(
         (_, ty::Array(..) | ty::Tuple(..) | ty::Adt(..)) if !ty.has_non_region_param() => {
             let ct = tcx.lift(ct).unwrap();
             let ty = tcx.lift(ty).unwrap();
-            if let Some(contents) = tcx.try_destructure_mir_constant_for_diagnostics(ct, ty) {
+            if let Some(contents) = tcx.try_destructure_mir_constant_for_user_output(ct, ty) {
                 let fields: Vec<(ConstValue<'_>, Ty<'_>)> = contents.fields.to_vec();
                 match *ty.kind() {
                     ty::Array(..) => {
@@ -1722,7 +1740,7 @@ fn pretty_print_const_value_tcx<'tcx>(
                         let args = tcx.lift(args).unwrap();
                         let mut cx = FmtPrinter::new(tcx, Namespace::ValueNS);
                         cx.print_alloc_ids = true;
-                        let cx = cx.print_value_path(variant_def.def_id, args)?;
+                        cx.print_value_path(variant_def.def_id, args)?;
                         fmt.write_str(&cx.into_buffer())?;
 
                         match variant_def.ctor_kind() {
@@ -1757,14 +1775,14 @@ fn pretty_print_const_value_tcx<'tcx>(
             let mut cx = FmtPrinter::new(tcx, Namespace::ValueNS);
             cx.print_alloc_ids = true;
             let ty = tcx.lift(ty).unwrap();
-            cx = cx.pretty_print_const_scalar(scalar, ty)?;
+            cx.pretty_print_const_scalar(scalar, ty)?;
             fmt.write_str(&cx.into_buffer())?;
             return Ok(());
         }
         (ConstValue::ZeroSized, ty::FnDef(d, s)) => {
             let mut cx = FmtPrinter::new(tcx, Namespace::ValueNS);
             cx.print_alloc_ids = true;
-            let cx = cx.print_value_path(*d, s)?;
+            cx.print_value_path(*d, s)?;
             fmt.write_str(&cx.into_buffer())?;
             return Ok(());
         }
diff --git a/compiler/rustc_middle/src/mir/query.rs b/compiler/rustc_middle/src/mir/query.rs
index c74a9536b..0540eb0ef 100644
--- a/compiler/rustc_middle/src/mir/query.rs
+++ b/compiler/rustc_middle/src/mir/query.rs
@@ -1,5 +1,6 @@
 //! Values computed by queries that use MIR.
 
+use crate::mir;
 use crate::ty::{self, OpaqueHiddenType, Ty, TyCtxt};
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_data_structures::unord::UnordSet;
@@ -132,11 +133,11 @@ pub struct UnsafetyCheckResult {
 rustc_index::newtype_index! {
     #[derive(HashStable)]
     #[debug_format = "_{}"]
-    pub struct GeneratorSavedLocal {}
+    pub struct CoroutineSavedLocal {}
 }
 
 #[derive(Clone, Debug, TyEncodable, TyDecodable, HashStable, TypeFoldable, TypeVisitable)]
-pub struct GeneratorSavedTy<'tcx> {
+pub struct CoroutineSavedTy<'tcx> {
     pub ty: Ty<'tcx>,
     /// Source info corresponding to the local in the original MIR body.
     pub source_info: SourceInfo,
@@ -144,18 +145,18 @@ pub struct GeneratorSavedTy<'tcx> {
     pub ignore_for_traits: bool,
 }
 
-/// The layout of generator state.
+/// The layout of coroutine state.
 #[derive(Clone, TyEncodable, TyDecodable, HashStable, TypeFoldable, TypeVisitable)]
-pub struct GeneratorLayout<'tcx> {
-    /// The type of every local stored inside the generator.
-    pub field_tys: IndexVec<GeneratorSavedLocal, GeneratorSavedTy<'tcx>>,
+pub struct CoroutineLayout<'tcx> {
+    /// The type of every local stored inside the coroutine.
+    pub field_tys: IndexVec<CoroutineSavedLocal, CoroutineSavedTy<'tcx>>,
 
     /// The name for debuginfo.
-    pub field_names: IndexVec<GeneratorSavedLocal, Option<Symbol>>,
+    pub field_names: IndexVec<CoroutineSavedLocal, Option<Symbol>>,
 
     /// Which of the above fields are in each variant. Note that one field may
     /// be stored in multiple variants.
-    pub variant_fields: IndexVec<VariantIdx, IndexVec<FieldIdx, GeneratorSavedLocal>>,
+    pub variant_fields: IndexVec<VariantIdx, IndexVec<FieldIdx, CoroutineSavedLocal>>,
 
     /// The source that led to each variant being created (usually, a yield or
     /// await).
@@ -166,10 +167,10 @@ pub struct GeneratorLayout<'tcx> {
     /// layout.
     #[type_foldable(identity)]
     #[type_visitable(ignore)]
-    pub storage_conflicts: BitMatrix<GeneratorSavedLocal, GeneratorSavedLocal>,
+    pub storage_conflicts: BitMatrix<CoroutineSavedLocal, CoroutineSavedLocal>,
 }
 
-impl Debug for GeneratorLayout<'_> {
+impl Debug for CoroutineLayout<'_> {
     fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
         /// Prints an iterator of (key, value) tuples as a map.
         struct MapPrinter<'a, K, V>(Cell<Option<Box<dyn Iterator<Item = (K, V)> + 'a>>>);
@@ -184,7 +185,7 @@ impl Debug for GeneratorLayout<'_> {
             }
         }
 
-        /// Prints the generator variant name.
+        /// Prints the coroutine variant name.
         struct GenVariantPrinter(VariantIdx);
         impl From<VariantIdx> for GenVariantPrinter {
             fn from(idx: VariantIdx) -> Self {
@@ -193,7 +194,7 @@ impl Debug for GeneratorLayout<'_> {
         }
         impl Debug for GenVariantPrinter {
             fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
-                let variant_name = ty::GeneratorArgs::variant_name(self.0);
+                let variant_name = ty::CoroutineArgs::variant_name(self.0);
                 if fmt.alternate() {
                     write!(fmt, "{:9}({:?})", variant_name, self.0)
                 } else {
@@ -210,7 +211,7 @@ impl Debug for GeneratorLayout<'_> {
             }
         }
 
-        fmt.debug_struct("GeneratorLayout")
+        fmt.debug_struct("CoroutineLayout")
             .field("field_tys", &MapPrinter::new(self.field_tys.iter_enumerated()))
             .field(
                 "variant_fields",
@@ -258,7 +259,7 @@ pub struct ConstQualifs {
 ///
 /// The requirements are listed as being between various `RegionVid`. The 0th
 /// region refers to `'static`; subsequent region vids refer to the free
-/// regions that appear in the closure (or generator's) type, in order of
+/// regions that appear in the closure (or coroutine's) type, in order of
 /// appearance. (This numbering is actually defined by the `UniversalRegions`
 /// struct in the NLL region checker. See for example
 /// `UniversalRegions::closure_mapping`.) Note the free regions in the
@@ -445,14 +446,19 @@ pub struct DestructuredConstant<'tcx> {
     pub fields: &'tcx [(ConstValue<'tcx>, Ty<'tcx>)],
 }
 
-/// Coverage information summarized from a MIR if instrumented for source code coverage (see
-/// compiler option `-Cinstrument-coverage`). This information is generated by the
-/// `InstrumentCoverage` MIR pass and can be retrieved via the `coverageinfo` query.
+/// Summarizes coverage IDs inserted by the `InstrumentCoverage` MIR pass
+/// (for compiler option `-Cinstrument-coverage`), after MIR optimizations
+/// have had a chance to potentially remove some of them.
+///
+/// Used by the `coverage_ids_info` query.
 #[derive(Clone, TyEncodable, TyDecodable, Debug, HashStable)]
-pub struct CoverageInfo {
-    /// The total number of coverage region counters added to the MIR `Body`.
-    pub num_counters: u32,
-
-    /// The total number of coverage region counter expressions added to the MIR `Body`.
-    pub num_expressions: u32,
+pub struct CoverageIdsInfo {
+    /// Coverage codegen needs to know the highest counter ID that is ever
+    /// incremented within a function, so that it can set the `num-counters`
+    /// argument of the `llvm.instrprof.increment` intrinsic.
+    ///
+    /// This may be less than the highest counter ID emitted by the
+    /// InstrumentCoverage MIR pass, if the highest-numbered counter increments
+    /// were removed by MIR optimizations.
+    pub max_counter_id: mir::coverage::CounterId,
 }
diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs
index 0b95fdfa1..7b0f27f9b 100644
--- a/compiler/rustc_middle/src/mir/syntax.rs
+++ b/compiler/rustc_middle/src/mir/syntax.rs
@@ -5,7 +5,7 @@
 
 use super::{BasicBlock, Const, Local, UserTypeProjection};
 
-use crate::mir::coverage::{CodeRegion, CoverageKind};
+use crate::mir::coverage::CoverageKind;
 use crate::traits::Reveal;
 use crate::ty::adjustment::PointerCoercion;
 use crate::ty::GenericArgsRef;
@@ -15,7 +15,7 @@ use crate::ty::{Region, UserTypeAnnotationIndex};
 use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
 use rustc_hir::def_id::DefId;
 use rustc_hir::{self as hir};
-use rustc_hir::{self, GeneratorKind};
+use rustc_hir::{self, CoroutineKind};
 use rustc_index::IndexVec;
 use rustc_target::abi::{FieldIdx, VariantIdx};
 
@@ -82,10 +82,10 @@ pub enum MirPhase {
     ///    that Rust itself has them. Where exactly these are is generally subject to change, and so we
     ///    don't document this here. Runtime MIR has most retags explicit (though implicit retags
     ///    can still occur at `Rvalue::{Ref,AddrOf}`).
-    ///  - Generator bodies: In analysis MIR, locals may actually be behind a pointer that user code has
-    ///    access to. This occurs in generator bodies. Such locals do not behave like other locals,
+    ///  - Coroutine bodies: In analysis MIR, locals may actually be behind a pointer that user code has
+    ///    access to. This occurs in coroutine bodies. Such locals do not behave like other locals,
     ///    because they eg may be aliased in surprising ways. Runtime MIR has no such special locals -
-    ///    all generator bodies are lowered and so all places that look like locals really are locals.
+    ///    all coroutine bodies are lowered and so all places that look like locals really are locals.
     ///
     /// Also note that the lint pass which reports eg `200_u8 + 200_u8` as an error is run as a part
     /// of analysis to runtime MIR lowering. To ensure lints are reported reliably, this means that
@@ -137,7 +137,7 @@ pub enum RuntimePhase {
     /// In addition to the semantic changes, beginning with this phase, the following variants are
     /// disallowed:
     /// * [`TerminatorKind::Yield`]
-    /// * [`TerminatorKind::GeneratorDrop`]
+    /// * [`TerminatorKind::CoroutineDrop`]
     /// * [`Rvalue::Aggregate`] for any `AggregateKind` except `Array`
     /// * [`PlaceElem::OpaqueCast`]
     ///
@@ -292,7 +292,7 @@ pub enum StatementKind<'tcx> {
 
     /// Write the discriminant for a variant to the enum Place.
     ///
-    /// This is permitted for both generators and ADTs. This does not necessarily write to the
+    /// This is permitted for both coroutines and ADTs. This does not necessarily write to the
     /// entire place; instead, it writes to the minimum set of bytes as required by the layout for
     /// the type.
     SetDiscriminant { place: Box<Place<'tcx>>, variant_index: VariantIdx },
@@ -361,11 +361,16 @@ pub enum StatementKind<'tcx> {
     /// Disallowed after drop elaboration.
     AscribeUserType(Box<(Place<'tcx>, UserTypeProjection)>, ty::Variance),
 
-    /// Marks the start of a "coverage region", injected with '-Cinstrument-coverage'. A
-    /// `Coverage` statement carries metadata about the coverage region, used to inject a coverage
-    /// map into the binary. If `Coverage::kind` is a `Counter`, the statement also generates
-    /// executable code, to increment a counter variable at runtime, each time the code region is
-    /// executed.
+    /// Carries control-flow-sensitive information injected by `-Cinstrument-coverage`,
+    /// such as where to generate physical coverage-counter-increments during codegen.
+    ///
+    /// Coverage statements are used in conjunction with the coverage mappings and other
+    /// information stored in the function's
+    /// [`mir::Body::function_coverage_info`](crate::mir::Body::function_coverage_info).
+    /// (For inlined MIR, take care to look up the *original function's* coverage info.)
+    ///
+    /// Interpreters and codegen backends that don't support coverage instrumentation
+    /// can usually treat this as a no-op.
     Coverage(Box<Coverage>),
 
     /// Denotes a call to an intrinsic that does not require an unwind path and always returns.
@@ -514,7 +519,6 @@ pub enum FakeReadCause {
 #[derive(TypeFoldable, TypeVisitable)]
 pub struct Coverage {
     pub kind: CoverageKind,
-    pub code_region: Option<CodeRegion>,
 }
 
 #[derive(Clone, Debug, PartialEq, TyEncodable, TyDecodable, Hash, HashStable)]
@@ -622,8 +626,8 @@ pub enum TerminatorKind<'tcx> {
     /// `dest = move _0`. It might additionally do other things, like have side-effects in the
     /// aliasing model.
     ///
-    /// If the body is a generator body, this has slightly different semantics; it instead causes a
-    /// `GeneratorState::Returned(_0)` to be created (as if by an `Aggregate` rvalue) and assigned
+    /// If the body is a coroutine body, this has slightly different semantics; it instead causes a
+    /// `CoroutineState::Returned(_0)` to be created (as if by an `Aggregate` rvalue) and assigned
     /// to the return place.
     Return,
 
@@ -705,14 +709,14 @@ pub enum TerminatorKind<'tcx> {
 
     /// Marks a suspend point.
     ///
-    /// Like `Return` terminators in generator bodies, this computes `value` and then a
-    /// `GeneratorState::Yielded(value)` as if by `Aggregate` rvalue. That value is then assigned to
+    /// Like `Return` terminators in coroutine bodies, this computes `value` and then a
+    /// `CoroutineState::Yielded(value)` as if by `Aggregate` rvalue. That value is then assigned to
     /// the return place of the function calling this one, and execution continues in the calling
     /// function. When next invoked with the same first argument, execution of this function
     /// continues at the `resume` basic block, with the second argument written to the `resume_arg`
-    /// place. If the generator is dropped before then, the `drop` basic block is invoked.
+    /// place. If the coroutine is dropped before then, the `drop` basic block is invoked.
     ///
-    /// Not permitted in bodies that are not generator bodies, or after generator lowering.
+    /// Not permitted in bodies that are not coroutine bodies, or after coroutine lowering.
     ///
     /// **Needs clarification**: What about the evaluation order of the `resume_arg` and `value`?
     Yield {
@@ -722,21 +726,21 @@ pub enum TerminatorKind<'tcx> {
         resume: BasicBlock,
         /// The place to store the resume argument in.
         resume_arg: Place<'tcx>,
-        /// Cleanup to be done if the generator is dropped at this suspend point.
+        /// Cleanup to be done if the coroutine is dropped at this suspend point.
         drop: Option<BasicBlock>,
     },
 
-    /// Indicates the end of dropping a generator.
+    /// Indicates the end of dropping a coroutine.
     ///
-    /// Semantically just a `return` (from the generators drop glue). Only permitted in the same situations
+    /// Semantically just a `return` (from the coroutines drop glue). Only permitted in the same situations
     /// as `yield`.
     ///
-    /// **Needs clarification**: Is that even correct? The generator drop code is always confusing
+    /// **Needs clarification**: Is that even correct? The coroutine drop code is always confusing
     /// to me, because it's not even really in the current body.
     ///
     /// **Needs clarification**: Are there type system constraints on these terminators? Should
     /// there be a "block type" like `cleanup` blocks for them?
-    GeneratorDrop,
+    CoroutineDrop,
 
     /// A block where control flow only ever takes one real path, but borrowck needs to be more
     /// conservative.
@@ -811,7 +815,7 @@ impl TerminatorKind<'_> {
             TerminatorKind::Call { .. } => "Call",
             TerminatorKind::Assert { .. } => "Assert",
             TerminatorKind::Yield { .. } => "Yield",
-            TerminatorKind::GeneratorDrop => "GeneratorDrop",
+            TerminatorKind::CoroutineDrop => "CoroutineDrop",
             TerminatorKind::FalseEdge { .. } => "FalseEdge",
             TerminatorKind::FalseUnwind { .. } => "FalseUnwind",
             TerminatorKind::InlineAsm { .. } => "InlineAsm",
@@ -879,8 +883,8 @@ pub enum AssertKind<O> {
     OverflowNeg(O),
     DivisionByZero(O),
     RemainderByZero(O),
-    ResumedAfterReturn(GeneratorKind),
-    ResumedAfterPanic(GeneratorKind),
+    ResumedAfterReturn(CoroutineKind),
+    ResumedAfterPanic(CoroutineKind),
     MisalignedPointerDereference { required: O, found: O },
 }
 
@@ -957,8 +961,8 @@ pub type AssertMessage<'tcx> = AssertKind<Operand<'tcx>>;
 ///    was unsized and so had metadata associated with it, then the metadata is retained if the
 ///    field is unsized and thrown out if it is sized.
 ///
-///    These projections are only legal for tuples, ADTs, closures, and generators. If the ADT or
-///    generator has more than one variant, the parent place's variant index must be set, indicating
+///    These projections are only legal for tuples, ADTs, closures, and coroutines. If the ADT or
+///    coroutine has more than one variant, the parent place's variant index must be set, indicating
 ///    which variant is being used. If it has just one variant, the variant index may or may not be
 ///    included - the single possible variant is inferred if it is not included.
 ///  - [`OpaqueCast`](ProjectionElem::OpaqueCast): This projection changes the place's type to the
@@ -986,18 +990,15 @@ pub type AssertMessage<'tcx> = AssertKind<Operand<'tcx>>;
 ///    pointee's type. The resulting address is the address that was stored in the pointer. If the
 ///    pointee type is unsized, the pointer additionally stored the value of the metadata.
 ///
-/// Computing a place may cause UB. One possibility is that the pointer used for a `Deref` may not
-/// be suitably aligned. Another possibility is that the place is not in bounds, meaning it does not
-/// point to an actual allocation.
-///
-/// However, if this is actually UB and when the UB kicks in is undecided. This is being discussed
-/// in [UCG#319]. The options include that every place must obey those rules, that only some places
-/// must obey them, or that places impose no rules of their own.
-///
-/// [UCG#319]: https://github.com/rust-lang/unsafe-code-guidelines/issues/319
-///
-/// Rust currently requires that every place obey those two rules. This is checked by MIRI and taken
-/// advantage of by codegen (via `gep inbounds`). That is possibly subject to change.
+/// The "validity invariant" of places is the same as that of raw pointers, meaning that e.g.
+/// `*ptr` on a dangling or unaligned pointer is never UB. (Later doing a load/store on that place
+/// or turning it into a reference can be UB though!) The only ways for a place computation can
+/// cause UB are:
+/// - On a `Deref` projection, we do an actual load of the inner place, with all the usual
+///   consequences (the inner place must be based on an aligned pointer, it must point to allocated
+///   memory, the aliasig model must allow reads, this must not be a data race).
+/// - For the projections that perform pointer arithmetic, the offset must in-bounds of an
+///   allocation (i.e., the preconditions of `ptr::offset` must be met).
 #[derive(Copy, Clone, PartialEq, Eq, Hash, TyEncodable, HashStable, TypeFoldable, TypeVisitable)]
 pub struct Place<'tcx> {
     pub local: Local,
@@ -1067,7 +1068,7 @@ pub enum ProjectionElem<V, T> {
         from_end: bool,
     },
 
-    /// "Downcast" to a variant of an enum or a generator.
+    /// "Downcast" to a variant of an enum or a coroutine.
     ///
     /// The included Symbol is the name of the variant, used for printing MIR.
     Downcast(Option<Symbol>, VariantIdx),
@@ -1277,8 +1278,8 @@ pub enum Rvalue<'tcx> {
     /// `dest = Foo { x: ..., y: ... }` from `dest.x = ...; dest.y = ...;` in the case that `Foo`
     /// has a destructor.
     ///
-    /// Disallowed after deaggregation for all aggregate kinds except `Array` and `Generator`. After
-    /// generator lowering, `Generator` aggregate kinds are disallowed too.
+    /// Disallowed after deaggregation for all aggregate kinds except `Array` and `Coroutine`. After
+    /// coroutine lowering, `Coroutine` aggregate kinds are disallowed too.
     Aggregate(Box<AggregateKind<'tcx>>, IndexVec<FieldIdx, Operand<'tcx>>),
 
     /// Transmutes a `*mut u8` into shallow-initialized `Box<T>`.
@@ -1343,7 +1344,7 @@ pub enum AggregateKind<'tcx> {
     Adt(DefId, VariantIdx, GenericArgsRef<'tcx>, Option<UserTypeAnnotationIndex>, Option<FieldIdx>),
 
     Closure(DefId, GenericArgsRef<'tcx>),
-    Generator(DefId, GenericArgsRef<'tcx>, hir::Movability),
+    Coroutine(DefId, GenericArgsRef<'tcx>, hir::Movability),
 }
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq, TyEncodable, TyDecodable, Hash, HashStable)]
@@ -1353,7 +1354,7 @@ pub enum NullOp<'tcx> {
     /// Returns the minimum alignment of a type
     AlignOf,
     /// Returns the offset of a field
-    OffsetOf(&'tcx List<FieldIdx>),
+    OffsetOf(&'tcx List<(VariantIdx, FieldIdx)>),
 }
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
diff --git a/compiler/rustc_middle/src/mir/tcx.rs b/compiler/rustc_middle/src/mir/tcx.rs
index 7df25fc5c..6ab2da23a 100644
--- a/compiler/rustc_middle/src/mir/tcx.rs
+++ b/compiler/rustc_middle/src/mir/tcx.rs
@@ -11,7 +11,7 @@ use rustc_target::abi::{FieldIdx, VariantIdx};
 #[derive(Copy, Clone, Debug, TypeFoldable, TypeVisitable)]
 pub struct PlaceTy<'tcx> {
     pub ty: Ty<'tcx>,
-    /// Downcast to a particular variant of an enum or a generator, if included.
+    /// Downcast to a particular variant of an enum or a coroutine, if included.
     pub variant_index: Option<VariantIdx>,
 }
 
@@ -205,8 +205,8 @@ impl<'tcx> Rvalue<'tcx> {
                 }
                 AggregateKind::Adt(did, _, args, _, _) => tcx.type_of(did).instantiate(tcx, args),
                 AggregateKind::Closure(did, args) => Ty::new_closure(tcx, did, args),
-                AggregateKind::Generator(did, args, movability) => {
-                    Ty::new_generator(tcx, did, args, movability)
+                AggregateKind::Coroutine(did, args, movability) => {
+                    Ty::new_coroutine(tcx, did, args, movability)
                 }
             },
             Rvalue::ShallowInitBox(_, ty) => Ty::new_box(tcx, ty),
diff --git a/compiler/rustc_middle/src/mir/terminator.rs b/compiler/rustc_middle/src/mir/terminator.rs
index 02aab4a89..9dfbe1733 100644
--- a/compiler/rustc_middle/src/mir/terminator.rs
+++ b/compiler/rustc_middle/src/mir/terminator.rs
@@ -3,12 +3,10 @@ use rustc_hir::LangItem;
 use smallvec::SmallVec;
 
 use super::{BasicBlock, InlineAsmOperand, Operand, SourceInfo, TerminatorKind, UnwindAction};
-pub use rustc_ast::Mutability;
 use rustc_macros::HashStable;
 use std::iter;
 use std::slice;
 
-pub use super::query::*;
 use super::*;
 
 impl SwitchTargets {
@@ -28,6 +26,15 @@ impl SwitchTargets {
         Self { values: smallvec![value], targets: smallvec![then, else_] }
     }
 
+    /// Inverse of `SwitchTargets::static_if`.
+    pub fn as_static_if(&self) -> Option<(u128, BasicBlock, BasicBlock)> {
+        if let &[value] = &self.values[..] && let &[then, else_] = &self.targets[..] {
+            Some((value, then, else_))
+        } else {
+            None
+        }
+    }
+
     /// Returns the fallback target that is jumped to when none of the values match the operand.
     pub fn otherwise(&self) -> BasicBlock {
         *self.targets.last().unwrap()
@@ -139,10 +146,17 @@ impl<O> AssertKind<O> {
             Overflow(op, _, _) => bug!("{:?} cannot overflow", op),
             DivisionByZero(_) => "attempt to divide by zero",
             RemainderByZero(_) => "attempt to calculate the remainder with a divisor of zero",
-            ResumedAfterReturn(GeneratorKind::Gen) => "generator resumed after completion",
-            ResumedAfterReturn(GeneratorKind::Async(_)) => "`async fn` resumed after completion",
-            ResumedAfterPanic(GeneratorKind::Gen) => "generator resumed after panicking",
-            ResumedAfterPanic(GeneratorKind::Async(_)) => "`async fn` resumed after panicking",
+            ResumedAfterReturn(CoroutineKind::Coroutine) => "coroutine resumed after completion",
+            ResumedAfterReturn(CoroutineKind::Async(_)) => "`async fn` resumed after completion",
+            ResumedAfterReturn(CoroutineKind::Gen(_)) => {
+                "`gen fn` should just keep returning `None` after completion"
+            }
+            ResumedAfterPanic(CoroutineKind::Coroutine) => "coroutine resumed after panicking",
+            ResumedAfterPanic(CoroutineKind::Async(_)) => "`async fn` resumed after panicking",
+            ResumedAfterPanic(CoroutineKind::Gen(_)) => {
+                "`gen fn` should just keep returning `None` after panicking"
+            }
+
             BoundsCheck { .. } | MisalignedPointerDereference { .. } => {
                 bug!("Unexpected AssertKind")
             }
@@ -228,10 +242,18 @@ impl<O> AssertKind<O> {
             OverflowNeg(_) => middle_assert_overflow_neg,
             DivisionByZero(_) => middle_assert_divide_by_zero,
             RemainderByZero(_) => middle_assert_remainder_by_zero,
-            ResumedAfterReturn(GeneratorKind::Async(_)) => middle_assert_async_resume_after_return,
-            ResumedAfterReturn(GeneratorKind::Gen) => middle_assert_generator_resume_after_return,
-            ResumedAfterPanic(GeneratorKind::Async(_)) => middle_assert_async_resume_after_panic,
-            ResumedAfterPanic(GeneratorKind::Gen) => middle_assert_generator_resume_after_panic,
+            ResumedAfterReturn(CoroutineKind::Async(_)) => middle_assert_async_resume_after_return,
+            ResumedAfterReturn(CoroutineKind::Gen(_)) => {
+                bug!("gen blocks can be resumed after they return and will keep returning `None`")
+            }
+            ResumedAfterReturn(CoroutineKind::Coroutine) => {
+                middle_assert_coroutine_resume_after_return
+            }
+            ResumedAfterPanic(CoroutineKind::Async(_)) => middle_assert_async_resume_after_panic,
+            ResumedAfterPanic(CoroutineKind::Gen(_)) => middle_assert_gen_resume_after_panic,
+            ResumedAfterPanic(CoroutineKind::Coroutine) => {
+                middle_assert_coroutine_resume_after_panic
+            }
 
             MisalignedPointerDereference { .. } => middle_assert_misaligned_ptr_deref,
         }
@@ -331,7 +353,7 @@ impl<'tcx> TerminatorKind<'tcx> {
             }
             UnwindResume
             | UnwindTerminate(_)
-            | GeneratorDrop
+            | CoroutineDrop
             | Return
             | Unreachable
             | Call { target: None, unwind: _, .. }
@@ -373,7 +395,7 @@ impl<'tcx> TerminatorKind<'tcx> {
             }
             UnwindResume
             | UnwindTerminate(_)
-            | GeneratorDrop
+            | CoroutineDrop
             | Return
             | Unreachable
             | Call { target: None, unwind: _, .. }
@@ -392,7 +414,7 @@ impl<'tcx> TerminatorKind<'tcx> {
             | TerminatorKind::UnwindTerminate(_)
             | TerminatorKind::Return
             | TerminatorKind::Unreachable
-            | TerminatorKind::GeneratorDrop
+            | TerminatorKind::CoroutineDrop
             | TerminatorKind::Yield { .. }
             | TerminatorKind::SwitchInt { .. }
             | TerminatorKind::FalseEdge { .. } => None,
@@ -411,7 +433,7 @@ impl<'tcx> TerminatorKind<'tcx> {
             | TerminatorKind::UnwindTerminate(_)
             | TerminatorKind::Return
             | TerminatorKind::Unreachable
-            | TerminatorKind::GeneratorDrop
+            | TerminatorKind::CoroutineDrop
             | TerminatorKind::Yield { .. }
             | TerminatorKind::SwitchInt { .. }
             | TerminatorKind::FalseEdge { .. } => None,
@@ -493,7 +515,7 @@ impl<'tcx> TerminatorKind<'tcx> {
     pub fn edges(&self) -> TerminatorEdges<'_, 'tcx> {
         use TerminatorKind::*;
         match *self {
-            Return | UnwindResume | UnwindTerminate(_) | GeneratorDrop | Unreachable => {
+            Return | UnwindResume | UnwindTerminate(_) | CoroutineDrop | Unreachable => {
                 TerminatorEdges::None
             }
 
diff --git a/compiler/rustc_middle/src/mir/type_foldable.rs b/compiler/rustc_middle/src/mir/type_foldable.rs
index 8d427fdb6..d5c81b6cd 100644
--- a/compiler/rustc_middle/src/mir/type_foldable.rs
+++ b/compiler/rustc_middle/src/mir/type_foldable.rs
@@ -19,8 +19,8 @@ TrivialTypeTraversalImpls! {
     hir::Movability,
     BasicBlock,
     SwitchTargets,
-    GeneratorKind,
-    GeneratorSavedLocal,
+    CoroutineKind,
+    CoroutineSavedLocal,
 }
 
 TrivialTypeTraversalImpls! {
diff --git a/compiler/rustc_middle/src/mir/visit.rs b/compiler/rustc_middle/src/mir/visit.rs
index f2745b32c..d47cfd571 100644
--- a/compiler/rustc_middle/src/mir/visit.rs
+++ b/compiler/rustc_middle/src/mir/visit.rs
@@ -184,6 +184,8 @@ macro_rules! make_mir_visitor {
 
             visit_place_fns!($($mutability)?);
 
+            /// This is called for every constant in the MIR body and every `required_consts`
+            /// (i.e., including consts that have been dead-code-eliminated).
             fn visit_constant(
                 &mut self,
                 constant: & $($mutability)? ConstOperand<'tcx>,
@@ -471,7 +473,7 @@ macro_rules! make_mir_visitor {
                     TerminatorKind::Goto { .. } |
                     TerminatorKind::UnwindResume |
                     TerminatorKind::UnwindTerminate(_) |
-                    TerminatorKind::GeneratorDrop |
+                    TerminatorKind::CoroutineDrop |
                     TerminatorKind::Unreachable |
                     TerminatorKind::FalseEdge { .. } |
                     TerminatorKind::FalseUnwind { .. } => {}
@@ -733,12 +735,12 @@ macro_rules! make_mir_visitor {
                             ) => {
                                 self.visit_args(closure_args, location);
                             }
-                            AggregateKind::Generator(
+                            AggregateKind::Coroutine(
                                 _,
-                                generator_args,
+                                coroutine_args,
                                 _movability,
                             ) => {
-                                self.visit_args(generator_args, location);
+                                self.visit_args(coroutine_args, location);
                             }
                         }
 
@@ -815,7 +817,6 @@ macro_rules! make_mir_visitor {
                     ty,
                     user_ty,
                     source_info,
-                    internal: _,
                     local_info: _,
                 } = local_decl;
 
@@ -991,7 +992,7 @@ macro_rules! extra_body_methods {
 macro_rules! super_body {
     ($self:ident, $body:ident, $($mutability:ident, $invalidate:tt)?) => {
         let span = $body.span;
-        if let Some(gen) = &$($mutability)? $body.generator {
+        if let Some(gen) = &$($mutability)? $body.coroutine {
             if let Some(yield_ty) = $(& $mutability)? gen.yield_ty {
                 $self.visit_ty(
                     yield_ty,
diff --git a/compiler/rustc_middle/src/query/erase.rs b/compiler/rustc_middle/src/query/erase.rs
index 8ba3764bc..cdde6a596 100644
--- a/compiler/rustc_middle/src/query/erase.rs
+++ b/compiler/rustc_middle/src/query/erase.rs
@@ -2,7 +2,8 @@ use crate::mir;
 use crate::query::CyclePlaceholder;
 use crate::traits;
 use crate::ty::{self, Ty};
-use std::mem::{size_of, transmute_copy, MaybeUninit};
+use std::intrinsics::transmute_unchecked;
+use std::mem::{size_of, MaybeUninit};
 
 #[derive(Copy, Clone)]
 pub struct Erased<T: Copy> {
@@ -29,8 +30,15 @@ pub fn erase<T: EraseType>(src: T) -> Erase<T> {
     };
 
     Erased::<<T as EraseType>::Result> {
+        // `transmute_unchecked` is needed here because it does not have `transmute`'s size check
+        // (and thus allows to transmute between `T` and `MaybeUninit<T::Result>`) (we do the size
+        // check ourselves in the `const` block above).
+        //
+        // `transmute_copy` is also commonly used for this (and it would work here since
+        // `EraseType: Copy`), but `transmute_unchecked` better explains the intent.
+        //
         // SAFETY: It is safe to transmute to MaybeUninit for types with the same sizes.
-        data: unsafe { transmute_copy(&src) },
+        data: unsafe { transmute_unchecked::<T, MaybeUninit<T::Result>>(src) },
     }
 }
 
@@ -38,22 +46,24 @@ pub fn erase<T: EraseType>(src: T) -> Erase<T> {
 #[inline(always)]
 pub fn restore<T: EraseType>(value: Erase<T>) -> T {
     let value: Erased<<T as EraseType>::Result> = value;
+    // See comment in `erase` for why we use `transmute_unchecked`.
+    //
     // SAFETY: Due to the use of impl Trait in `Erase` the only way to safely create an instance
     // of `Erase` is to call `erase`, so we know that `value.data` is a valid instance of `T` of
     // the right size.
-    unsafe { transmute_copy(&value.data) }
+    unsafe { transmute_unchecked::<MaybeUninit<T::Result>, T>(value.data) }
 }
 
 impl<T> EraseType for &'_ T {
-    type Result = [u8; size_of::<*const ()>()];
+    type Result = [u8; size_of::<&'static ()>()];
 }
 
 impl<T> EraseType for &'_ [T] {
-    type Result = [u8; size_of::<*const [()]>()];
+    type Result = [u8; size_of::<&'static [()]>()];
 }
 
 impl<T> EraseType for &'_ ty::List<T> {
-    type Result = [u8; size_of::<*const ()>()];
+    type Result = [u8; size_of::<&'static ty::List<()>>()];
 }
 
 impl<I: rustc_index::Idx, T> EraseType for &'_ rustc_index::IndexSlice<I, T> {
@@ -210,7 +220,7 @@ trivial! {
     Option<rustc_attr::Stability>,
     Option<rustc_data_structures::svh::Svh>,
     Option<rustc_hir::def::DefKind>,
-    Option<rustc_hir::GeneratorKind>,
+    Option<rustc_hir::CoroutineKind>,
     Option<rustc_hir::HirId>,
     Option<rustc_middle::middle::stability::DeprecationEntry>,
     Option<rustc_middle::ty::Destructor>,
@@ -239,7 +249,7 @@ trivial! {
     rustc_hir::def::DefKind,
     rustc_hir::Defaultness,
     rustc_hir::definitions::DefKey,
-    rustc_hir::GeneratorKind,
+    rustc_hir::CoroutineKind,
     rustc_hir::HirId,
     rustc_hir::IsAsync,
     rustc_hir::ItemLocalId,
diff --git a/compiler/rustc_middle/src/query/keys.rs b/compiler/rustc_middle/src/query/keys.rs
index b1f837968..113763450 100644
--- a/compiler/rustc_middle/src/query/keys.rs
+++ b/compiler/rustc_middle/src/query/keys.rs
@@ -12,7 +12,6 @@ use rustc_hir::hir_id::{HirId, OwnerId};
 use rustc_query_system::query::{DefaultCacheSelector, SingleCacheSelector, VecCacheSelector};
 use rustc_span::symbol::{Ident, Symbol};
 use rustc_span::{Span, DUMMY_SP};
-use rustc_target::abi::FieldIdx;
 
 /// Placeholder for `CrateNum`'s "local" counterpart
 #[derive(Copy, Clone, Debug)]
@@ -360,30 +359,6 @@ impl<'tcx> Key for (ty::ParamEnv<'tcx>, ty::TraitRef<'tcx>) {
     }
 }
 
-impl<'tcx> Key for (ty::Const<'tcx>, FieldIdx) {
-    type CacheSelector = DefaultCacheSelector<Self>;
-
-    fn default_span(&self, _: TyCtxt<'_>) -> Span {
-        DUMMY_SP
-    }
-}
-
-impl<'tcx> Key for (mir::ConstValue<'tcx>, Ty<'tcx>) {
-    type CacheSelector = DefaultCacheSelector<Self>;
-
-    fn default_span(&self, _: TyCtxt<'_>) -> Span {
-        DUMMY_SP
-    }
-}
-
-impl<'tcx> Key for mir::ConstAlloc<'tcx> {
-    type CacheSelector = DefaultCacheSelector<Self>;
-
-    fn default_span(&self, _: TyCtxt<'_>) -> Span {
-        DUMMY_SP
-    }
-}
-
 impl<'tcx> Key for ty::PolyTraitRef<'tcx> {
     type CacheSelector = DefaultCacheSelector<Self>;
 
@@ -416,14 +391,6 @@ impl<'tcx> Key for GenericArg<'tcx> {
     }
 }
 
-impl<'tcx> Key for mir::Const<'tcx> {
-    type CacheSelector = DefaultCacheSelector<Self>;
-
-    fn default_span(&self, _: TyCtxt<'_>) -> Span {
-        DUMMY_SP
-    }
-}
-
 impl<'tcx> Key for ty::Const<'tcx> {
     type CacheSelector = DefaultCacheSelector<Self>;
 
diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs
index 340c5a769..f9ec36836 100644
--- a/compiler/rustc_middle/src/query/mod.rs
+++ b/compiler/rustc_middle/src/query/mod.rs
@@ -25,7 +25,9 @@ use crate::mir::interpret::{
 use crate::mir::interpret::{LitToConstError, LitToConstInput};
 use crate::mir::mono::CodegenUnit;
 use crate::query::erase::{erase, restore, Erase};
-use crate::query::plumbing::{query_ensure, query_get_at, CyclePlaceholder, DynamicQuery};
+use crate::query::plumbing::{
+    query_ensure, query_ensure_error_guaranteed, query_get_at, CyclePlaceholder, DynamicQuery,
+};
 use crate::thir;
 use crate::traits::query::{
     CanonicalPredicateGoal, CanonicalProjectionGoal, CanonicalTyGoal,
@@ -249,6 +251,7 @@ rustc_queries! {
             "computing type of opaque `{path}`",
             path = tcx.def_path_str(key),
         }
+        cycle_stash
     }
 
     query type_alias_is_lazy(key: DefId) -> bool {
@@ -339,7 +342,7 @@ rustc_queries! {
 
     query opaque_types_defined_by(
         key: LocalDefId
-    ) -> &'tcx [LocalDefId] {
+    ) -> &'tcx ty::List<LocalDefId> {
         desc {
             |tcx| "computing the opaque types defined by `{}`",
             tcx.def_path_str(key.to_def_id())
@@ -541,28 +544,28 @@ rustc_queries! {
         }
     }
 
-    /// Returns names of captured upvars for closures and generators.
+    /// Returns names of captured upvars for closures and coroutines.
     ///
     /// Here are some examples:
     ///  - `name__field1__field2` when the upvar is captured by value.
     ///  - `_ref__name__field` when the upvar is captured by reference.
     ///
-    /// For generators this only contains upvars that are shared by all states.
+    /// For coroutines this only contains upvars that are shared by all states.
     query closure_saved_names_of_captured_variables(def_id: DefId) -> &'tcx IndexVec<abi::FieldIdx, Symbol> {
         arena_cache
         desc { |tcx| "computing debuginfo for closure `{}`", tcx.def_path_str(def_id) }
         separate_provide_extern
     }
 
-    query mir_generator_witnesses(key: DefId) -> &'tcx Option<mir::GeneratorLayout<'tcx>> {
+    query mir_coroutine_witnesses(key: DefId) -> &'tcx Option<mir::CoroutineLayout<'tcx>> {
         arena_cache
-        desc { |tcx| "generator witness types for `{}`", tcx.def_path_str(key) }
+        desc { |tcx| "coroutine witness types for `{}`", tcx.def_path_str(key) }
         cache_on_disk_if { key.is_local() }
         separate_provide_extern
     }
 
-    query check_generator_obligations(key: LocalDefId) {
-        desc { |tcx| "verify auto trait bounds for generator interior type `{}`", tcx.def_path_str(key) }
+    query check_coroutine_obligations(key: LocalDefId) {
+        desc { |tcx| "verify auto trait bounds for coroutine interior type `{}`", tcx.def_path_str(key) }
     }
 
     /// MIR after our optimization passes have run. This is MIR that is ready
@@ -573,22 +576,12 @@ rustc_queries! {
         separate_provide_extern
     }
 
-    /// Returns coverage summary info for a function, after executing the `InstrumentCoverage`
-    /// MIR pass (assuming the -Cinstrument-coverage option is enabled).
-    query coverageinfo(key: ty::InstanceDef<'tcx>) -> &'tcx mir::CoverageInfo {
-        desc { |tcx| "retrieving coverage info from MIR for `{}`", tcx.def_path_str(key.def_id()) }
-        arena_cache
-    }
-
-    /// Returns the `CodeRegions` for a function that has instrumented coverage, in case the
-    /// function was optimized out before codegen, and before being added to the Coverage Map.
-    query covered_code_regions(key: DefId) -> &'tcx Vec<&'tcx mir::coverage::CodeRegion> {
-        desc {
-            |tcx| "retrieving the covered `CodeRegion`s, if instrumented, for `{}`",
-            tcx.def_path_str(key)
-        }
+    /// Summarizes coverage IDs inserted by the `InstrumentCoverage` MIR pass
+    /// (for compiler option `-Cinstrument-coverage`), after MIR optimizations
+    /// have had a chance to potentially remove some of them.
+    query coverage_ids_info(key: ty::InstanceDef<'tcx>) -> &'tcx mir::CoverageIdsInfo {
+        desc { |tcx| "retrieving coverage IDs info from MIR for `{}`", tcx.def_path_str(key.def_id()) }
         arena_cache
-        cache_on_disk_if { key.is_local() }
     }
 
     /// The `DefId` is the `DefId` of the containing MIR body. Promoteds do not have their own
@@ -753,9 +746,9 @@ rustc_queries! {
         desc { |tcx| "checking if item is promotable: `{}`", tcx.def_path_str(key) }
     }
 
-    /// Returns `Some(generator_kind)` if the node pointed to by `def_id` is a generator.
-    query generator_kind(def_id: DefId) -> Option<hir::GeneratorKind> {
-        desc { |tcx| "looking up generator kind of `{}`", tcx.def_path_str(def_id) }
+    /// Returns `Some(coroutine_kind)` if the node pointed to by `def_id` is a coroutine.
+    query coroutine_kind(def_id: DefId) -> Option<hir::CoroutineKind> {
+        desc { |tcx| "looking up coroutine kind of `{}`", tcx.def_path_str(def_id) }
         separate_provide_extern
     }
 
@@ -975,8 +968,9 @@ rustc_queries! {
         desc { |tcx| "checking that impls are well-formed in {}", describe_as_module(key, tcx) }
     }
 
-    query check_mod_type_wf(key: LocalModDefId) -> () {
+    query check_mod_type_wf(key: LocalModDefId) -> Result<(), ErrorGuaranteed> {
         desc { |tcx| "checking that types are well-formed in {}", describe_as_module(key, tcx) }
+        ensure_forwards_result_if_red
     }
 
     query collect_mod_item_types(key: LocalModDefId) -> () {
@@ -1107,10 +1101,6 @@ rustc_queries! {
         desc { "destructuring type level constant"}
     }
 
-    query const_caller_location(key: (rustc_span::Symbol, u32, u32)) -> mir::ConstValue<'tcx> {
-        desc { "getting a &core::panic::Location referring to a span" }
-    }
-
     // FIXME get rid of this with valtrees
     query lit_to_const(
         key: LitToConstInput<'tcx>
@@ -1509,8 +1499,9 @@ rustc_queries! {
         feedable
     }
 
-    query check_well_formed(key: hir::OwnerId) -> () {
+    query check_well_formed(key: hir::OwnerId) -> Result<(), ErrorGuaranteed> {
         desc { |tcx| "checking that `{}` is well-formed", tcx.def_path_str(key) }
+        ensure_forwards_result_if_red
     }
 
     // The `DefId`s of all non-generic functions and statics in the given crate
@@ -1892,12 +1883,6 @@ rustc_queries! {
         desc { |tcx| "determining whether `{}` needs codegen", tcx.def_path_str(def_id) }
     }
 
-    /// All items participating in code generation together with items inlined into them.
-    query codegened_and_inlined_items(_: ()) -> &'tcx DefIdSet {
-        eval_always
-        desc { "collecting codegened and inlined items" }
-    }
-
     query codegen_unit(sym: Symbol) -> &'tcx CodegenUnit<'tcx> {
         desc { "getting codegen unit `{sym}`" }
     }
@@ -2202,6 +2187,11 @@ rustc_queries! {
     query generics_require_sized_self(def_id: DefId) -> bool {
         desc { "check whether the item has a `where Self: Sized` bound" }
     }
+
+    query cross_crate_inlinable(def_id: DefId) -> bool {
+        desc { "whether the item should be made inlinable across crates" }
+        separate_provide_extern
+    }
 }
 
 rustc_query_append! { define_callbacks! }
diff --git a/compiler/rustc_middle/src/query/plumbing.rs b/compiler/rustc_middle/src/query/plumbing.rs
index 34e5b02ba..f4a8ada8f 100644
--- a/compiler/rustc_middle/src/query/plumbing.rs
+++ b/compiler/rustc_middle/src/query/plumbing.rs
@@ -173,6 +173,45 @@ pub fn query_ensure<'tcx, Cache>(
     }
 }
 
+#[inline]
+pub fn query_ensure_error_guaranteed<'tcx, Cache>(
+    tcx: TyCtxt<'tcx>,
+    execute_query: fn(TyCtxt<'tcx>, Span, Cache::Key, QueryMode) -> Option<Cache::Value>,
+    query_cache: &Cache,
+    key: Cache::Key,
+    check_cache: bool,
+) -> Result<(), ErrorGuaranteed>
+where
+    Cache: QueryCache<Value = super::erase::Erase<Result<(), ErrorGuaranteed>>>,
+{
+    let key = key.into_query_param();
+    if let Some(res) = try_get_cached(tcx, query_cache, &key) {
+        super::erase::restore(res)
+    } else {
+        execute_query(tcx, DUMMY_SP, key, QueryMode::Ensure { check_cache })
+            .map(super::erase::restore)
+            // Either we actually executed the query, which means we got a full `Result`,
+            // or we can just assume the query succeeded, because it was green in the
+            // incremental cache. If it is green, that means that the previous compilation
+            // that wrote to the incremental cache compiles successfully. That is only
+            // possible if the cache entry was `Ok(())`, so we emit that here, without
+            // actually encoding the `Result` in the cache or loading it from there.
+            .unwrap_or(Ok(()))
+    }
+}
+
+macro_rules! query_ensure {
+    ([]$($args:tt)*) => {
+        query_ensure($($args)*)
+    };
+    ([(ensure_forwards_result_if_red) $($rest:tt)*]$($args:tt)*) => {
+        query_ensure_error_guaranteed($($args)*)
+    };
+    ([$other:tt $($modifiers:tt)*]$($args:tt)*) => {
+        query_ensure!([$($modifiers)*]$($args)*)
+    };
+}
+
 macro_rules! query_helper_param_ty {
     (DefId) => { impl IntoQueryParam<DefId> };
     (LocalDefId) => { impl IntoQueryParam<LocalDefId> };
@@ -220,6 +259,18 @@ macro_rules! separate_provide_extern_decl {
     };
 }
 
+macro_rules! ensure_result {
+    ([][$ty:ty]) => {
+        ()
+    };
+    ([(ensure_forwards_result_if_red) $($rest:tt)*][$ty:ty]) => {
+        Result<(), ErrorGuaranteed>
+    };
+    ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
+        ensure_result!([$($modifiers)*][$($args)*])
+    };
+}
+
 macro_rules! separate_provide_extern_default {
     ([][$name:ident]) => {
         ()
@@ -343,14 +394,15 @@ macro_rules! define_callbacks {
         impl<'tcx> TyCtxtEnsure<'tcx> {
             $($(#[$attr])*
             #[inline(always)]
-            pub fn $name(self, key: query_helper_param_ty!($($K)*)) {
-                query_ensure(
+            pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> ensure_result!([$($modifiers)*][$V]) {
+                query_ensure!(
+                    [$($modifiers)*]
                     self.tcx,
                     self.tcx.query_system.fns.engine.$name,
                     &self.tcx.query_system.caches.$name,
                     key.into_query_param(),
                     false,
-                );
+                )
             })*
         }
 
diff --git a/compiler/rustc_middle/src/thir.rs b/compiler/rustc_middle/src/thir.rs
index 89934e435..3086082fe 100644
--- a/compiler/rustc_middle/src/thir.rs
+++ b/compiler/rustc_middle/src/thir.rs
@@ -16,16 +16,19 @@ use rustc_hir::RangeEnd;
 use rustc_index::newtype_index;
 use rustc_index::IndexVec;
 use rustc_middle::middle::region;
-use rustc_middle::mir::interpret::AllocId;
+use rustc_middle::mir::interpret::{AllocId, Scalar};
 use rustc_middle::mir::{self, BinOp, BorrowKind, FakeReadCause, Mutability, UnOp};
 use rustc_middle::ty::adjustment::PointerCoercion;
-use rustc_middle::ty::GenericArgsRef;
-use rustc_middle::ty::{self, AdtDef, FnSig, List, Ty, UpvarArgs};
-use rustc_middle::ty::{CanonicalUserType, CanonicalUserTypeAnnotation};
+use rustc_middle::ty::layout::IntegerExt;
+use rustc_middle::ty::{
+    self, AdtDef, CanonicalUserType, CanonicalUserTypeAnnotation, FnSig, GenericArgsRef, List, Ty,
+    TyCtxt, UpvarArgs,
+};
 use rustc_span::def_id::LocalDefId;
-use rustc_span::{sym, Span, Symbol, DUMMY_SP};
-use rustc_target::abi::{FieldIdx, VariantIdx};
+use rustc_span::{sym, ErrorGuaranteed, Span, Symbol, DUMMY_SP};
+use rustc_target::abi::{FieldIdx, Integer, Size, VariantIdx};
 use rustc_target::asm::InlineAsmRegOrRegClass;
+use std::cmp::Ordering;
 use std::fmt;
 use std::ops::Index;
 
@@ -381,9 +384,9 @@ pub enum ExprKind<'tcx> {
     VarRef {
         id: LocalVarId,
     },
-    /// Used to represent upvars mentioned in a closure/generator
+    /// Used to represent upvars mentioned in a closure/coroutine
     UpvarRef {
-        /// DefId of the closure/generator
+        /// DefId of the closure/coroutine
         closure_def_id: DefId,
 
         /// HirId of the root variable
@@ -489,7 +492,7 @@ pub enum ExprKind<'tcx> {
     /// Field offset (`offset_of!`)
     OffsetOf {
         container: Ty<'tcx>,
-        fields: &'tcx List<FieldIdx>,
+        fields: &'tcx List<(VariantIdx, FieldIdx)>,
     },
     /// An expression taking a reference to a thread local.
     ThreadLocalRef(DefId),
@@ -581,13 +584,13 @@ pub enum BindingMode {
     ByRef(BorrowKind),
 }
 
-#[derive(Clone, Debug, HashStable)]
+#[derive(Clone, Debug, HashStable, TypeVisitable)]
 pub struct FieldPat<'tcx> {
     pub field: FieldIdx,
     pub pattern: Box<Pat<'tcx>>,
 }
 
-#[derive(Clone, Debug, HashStable)]
+#[derive(Clone, Debug, HashStable, TypeVisitable)]
 pub struct Pat<'tcx> {
     pub ty: Ty<'tcx>,
     pub span: Span,
@@ -632,10 +635,11 @@ impl<'tcx> Pat<'tcx> {
 
         use PatKind::*;
         match &self.kind {
-            Wild | Range(..) | Binding { subpattern: None, .. } | Constant { .. } => {}
+            Wild | Range(..) | Binding { subpattern: None, .. } | Constant { .. } | Error(_) => {}
             AscribeUserType { subpattern, .. }
             | Binding { subpattern: Some(subpattern), .. }
-            | Deref { subpattern } => subpattern.walk_(it),
+            | Deref { subpattern }
+            | InlineConstant { subpattern, .. } => subpattern.walk_(it),
             Leaf { subpatterns } | Variant { subpatterns, .. } => {
                 subpatterns.iter().for_each(|field| field.pattern.walk_(it))
             }
@@ -647,6 +651,21 @@ impl<'tcx> Pat<'tcx> {
         }
     }
 
+    /// Whether the pattern has a `PatKind::Error` nested within.
+    pub fn pat_error_reported(&self) -> Result<(), ErrorGuaranteed> {
+        let mut error = None;
+        self.walk(|pat| {
+            if let PatKind::Error(e) = pat.kind && error.is_none() {
+                error = Some(e);
+            }
+            error.is_none()
+        });
+        match error {
+            None => Ok(()),
+            Some(e) => Err(e),
+        }
+    }
+
     /// Walk the pattern in left-to-right order.
     ///
     /// If you always want to recurse, prefer this method over `walk`.
@@ -664,7 +683,7 @@ impl<'tcx> IntoDiagnosticArg for Pat<'tcx> {
     }
 }
 
-#[derive(Clone, Debug, HashStable)]
+#[derive(Clone, Debug, HashStable, TypeVisitable)]
 pub struct Ascription<'tcx> {
     pub annotation: CanonicalUserTypeAnnotation<'tcx>,
     /// Variance to use when relating the `user_ty` to the **type of the value being
@@ -688,7 +707,7 @@ pub struct Ascription<'tcx> {
     pub variance: ty::Variance,
 }
 
-#[derive(Clone, Debug, HashStable)]
+#[derive(Clone, Debug, HashStable, TypeVisitable)]
 pub enum PatKind<'tcx> {
     /// A wildcard pattern: `_`.
     Wild,
@@ -702,7 +721,9 @@ pub enum PatKind<'tcx> {
     Binding {
         mutability: Mutability,
         name: Symbol,
+        #[type_visitable(ignore)]
         mode: BindingMode,
+        #[type_visitable(ignore)]
         var: LocalVarId,
         ty: Ty<'tcx>,
         subpattern: Option<Box<Pat<'tcx>>>,
@@ -746,6 +767,22 @@ pub enum PatKind<'tcx> {
         value: mir::Const<'tcx>,
     },
 
+    /// Inline constant found while lowering a pattern.
+    InlineConstant {
+        /// [LocalDefId] of the constant, we need this so that we have a
+        /// reference that can be used by unsafety checking to visit nested
+        /// unevaluated constants.
+        def: LocalDefId,
+        /// If the inline constant is used in a range pattern, this subpattern
+        /// represents the range (if both ends are inline constants, there will
+        /// be multiple InlineConstant wrappers).
+        ///
+        /// Otherwise, the actual pattern that the constant lowered to. As with
+        /// other constants, inline constants are matched structurally where
+        /// possible.
+        subpattern: Box<Pat<'tcx>>,
+    },
+
     Range(Box<PatRange<'tcx>>),
 
     /// Matches against a slice, checking the length and extracting elements.
@@ -769,13 +806,249 @@ pub enum PatKind<'tcx> {
     Or {
         pats: Box<[Box<Pat<'tcx>>]>,
     },
+
+    /// An error has been encountered during lowering. We probably shouldn't report more lints
+    /// related to this pattern.
+    Error(ErrorGuaranteed),
 }
 
-#[derive(Clone, Debug, PartialEq, HashStable)]
+/// A range pattern.
+/// The boundaries must be of the same type and that type must be numeric.
+#[derive(Clone, Debug, PartialEq, HashStable, TypeVisitable)]
 pub struct PatRange<'tcx> {
-    pub lo: mir::Const<'tcx>,
-    pub hi: mir::Const<'tcx>,
+    pub lo: PatRangeBoundary<'tcx>,
+    pub hi: PatRangeBoundary<'tcx>,
+    #[type_visitable(ignore)]
     pub end: RangeEnd,
+    pub ty: Ty<'tcx>,
+}
+
+impl<'tcx> PatRange<'tcx> {
+    /// Whether this range covers the full extent of possible values (best-effort, we ignore floats).
+    #[inline]
+    pub fn is_full_range(&self, tcx: TyCtxt<'tcx>) -> Option<bool> {
+        let (min, max, size, bias) = match *self.ty.kind() {
+            ty::Char => (0, std::char::MAX as u128, Size::from_bits(32), 0),
+            ty::Int(ity) => {
+                let size = Integer::from_int_ty(&tcx, ity).size();
+                let max = size.truncate(u128::MAX);
+                let bias = 1u128 << (size.bits() - 1);
+                (0, max, size, bias)
+            }
+            ty::Uint(uty) => {
+                let size = Integer::from_uint_ty(&tcx, uty).size();
+                let max = size.unsigned_int_max();
+                (0, max, size, 0)
+            }
+            _ => return None,
+        };
+
+        // We want to compare ranges numerically, but the order of the bitwise representation of
+        // signed integers does not match their numeric order. Thus, to correct the ordering, we
+        // need to shift the range of signed integers to correct the comparison. This is achieved by
+        // XORing with a bias (see pattern/deconstruct_pat.rs for another pertinent example of this
+        // pattern).
+        //
+        // Also, for performance, it's important to only do the second `try_to_bits` if necessary.
+        let lo_is_min = match self.lo {
+            PatRangeBoundary::NegInfinity => true,
+            PatRangeBoundary::Finite(value) => {
+                let lo = value.try_to_bits(size).unwrap() ^ bias;
+                lo <= min
+            }
+            PatRangeBoundary::PosInfinity => false,
+        };
+        if lo_is_min {
+            let hi_is_max = match self.hi {
+                PatRangeBoundary::NegInfinity => false,
+                PatRangeBoundary::Finite(value) => {
+                    let hi = value.try_to_bits(size).unwrap() ^ bias;
+                    hi > max || hi == max && self.end == RangeEnd::Included
+                }
+                PatRangeBoundary::PosInfinity => true,
+            };
+            if hi_is_max {
+                return Some(true);
+            }
+        }
+        Some(false)
+    }
+
+    #[inline]
+    pub fn contains(
+        &self,
+        value: mir::Const<'tcx>,
+        tcx: TyCtxt<'tcx>,
+        param_env: ty::ParamEnv<'tcx>,
+    ) -> Option<bool> {
+        use Ordering::*;
+        debug_assert_eq!(self.ty, value.ty());
+        let ty = self.ty;
+        let value = PatRangeBoundary::Finite(value);
+        // For performance, it's important to only do the second comparison if necessary.
+        Some(
+            match self.lo.compare_with(value, ty, tcx, param_env)? {
+                Less | Equal => true,
+                Greater => false,
+            } && match value.compare_with(self.hi, ty, tcx, param_env)? {
+                Less => true,
+                Equal => self.end == RangeEnd::Included,
+                Greater => false,
+            },
+        )
+    }
+
+    #[inline]
+    pub fn overlaps(
+        &self,
+        other: &Self,
+        tcx: TyCtxt<'tcx>,
+        param_env: ty::ParamEnv<'tcx>,
+    ) -> Option<bool> {
+        use Ordering::*;
+        debug_assert_eq!(self.ty, other.ty);
+        // For performance, it's important to only do the second comparison if necessary.
+        Some(
+            match other.lo.compare_with(self.hi, self.ty, tcx, param_env)? {
+                Less => true,
+                Equal => self.end == RangeEnd::Included,
+                Greater => false,
+            } && match self.lo.compare_with(other.hi, self.ty, tcx, param_env)? {
+                Less => true,
+                Equal => other.end == RangeEnd::Included,
+                Greater => false,
+            },
+        )
+    }
+}
+
+impl<'tcx> fmt::Display for PatRange<'tcx> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        if let PatRangeBoundary::Finite(value) = &self.lo {
+            write!(f, "{value}")?;
+        }
+        if let PatRangeBoundary::Finite(value) = &self.hi {
+            write!(f, "{}", self.end)?;
+            write!(f, "{value}")?;
+        } else {
+            // `0..` is parsed as an inclusive range, we must display it correctly.
+            write!(f, "..")?;
+        }
+        Ok(())
+    }
+}
+
+/// A (possibly open) boundary of a range pattern.
+/// If present, the const must be of a numeric type.
+#[derive(Copy, Clone, Debug, PartialEq, HashStable, TypeVisitable)]
+pub enum PatRangeBoundary<'tcx> {
+    Finite(mir::Const<'tcx>),
+    NegInfinity,
+    PosInfinity,
+}
+
+impl<'tcx> PatRangeBoundary<'tcx> {
+    #[inline]
+    pub fn is_finite(self) -> bool {
+        matches!(self, Self::Finite(..))
+    }
+    #[inline]
+    pub fn as_finite(self) -> Option<mir::Const<'tcx>> {
+        match self {
+            Self::Finite(value) => Some(value),
+            Self::NegInfinity | Self::PosInfinity => None,
+        }
+    }
+    #[inline]
+    pub fn to_const(self, ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> mir::Const<'tcx> {
+        match self {
+            Self::Finite(value) => value,
+            Self::NegInfinity => {
+                // Unwrap is ok because the type is known to be numeric.
+                let c = ty.numeric_min_val(tcx).unwrap();
+                mir::Const::from_ty_const(c, tcx)
+            }
+            Self::PosInfinity => {
+                // Unwrap is ok because the type is known to be numeric.
+                let c = ty.numeric_max_val(tcx).unwrap();
+                mir::Const::from_ty_const(c, tcx)
+            }
+        }
+    }
+    pub fn eval_bits(self, ty: Ty<'tcx>, tcx: TyCtxt<'tcx>, param_env: ty::ParamEnv<'tcx>) -> u128 {
+        match self {
+            Self::Finite(value) => value.eval_bits(tcx, param_env),
+            Self::NegInfinity => {
+                // Unwrap is ok because the type is known to be numeric.
+                ty.numeric_min_and_max_as_bits(tcx).unwrap().0
+            }
+            Self::PosInfinity => {
+                // Unwrap is ok because the type is known to be numeric.
+                ty.numeric_min_and_max_as_bits(tcx).unwrap().1
+            }
+        }
+    }
+
+    #[instrument(skip(tcx, param_env), level = "debug", ret)]
+    pub fn compare_with(
+        self,
+        other: Self,
+        ty: Ty<'tcx>,
+        tcx: TyCtxt<'tcx>,
+        param_env: ty::ParamEnv<'tcx>,
+    ) -> Option<Ordering> {
+        use PatRangeBoundary::*;
+        match (self, other) {
+            // When comparing with infinities, we must remember that `0u8..` and `0u8..=255`
+            // describe the same range. These two shortcuts are ok, but for the rest we must check
+            // bit values.
+            (PosInfinity, PosInfinity) => return Some(Ordering::Equal),
+            (NegInfinity, NegInfinity) => return Some(Ordering::Equal),
+
+            // This code is hot when compiling matches with many ranges. So we
+            // special-case extraction of evaluated scalars for speed, for types where
+            // raw data comparisons are appropriate. E.g. `unicode-normalization` has
+            // many ranges such as '\u{037A}'..='\u{037F}', and chars can be compared
+            // in this way.
+            (Finite(mir::Const::Ty(a)), Finite(mir::Const::Ty(b)))
+                if matches!(ty.kind(), ty::Uint(_) | ty::Char) =>
+            {
+                return Some(a.kind().cmp(&b.kind()));
+            }
+            (
+                Finite(mir::Const::Val(mir::ConstValue::Scalar(Scalar::Int(a)), _)),
+                Finite(mir::Const::Val(mir::ConstValue::Scalar(Scalar::Int(b)), _)),
+            ) if matches!(ty.kind(), ty::Uint(_) | ty::Char) => return Some(a.cmp(&b)),
+            _ => {}
+        }
+
+        let a = self.eval_bits(ty, tcx, param_env);
+        let b = other.eval_bits(ty, tcx, param_env);
+
+        match ty.kind() {
+            ty::Float(ty::FloatTy::F32) => {
+                use rustc_apfloat::Float;
+                let a = rustc_apfloat::ieee::Single::from_bits(a);
+                let b = rustc_apfloat::ieee::Single::from_bits(b);
+                a.partial_cmp(&b)
+            }
+            ty::Float(ty::FloatTy::F64) => {
+                use rustc_apfloat::Float;
+                let a = rustc_apfloat::ieee::Double::from_bits(a);
+                let b = rustc_apfloat::ieee::Double::from_bits(b);
+                a.partial_cmp(&b)
+            }
+            ty::Int(ity) => {
+                use rustc_middle::ty::layout::IntegerExt;
+                let size = rustc_target::abi::Integer::from_int_ty(&tcx, *ity).size();
+                let a = size.sign_extend(a) as i128;
+                let b = size.sign_extend(b) as i128;
+                Some(a.cmp(&b))
+            }
+            ty::Uint(_) | ty::Char => Some(a.cmp(&b)),
+            _ => bug!(),
+        }
+    }
 }
 
 impl<'tcx> fmt::Display for Pat<'tcx> {
@@ -901,11 +1174,10 @@ impl<'tcx> fmt::Display for Pat<'tcx> {
                 write!(f, "{subpattern}")
             }
             PatKind::Constant { value } => write!(f, "{value}"),
-            PatKind::Range(box PatRange { lo, hi, end }) => {
-                write!(f, "{lo}")?;
-                write!(f, "{end}")?;
-                write!(f, "{hi}")
+            PatKind::InlineConstant { def: _, ref subpattern } => {
+                write!(f, "{} (from inline const)", subpattern)
             }
+            PatKind::Range(ref range) => write!(f, "{range}"),
             PatKind::Slice { ref prefix, ref slice, ref suffix }
             | PatKind::Array { ref prefix, ref slice, ref suffix } => {
                 write!(f, "[")?;
@@ -931,6 +1203,7 @@ impl<'tcx> fmt::Display for Pat<'tcx> {
                 }
                 Ok(())
             }
+            PatKind::Error(_) => write!(f, "<error>"),
         }
     }
 }
diff --git a/compiler/rustc_middle/src/thir/visit.rs b/compiler/rustc_middle/src/thir/visit.rs
index b84e15688..8feefb4c0 100644
--- a/compiler/rustc_middle/src/thir/visit.rs
+++ b/compiler/rustc_middle/src/thir/visit.rs
@@ -66,8 +66,9 @@ pub fn walk_expr<'a, 'tcx: 'a, V: Visitor<'a, 'tcx>>(visitor: &mut V, expr: &Exp
         Use { source } => visitor.visit_expr(&visitor.thir()[source]),
         NeverToAny { source } => visitor.visit_expr(&visitor.thir()[source]),
         PointerCoercion { source, cast: _ } => visitor.visit_expr(&visitor.thir()[source]),
-        Let { expr, .. } => {
+        Let { expr, ref pat } => {
             visitor.visit_expr(&visitor.thir()[expr]);
+            visitor.visit_pat(pat);
         }
         Loop { body } => visitor.visit_expr(&visitor.thir()[body]),
         Match { scrutinee, ref arms, .. } => {
@@ -226,23 +227,24 @@ pub fn walk_pat<'a, 'tcx: 'a, V: Visitor<'a, 'tcx>>(visitor: &mut V, pat: &Pat<'
             is_primary: _,
             name: _,
         } => visitor.visit_pat(&subpattern),
-        Binding { .. } | Wild => {}
+        Binding { .. } | Wild | Error(_) => {}
         Variant { subpatterns, adt_def: _, args: _, variant_index: _ } | Leaf { subpatterns } => {
             for subpattern in subpatterns {
                 visitor.visit_pat(&subpattern.pattern);
             }
         }
         Constant { value: _ } => {}
+        InlineConstant { def: _, subpattern } => visitor.visit_pat(subpattern),
         Range(_) => {}
         Slice { prefix, slice, suffix } | Array { prefix, slice, suffix } => {
             for subpattern in prefix.iter() {
-                visitor.visit_pat(&subpattern);
+                visitor.visit_pat(subpattern);
             }
             if let Some(pat) = slice {
-                visitor.visit_pat(&pat);
+                visitor.visit_pat(pat);
             }
             for subpattern in suffix.iter() {
-                visitor.visit_pat(&subpattern);
+                visitor.visit_pat(subpattern);
             }
         }
         Or { pats } => {
diff --git a/compiler/rustc_middle/src/traits/mod.rs b/compiler/rustc_middle/src/traits/mod.rs
index 99b750c9a..6cd75e087 100644
--- a/compiler/rustc_middle/src/traits/mod.rs
+++ b/compiler/rustc_middle/src/traits/mod.rs
@@ -301,13 +301,18 @@ pub enum ObligationCauseCode<'tcx> {
     InlineAsmSized,
     /// Captured closure type must be `Sized`.
     SizedClosureCapture(LocalDefId),
-    /// Types live across generator yields must be `Sized`.
-    SizedGeneratorInterior(LocalDefId),
+    /// Types live across coroutine yields must be `Sized`.
+    SizedCoroutineInterior(LocalDefId),
     /// `[expr; N]` requires `type_of(expr): Copy`.
     RepeatElementCopy {
-        /// If element is a `const fn` we display a help message suggesting to move the
-        /// function call to a new `const` item while saying that `T` doesn't implement `Copy`.
-        is_const_fn: bool,
+        /// If element is a `const fn` or const ctor we display a help message suggesting
+        /// to move it to a new `const` item while saying that `T` doesn't implement `Copy`.
+        is_constable: IsConstable,
+        elt_type: Ty<'tcx>,
+        elt_span: Span,
+        /// Span of the statement/item in which the repeat expression occurs. We can use this to
+        /// place a `const` declaration before it
+        elt_stmt_span: Span,
     },
 
     /// Types of fields (other than the last, except for packed structs) in a struct must be sized.
@@ -455,6 +460,21 @@ pub enum ObligationCauseCode<'tcx> {
     TypeAlias(InternedObligationCauseCode<'tcx>, Span, DefId),
 }
 
+/// Whether a value can be extracted into a const.
+/// Used for diagnostics around array repeat expressions.
+#[derive(Copy, Clone, Debug, PartialEq, Eq, HashStable, TyEncodable, TyDecodable)]
+pub enum IsConstable {
+    No,
+    /// Call to a const fn
+    Fn,
+    /// Use of a const ctor
+    Ctor,
+}
+
+crate::TrivialTypeTraversalAndLiftImpls! {
+    IsConstable,
+}
+
 /// The 'location' at which we try to perform HIR-based wf checking.
 /// This information is used to obtain an `hir::Ty`, which
 /// we can walk in order to obtain precise spans for any
@@ -541,6 +561,7 @@ pub struct MatchExpressionArmCause<'tcx> {
     pub prior_arm_ty: Ty<'tcx>,
     pub prior_arm_span: Span,
     pub scrut_span: Span,
+    pub scrut_hir_id: hir::HirId,
     pub source: hir::MatchSource,
     pub prior_arms: Vec<Span>,
     pub opt_suggest_box_span: Option<Span>,
diff --git a/compiler/rustc_middle/src/traits/query.rs b/compiler/rustc_middle/src/traits/query.rs
index 975e3e3ac..3cceb8b2c 100644
--- a/compiler/rustc_middle/src/traits/query.rs
+++ b/compiler/rustc_middle/src/traits/query.rs
@@ -10,7 +10,7 @@ use crate::infer::canonical::{Canonical, QueryResponse};
 use crate::ty::error::TypeError;
 use crate::ty::GenericArg;
 use crate::ty::{self, Ty, TyCtxt};
-use rustc_span::source_map::Span;
+use rustc_span::Span;
 
 pub mod type_op {
     use crate::ty::fold::TypeFoldable;
diff --git a/compiler/rustc_middle/src/traits/select.rs b/compiler/rustc_middle/src/traits/select.rs
index 90bc5dd8f..f33421bba 100644
--- a/compiler/rustc_middle/src/traits/select.rs
+++ b/compiler/rustc_middle/src/traits/select.rs
@@ -136,14 +136,18 @@ pub enum SelectionCandidate<'tcx> {
         is_const: bool,
     },
 
-    /// Implementation of a `Generator` trait by one of the anonymous types
-    /// generated for a generator.
-    GeneratorCandidate,
+    /// Implementation of a `Coroutine` trait by one of the anonymous types
+    /// generated for a coroutine.
+    CoroutineCandidate,
 
-    /// Implementation of a `Future` trait by one of the generator types
+    /// Implementation of a `Future` trait by one of the coroutine types
     /// generated for an async construct.
     FutureCandidate,
 
+    /// Implementation of an `Iterator` trait by one of the generator types
+    /// generated for a gen construct.
+    IteratorCandidate,
+
     /// Implementation of a `Fn`-family trait by one of the anonymous
     /// types generated for a fn pointer type (e.g., `fn(int) -> int`)
     FnPointerCandidate {
diff --git a/compiler/rustc_middle/src/traits/solve/cache.rs b/compiler/rustc_middle/src/traits/solve/cache.rs
index 9898b0019..e9e9cc418 100644
--- a/compiler/rustc_middle/src/traits/solve/cache.rs
+++ b/compiler/rustc_middle/src/traits/solve/cache.rs
@@ -1,4 +1,4 @@
-use super::{CanonicalInput, QueryResult};
+use super::{inspect, CanonicalInput, QueryResult};
 use crate::ty::TyCtxt;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::sync::Lock;
@@ -14,8 +14,10 @@ pub struct EvaluationCache<'tcx> {
     map: Lock<FxHashMap<CanonicalInput<'tcx>, CacheEntry<'tcx>>>,
 }
 
+#[derive(PartialEq, Eq)]
 pub struct CacheData<'tcx> {
     pub result: QueryResult<'tcx>,
+    pub proof_tree: Option<&'tcx [inspect::GoalEvaluationStep<'tcx>]>,
     pub reached_depth: usize,
     pub encountered_overflow: bool,
 }
@@ -24,22 +26,33 @@ impl<'tcx> EvaluationCache<'tcx> {
     /// Insert a final result into the global cache.
     pub fn insert(
         &self,
+        tcx: TyCtxt<'tcx>,
         key: CanonicalInput<'tcx>,
+        proof_tree: Option<&'tcx [inspect::GoalEvaluationStep<'tcx>]>,
         reached_depth: usize,
-        did_overflow: bool,
+        encountered_overflow: bool,
         cycle_participants: FxHashSet<CanonicalInput<'tcx>>,
         dep_node: DepNodeIndex,
         result: QueryResult<'tcx>,
     ) {
         let mut map = self.map.borrow_mut();
         let entry = map.entry(key).or_default();
-        let data = WithDepNode::new(dep_node, result);
+        let data = WithDepNode::new(dep_node, QueryData { result, proof_tree });
         entry.cycle_participants.extend(cycle_participants);
-        if did_overflow {
+        if encountered_overflow {
             entry.with_overflow.insert(reached_depth, data);
         } else {
             entry.success = Some(Success { data, reached_depth });
         }
+
+        if cfg!(debug_assertions) {
+            drop(map);
+            if Some(CacheData { result, proof_tree, reached_depth, encountered_overflow })
+                != self.get(tcx, key, |_| false, Limit(reached_depth))
+            {
+                bug!("unable to retrieve inserted element from cache: {key:?}");
+            }
+        }
     }
 
     /// Try to fetch a cached result, checking the recursion limit
@@ -62,27 +75,39 @@ impl<'tcx> EvaluationCache<'tcx> {
 
         if let Some(ref success) = entry.success {
             if available_depth.value_within_limit(success.reached_depth) {
+                let QueryData { result, proof_tree } = success.data.get(tcx);
                 return Some(CacheData {
-                    result: success.data.get(tcx),
+                    result,
+                    proof_tree,
                     reached_depth: success.reached_depth,
                     encountered_overflow: false,
                 });
             }
         }
 
-        entry.with_overflow.get(&available_depth.0).map(|e| CacheData {
-            result: e.get(tcx),
-            reached_depth: available_depth.0,
-            encountered_overflow: true,
+        entry.with_overflow.get(&available_depth.0).map(|e| {
+            let QueryData { result, proof_tree } = e.get(tcx);
+            CacheData {
+                result,
+                proof_tree,
+                reached_depth: available_depth.0,
+                encountered_overflow: true,
+            }
         })
     }
 }
 
 struct Success<'tcx> {
-    data: WithDepNode<QueryResult<'tcx>>,
+    data: WithDepNode<QueryData<'tcx>>,
     reached_depth: usize,
 }
 
+#[derive(Clone, Copy)]
+pub struct QueryData<'tcx> {
+    pub result: QueryResult<'tcx>,
+    pub proof_tree: Option<&'tcx [inspect::GoalEvaluationStep<'tcx>]>,
+}
+
 /// The cache entry for a goal `CanonicalInput`.
 ///
 /// This contains results whose computation never hit the
@@ -96,5 +121,5 @@ struct CacheEntry<'tcx> {
     /// See the doc comment of `StackEntry::cycle_participants` for more
     /// details.
     cycle_participants: FxHashSet<CanonicalInput<'tcx>>,
-    with_overflow: FxHashMap<usize, WithDepNode<QueryResult<'tcx>>>,
+    with_overflow: FxHashMap<usize, WithDepNode<QueryData<'tcx>>>,
 }
diff --git a/compiler/rustc_middle/src/traits/solve/inspect.rs b/compiler/rustc_middle/src/traits/solve/inspect.rs
index e7e40bee6..a5916c4ab 100644
--- a/compiler/rustc_middle/src/traits/solve/inspect.rs
+++ b/compiler/rustc_middle/src/traits/solve/inspect.rs
@@ -42,12 +42,6 @@ pub struct State<'tcx, T> {
 
 pub type CanonicalState<'tcx, T> = Canonical<'tcx, State<'tcx, T>>;
 
-#[derive(Debug, Eq, PartialEq)]
-pub enum CacheHit {
-    Provisional,
-    Global,
-}
-
 /// When evaluating the root goals we also store the
 /// original values for the `CanonicalVarValues` of the
 /// canonicalized goal. We use this to map any [CanonicalState]
@@ -78,8 +72,8 @@ pub struct CanonicalGoalEvaluation<'tcx> {
 #[derive(Eq, PartialEq)]
 pub enum CanonicalGoalEvaluationKind<'tcx> {
     Overflow,
-    CacheHit(CacheHit),
-    Uncached { revisions: Vec<GoalEvaluationStep<'tcx>> },
+    CycleInStack,
+    Evaluation { revisions: &'tcx [GoalEvaluationStep<'tcx>] },
 }
 impl Debug for GoalEvaluation<'_> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
diff --git a/compiler/rustc_middle/src/traits/solve/inspect/format.rs b/compiler/rustc_middle/src/traits/solve/inspect/format.rs
index 5733be00a..4b73d8e41 100644
--- a/compiler/rustc_middle/src/traits/solve/inspect/format.rs
+++ b/compiler/rustc_middle/src/traits/solve/inspect/format.rs
@@ -74,13 +74,10 @@ impl<'a, 'b> ProofTreeFormatter<'a, 'b> {
             CanonicalGoalEvaluationKind::Overflow => {
                 writeln!(self.f, "OVERFLOW: {:?}", eval.result)
             }
-            CanonicalGoalEvaluationKind::CacheHit(CacheHit::Global) => {
-                writeln!(self.f, "GLOBAL CACHE HIT: {:?}", eval.result)
+            CanonicalGoalEvaluationKind::CycleInStack => {
+                writeln!(self.f, "CYCLE IN STACK: {:?}", eval.result)
             }
-            CanonicalGoalEvaluationKind::CacheHit(CacheHit::Provisional) => {
-                writeln!(self.f, "PROVISIONAL CACHE HIT: {:?}", eval.result)
-            }
-            CanonicalGoalEvaluationKind::Uncached { revisions } => {
+            CanonicalGoalEvaluationKind::Evaluation { revisions } => {
                 for (n, step) in revisions.iter().enumerate() {
                     writeln!(self.f, "REVISION {n}")?;
                     self.nested(|this| this.format_evaluation_step(step))?;
diff --git a/compiler/rustc_middle/src/traits/util.rs b/compiler/rustc_middle/src/traits/util.rs
index 05c06efaf..b4054f8ff 100644
--- a/compiler/rustc_middle/src/traits/util.rs
+++ b/compiler/rustc_middle/src/traits/util.rs
@@ -3,9 +3,10 @@ use rustc_data_structures::fx::FxHashSet;
 use crate::ty::{PolyTraitRef, TyCtxt};
 
 /// Given a PolyTraitRef, get the PolyTraitRefs of the trait's (transitive) supertraits.
-///
-/// A simplified version of the same function at `rustc_infer::traits::util::supertraits`.
-pub fn supertraits<'tcx>(
+/// This only exists in `rustc_middle` because the more powerful elaborator depends on
+/// `rustc_infer` for elaborating outlives bounds -- this should only be used for pretty
+/// printing.
+pub fn supertraits_for_pretty_printing<'tcx>(
     tcx: TyCtxt<'tcx>,
     trait_ref: PolyTraitRef<'tcx>,
 ) -> impl Iterator<Item = PolyTraitRef<'tcx>> {
diff --git a/compiler/rustc_middle/src/ty/_match.rs b/compiler/rustc_middle/src/ty/_match.rs
index 09517200b..85181720d 100644
--- a/compiler/rustc_middle/src/ty/_match.rs
+++ b/compiler/rustc_middle/src/ty/_match.rs
@@ -18,20 +18,20 @@ use crate::ty::{self, InferConst, Ty, TyCtxt};
 /// Like subtyping, matching is really a binary relation, so the only
 /// important thing about the result is Ok/Err. Also, matching never
 /// affects any type variables or unification state.
-pub struct Match<'tcx> {
+pub struct MatchAgainstFreshVars<'tcx> {
     tcx: TyCtxt<'tcx>,
     param_env: ty::ParamEnv<'tcx>,
 }
 
-impl<'tcx> Match<'tcx> {
-    pub fn new(tcx: TyCtxt<'tcx>, param_env: ty::ParamEnv<'tcx>) -> Match<'tcx> {
-        Match { tcx, param_env }
+impl<'tcx> MatchAgainstFreshVars<'tcx> {
+    pub fn new(tcx: TyCtxt<'tcx>, param_env: ty::ParamEnv<'tcx>) -> MatchAgainstFreshVars<'tcx> {
+        MatchAgainstFreshVars { tcx, param_env }
     }
 }
 
-impl<'tcx> TypeRelation<'tcx> for Match<'tcx> {
+impl<'tcx> TypeRelation<'tcx> for MatchAgainstFreshVars<'tcx> {
     fn tag(&self) -> &'static str {
-        "Match"
+        "MatchAgainstFreshVars"
     }
     fn tcx(&self) -> TyCtxt<'tcx> {
         self.tcx
diff --git a/compiler/rustc_middle/src/ty/assoc.rs b/compiler/rustc_middle/src/ty/assoc.rs
index f77a8c671..94a5ff131 100644
--- a/compiler/rustc_middle/src/ty/assoc.rs
+++ b/compiler/rustc_middle/src/ty/assoc.rs
@@ -1,5 +1,3 @@
-pub use self::AssocItemContainer::*;
-
 use crate::ty;
 use rustc_data_structures::sorted_map::SortedIndexMultiMap;
 use rustc_hir as hir;
diff --git a/compiler/rustc_middle/src/ty/codec.rs b/compiler/rustc_middle/src/ty/codec.rs
index dff7ff8c6..8b67e3966 100644
--- a/compiler/rustc_middle/src/ty/codec.rs
+++ b/compiler/rustc_middle/src/ty/codec.rs
@@ -19,7 +19,7 @@ use rustc_data_structures::fx::FxHashMap;
 use rustc_middle::ty::TyCtxt;
 use rustc_serialize::{Decodable, Encodable};
 use rustc_span::Span;
-use rustc_target::abi::FieldIdx;
+use rustc_target::abi::{FieldIdx, VariantIdx};
 pub use rustc_type_ir::{TyDecoder, TyEncoder};
 use std::hash::Hash;
 use std::intrinsics;
@@ -230,9 +230,9 @@ impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> Decodable<D>
                 assert!(pos >= SHORTHAND_OFFSET);
                 let shorthand = pos - SHORTHAND_OFFSET;
 
-                decoder.with_position(shorthand, ty::PredicateKind::decode)
+                decoder.with_position(shorthand, <ty::PredicateKind<'tcx> as Decodable<D>>::decode)
             } else {
-                ty::PredicateKind::decode(decoder)
+                <ty::PredicateKind<'tcx> as Decodable<D>>::decode(decoder)
             },
             bound_vars,
         )
@@ -348,9 +348,10 @@ impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> Decodable<D> for ty::Const<'tcx> {
 
 impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D> for [ty::ValTree<'tcx>] {
     fn decode(decoder: &mut D) -> &'tcx Self {
-        decoder.interner().arena.alloc_from_iter(
-            (0..decoder.read_usize()).map(|_| Decodable::decode(decoder)).collect::<Vec<_>>(),
-        )
+        decoder
+            .interner()
+            .arena
+            .alloc_from_iter((0..decoder.read_usize()).map(|_| Decodable::decode(decoder)))
     }
 }
 
@@ -368,9 +369,10 @@ impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> Decodable<D> for AdtDef<'tcx> {
 
 impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D> for [(ty::Clause<'tcx>, Span)] {
     fn decode(decoder: &mut D) -> &'tcx Self {
-        decoder.interner().arena.alloc_from_iter(
-            (0..decoder.read_usize()).map(|_| Decodable::decode(decoder)).collect::<Vec<_>>(),
-        )
+        decoder
+            .interner()
+            .arena
+            .alloc_from_iter((0..decoder.read_usize()).map(|_| Decodable::decode(decoder)))
     }
 }
 
@@ -412,6 +414,17 @@ impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D> for ty::List<Fi
     }
 }
 
+impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D>
+    for ty::List<(VariantIdx, FieldIdx)>
+{
+    fn decode(decoder: &mut D) -> &'tcx Self {
+        let len = decoder.read_usize();
+        decoder.interner().mk_offset_of_from_iter(
+            (0..len).map::<(VariantIdx, FieldIdx), _>(|_| Decodable::decode(decoder)),
+        )
+    }
+}
+
 impl_decodable_via_ref! {
     &'tcx ty::TypeckResults<'tcx>,
     &'tcx ty::List<Ty<'tcx>>,
@@ -424,6 +437,7 @@ impl_decodable_via_ref! {
     &'tcx ty::List<ty::BoundVariableKind>,
     &'tcx ty::List<ty::Clause<'tcx>>,
     &'tcx ty::List<FieldIdx>,
+    &'tcx ty::List<(VariantIdx, FieldIdx)>,
 }
 
 #[macro_export]
diff --git a/compiler/rustc_middle/src/ty/consts.rs b/compiler/rustc_middle/src/ty/consts.rs
index 2518f0cf2..af5ffc20d 100644
--- a/compiler/rustc_middle/src/ty/consts.rs
+++ b/compiler/rustc_middle/src/ty/consts.rs
@@ -57,7 +57,7 @@ impl<'tcx> Const<'tcx> {
     }
 
     #[inline]
-    pub fn new_var(tcx: TyCtxt<'tcx>, infer: ty::ConstVid<'tcx>, ty: Ty<'tcx>) -> Const<'tcx> {
+    pub fn new_var(tcx: TyCtxt<'tcx>, infer: ty::ConstVid, ty: Ty<'tcx>) -> Const<'tcx> {
         Const::new(tcx, ty::ConstKind::Infer(ty::InferConst::Var(infer)), ty)
     }
 
@@ -67,7 +67,7 @@ impl<'tcx> Const<'tcx> {
     }
 
     #[inline]
-    pub fn new_infer(tcx: TyCtxt<'tcx>, infer: ty::InferConst<'tcx>, ty: Ty<'tcx>) -> Const<'tcx> {
+    pub fn new_infer(tcx: TyCtxt<'tcx>, infer: ty::InferConst, ty: Ty<'tcx>) -> Const<'tcx> {
         Const::new(tcx, ty::ConstKind::Infer(infer), ty)
     }
 
@@ -84,7 +84,7 @@ impl<'tcx> Const<'tcx> {
     #[inline]
     pub fn new_placeholder(
         tcx: TyCtxt<'tcx>,
-        placeholder: ty::PlaceholderConst<'tcx>,
+        placeholder: ty::PlaceholderConst,
         ty: Ty<'tcx>,
     ) -> Const<'tcx> {
         Const::new(tcx, ty::ConstKind::Placeholder(placeholder), ty)
diff --git a/compiler/rustc_middle/src/ty/consts/kind.rs b/compiler/rustc_middle/src/ty/consts/kind.rs
index 749b54ca0..4af841fcf 100644
--- a/compiler/rustc_middle/src/ty/consts/kind.rs
+++ b/compiler/rustc_middle/src/ty/consts/kind.rs
@@ -80,19 +80,19 @@ static_assert_size!(super::ConstKind<'_>, 32);
 
 /// An inference variable for a const, for use in const generics.
 #[derive(Copy, Clone, Eq, PartialEq, PartialOrd, Ord, TyEncodable, TyDecodable, Hash)]
-pub enum InferConst<'tcx> {
+pub enum InferConst {
     /// Infer the value of the const.
-    Var(ty::ConstVid<'tcx>),
+    Var(ty::ConstVid),
     /// Infer the value of the effect.
     ///
     /// For why this is separate from the `Var` variant above, see the
     /// documentation on `EffectVid`.
-    EffectVar(ty::EffectVid<'tcx>),
+    EffectVar(ty::EffectVid),
     /// A fresh const variable. See `infer::freshen` for more details.
     Fresh(u32),
 }
 
-impl<CTX> HashStable<CTX> for InferConst<'_> {
+impl<CTX> HashStable<CTX> for InferConst {
     fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) {
         match self {
             InferConst::Var(_) | InferConst::EffectVar(_) => {
diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index c06b8b2df..551c4a15d 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -6,7 +6,7 @@ pub mod tls;
 
 use crate::arena::Arena;
 use crate::dep_graph::{DepGraph, DepKindStruct};
-use crate::infer::canonical::CanonicalVarInfo;
+use crate::infer::canonical::{CanonicalVarInfo, CanonicalVarInfos};
 use crate::lint::struct_lint_level;
 use crate::metadata::ModChild;
 use crate::middle::codegen_fn_attrs::CodegenFnAttrs;
@@ -65,7 +65,7 @@ use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::{Span, DUMMY_SP};
 use rustc_target::abi::{FieldIdx, Layout, LayoutS, TargetDataLayout, VariantIdx};
 use rustc_target::spec::abi;
-use rustc_type_ir::sty::TyKind::*;
+use rustc_type_ir::TyKind::*;
 use rustc_type_ir::WithCachedTypeInfo;
 use rustc_type_ir::{CollectAndApply, Interner, TypeFlags};
 
@@ -80,54 +80,59 @@ use std::ops::{Bound, Deref};
 
 #[allow(rustc::usage_of_ty_tykind)]
 impl<'tcx> Interner for TyCtxt<'tcx> {
+    type DefId = DefId;
     type AdtDef = ty::AdtDef<'tcx>;
-    type GenericArgsRef = ty::GenericArgsRef<'tcx>;
+    type GenericArgs = ty::GenericArgsRef<'tcx>;
     type GenericArg = ty::GenericArg<'tcx>;
-    type DefId = DefId;
+    type Term = ty::Term<'tcx>;
+
     type Binder<T> = Binder<'tcx, T>;
-    type Ty = Ty<'tcx>;
-    type Const = ty::Const<'tcx>;
-    type Region = Region<'tcx>;
-    type Predicate = Predicate<'tcx>;
     type TypeAndMut = TypeAndMut<'tcx>;
-    type Mutability = hir::Mutability;
-    type Movability = hir::Movability;
-    type PolyFnSig = PolyFnSig<'tcx>;
-    type ListBinderExistentialPredicate = &'tcx List<PolyExistentialPredicate<'tcx>>;
-    type BinderListTy = Binder<'tcx, &'tcx List<Ty<'tcx>>>;
-    type ListTy = &'tcx List<Ty<'tcx>>;
+    type CanonicalVars = CanonicalVarInfos<'tcx>;
+
+    type Ty = Ty<'tcx>;
+    type Tys = &'tcx List<Ty<'tcx>>;
     type AliasTy = ty::AliasTy<'tcx>;
     type ParamTy = ParamTy;
     type BoundTy = ty::BoundTy;
-    type PlaceholderType = ty::PlaceholderType;
+    type PlaceholderTy = ty::PlaceholderType;
     type InferTy = InferTy;
+
     type ErrorGuaranteed = ErrorGuaranteed;
-    type PredicateKind = ty::PredicateKind<'tcx>;
+    type BoundExistentialPredicates = &'tcx List<PolyExistentialPredicate<'tcx>>;
+    type PolyFnSig = PolyFnSig<'tcx>;
     type AllocId = crate::mir::interpret::AllocId;
 
-    type InferConst = ty::InferConst<'tcx>;
+    type Const = ty::Const<'tcx>;
+    type InferConst = ty::InferConst;
     type AliasConst = ty::UnevaluatedConst<'tcx>;
+    type PlaceholderConst = ty::PlaceholderConst;
     type ParamConst = ty::ParamConst;
     type BoundConst = ty::BoundVar;
-    type PlaceholderConst = ty::PlaceholderConst<'tcx>;
     type ValueConst = ty::ValTree<'tcx>;
     type ExprConst = ty::Expr<'tcx>;
 
+    type Region = Region<'tcx>;
     type EarlyBoundRegion = ty::EarlyBoundRegion;
     type BoundRegion = ty::BoundRegion;
     type FreeRegion = ty::FreeRegion;
-    type RegionVid = ty::RegionVid;
+    type InferRegion = ty::RegionVid;
     type PlaceholderRegion = ty::PlaceholderRegion;
 
+    type Predicate = Predicate<'tcx>;
+    type TraitPredicate = ty::TraitPredicate<'tcx>;
+    type RegionOutlivesPredicate = ty::RegionOutlivesPredicate<'tcx>;
+    type TypeOutlivesPredicate = ty::TypeOutlivesPredicate<'tcx>;
+    type ProjectionPredicate = ty::ProjectionPredicate<'tcx>;
+    type SubtypePredicate = ty::SubtypePredicate<'tcx>;
+    type CoercePredicate = ty::CoercePredicate<'tcx>;
+    type ClosureKind = ty::ClosureKind;
+
     fn ty_and_mut_to_parts(
         TypeAndMut { ty, mutbl }: TypeAndMut<'tcx>,
-    ) -> (Self::Ty, Self::Mutability) {
+    ) -> (Self::Ty, ty::Mutability) {
         (ty, mutbl)
     }
-
-    fn mutability_is_mut(mutbl: Self::Mutability) -> bool {
-        mutbl.is_mut()
-    }
 }
 
 type InternedSet<'tcx, T> = ShardedHashMap<InternedInSet<'tcx, T>, ()>;
@@ -152,11 +157,13 @@ pub struct CtxtInterners<'tcx> {
     const_: InternedSet<'tcx, ConstData<'tcx>>,
     const_allocation: InternedSet<'tcx, Allocation>,
     bound_variable_kinds: InternedSet<'tcx, List<ty::BoundVariableKind>>,
-    layout: InternedSet<'tcx, LayoutS>,
+    layout: InternedSet<'tcx, LayoutS<FieldIdx, VariantIdx>>,
     adt_def: InternedSet<'tcx, AdtDefData>,
     external_constraints: InternedSet<'tcx, ExternalConstraintsData<'tcx>>,
     predefined_opaques_in_body: InternedSet<'tcx, PredefinedOpaquesData<'tcx>>,
     fields: InternedSet<'tcx, List<FieldIdx>>,
+    local_def_ids: InternedSet<'tcx, List<LocalDefId>>,
+    offset_of: InternedSet<'tcx, List<(VariantIdx, FieldIdx)>>,
 }
 
 impl<'tcx> CtxtInterners<'tcx> {
@@ -182,6 +189,8 @@ impl<'tcx> CtxtInterners<'tcx> {
             external_constraints: Default::default(),
             predefined_opaques_in_body: Default::default(),
             fields: Default::default(),
+            local_def_ids: Default::default(),
+            offset_of: Default::default(),
         }
     }
 
@@ -770,9 +779,20 @@ impl<'tcx> TyCtxt<'tcx> {
         self.diagnostic_items(did.krate).name_to_id.get(&name) == Some(&did)
     }
 
-    /// Returns `true` if the node pointed to by `def_id` is a generator for an async construct.
-    pub fn generator_is_async(self, def_id: DefId) -> bool {
-        matches!(self.generator_kind(def_id), Some(hir::GeneratorKind::Async(_)))
+    /// Returns `true` if the node pointed to by `def_id` is a coroutine for an async construct.
+    pub fn coroutine_is_async(self, def_id: DefId) -> bool {
+        matches!(self.coroutine_kind(def_id), Some(hir::CoroutineKind::Async(_)))
+    }
+
+    /// Returns `true` if the node pointed to by `def_id` is a general coroutine that implements `Coroutine`.
+    /// This means it is neither an `async` or `gen` construct.
+    pub fn is_general_coroutine(self, def_id: DefId) -> bool {
+        matches!(self.coroutine_kind(def_id), Some(hir::CoroutineKind::Coroutine))
+    }
+
+    /// Returns `true` if the node pointed to by `def_id` is a coroutine for a gen construct.
+    pub fn coroutine_is_gen(self, def_id: DefId) -> bool {
+        matches!(self.coroutine_kind(def_id), Some(hir::CoroutineKind::Gen(_)))
     }
 
     pub fn stability(self) -> &'tcx stability::Index {
@@ -960,7 +980,7 @@ impl<'tcx> TyCtxt<'tcx> {
         self.dep_graph.read_index(DepNodeIndex::FOREVER_RED_NODE);
 
         let definitions = &self.untracked.definitions;
-        std::iter::from_generator(|| {
+        std::iter::from_coroutine(|| {
             let mut i = 0;
 
             // Recompute the number of definitions each time, because our caller may be creating
@@ -1057,16 +1077,21 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     /// Returns the `DefId` and the `BoundRegionKind` corresponding to the given region.
-    pub fn is_suitable_region(self, region: Region<'tcx>) -> Option<FreeRegionInfo> {
-        let (suitable_region_binding_scope, bound_region) = match *region {
-            ty::ReFree(ref free_region) => {
-                (free_region.scope.expect_local(), free_region.bound_region)
+    pub fn is_suitable_region(self, mut region: Region<'tcx>) -> Option<FreeRegionInfo> {
+        let (suitable_region_binding_scope, bound_region) = loop {
+            let def_id = match region.kind() {
+                ty::ReFree(fr) => fr.bound_region.get_id()?.as_local()?,
+                ty::ReEarlyBound(ebr) => ebr.def_id.expect_local(),
+                _ => return None, // not a free region
+            };
+            let scope = self.local_parent(def_id);
+            if self.def_kind(scope) == DefKind::OpaqueTy {
+                // Lifetime params of opaque types are synthetic and thus irrelevant to
+                // diagnostics. Map them back to their origin!
+                region = self.map_rpit_lifetime_to_fn_lifetime(def_id);
+                continue;
             }
-            ty::ReEarlyBound(ref ebr) => (
-                self.local_parent(ebr.def_id.expect_local()),
-                ty::BoundRegionKind::BrNamed(ebr.def_id, ebr.name),
-            ),
-            _ => return None, // not a free region
+            break (scope, ty::BrNamed(def_id.into(), self.item_name(def_id.into())));
         };
 
         let is_impl_item = match self.hir().find_by_def_id(suitable_region_binding_scope) {
@@ -1074,7 +1099,7 @@ impl<'tcx> TyCtxt<'tcx> {
             Some(Node::ImplItem(..)) => {
                 self.is_bound_region_in_impl_item(suitable_region_binding_scope)
             }
-            _ => return None,
+            _ => false,
         };
 
         Some(FreeRegionInfo {
@@ -1121,7 +1146,11 @@ impl<'tcx> TyCtxt<'tcx> {
         {
             v.visit_ty(alias_ty);
             if !v.0.is_empty() {
-                return Some((v.0, alias_generics.span, alias_generics.span_for_lifetime_suggestion()));
+                return Some((
+                    v.0,
+                    alias_generics.span,
+                    alias_generics.span_for_lifetime_suggestion(),
+                ));
             }
         }
         return None;
@@ -1382,8 +1411,8 @@ impl<'tcx> TyCtxt<'tcx> {
                     FnDef,
                     FnPtr,
                     Placeholder,
-                    Generator,
-                    GeneratorWitness,
+                    Coroutine,
+                    CoroutineWitness,
                     Dynamic,
                     Closure,
                     Tuple,
@@ -1521,7 +1550,7 @@ direct_interners! {
     region: pub(crate) intern_region(RegionKind<'tcx>): Region -> Region<'tcx>,
     const_: intern_const(ConstData<'tcx>): Const -> Const<'tcx>,
     const_allocation: pub mk_const_alloc(Allocation): ConstAllocation -> ConstAllocation<'tcx>,
-    layout: pub mk_layout(LayoutS): Layout -> Layout<'tcx>,
+    layout: pub mk_layout(LayoutS<FieldIdx, VariantIdx>): Layout -> Layout<'tcx>,
     adt_def: pub mk_adt_def_from_data(AdtDefData): AdtDef -> AdtDef<'tcx>,
     external_constraints: pub mk_external_constraints(ExternalConstraintsData<'tcx>):
         ExternalConstraints -> ExternalConstraints<'tcx>,
@@ -1559,6 +1588,8 @@ slice_interners!(
     place_elems: pub mk_place_elems(PlaceElem<'tcx>),
     bound_variable_kinds: pub mk_bound_variable_kinds(ty::BoundVariableKind),
     fields: pub mk_fields(FieldIdx),
+    local_def_ids: intern_local_def_ids(LocalDefId),
+    offset_of: pub mk_offset_of((VariantIdx, FieldIdx)),
 );
 
 impl<'tcx> TyCtxt<'tcx> {
@@ -1678,7 +1709,6 @@ impl<'tcx> TyCtxt<'tcx> {
                 && let DefKind::Impl { of_trait: false } = self.def_kind(self.parent(_def_id))
             {
                 // If this is an inherent projection.
-
                 generics.params.len() + 1
             } else {
                 generics.count()
@@ -1789,6 +1819,13 @@ impl<'tcx> TyCtxt<'tcx> {
         self.intern_clauses(clauses)
     }
 
+    pub fn mk_local_def_ids(self, clauses: &[LocalDefId]) -> &'tcx List<LocalDefId> {
+        // FIXME consider asking the input slice to be sorted to avoid
+        // re-interning permutations, in which case that would be asserted
+        // here.
+        self.intern_local_def_ids(clauses)
+    }
+
     pub fn mk_const_list_from_iter<I, T>(self, iter: I) -> T::Output
     where
         I: Iterator<Item = T>,
@@ -1880,6 +1917,14 @@ impl<'tcx> TyCtxt<'tcx> {
         T::collect_and_apply(iter, |xs| self.mk_fields(xs))
     }
 
+    pub fn mk_offset_of_from_iter<I, T>(self, iter: I) -> T::Output
+    where
+        I: Iterator<Item = T>,
+        T: CollectAndApply<(VariantIdx, FieldIdx), &'tcx List<(VariantIdx, FieldIdx)>>,
+    {
+        T::collect_and_apply(iter, |xs| self.mk_offset_of(xs))
+    }
+
     pub fn mk_args_trait(
         self,
         self_ty: Ty<'tcx>,
@@ -1888,15 +1933,6 @@ impl<'tcx> TyCtxt<'tcx> {
         self.mk_args_from_iter(iter::once(self_ty.into()).chain(rest))
     }
 
-    pub fn mk_alias_ty(
-        self,
-        def_id: DefId,
-        args: impl IntoIterator<Item: Into<GenericArg<'tcx>>>,
-    ) -> ty::AliasTy<'tcx> {
-        let args = self.check_and_mk_args(def_id, args);
-        ty::AliasTy { def_id, args, _use_mk_alias_ty_instead: () }
-    }
-
     pub fn mk_bound_variable_kinds_from_iter<I, T>(self, iter: I) -> T::Output
     where
         I: Iterator<Item = T>,
diff --git a/compiler/rustc_middle/src/ty/diagnostics.rs b/compiler/rustc_middle/src/ty/diagnostics.rs
index f03813a45..0094825fc 100644
--- a/compiler/rustc_middle/src/ty/diagnostics.rs
+++ b/compiler/rustc_middle/src/ty/diagnostics.rs
@@ -17,7 +17,7 @@ use rustc_hir::def::DefKind;
 use rustc_hir::def_id::DefId;
 use rustc_hir::{PredicateOrigin, WherePredicate};
 use rustc_span::{BytePos, Span};
-use rustc_type_ir::sty::TyKind::*;
+use rustc_type_ir::TyKind::*;
 
 impl<'tcx> IntoDiagnosticArg for Ty<'tcx> {
     fn into_diagnostic_arg(self) -> DiagnosticArgValue<'static> {
@@ -274,6 +274,8 @@ pub fn suggest_constraining_type_params<'a>(
                 span,
                 if span_to_replace.is_some() {
                     constraint.clone()
+                } else if constraint.starts_with('<') {
+                    constraint.to_string()
                 } else if bound_list_non_empty {
                     format!(" + {constraint}")
                 } else {
@@ -482,8 +484,8 @@ impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for IsSuggestableVisitor<'tcx> {
             FnDef(..)
             | Closure(..)
             | Infer(..)
-            | Generator(..)
-            | GeneratorWitness(..)
+            | Coroutine(..)
+            | CoroutineWitness(..)
             | Bound(_, _)
             | Placeholder(_)
             | Error(_) => {
@@ -494,7 +496,8 @@ impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for IsSuggestableVisitor<'tcx> {
                 let parent = self.tcx.parent(def_id);
                 let parent_ty = self.tcx.type_of(parent).instantiate_identity();
                 if let DefKind::TyAlias | DefKind::AssocTy = self.tcx.def_kind(parent)
-                    && let Alias(Opaque, AliasTy { def_id: parent_opaque_def_id, .. }) = *parent_ty.kind()
+                    && let Alias(Opaque, AliasTy { def_id: parent_opaque_def_id, .. }) =
+                        *parent_ty.kind()
                     && parent_opaque_def_id == def_id
                 {
                     // Okay
@@ -566,8 +569,8 @@ impl<'tcx> FallibleTypeFolder<TyCtxt<'tcx>> for MakeSuggestableFolder<'tcx> {
             // FIXME(compiler-errors): We could replace these with infer, I guess.
             Closure(..)
             | Infer(..)
-            | Generator(..)
-            | GeneratorWitness(..)
+            | Coroutine(..)
+            | CoroutineWitness(..)
             | Bound(_, _)
             | Placeholder(_)
             | Error(_) => {
@@ -577,8 +580,10 @@ impl<'tcx> FallibleTypeFolder<TyCtxt<'tcx>> for MakeSuggestableFolder<'tcx> {
             Alias(Opaque, AliasTy { def_id, .. }) => {
                 let parent = self.tcx.parent(def_id);
                 let parent_ty = self.tcx.type_of(parent).instantiate_identity();
-                if let hir::def::DefKind::TyAlias | hir::def::DefKind::AssocTy = self.tcx.def_kind(parent)
-                    && let Alias(Opaque, AliasTy { def_id: parent_opaque_def_id, .. }) = *parent_ty.kind()
+                if let hir::def::DefKind::TyAlias | hir::def::DefKind::AssocTy =
+                    self.tcx.def_kind(parent)
+                    && let Alias(Opaque, AliasTy { def_id: parent_opaque_def_id, .. }) =
+                        *parent_ty.kind()
                     && parent_opaque_def_id == def_id
                 {
                     t
diff --git a/compiler/rustc_middle/src/ty/error.rs b/compiler/rustc_middle/src/ty/error.rs
index 459c8dfb5..738bb5e8b 100644
--- a/compiler/rustc_middle/src/ty/error.rs
+++ b/compiler/rustc_middle/src/ty/error.rs
@@ -241,8 +241,10 @@ impl<'tcx> Ty<'tcx> {
             }
             ty::Dynamic(..) => "trait object".into(),
             ty::Closure(..) => "closure".into(),
-            ty::Generator(def_id, ..) => tcx.generator_kind(def_id).unwrap().descr().into(),
-            ty::GeneratorWitness(..) => "generator witness".into(),
+            ty::Coroutine(def_id, ..) => {
+                format!("{:#}", tcx.coroutine_kind(def_id).unwrap()).into()
+            }
+            ty::CoroutineWitness(..) => "coroutine witness".into(),
             ty::Infer(ty::TyVar(_)) => "inferred type".into(),
             ty::Infer(ty::IntVar(_)) => "integer".into(),
             ty::Infer(ty::FloatVar(_)) => "floating-point number".into(),
@@ -253,7 +255,13 @@ impl<'tcx> Ty<'tcx> {
             ty::Infer(ty::FreshFloatTy(_)) => "fresh floating-point type".into(),
             ty::Alias(ty::Projection | ty::Inherent, _) => "associated type".into(),
             ty::Param(p) => format!("type parameter `{p}`").into(),
-            ty::Alias(ty::Opaque, ..) => if tcx.ty_is_opaque_future(self) { "future".into() } else { "opaque type".into() },
+            ty::Alias(ty::Opaque, ..) => {
+                if tcx.ty_is_opaque_future(self) {
+                    "future".into()
+                } else {
+                    "opaque type".into()
+                }
+            }
             ty::Error(_) => "type error".into(),
             _ => {
                 let width = tcx.sess.diagnostic_width();
@@ -293,8 +301,10 @@ impl<'tcx> Ty<'tcx> {
             ty::FnPtr(_) => "fn pointer".into(),
             ty::Dynamic(..) => "trait object".into(),
             ty::Closure(..) => "closure".into(),
-            ty::Generator(def_id, ..) => tcx.generator_kind(def_id).unwrap().descr().into(),
-            ty::GeneratorWitness(..) => "generator witness".into(),
+            ty::Coroutine(def_id, ..) => {
+                format!("{:#}", tcx.coroutine_kind(def_id).unwrap()).into()
+            }
+            ty::CoroutineWitness(..) => "coroutine witness".into(),
             ty::Tuple(..) => "tuple".into(),
             ty::Placeholder(..) => "higher-ranked type".into(),
             ty::Bound(..) => "bound type variable".into(),
@@ -309,26 +319,25 @@ impl<'tcx> Ty<'tcx> {
 impl<'tcx> TyCtxt<'tcx> {
     pub fn ty_string_with_limit(self, ty: Ty<'tcx>, length_limit: usize) -> String {
         let mut type_limit = 50;
-        let regular = FmtPrinter::new(self, hir::def::Namespace::TypeNS)
-            .pretty_print_type(ty)
-            .expect("could not write to `String`")
-            .into_buffer();
+        let regular = FmtPrinter::print_string(self, hir::def::Namespace::TypeNS, |cx| {
+            cx.pretty_print_type(ty)
+        })
+        .expect("could not write to `String`");
         if regular.len() <= length_limit {
             return regular;
         }
         let mut short;
         loop {
             // Look for the longest properly trimmed path that still fits in length_limit.
-            short = with_forced_trimmed_paths!(
-                FmtPrinter::new_with_limit(
+            short = with_forced_trimmed_paths!({
+                let mut cx = FmtPrinter::new_with_limit(
                     self,
                     hir::def::Namespace::TypeNS,
                     rustc_session::Limit(type_limit),
-                )
-                .pretty_print_type(ty)
-                .expect("could not write to `String`")
-                .into_buffer()
-            );
+                );
+                cx.pretty_print_type(ty).expect("could not write to `String`");
+                cx.into_buffer()
+            });
             if short.len() <= length_limit || type_limit == 0 {
                 break;
             }
@@ -338,10 +347,10 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     pub fn short_ty_string(self, ty: Ty<'tcx>) -> (String, Option<PathBuf>) {
-        let regular = FmtPrinter::new(self, hir::def::Namespace::TypeNS)
-            .pretty_print_type(ty)
-            .expect("could not write to `String`")
-            .into_buffer();
+        let regular = FmtPrinter::print_string(self, hir::def::Namespace::TypeNS, |cx| {
+            cx.pretty_print_type(ty)
+        })
+        .expect("could not write to `String`");
 
         if !self.sess.opts.unstable_opts.write_long_types_to_disk {
             return (regular, None);
diff --git a/compiler/rustc_middle/src/ty/fast_reject.rs b/compiler/rustc_middle/src/ty/fast_reject.rs
index 16935d5b3..75ea53195 100644
--- a/compiler/rustc_middle/src/ty/fast_reject.rs
+++ b/compiler/rustc_middle/src/ty/fast_reject.rs
@@ -28,8 +28,8 @@ pub enum SimplifiedType {
     MarkerTraitObject,
     Trait(DefId),
     Closure(DefId),
-    Generator(DefId),
-    GeneratorWitness(DefId),
+    Coroutine(DefId),
+    CoroutineWitness(DefId),
     Function(usize),
     Placeholder,
 }
@@ -128,8 +128,8 @@ pub fn simplify_type<'tcx>(
         },
         ty::Ref(_, _, mutbl) => Some(SimplifiedType::Ref(mutbl)),
         ty::FnDef(def_id, _) | ty::Closure(def_id, _) => Some(SimplifiedType::Closure(def_id)),
-        ty::Generator(def_id, _, _) => Some(SimplifiedType::Generator(def_id)),
-        ty::GeneratorWitness(def_id, _) => Some(SimplifiedType::GeneratorWitness(def_id)),
+        ty::Coroutine(def_id, _, _) => Some(SimplifiedType::Coroutine(def_id)),
+        ty::CoroutineWitness(def_id, _) => Some(SimplifiedType::CoroutineWitness(def_id)),
         ty::Never => Some(SimplifiedType::Never),
         ty::Tuple(tys) => Some(SimplifiedType::Tuple(tys.len())),
         ty::FnPtr(f) => Some(SimplifiedType::Function(f.skip_binder().inputs().len())),
@@ -164,8 +164,8 @@ impl SimplifiedType {
             | SimplifiedType::Foreign(d)
             | SimplifiedType::Trait(d)
             | SimplifiedType::Closure(d)
-            | SimplifiedType::Generator(d)
-            | SimplifiedType::GeneratorWitness(d) => Some(d),
+            | SimplifiedType::Coroutine(d)
+            | SimplifiedType::CoroutineWitness(d) => Some(d),
             _ => None,
         }
     }
@@ -234,8 +234,8 @@ impl DeepRejectCtxt {
             | ty::Foreign(..) => {}
             ty::FnDef(..)
             | ty::Closure(..)
-            | ty::Generator(..)
-            | ty::GeneratorWitness(..)
+            | ty::Coroutine(..)
+            | ty::CoroutineWitness(..)
             | ty::Placeholder(..)
             | ty::Bound(..)
             | ty::Infer(_) => bug!("unexpected impl_ty: {impl_ty}"),
@@ -310,7 +310,7 @@ impl DeepRejectCtxt {
             },
 
             // Impls cannot contain these types as these cannot be named directly.
-            ty::FnDef(..) | ty::Closure(..) | ty::Generator(..) => false,
+            ty::FnDef(..) | ty::Closure(..) | ty::Coroutine(..) => false,
 
             // Placeholder types don't unify with anything on their own
             ty::Placeholder(..) | ty::Bound(..) => false,
@@ -337,7 +337,7 @@ impl DeepRejectCtxt {
 
             ty::Error(_) => true,
 
-            ty::GeneratorWitness(..) => {
+            ty::CoroutineWitness(..) => {
                 bug!("unexpected obligation type: {:?}", obligation_ty)
             }
         }
diff --git a/compiler/rustc_middle/src/ty/flags.rs b/compiler/rustc_middle/src/ty/flags.rs
index c23d553d9..ec36bdc5a 100644
--- a/compiler/rustc_middle/src/ty/flags.rs
+++ b/compiler/rustc_middle/src/ty/flags.rs
@@ -95,8 +95,8 @@ impl FlagComputation {
                 self.add_flags(TypeFlags::STILL_FURTHER_SPECIALIZABLE);
             }
 
-            ty::Generator(_, args, _) => {
-                let args = args.as_generator();
+            ty::Coroutine(_, args, _) => {
+                let args = args.as_coroutine();
                 let should_remove_further_specializable =
                     !self.flags.contains(TypeFlags::STILL_FURTHER_SPECIALIZABLE);
                 self.add_args(args.parent_args());
@@ -111,14 +111,14 @@ impl FlagComputation {
                 self.add_ty(args.tupled_upvars_ty());
             }
 
-            ty::GeneratorWitness(_, args) => {
+            ty::CoroutineWitness(_, args) => {
                 let should_remove_further_specializable =
                     !self.flags.contains(TypeFlags::STILL_FURTHER_SPECIALIZABLE);
                 self.add_args(args);
                 if should_remove_further_specializable {
                     self.flags -= TypeFlags::STILL_FURTHER_SPECIALIZABLE;
                 }
-                self.add_flags(TypeFlags::HAS_TY_GENERATOR);
+                self.add_flags(TypeFlags::HAS_TY_COROUTINE);
             }
 
             &ty::Closure(_, args) => {
diff --git a/compiler/rustc_middle/src/ty/generic_args.rs b/compiler/rustc_middle/src/ty/generic_args.rs
index 72390e4bb..41a1bf04e 100644
--- a/compiler/rustc_middle/src/ty/generic_args.rs
+++ b/compiler/rustc_middle/src/ty/generic_args.rs
@@ -2,7 +2,7 @@
 
 use crate::ty::codec::{TyDecoder, TyEncoder};
 use crate::ty::fold::{FallibleTypeFolder, TypeFoldable, TypeFolder, TypeSuperFoldable};
-use crate::ty::sty::{ClosureArgs, GeneratorArgs, InlineConstArgs};
+use crate::ty::sty::{ClosureArgs, CoroutineArgs, InlineConstArgs};
 use crate::ty::visit::{TypeVisitable, TypeVisitableExt, TypeVisitor};
 use crate::ty::{self, Lift, List, ParamConst, Ty, TyCtxt};
 
@@ -11,7 +11,6 @@ use rustc_errors::{DiagnosticArgValue, IntoDiagnosticArg};
 use rustc_hir::def_id::DefId;
 use rustc_macros::HashStable;
 use rustc_serialize::{self, Decodable, Encodable};
-use rustc_span::sym;
 use rustc_type_ir::WithCachedTypeInfo;
 use smallvec::SmallVec;
 
@@ -267,12 +266,12 @@ impl<'tcx> GenericArgs<'tcx> {
         ClosureArgs { args: self }
     }
 
-    /// Interpret these generic args as the args of a generator type.
-    /// Generator args have a particular structure controlled by the
-    /// compiler that encodes information like the signature and generator kind;
-    /// see `ty::GeneratorArgs` struct for more comments.
-    pub fn as_generator(&'tcx self) -> GeneratorArgs<'tcx> {
-        GeneratorArgs { args: self }
+    /// Interpret these generic args as the args of a coroutine type.
+    /// Coroutine args have a particular structure controlled by the
+    /// compiler that encodes information like the signature and coroutine kind;
+    /// see `ty::CoroutineArgs` struct for more comments.
+    pub fn as_coroutine(&'tcx self) -> CoroutineArgs<'tcx> {
+        CoroutineArgs { args: self }
     }
 
     /// Interpret these generic args as the args of an inline const.
@@ -452,10 +451,6 @@ impl<'tcx> GenericArgs<'tcx> {
         tcx.mk_args_from_iter(self.iter().take(generics.count()))
     }
 
-    pub fn host_effect_param(&'tcx self) -> Option<ty::Const<'tcx>> {
-        self.consts().rfind(|x| matches!(x.kind(), ty::ConstKind::Param(p) if p.name == sym::host))
-    }
-
     pub fn print_as_list(&self) -> String {
         let v = self.iter().map(|arg| arg.to_string()).collect::<Vec<_>>();
         format!("[{}]", v.join(", "))
diff --git a/compiler/rustc_middle/src/ty/generics.rs b/compiler/rustc_middle/src/ty/generics.rs
index 8e6c1cd4b..888ee1d23 100644
--- a/compiler/rustc_middle/src/ty/generics.rs
+++ b/compiler/rustc_middle/src/ty/generics.rs
@@ -79,6 +79,10 @@ impl GenericParamDef {
         }
     }
 
+    pub fn is_host_effect(&self) -> bool {
+        matches!(self.kind, GenericParamDefKind::Const { is_host_effect: true, .. })
+    }
+
     pub fn default_value<'tcx>(
         &self,
         tcx: TyCtxt<'tcx>,
@@ -233,6 +237,20 @@ impl<'tcx> Generics {
         }
     }
 
+    /// Returns the `GenericParamDef` with the given index if available.
+    pub fn opt_param_at(
+        &'tcx self,
+        param_index: usize,
+        tcx: TyCtxt<'tcx>,
+    ) -> Option<&'tcx GenericParamDef> {
+        if let Some(index) = param_index.checked_sub(self.parent_count) {
+            self.params.get(index)
+        } else {
+            tcx.generics_of(self.parent.expect("parent_count > 0 but no parent?"))
+                .opt_param_at(param_index, tcx)
+        }
+    }
+
     pub fn params_to(&'tcx self, param_index: usize, tcx: TyCtxt<'tcx>) -> &'tcx [GenericParamDef] {
         if let Some(index) = param_index.checked_sub(self.parent_count) {
             &self.params[..index]
@@ -264,6 +282,20 @@ impl<'tcx> Generics {
         }
     }
 
+    /// Returns the `GenericParamDef` associated with this `ParamTy` if it belongs to this
+    /// `Generics`.
+    pub fn opt_type_param(
+        &'tcx self,
+        param: &ParamTy,
+        tcx: TyCtxt<'tcx>,
+    ) -> Option<&'tcx GenericParamDef> {
+        let param = self.opt_param_at(param.index as usize, tcx)?;
+        match param.kind {
+            GenericParamDefKind::Type { .. } => Some(param),
+            _ => None,
+        }
+    }
+
     /// Returns the `GenericParamDef` associated with this `ParamConst`.
     pub fn const_param(&'tcx self, param: &ParamConst, tcx: TyCtxt<'tcx>) -> &GenericParamDef {
         let param = self.param_at(param.index as usize, tcx);
diff --git a/compiler/rustc_middle/src/ty/inhabitedness/mod.rs b/compiler/rustc_middle/src/ty/inhabitedness/mod.rs
index 4dac6891b..68ac54e89 100644
--- a/compiler/rustc_middle/src/ty/inhabitedness/mod.rs
+++ b/compiler/rustc_middle/src/ty/inhabitedness/mod.rs
@@ -47,7 +47,7 @@ use crate::query::Providers;
 use crate::ty::context::TyCtxt;
 use crate::ty::{self, DefId, Ty, VariantDef, Visibility};
 
-use rustc_type_ir::sty::TyKind::*;
+use rustc_type_ir::TyKind::*;
 
 pub mod inhabited_predicate;
 
diff --git a/compiler/rustc_middle/src/ty/instance.rs b/compiler/rustc_middle/src/ty/instance.rs
index 0b0a708e4..cebefbccc 100644
--- a/compiler/rustc_middle/src/ty/instance.rs
+++ b/compiler/rustc_middle/src/ty/instance.rs
@@ -36,7 +36,7 @@ pub enum InstanceDef<'tcx> {
     /// This includes:
     /// - `fn` items
     /// - closures
-    /// - generators
+    /// - coroutines
     Item(DefId),
 
     /// An intrinsic `fn` item (with `"rust-intrinsic"` or `"platform-intrinsic"` ABI).
@@ -245,16 +245,15 @@ impl<'tcx> InstanceDef<'tcx> {
             // drops of `Option::None` before LTO. We also respect the intent of
             // `#[inline]` on `Drop::drop` implementations.
             return ty.ty_adt_def().map_or(true, |adt_def| {
-                adt_def.destructor(tcx).map_or_else(
-                    || adt_def.is_enum(),
-                    |dtor| tcx.codegen_fn_attrs(dtor.did).requests_inline(),
-                )
+                adt_def
+                    .destructor(tcx)
+                    .map_or_else(|| adt_def.is_enum(), |dtor| tcx.cross_crate_inlinable(dtor.did))
             });
         }
         if let ty::InstanceDef::ThreadLocalShim(..) = *self {
             return false;
         }
-        tcx.codegen_fn_attrs(self.def_id()).requests_inline()
+        tcx.cross_crate_inlinable(self.def_id())
     }
 
     pub fn requires_caller_location(&self, tcx: TyCtxt<'_>) -> bool {
@@ -299,9 +298,9 @@ fn fmt_instance(
     ty::tls::with(|tcx| {
         let args = tcx.lift(instance.args).expect("could not lift for printing");
 
-        let s = FmtPrinter::new_with_limit(tcx, Namespace::ValueNS, type_length)
-            .print_def_path(instance.def_id(), args)?
-            .into_buffer();
+        let mut cx = FmtPrinter::new_with_limit(tcx, Namespace::ValueNS, type_length);
+        cx.print_def_path(instance.def_id(), args)?;
+        let s = cx.into_buffer();
         f.write_str(&s)
     })?;
 
@@ -617,12 +616,17 @@ impl<'tcx> Instance<'tcx> {
         v: EarlyBinder<T>,
     ) -> Result<T, NormalizationError<'tcx>>
     where
-        T: TypeFoldable<TyCtxt<'tcx>> + Clone,
+        T: TypeFoldable<TyCtxt<'tcx>>,
     {
         if let Some(args) = self.args_for_mir_body() {
             tcx.try_instantiate_and_normalize_erasing_regions(args, param_env, v)
         } else {
-            tcx.try_normalize_erasing_regions(param_env, v.skip_binder())
+            // We're using `instantiate_identity` as e.g.
+            // `FnPtrShim` is separately generated for every
+            // instantiation of the `FnDef`, so the MIR body
+            // is already instantiated. Any generic parameters it
+            // contains are generic parameters from the caller.
+            tcx.try_normalize_erasing_regions(param_env, v.instantiate_identity())
         }
     }
 
@@ -649,15 +653,15 @@ fn polymorphize<'tcx>(
     let unused = tcx.unused_generic_params(instance);
     debug!("polymorphize: unused={:?}", unused);
 
-    // If this is a closure or generator then we need to handle the case where another closure
+    // If this is a closure or coroutine then we need to handle the case where another closure
     // from the function is captured as an upvar and hasn't been polymorphized. In this case,
     // the unpolymorphized upvar closure would result in a polymorphized closure producing
     // multiple mono items (and eventually symbol clashes).
     let def_id = instance.def_id();
     let upvars_ty = if tcx.is_closure(def_id) {
         Some(args.as_closure().tupled_upvars_ty())
-    } else if tcx.type_of(def_id).skip_binder().is_generator() {
-        Some(args.as_generator().tupled_upvars_ty())
+    } else if tcx.type_of(def_id).skip_binder().is_coroutine() {
+        Some(args.as_coroutine().tupled_upvars_ty())
     } else {
         None
     };
@@ -685,13 +689,13 @@ fn polymorphize<'tcx>(
                         Ty::new_closure(self.tcx, def_id, polymorphized_args)
                     }
                 }
-                ty::Generator(def_id, args, movability) => {
+                ty::Coroutine(def_id, args, movability) => {
                     let polymorphized_args =
                         polymorphize(self.tcx, ty::InstanceDef::Item(def_id), args);
                     if args == polymorphized_args {
                         ty
                     } else {
-                        Ty::new_generator(self.tcx, def_id, polymorphized_args, movability)
+                        Ty::new_coroutine(self.tcx, def_id, polymorphized_args, movability)
                     }
                 }
                 _ => ty.super_fold_with(self),
@@ -711,7 +715,7 @@ fn polymorphize<'tcx>(
                 upvars_ty == Some(args[param.index as usize].expect_ty()) => {
                     // ..then double-check that polymorphization marked it used..
                     debug_assert!(!is_unused);
-                    // ..and polymorphize any closures/generators captured as upvars.
+                    // ..and polymorphize any closures/coroutines captured as upvars.
                     let upvars_ty = upvars_ty.unwrap();
                     let polymorphized_upvars_ty = upvars_ty.fold_with(
                         &mut PolymorphizationFolder { tcx });
diff --git a/compiler/rustc_middle/src/ty/layout.rs b/compiler/rustc_middle/src/ty/layout.rs
index bccf5e839..4223e503f 100644
--- a/compiler/rustc_middle/src/ty/layout.rs
+++ b/compiler/rustc_middle/src/ty/layout.rs
@@ -809,7 +809,7 @@ where
                 | ty::FnPtr(_)
                 | ty::Never
                 | ty::FnDef(..)
-                | ty::GeneratorWitness(..)
+                | ty::CoroutineWitness(..)
                 | ty::Foreign(..)
                 | ty::Dynamic(_, _, ty::Dyn) => {
                     bug!("TyAndLayout::field({:?}): not applicable", this)
@@ -868,7 +868,7 @@ where
                     {
                         let metadata = tcx.normalize_erasing_regions(
                             cx.param_env(),
-                            Ty::new_projection(tcx,metadata_def_id, [pointee]),
+                            Ty::new_projection(tcx, metadata_def_id, [pointee]),
                         );
 
                         // Map `Metadata = DynMetadata<dyn Trait>` back to a vtable, since it
@@ -898,16 +898,16 @@ where
                 ty::Array(element, _) | ty::Slice(element) => TyMaybeWithLayout::Ty(element),
                 ty::Str => TyMaybeWithLayout::Ty(tcx.types.u8),
 
-                // Tuples, generators and closures.
+                // Tuples, coroutines and closures.
                 ty::Closure(_, ref args) => field_ty_or_layout(
                     TyAndLayout { ty: args.as_closure().tupled_upvars_ty(), ..this },
                     cx,
                     i,
                 ),
 
-                ty::Generator(def_id, ref args, _) => match this.variants {
+                ty::Coroutine(def_id, ref args, _) => match this.variants {
                     Variants::Single { index } => TyMaybeWithLayout::Ty(
-                        args.as_generator()
+                        args.as_coroutine()
                             .state_tys(def_id, tcx)
                             .nth(index.as_usize())
                             .unwrap()
@@ -918,7 +918,7 @@ where
                         if i == tag_field {
                             return TyMaybeWithLayout::TyAndLayout(tag_layout(tag));
                         }
-                        TyMaybeWithLayout::Ty(args.as_generator().prefix_tys()[i])
+                        TyMaybeWithLayout::Ty(args.as_coroutine().prefix_tys()[i])
                     }
                 },
 
diff --git a/compiler/rustc_middle/src/ty/list.rs b/compiler/rustc_middle/src/ty/list.rs
index 7a32cfb10..4f9c9d857 100644
--- a/compiler/rustc_middle/src/ty/list.rs
+++ b/compiler/rustc_middle/src/ty/list.rs
@@ -1,7 +1,7 @@
 use crate::arena::Arena;
 use rustc_data_structures::aligned::{align_of, Aligned};
 use rustc_serialize::{Encodable, Encoder};
-use rustc_type_ir::{InferCtxtLike, OptWithInfcx};
+use rustc_type_ir::{InferCtxtLike, WithInfcx};
 use std::alloc::Layout;
 use std::cmp::Ordering;
 use std::fmt;
@@ -121,8 +121,8 @@ impl<T: fmt::Debug> fmt::Debug for List<T> {
     }
 }
 impl<'tcx, T: super::DebugWithInfcx<TyCtxt<'tcx>>> super::DebugWithInfcx<TyCtxt<'tcx>> for List<T> {
-    fn fmt<InfCtx: InferCtxtLike<TyCtxt<'tcx>>>(
-        this: OptWithInfcx<'_, TyCtxt<'tcx>, InfCtx, &Self>,
+    fn fmt<Infcx: InferCtxtLike<Interner = TyCtxt<'tcx>>>(
+        this: WithInfcx<'_, Infcx, &Self>,
         f: &mut core::fmt::Formatter<'_>,
     ) -> core::fmt::Result {
         fmt::Debug::fmt(&this.map(|this| this.as_slice()), f)
diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs
index aa1e7f216..e1c616ba0 100644
--- a/compiler/rustc_middle/src/ty/mod.rs
+++ b/compiler/rustc_middle/src/ty/mod.rs
@@ -20,7 +20,7 @@ pub use self::Variance::*;
 use crate::error::{OpaqueHiddenTypeMismatch, TypeMismatchReason};
 use crate::metadata::ModChild;
 use crate::middle::privacy::EffectiveVisibilities;
-use crate::mir::{Body, GeneratorLayout};
+use crate::mir::{Body, CoroutineLayout};
 use crate::query::Providers;
 use crate::traits::{self, Reveal};
 use crate::ty;
@@ -54,7 +54,7 @@ use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::{ExpnId, ExpnKind, Span};
 use rustc_target::abi::{Align, FieldIdx, Integer, IntegerType, VariantIdx};
 pub use rustc_target::abi::{ReprFlags, ReprOptions};
-pub use rustc_type_ir::{DebugWithInfcx, InferCtxtLike, OptWithInfcx};
+pub use rustc_type_ir::{DebugWithInfcx, InferCtxtLike, WithInfcx};
 pub use vtable::*;
 
 use std::fmt::Debug;
@@ -97,17 +97,17 @@ pub use self::rvalue_scopes::RvalueScopes;
 pub use self::sty::BoundRegionKind::*;
 pub use self::sty::{
     AliasTy, Article, Binder, BoundRegion, BoundRegionKind, BoundTy, BoundTyKind, BoundVar,
-    BoundVariableKind, CanonicalPolyFnSig, ClosureArgs, ClosureArgsParts, ConstKind, ConstVid,
-    EarlyBoundRegion, EffectVid, ExistentialPredicate, ExistentialProjection, ExistentialTraitRef,
-    FnSig, FreeRegion, GenSig, GeneratorArgs, GeneratorArgsParts, InlineConstArgs,
+    BoundVariableKind, CanonicalPolyFnSig, ClauseKind, ClosureArgs, ClosureArgsParts, ConstKind,
+    ConstVid, CoroutineArgs, CoroutineArgsParts, EarlyBoundRegion, EffectVid, ExistentialPredicate,
+    ExistentialProjection, ExistentialTraitRef, FnSig, FreeRegion, GenSig, InlineConstArgs,
     InlineConstArgsParts, ParamConst, ParamTy, PolyExistentialPredicate, PolyExistentialProjection,
-    PolyExistentialTraitRef, PolyFnSig, PolyGenSig, PolyTraitRef, Region, RegionKind, RegionVid,
-    TraitRef, TyKind, TypeAndMut, UpvarArgs, VarianceDiagInfo,
+    PolyExistentialTraitRef, PolyFnSig, PolyGenSig, PolyTraitRef, PredicateKind, Region,
+    RegionKind, RegionVid, TraitRef, TyKind, TypeAndMut, UpvarArgs, VarianceDiagInfo,
 };
 pub use self::trait_def::TraitDef;
 pub use self::typeck_results::{
-    CanonicalUserType, CanonicalUserTypeAnnotation, CanonicalUserTypeAnnotations, TypeckResults,
-    UserType, UserTypeAnnotationIndex,
+    CanonicalUserType, CanonicalUserTypeAnnotation, CanonicalUserTypeAnnotations, IsIdentity,
+    TypeckResults, UserType, UserTypeAnnotationIndex,
 };
 
 pub mod _match;
@@ -233,6 +233,7 @@ impl MainDefinition {
 #[derive(Clone, Debug, TypeFoldable, TypeVisitable)]
 pub struct ImplHeader<'tcx> {
     pub impl_def_id: DefId,
+    pub impl_args: ty::GenericArgsRef<'tcx>,
     pub self_ty: Ty<'tcx>,
     pub trait_ref: Option<TraitRef<'tcx>>,
     pub predicates: Vec<Predicate<'tcx>>,
@@ -578,11 +579,6 @@ impl rustc_errors::IntoDiagnosticArg for Clause<'_> {
 pub struct Clause<'tcx>(Interned<'tcx, WithCachedTypeInfo<ty::Binder<'tcx, PredicateKind<'tcx>>>>);
 
 impl<'tcx> Clause<'tcx> {
-    pub fn from_projection_clause(tcx: TyCtxt<'tcx>, pred: PolyProjectionPredicate<'tcx>) -> Self {
-        let pred: Predicate<'tcx> = pred.to_predicate(tcx);
-        pred.expect_clause()
-    }
-
     pub fn as_predicate(self) -> Predicate<'tcx> {
         Predicate(self.0)
     }
@@ -631,98 +627,6 @@ impl<'tcx> Clause<'tcx> {
     }
 }
 
-#[derive(Clone, Copy, PartialEq, Eq, Hash, TyEncodable, TyDecodable)]
-#[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
-/// A clause is something that can appear in where bounds or be inferred
-/// by implied bounds.
-pub enum ClauseKind<'tcx> {
-    /// Corresponds to `where Foo: Bar<A, B, C>`. `Foo` here would be
-    /// the `Self` type of the trait reference and `A`, `B`, and `C`
-    /// would be the type parameters.
-    Trait(TraitPredicate<'tcx>),
-
-    /// `where 'a: 'b`
-    RegionOutlives(RegionOutlivesPredicate<'tcx>),
-
-    /// `where T: 'a`
-    TypeOutlives(TypeOutlivesPredicate<'tcx>),
-
-    /// `where <T as TraitRef>::Name == X`, approximately.
-    /// See the `ProjectionPredicate` struct for details.
-    Projection(ProjectionPredicate<'tcx>),
-
-    /// Ensures that a const generic argument to a parameter `const N: u8`
-    /// is of type `u8`.
-    ConstArgHasType(Const<'tcx>, Ty<'tcx>),
-
-    /// No syntax: `T` well-formed.
-    WellFormed(GenericArg<'tcx>),
-
-    /// Constant initializer must evaluate successfully.
-    ConstEvaluatable(ty::Const<'tcx>),
-}
-
-#[derive(Clone, Copy, PartialEq, Eq, Hash, TyEncodable, TyDecodable)]
-#[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
-pub enum PredicateKind<'tcx> {
-    /// Prove a clause
-    Clause(ClauseKind<'tcx>),
-
-    /// Trait must be object-safe.
-    ObjectSafe(DefId),
-
-    /// No direct syntax. May be thought of as `where T: FnFoo<...>`
-    /// for some generic args `...` and `T` being a closure type.
-    /// Satisfied (or refuted) once we know the closure's kind.
-    ClosureKind(DefId, GenericArgsRef<'tcx>, ClosureKind),
-
-    /// `T1 <: T2`
-    ///
-    /// This obligation is created most often when we have two
-    /// unresolved type variables and hence don't have enough
-    /// information to process the subtyping obligation yet.
-    Subtype(SubtypePredicate<'tcx>),
-
-    /// `T1` coerced to `T2`
-    ///
-    /// Like a subtyping obligation, this is created most often
-    /// when we have two unresolved type variables and hence
-    /// don't have enough information to process the coercion
-    /// obligation yet. At the moment, we actually process coercions
-    /// very much like subtyping and don't handle the full coercion
-    /// logic.
-    Coerce(CoercePredicate<'tcx>),
-
-    /// Constants must be equal. The first component is the const that is expected.
-    ConstEquate(Const<'tcx>, Const<'tcx>),
-
-    /// A marker predicate that is always ambiguous.
-    /// Used for coherence to mark opaque types as possibly equal to each other but ambiguous.
-    Ambiguous,
-
-    /// Separate from `ClauseKind::Projection` which is used for normalization in new solver.
-    /// This predicate requires two terms to be equal to eachother.
-    ///
-    /// Only used for new solver
-    AliasRelate(Term<'tcx>, Term<'tcx>, AliasRelationDirection),
-}
-
-#[derive(Clone, Copy, PartialEq, Eq, Hash, TyEncodable, TyDecodable)]
-#[derive(HashStable, Debug)]
-pub enum AliasRelationDirection {
-    Equate,
-    Subtype,
-}
-
-impl std::fmt::Display for AliasRelationDirection {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            AliasRelationDirection::Equate => write!(f, "=="),
-            AliasRelationDirection::Subtype => write!(f, "<:"),
-        }
-    }
-}
-
 /// The crate outlives map is computed during typeck and contains the
 /// outlives of every item in the local crate. You should not use it
 /// directly, because to do so will make your pass dependent on the
@@ -1028,7 +932,7 @@ impl<'tcx> Term<'tcx> {
                 _ => None,
             },
             TermKind::Const(ct) => match ct.kind() {
-                ConstKind::Unevaluated(uv) => Some(tcx.mk_alias_ty(uv.def, uv.args)),
+                ConstKind::Unevaluated(uv) => Some(AliasTy::new(tcx, uv.def, uv.args)),
                 _ => None,
             },
         }
@@ -1089,19 +993,19 @@ impl ParamTerm {
 }
 
 #[derive(Copy, Clone, Eq, PartialEq, Debug)]
-pub enum TermVid<'tcx> {
+pub enum TermVid {
     Ty(ty::TyVid),
-    Const(ty::ConstVid<'tcx>),
+    Const(ty::ConstVid),
 }
 
-impl From<ty::TyVid> for TermVid<'_> {
+impl From<ty::TyVid> for TermVid {
     fn from(value: ty::TyVid) -> Self {
         TermVid::Ty(value)
     }
 }
 
-impl<'tcx> From<ty::ConstVid<'tcx>> for TermVid<'tcx> {
-    fn from(value: ty::ConstVid<'tcx>) -> Self {
+impl From<ty::ConstVid> for TermVid {
+    fn from(value: ty::ConstVid) -> Self {
         TermVid::Const(value)
     }
 }
@@ -1296,12 +1200,25 @@ impl<'tcx> ToPredicate<'tcx, PolyTraitPredicate<'tcx>> for Binder<'tcx, TraitRef
     }
 }
 
+impl<'tcx> ToPredicate<'tcx> for TraitPredicate<'tcx> {
+    fn to_predicate(self, tcx: TyCtxt<'tcx>) -> Predicate<'tcx> {
+        PredicateKind::Clause(ClauseKind::Trait(self)).to_predicate(tcx)
+    }
+}
+
 impl<'tcx> ToPredicate<'tcx> for PolyTraitPredicate<'tcx> {
     fn to_predicate(self, tcx: TyCtxt<'tcx>) -> Predicate<'tcx> {
         self.map_bound(|p| PredicateKind::Clause(ClauseKind::Trait(p))).to_predicate(tcx)
     }
 }
 
+impl<'tcx> ToPredicate<'tcx, Clause<'tcx>> for TraitPredicate<'tcx> {
+    fn to_predicate(self, tcx: TyCtxt<'tcx>) -> Clause<'tcx> {
+        let p: Predicate<'tcx> = self.to_predicate(tcx);
+        p.expect_clause()
+    }
+}
+
 impl<'tcx> ToPredicate<'tcx, Clause<'tcx>> for PolyTraitPredicate<'tcx> {
     fn to_predicate(self, tcx: TyCtxt<'tcx>) -> Clause<'tcx> {
         let p: Predicate<'tcx> = self.to_predicate(tcx);
@@ -1340,9 +1257,10 @@ impl<'tcx> ToPredicate<'tcx, Clause<'tcx>> for ProjectionPredicate<'tcx> {
     }
 }
 
-impl<'tcx> ToPredicate<'tcx> for TraitPredicate<'tcx> {
-    fn to_predicate(self, tcx: TyCtxt<'tcx>) -> Predicate<'tcx> {
-        PredicateKind::Clause(ClauseKind::Trait(self)).to_predicate(tcx)
+impl<'tcx> ToPredicate<'tcx, Clause<'tcx>> for PolyProjectionPredicate<'tcx> {
+    fn to_predicate(self, tcx: TyCtxt<'tcx>) -> Clause<'tcx> {
+        let p: Predicate<'tcx> = self.to_predicate(tcx);
+        p.expect_clause()
     }
 }
 
@@ -1609,7 +1527,7 @@ pub struct BoundConst<'tcx> {
     pub ty: Ty<'tcx>,
 }
 
-pub type PlaceholderConst<'tcx> = Placeholder<BoundVar>;
+pub type PlaceholderConst = Placeholder<BoundVar>;
 
 /// When type checking, we use the `ParamEnv` to track
 /// details about the set of where-clauses that are in scope at this
@@ -1743,30 +1661,9 @@ impl<'tcx> ParamEnv<'tcx> {
         Self::new(List::empty(), self.reveal())
     }
 
-    /// Creates a suitable environment in which to perform trait
-    /// queries on the given value. When type-checking, this is simply
-    /// the pair of the environment plus value. But when reveal is set to
-    /// All, then if `value` does not reference any type parameters, we will
-    /// pair it with the empty environment. This improves caching and is generally
-    /// invisible.
-    ///
-    /// N.B., we preserve the environment when type-checking because it
-    /// is possible for the user to have wacky where-clauses like
-    /// `where Box<u32>: Copy`, which are clearly never
-    /// satisfiable. We generally want to behave as if they were true,
-    /// although the surrounding function is never reachable.
+    /// Creates a pair of param-env and value for use in queries.
     pub fn and<T: TypeVisitable<TyCtxt<'tcx>>>(self, value: T) -> ParamEnvAnd<'tcx, T> {
-        match self.reveal() {
-            Reveal::UserFacing => ParamEnvAnd { param_env: self, value },
-
-            Reveal::All => {
-                if value.is_global() {
-                    ParamEnvAnd { param_env: self.without_caller_bounds(), value }
-                } else {
-                    ParamEnvAnd { param_env: self, value }
-                }
-            }
-        }
+        ParamEnvAnd { param_env: self, value }
     }
 }
 
@@ -2114,7 +2011,7 @@ impl<'tcx> TyCtxt<'tcx> {
 
         // Generate a deterministically-derived seed from the item's path hash
         // to allow for cross-crate compilation to actually work
-        let mut field_shuffle_seed = self.def_path_hash(did).0.to_smaller_hash();
+        let mut field_shuffle_seed = self.def_path_hash(did).0.to_smaller_hash().as_u64();
 
         // If the user defined a custom seed for layout randomization, xor the item's
         // path hash with the user defined seed, this will allowing determinism while
@@ -2433,10 +2330,10 @@ impl<'tcx> TyCtxt<'tcx> {
         self.def_kind(trait_def_id) == DefKind::TraitAlias
     }
 
-    /// Returns layout of a generator. Layout might be unavailable if the
-    /// generator is tainted by errors.
-    pub fn generator_layout(self, def_id: DefId) -> Option<&'tcx GeneratorLayout<'tcx>> {
-        self.optimized_mir(def_id).generator_layout()
+    /// Returns layout of a coroutine. Layout might be unavailable if the
+    /// coroutine is tainted by errors.
+    pub fn coroutine_layout(self, def_id: DefId) -> Option<&'tcx CoroutineLayout<'tcx>> {
+        self.optimized_mir(def_id).coroutine_layout()
     }
 
     /// Given the `DefId` of an impl, returns the `DefId` of the trait it implements.
diff --git a/compiler/rustc_middle/src/ty/opaque_types.rs b/compiler/rustc_middle/src/ty/opaque_types.rs
index 6491936c2..8d895732d 100644
--- a/compiler/rustc_middle/src/ty/opaque_types.rs
+++ b/compiler/rustc_middle/src/ty/opaque_types.rs
@@ -152,14 +152,14 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for ReverseMapper<'tcx> {
                 Ty::new_closure(self.tcx, def_id, args)
             }
 
-            ty::Generator(def_id, args, movability) => {
+            ty::Coroutine(def_id, args, movability) => {
                 let args = self.fold_closure_args(def_id, args);
-                Ty::new_generator(self.tcx, def_id, args, movability)
+                Ty::new_coroutine(self.tcx, def_id, args, movability)
             }
 
-            ty::GeneratorWitness(def_id, args) => {
+            ty::CoroutineWitness(def_id, args) => {
                 let args = self.fold_closure_args(def_id, args);
-                Ty::new_generator_witness(self.tcx, def_id, args)
+                Ty::new_coroutine_witness(self.tcx, def_id, args)
             }
 
             ty::Param(param) => {
diff --git a/compiler/rustc_middle/src/ty/parameterized.rs b/compiler/rustc_middle/src/ty/parameterized.rs
index 9aa673e44..9afa50cf5 100644
--- a/compiler/rustc_middle/src/ty/parameterized.rs
+++ b/compiler/rustc_middle/src/ty/parameterized.rs
@@ -82,7 +82,7 @@ trivially_parameterized_over_tcx! {
     rustc_attr::Stability,
     rustc_hir::Constness,
     rustc_hir::Defaultness,
-    rustc_hir::GeneratorKind,
+    rustc_hir::CoroutineKind,
     rustc_hir::IsAsync,
     rustc_hir::LangItem,
     rustc_hir::def::DefKind,
@@ -123,7 +123,7 @@ macro_rules! parameterized_over_tcx {
 parameterized_over_tcx! {
     crate::middle::exported_symbols::ExportedSymbol,
     crate::mir::Body,
-    crate::mir::GeneratorLayout,
+    crate::mir::CoroutineLayout,
     ty::Ty,
     ty::FnSig,
     ty::GenericPredicates,
diff --git a/compiler/rustc_middle/src/ty/print/mod.rs b/compiler/rustc_middle/src/ty/print/mod.rs
index aa8e2e307..6bbc8f70f 100644
--- a/compiler/rustc_middle/src/ty/print/mod.rs
+++ b/compiler/rustc_middle/src/ty/print/mod.rs
@@ -10,13 +10,10 @@ use rustc_hir::definitions::{DefPathData, DisambiguatedDefPathData};
 mod pretty;
 pub use self::pretty::*;
 
-// FIXME(eddyb) false positive, the lifetime parameters are used with `P:  Printer<...>`.
-#[allow(unused_lifetimes)]
-pub trait Print<'tcx, P> {
-    type Output;
-    type Error;
+pub type PrintError = std::fmt::Error;
 
-    fn print(&self, cx: P) -> Result<Self::Output, Self::Error>;
+pub trait Print<'tcx, P> {
+    fn print(&self, cx: &mut P) -> Result<(), PrintError>;
 }
 
 /// Interface for outputting user-facing "type-system entities"
@@ -29,81 +26,73 @@ pub trait Print<'tcx, P> {
 //
 // FIXME(eddyb) find a better name; this is more general than "printing".
 pub trait Printer<'tcx>: Sized {
-    type Error;
-
-    type Path;
-    type Region;
-    type Type;
-    type DynExistential;
-    type Const;
-
     fn tcx<'a>(&'a self) -> TyCtxt<'tcx>;
 
     fn print_def_path(
-        self,
+        &mut self,
         def_id: DefId,
         args: &'tcx [GenericArg<'tcx>],
-    ) -> Result<Self::Path, Self::Error> {
+    ) -> Result<(), PrintError> {
         self.default_print_def_path(def_id, args)
     }
 
     fn print_impl_path(
-        self,
+        &mut self,
         impl_def_id: DefId,
         args: &'tcx [GenericArg<'tcx>],
         self_ty: Ty<'tcx>,
         trait_ref: Option<ty::TraitRef<'tcx>>,
-    ) -> Result<Self::Path, Self::Error> {
+    ) -> Result<(), PrintError> {
         self.default_print_impl_path(impl_def_id, args, self_ty, trait_ref)
     }
 
-    fn print_region(self, region: ty::Region<'tcx>) -> Result<Self::Region, Self::Error>;
+    fn print_region(&mut self, region: ty::Region<'tcx>) -> Result<(), PrintError>;
 
-    fn print_type(self, ty: Ty<'tcx>) -> Result<Self::Type, Self::Error>;
+    fn print_type(&mut self, ty: Ty<'tcx>) -> Result<(), PrintError>;
 
     fn print_dyn_existential(
-        self,
+        &mut self,
         predicates: &'tcx ty::List<ty::PolyExistentialPredicate<'tcx>>,
-    ) -> Result<Self::DynExistential, Self::Error>;
+    ) -> Result<(), PrintError>;
 
-    fn print_const(self, ct: ty::Const<'tcx>) -> Result<Self::Const, Self::Error>;
+    fn print_const(&mut self, ct: ty::Const<'tcx>) -> Result<(), PrintError>;
 
-    fn path_crate(self, cnum: CrateNum) -> Result<Self::Path, Self::Error>;
+    fn path_crate(&mut self, cnum: CrateNum) -> Result<(), PrintError>;
 
     fn path_qualified(
-        self,
+        &mut self,
         self_ty: Ty<'tcx>,
         trait_ref: Option<ty::TraitRef<'tcx>>,
-    ) -> Result<Self::Path, Self::Error>;
+    ) -> Result<(), PrintError>;
 
     fn path_append_impl(
-        self,
-        print_prefix: impl FnOnce(Self) -> Result<Self::Path, Self::Error>,
+        &mut self,
+        print_prefix: impl FnOnce(&mut Self) -> Result<(), PrintError>,
         disambiguated_data: &DisambiguatedDefPathData,
         self_ty: Ty<'tcx>,
         trait_ref: Option<ty::TraitRef<'tcx>>,
-    ) -> Result<Self::Path, Self::Error>;
+    ) -> Result<(), PrintError>;
 
     fn path_append(
-        self,
-        print_prefix: impl FnOnce(Self) -> Result<Self::Path, Self::Error>,
+        &mut self,
+        print_prefix: impl FnOnce(&mut Self) -> Result<(), PrintError>,
         disambiguated_data: &DisambiguatedDefPathData,
-    ) -> Result<Self::Path, Self::Error>;
+    ) -> Result<(), PrintError>;
 
     fn path_generic_args(
-        self,
-        print_prefix: impl FnOnce(Self) -> Result<Self::Path, Self::Error>,
+        &mut self,
+        print_prefix: impl FnOnce(&mut Self) -> Result<(), PrintError>,
         args: &[GenericArg<'tcx>],
-    ) -> Result<Self::Path, Self::Error>;
+    ) -> Result<(), PrintError>;
 
     // Defaults (should not be overridden):
 
     #[instrument(skip(self), level = "debug")]
     fn default_print_def_path(
-        self,
+        &mut self,
         def_id: DefId,
         args: &'tcx [GenericArg<'tcx>],
-    ) -> Result<Self::Path, Self::Error> {
+    ) -> Result<(), PrintError> {
         let key = self.tcx().def_key(def_id);
         debug!(?key);
 
@@ -170,7 +159,7 @@ pub trait Printer<'tcx>: Sized {
                 }
 
                 self.path_append(
-                    |cx: Self| {
+                    |cx: &mut Self| {
                         if trait_qualify_parent {
                             let trait_ref = ty::TraitRef::new(
                                 cx.tcx(),
@@ -189,12 +178,12 @@ pub trait Printer<'tcx>: Sized {
     }
 
     fn default_print_impl_path(
-        self,
+        &mut self,
         impl_def_id: DefId,
         _args: &'tcx [GenericArg<'tcx>],
         self_ty: Ty<'tcx>,
         impl_trait_ref: Option<ty::TraitRef<'tcx>>,
-    ) -> Result<Self::Path, Self::Error> {
+    ) -> Result<(), PrintError> {
         debug!(
             "default_print_impl_path: impl_def_id={:?}, self_ty={}, impl_trait_ref={:?}",
             impl_def_id, self_ty, impl_trait_ref
@@ -270,8 +259,8 @@ fn characteristic_def_id_of_type_cached<'a>(
 
         ty::FnDef(def_id, _)
         | ty::Closure(def_id, _)
-        | ty::Generator(def_id, _, _)
-        | ty::GeneratorWitness(def_id, _)
+        | ty::Coroutine(def_id, _, _)
+        | ty::CoroutineWitness(def_id, _)
         | ty::Foreign(def_id) => Some(def_id),
 
         ty::Bool
@@ -295,34 +284,25 @@ pub fn characteristic_def_id_of_type(ty: Ty<'_>) -> Option<DefId> {
 }
 
 impl<'tcx, P: Printer<'tcx>> Print<'tcx, P> for ty::Region<'tcx> {
-    type Output = P::Region;
-    type Error = P::Error;
-    fn print(&self, cx: P) -> Result<Self::Output, Self::Error> {
+    fn print(&self, cx: &mut P) -> Result<(), PrintError> {
         cx.print_region(*self)
     }
 }
 
 impl<'tcx, P: Printer<'tcx>> Print<'tcx, P> for Ty<'tcx> {
-    type Output = P::Type;
-    type Error = P::Error;
-
-    fn print(&self, cx: P) -> Result<Self::Output, Self::Error> {
+    fn print(&self, cx: &mut P) -> Result<(), PrintError> {
         cx.print_type(*self)
     }
 }
 
 impl<'tcx, P: Printer<'tcx>> Print<'tcx, P> for &'tcx ty::List<ty::PolyExistentialPredicate<'tcx>> {
-    type Output = P::DynExistential;
-    type Error = P::Error;
-    fn print(&self, cx: P) -> Result<Self::Output, Self::Error> {
+    fn print(&self, cx: &mut P) -> Result<(), PrintError> {
         cx.print_dyn_existential(self)
     }
 }
 
 impl<'tcx, P: Printer<'tcx>> Print<'tcx, P> for ty::Const<'tcx> {
-    type Output = P::Const;
-    type Error = P::Error;
-    fn print(&self, cx: P) -> Result<Self::Output, Self::Error> {
+    fn print(&self, cx: &mut P) -> Result<(), PrintError> {
         cx.print_const(*self)
     }
 }
diff --git a/compiler/rustc_middle/src/ty/print/pretty.rs b/compiler/rustc_middle/src/ty/print/pretty.rs
index 2d7350387..baf160bcc 100644
--- a/compiler/rustc_middle/src/ty/print/pretty.rs
+++ b/compiler/rustc_middle/src/ty/print/pretty.rs
@@ -1,6 +1,7 @@
 use crate::mir::interpret::{AllocRange, GlobalAlloc, Pointer, Provenance, Scalar};
 use crate::query::IntoQueryParam;
 use crate::query::Providers;
+use crate::traits::util::supertraits_for_pretty_printing;
 use crate::ty::{
     self, ConstInt, ParamConst, ScalarInt, Term, TermKind, Ty, TyCtxt, TypeFoldable,
     TypeSuperFoldable, TypeSuperVisitable, TypeVisitable, TypeVisitableExt,
@@ -41,10 +42,10 @@ macro_rules! p {
         write!(scoped_cx!(), $($data),+)?
     };
     (@print($x:expr)) => {
-        scoped_cx!() = $x.print(scoped_cx!())?
+        $x.print(scoped_cx!())?
     };
     (@$method:ident($($arg:expr),*)) => {
-        scoped_cx!() = scoped_cx!().$method($($arg),*)?
+        scoped_cx!().$method($($arg),*)?
     };
     ($($elem:tt $(($($args:tt)*))?),+) => {{
         $(p!(@ $elem $(($($args)*))?);)+
@@ -52,7 +53,6 @@ macro_rules! p {
 }
 macro_rules! define_scoped_cx {
     ($cx:ident) => {
-        #[allow(unused_macros)]
         macro_rules! scoped_cx {
             () => {
                 $cx
@@ -205,79 +205,69 @@ impl<'tcx> RegionHighlightMode<'tcx> {
 }
 
 /// Trait for printers that pretty-print using `fmt::Write` to the printer.
-pub trait PrettyPrinter<'tcx>:
-    Printer<
-        'tcx,
-        Error = fmt::Error,
-        Path = Self,
-        Region = Self,
-        Type = Self,
-        DynExistential = Self,
-        Const = Self,
-    > + fmt::Write
-{
+pub trait PrettyPrinter<'tcx>: Printer<'tcx> + fmt::Write {
     /// Like `print_def_path` but for value paths.
     fn print_value_path(
-        self,
+        &mut self,
         def_id: DefId,
         args: &'tcx [GenericArg<'tcx>],
-    ) -> Result<Self::Path, Self::Error> {
+    ) -> Result<(), PrintError> {
         self.print_def_path(def_id, args)
     }
 
-    fn in_binder<T>(self, value: &ty::Binder<'tcx, T>) -> Result<Self, Self::Error>
+    fn in_binder<T>(&mut self, value: &ty::Binder<'tcx, T>) -> Result<(), PrintError>
     where
-        T: Print<'tcx, Self, Output = Self, Error = Self::Error> + TypeFoldable<TyCtxt<'tcx>>,
+        T: Print<'tcx, Self> + TypeFoldable<TyCtxt<'tcx>>,
     {
         value.as_ref().skip_binder().print(self)
     }
 
-    fn wrap_binder<T, F: FnOnce(&T, Self) -> Result<Self, fmt::Error>>(
-        self,
+    fn wrap_binder<T, F: FnOnce(&T, &mut Self) -> Result<(), fmt::Error>>(
+        &mut self,
         value: &ty::Binder<'tcx, T>,
         f: F,
-    ) -> Result<Self, Self::Error>
+    ) -> Result<(), PrintError>
     where
-        T: Print<'tcx, Self, Output = Self, Error = Self::Error> + TypeFoldable<TyCtxt<'tcx>>,
+        T: Print<'tcx, Self> + TypeFoldable<TyCtxt<'tcx>>,
     {
         f(value.as_ref().skip_binder(), self)
     }
 
     /// Prints comma-separated elements.
-    fn comma_sep<T>(mut self, mut elems: impl Iterator<Item = T>) -> Result<Self, Self::Error>
+    fn comma_sep<T>(&mut self, mut elems: impl Iterator<Item = T>) -> Result<(), PrintError>
     where
-        T: Print<'tcx, Self, Output = Self, Error = Self::Error>,
+        T: Print<'tcx, Self>,
     {
         if let Some(first) = elems.next() {
-            self = first.print(self)?;
+            first.print(self)?;
             for elem in elems {
                 self.write_str(", ")?;
-                self = elem.print(self)?;
+                elem.print(self)?;
             }
         }
-        Ok(self)
+        Ok(())
     }
 
     /// Prints `{f: t}` or `{f as t}` depending on the `cast` argument
     fn typed_value(
-        mut self,
-        f: impl FnOnce(Self) -> Result<Self, Self::Error>,
-        t: impl FnOnce(Self) -> Result<Self, Self::Error>,
+        &mut self,
+        f: impl FnOnce(&mut Self) -> Result<(), PrintError>,
+        t: impl FnOnce(&mut Self) -> Result<(), PrintError>,
         conversion: &str,
-    ) -> Result<Self::Const, Self::Error> {
+    ) -> Result<(), PrintError> {
         self.write_str("{")?;
-        self = f(self)?;
+        f(self)?;
         self.write_str(conversion)?;
-        self = t(self)?;
+        t(self)?;
         self.write_str("}")?;
-        Ok(self)
+        Ok(())
     }
 
     /// Prints `<...>` around what `f` prints.
     fn generic_delimiters(
-        self,
-        f: impl FnOnce(Self) -> Result<Self, Self::Error>,
-    ) -> Result<Self, Self::Error>;
+        &mut self,
+        f: impl FnOnce(&mut Self) -> Result<(), PrintError>,
+    ) -> Result<(), PrintError>;
 
     /// Returns `true` if the region should be printed in
     /// optional positions, e.g., `&'a T` or `dyn Tr + 'b`.
@@ -291,9 +281,9 @@ pub trait PrettyPrinter<'tcx>:
     /// If possible, this returns a global path resolving to `def_id` that is visible
     /// from at least one local module, and returns `true`. If the crate defining `def_id` is
     /// declared with an `extern crate`, the path is guaranteed to use the `extern crate`.
-    fn try_print_visible_def_path(self, def_id: DefId) -> Result<(Self, bool), Self::Error> {
+    fn try_print_visible_def_path(&mut self, def_id: DefId) -> Result<bool, PrintError> {
         if NO_VISIBLE_PATH.with(|flag| flag.get()) {
-            return Ok((self, false));
+            return Ok(false);
         }
 
         let mut callers = Vec::new();
@@ -305,10 +295,7 @@ pub trait PrettyPrinter<'tcx>:
     // For enum variants, if they have an unique name, then we only print the name, otherwise we
     // print the enum name and the variant name. Otherwise, we do not print anything and let the
     // caller use the `print_def_path` fallback.
-    fn force_print_trimmed_def_path(
-        mut self,
-        def_id: DefId,
-    ) -> Result<(Self::Path, bool), Self::Error> {
+    fn force_print_trimmed_def_path(&mut self, def_id: DefId) -> Result<bool, PrintError> {
         let key = self.tcx().def_key(def_id);
         let visible_parent_map = self.tcx().visible_parent_map(());
         let kind = self.tcx().def_kind(def_id);
@@ -319,8 +306,7 @@ pub trait PrettyPrinter<'tcx>:
                 && let DefPathData::TypeNs(_) = key.disambiguated_data.data
                 && Some(*visible_parent) != actual_parent
             {
-                this
-                    .tcx()
+                this.tcx()
                     // FIXME(typed_def_id): Further propagate ModDefId
                     .module_children(ModDefId::new_unchecked(*visible_parent))
                     .iter()
@@ -337,7 +323,7 @@ pub trait PrettyPrinter<'tcx>:
         {
             // If `Assoc` is unique, we don't want to talk about `Trait::Assoc`.
             self.write_str(get_local_name(&self, *symbol, def_id, key).as_str())?;
-            return Ok((self, true));
+            return Ok(true);
         }
         if let Some(symbol) = key.get_opt_name() {
             if let DefKind::AssocConst | DefKind::AssocFn | DefKind::AssocTy = kind
@@ -359,44 +345,46 @@ pub trait PrettyPrinter<'tcx>:
                 // the parent type in the path. For example, `Iterator::Item`.
                 self.write_str(get_local_name(&self, symbol, parent, parent_key).as_str())?;
                 self.write_str("::")?;
-            } else if let DefKind::Struct | DefKind::Union | DefKind::Enum | DefKind::Trait
-                | DefKind::TyAlias | DefKind::Fn | DefKind::Const | DefKind::Static(_) = kind
+            } else if let DefKind::Struct
+            | DefKind::Union
+            | DefKind::Enum
+            | DefKind::Trait
+            | DefKind::TyAlias
+            | DefKind::Fn
+            | DefKind::Const
+            | DefKind::Static(_) = kind
             {
             } else {
                 // If not covered above, like for example items out of `impl` blocks, fallback.
-                return Ok((self, false));
+                return Ok(false);
             }
             self.write_str(get_local_name(&self, symbol, def_id, key).as_str())?;
-            return Ok((self, true));
+            return Ok(true);
         }
-        Ok((self, false))
+        Ok(false)
     }
 
     /// Try to see if this path can be trimmed to a unique symbol name.
-    fn try_print_trimmed_def_path(
-        mut self,
-        def_id: DefId,
-    ) -> Result<(Self::Path, bool), Self::Error> {
+    fn try_print_trimmed_def_path(&mut self, def_id: DefId) -> Result<bool, PrintError> {
         if FORCE_TRIMMED_PATH.with(|flag| flag.get()) {
-            let (s, trimmed) = self.force_print_trimmed_def_path(def_id)?;
+            let trimmed = self.force_print_trimmed_def_path(def_id)?;
             if trimmed {
-                return Ok((s, true));
+                return Ok(true);
             }
-            self = s;
         }
         if !self.tcx().sess.opts.unstable_opts.trim_diagnostic_paths
             || matches!(self.tcx().sess.opts.trimmed_def_paths, TrimmedDefPaths::Never)
             || NO_TRIMMED_PATH.with(|flag| flag.get())
             || SHOULD_PREFIX_WITH_CRATE.with(|flag| flag.get())
         {
-            return Ok((self, false));
+            return Ok(false);
         }
 
         match self.tcx().trimmed_def_paths(()).get(&def_id) {
-            None => Ok((self, false)),
+            None => Ok(false),
             Some(symbol) => {
                 write!(self, "{}", Ident::with_dummy_span(*symbol))?;
-                Ok((self, true))
+                Ok(true)
             }
         }
     }
@@ -415,19 +403,18 @@ pub trait PrettyPrinter<'tcx>:
     /// This method returns false if we can't print the visible path, so
     /// `print_def_path` can fall back on the item's real definition path.
     fn try_print_visible_def_path_recur(
-        mut self,
+        &mut self,
         def_id: DefId,
         callers: &mut Vec<DefId>,
-    ) -> Result<(Self, bool), Self::Error> {
-        define_scoped_cx!(self);
-
+    ) -> Result<bool, PrintError> {
         debug!("try_print_visible_def_path: def_id={:?}", def_id);
 
         // If `def_id` is a direct or injected extern crate, return the
         // path to the crate followed by the path to the item within the crate.
         if let Some(cnum) = def_id.as_crate_root() {
             if cnum == LOCAL_CRATE {
-                return Ok((self.path_crate(cnum)?, true));
+                self.path_crate(cnum)?;
+                return Ok(true);
             }
 
             // In local mode, when we encounter a crate other than
@@ -450,7 +437,8 @@ pub trait PrettyPrinter<'tcx>:
                         // or avoid ending up with `ExternCrateSource::Extern`,
                         // for the injected `std`/`core`.
                         if span.is_dummy() {
-                            return Ok((self.path_crate(cnum)?, true));
+                            self.path_crate(cnum)?;
+                            return Ok(true);
                         }
 
                         // Disable `try_print_trimmed_def_path` behavior within
@@ -458,23 +446,25 @@ pub trait PrettyPrinter<'tcx>:
                         // in cases where the `extern crate foo` has non-trivial
                         // parents, e.g. it's nested in `impl foo::Trait for Bar`
                         // (see also issues #55779 and #87932).
-                        self = with_no_visible_paths!(self.print_def_path(def_id, &[])?);
+                        with_no_visible_paths!(self.print_def_path(def_id, &[])?);
 
-                        return Ok((self, true));
+                        return Ok(true);
                     }
                     (ExternCrateSource::Path, LOCAL_CRATE) => {
-                        return Ok((self.path_crate(cnum)?, true));
+                        self.path_crate(cnum)?;
+                        return Ok(true);
                     }
                     _ => {}
                 },
                 None => {
-                    return Ok((self.path_crate(cnum)?, true));
+                    self.path_crate(cnum)?;
+                    return Ok(true);
                 }
             }
         }
 
         if def_id.is_local() {
-            return Ok((self, false));
+            return Ok(false);
         }
 
         let visible_parent_map = self.tcx().visible_parent_map(());
@@ -495,7 +485,7 @@ pub trait PrettyPrinter<'tcx>:
         }
 
         let Some(visible_parent) = visible_parent_map.get(&def_id).cloned() else {
-            return Ok((self, false));
+            return Ok(false);
         };
 
         let actual_parent = self.tcx().opt_parent(def_id);
@@ -558,7 +548,7 @@ pub trait PrettyPrinter<'tcx>:
                     *name = new_name;
                 } else {
                     // There is no name that is public and isn't `_`, so bail.
-                    return Ok((self, false));
+                    return Ok(false);
                 }
             }
             // Re-exported `extern crate` (#43189).
@@ -570,7 +560,7 @@ pub trait PrettyPrinter<'tcx>:
         debug!("try_print_visible_def_path: data={:?}", data);
 
         if callers.contains(&visible_parent) {
-            return Ok((self, false));
+            return Ok(false);
         }
         callers.push(visible_parent);
         // HACK(eddyb) this bypasses `path_append`'s prefix printing to avoid
@@ -578,19 +568,19 @@ pub trait PrettyPrinter<'tcx>:
         // To support printers that do not implement `PrettyPrinter`, a `Vec` or
         // linked list on the stack would need to be built, before any printing.
         match self.try_print_visible_def_path_recur(visible_parent, callers)? {
-            (cx, false) => return Ok((cx, false)),
-            (cx, true) => self = cx,
+            false => return Ok(false),
+            true => {}
         }
         callers.pop();
-
-        Ok((self.path_append(Ok, &DisambiguatedDefPathData { data, disambiguator: 0 })?, true))
+        self.path_append(|_| Ok(()), &DisambiguatedDefPathData { data, disambiguator: 0 })?;
+        Ok(true)
     }
 
     fn pretty_path_qualified(
-        self,
+        &mut self,
         self_ty: Ty<'tcx>,
         trait_ref: Option<ty::TraitRef<'tcx>>,
-    ) -> Result<Self::Path, Self::Error> {
+    ) -> Result<(), PrintError> {
         if trait_ref.is_none() {
             // Inherent impls. Try to print `Foo::bar` for an inherent
             // impl on `Foo`, but fallback to `<Foo>::bar` if self-type is
@@ -611,26 +601,26 @@ pub trait PrettyPrinter<'tcx>:
             }
         }
 
-        self.generic_delimiters(|mut cx| {
+        self.generic_delimiters(|cx| {
             define_scoped_cx!(cx);
 
             p!(print(self_ty));
             if let Some(trait_ref) = trait_ref {
                 p!(" as ", print(trait_ref.print_only_trait_path()));
             }
-            Ok(cx)
+            Ok(())
         })
     }
 
     fn pretty_path_append_impl(
-        mut self,
-        print_prefix: impl FnOnce(Self) -> Result<Self::Path, Self::Error>,
+        &mut self,
+        print_prefix: impl FnOnce(&mut Self) -> Result<(), PrintError>,
         self_ty: Ty<'tcx>,
         trait_ref: Option<ty::TraitRef<'tcx>>,
-    ) -> Result<Self::Path, Self::Error> {
-        self = print_prefix(self)?;
+    ) -> Result<(), PrintError> {
+        print_prefix(self)?;
 
-        self.generic_delimiters(|mut cx| {
+        self.generic_delimiters(|cx| {
             define_scoped_cx!(cx);
 
             p!("impl ");
@@ -639,11 +629,11 @@ pub trait PrettyPrinter<'tcx>:
             }
             p!(print(self_ty));
 
-            Ok(cx)
+            Ok(())
         })
     }
 
-    fn pretty_print_type(mut self, ty: Ty<'tcx>) -> Result<Self::Type, Self::Error> {
+    fn pretty_print_type(&mut self, ty: Ty<'tcx>) -> Result<(), PrintError> {
         define_scoped_cx!(self);
 
         match *ty.kind() {
@@ -689,7 +679,7 @@ pub trait PrettyPrinter<'tcx>:
             ty::Infer(infer_ty) => {
                 if self.should_print_verbose() {
                     p!(write("{:?}", ty.kind()));
-                    return Ok(self);
+                    return Ok(());
                 }
 
                 if let ty::TyVar(ty_vid) = infer_ty {
@@ -706,7 +696,7 @@ pub trait PrettyPrinter<'tcx>:
             ty::Param(ref param_ty) => p!(print(param_ty)),
             ty::Bound(debruijn, bound_ty) => match bound_ty.kind {
                 ty::BoundTyKind::Anon => {
-                    rustc_type_ir::debug_bound_var(&mut self, debruijn, bound_ty.var)?
+                    rustc_type_ir::debug_bound_var(self, debruijn, bound_ty.var)?
                 }
                 ty::BoundTyKind::Param(_, s) => match self.should_print_verbose() {
                     true => p!(write("{:?}", ty.kind())),
@@ -761,7 +751,7 @@ pub trait PrettyPrinter<'tcx>:
                 if self.should_print_verbose() {
                     // FIXME(eddyb) print this with `print_def_path`.
                     p!(write("Opaque({:?}, {})", def_id, args.print_as_list()));
-                    return Ok(self);
+                    return Ok(());
                 }
 
                 let parent = self.tcx().parent(def_id);
@@ -776,17 +766,17 @@ pub trait PrettyPrinter<'tcx>:
                                 // If the type alias directly starts with the `impl` of the
                                 // opaque type we're printing, then skip the `::{opaque#1}`.
                                 p!(print_def_path(parent, args));
-                                return Ok(self);
+                                return Ok(());
                             }
                         }
                         // Complex opaque type, e.g. `type Foo = (i32, impl Debug);`
                         p!(print_def_path(def_id, args));
-                        return Ok(self);
+                        return Ok(());
                     }
                     _ => {
                         if with_no_queries() {
                             p!(print_def_path(def_id, &[]));
-                            return Ok(self);
+                            return Ok(());
                         } else {
                             return self.pretty_print_opaque_impl_type(def_id, args);
                         }
@@ -794,11 +784,11 @@ pub trait PrettyPrinter<'tcx>:
                 }
             }
             ty::Str => p!("str"),
-            ty::Generator(did, args, movability) => {
+            ty::Coroutine(did, args, movability) => {
                 p!(write("{{"));
-                let generator_kind = self.tcx().generator_kind(did).unwrap();
+                let coroutine_kind = self.tcx().coroutine_kind(did).unwrap();
                 let should_print_movability =
-                    self.should_print_verbose() || generator_kind == hir::GeneratorKind::Gen;
+                    self.should_print_verbose() || coroutine_kind == hir::CoroutineKind::Coroutine;
 
                 if should_print_movability {
                     match movability {
@@ -808,7 +798,7 @@ pub trait PrettyPrinter<'tcx>:
                 }
 
                 if !self.should_print_verbose() {
-                    p!(write("{}", generator_kind));
+                    p!(write("{}", coroutine_kind));
                     // FIXME(eddyb) should use `def_span`.
                     if let Some(did) = did.as_local() {
                         let span = self.tcx().def_span(did);
@@ -824,24 +814,24 @@ pub trait PrettyPrinter<'tcx>:
                 } else {
                     p!(print_def_path(did, args));
                     p!(" upvar_tys=(");
-                    if !args.as_generator().is_valid() {
+                    if !args.as_coroutine().is_valid() {
                         p!("unavailable");
                     } else {
-                        self = self.comma_sep(args.as_generator().upvar_tys().iter())?;
+                        self.comma_sep(args.as_coroutine().upvar_tys().iter())?;
                     }
                     p!(")");
 
-                    if args.as_generator().is_valid() {
-                        p!(" ", print(args.as_generator().witness()));
+                    if args.as_coroutine().is_valid() {
+                        p!(" ", print(args.as_coroutine().witness()));
                     }
                 }
 
                 p!("}}")
             }
-            ty::GeneratorWitness(did, args) => {
+            ty::CoroutineWitness(did, args) => {
                 p!(write("{{"));
                 if !self.tcx().sess.verbose() {
-                    p!("generator witness");
+                    p!("coroutine witness");
                     // FIXME(eddyb) should use `def_span`.
                     if let Some(did) = did.as_local() {
                         let span = self.tcx().def_span(did);
@@ -897,7 +887,7 @@ pub trait PrettyPrinter<'tcx>:
                             print(args.as_closure().sig_as_fn_ptr_ty())
                         );
                         p!(" upvar_tys=(");
-                        self = self.comma_sep(args.as_closure().upvar_tys().iter())?;
+                        self.comma_sep(args.as_closure().upvar_tys().iter())?;
                         p!(")");
                     }
                 }
@@ -907,14 +897,14 @@ pub trait PrettyPrinter<'tcx>:
             ty::Slice(ty) => p!("[", print(ty), "]"),
         }
 
-        Ok(self)
+        Ok(())
     }
 
     fn pretty_print_opaque_impl_type(
-        mut self,
+        &mut self,
         def_id: DefId,
         args: &'tcx ty::List<ty::GenericArg<'tcx>>,
-    ) -> Result<Self::Type, Self::Error> {
+    ) -> Result<(), PrintError> {
         let tcx = self.tcx();
 
         // Grab the "TraitA + TraitB" from `impl TraitA + TraitB`,
@@ -972,7 +962,7 @@ pub trait PrettyPrinter<'tcx>:
             write!(self, "{}", if first { "" } else { " + " })?;
             write!(self, "{}", if paren_needed { "(" } else { "" })?;
 
-            self = self.wrap_binder(&fn_once_trait_ref, |trait_ref, mut cx| {
+            self.wrap_binder(&fn_once_trait_ref, |trait_ref, cx| {
                 define_scoped_cx!(cx);
                 // Get the (single) generic ty (the args) of this FnOnce trait ref.
                 let generics = tcx.generics_of(trait_ref.def_id);
@@ -1029,7 +1019,7 @@ pub trait PrettyPrinter<'tcx>:
                     }
                 }
 
-                Ok(cx)
+                Ok(())
             })?;
         }
 
@@ -1037,7 +1027,7 @@ pub trait PrettyPrinter<'tcx>:
         for (trait_ref, assoc_items) in traits {
             write!(self, "{}", if first { "" } else { " + " })?;
 
-            self = self.wrap_binder(&trait_ref, |trait_ref, mut cx| {
+            self.wrap_binder(&trait_ref, |trait_ref, cx| {
                 define_scoped_cx!(cx);
                 p!(print(trait_ref.print_only_trait_name()));
 
@@ -1058,16 +1048,16 @@ pub trait PrettyPrinter<'tcx>:
                     }
 
                     for (assoc_item_def_id, term) in assoc_items {
-                        // Skip printing `<{generator@} as Generator<_>>::Return` from async blocks,
-                        // unless we can find out what generator return type it comes from.
+                        // Skip printing `<{coroutine@} as Coroutine<_>>::Return` from async blocks,
+                        // unless we can find out what coroutine return type it comes from.
                         let term = if let Some(ty) = term.skip_binder().ty()
                             && let ty::Alias(ty::Projection, proj) = ty.kind()
                             && let Some(assoc) = tcx.opt_associated_item(proj.def_id)
-                            && assoc.trait_container(tcx) == tcx.lang_items().gen_trait()
+                            && assoc.trait_container(tcx) == tcx.lang_items().coroutine_trait()
                             && assoc.name == rustc_span::sym::Return
                         {
-                            if let ty::Generator(_, args, _) = args.type_at(0).kind() {
-                                let return_ty = args.as_generator().return_ty();
+                            if let ty::Coroutine(_, args, _) = args.type_at(0).kind() {
+                                let return_ty = args.as_coroutine().return_ty();
                                 if !return_ty.is_ty_var() {
                                     return_ty.into()
                                 } else {
@@ -1101,7 +1091,7 @@ pub trait PrettyPrinter<'tcx>:
                 }
 
                 first = false;
-                Ok(cx)
+                Ok(())
             })?;
         }
 
@@ -1114,22 +1104,24 @@ pub trait PrettyPrinter<'tcx>:
         if !FORCE_TRIMMED_PATH.with(|flag| flag.get()) {
             for re in lifetimes {
                 write!(self, " + ")?;
-                self = self.print_region(re)?;
+                self.print_region(re)?;
             }
         }
 
         if self.tcx().features().return_type_notation
-            && let Some(ty::ImplTraitInTraitData::Trait { fn_def_id, .. }) = self.tcx().opt_rpitit_info(def_id)
-            && let ty::Alias(_, alias_ty) = self.tcx().fn_sig(fn_def_id).skip_binder().output().skip_binder().kind()
+            && let Some(ty::ImplTraitInTraitData::Trait { fn_def_id, .. }) =
+                self.tcx().opt_rpitit_info(def_id)
+            && let ty::Alias(_, alias_ty) =
+                self.tcx().fn_sig(fn_def_id).skip_binder().output().skip_binder().kind()
             && alias_ty.def_id == def_id
         {
             let num_args = self.tcx().generics_of(fn_def_id).count();
             write!(self, " {{ ")?;
-            self = self.print_def_path(fn_def_id, &args[..num_args])?;
+            self.print_def_path(fn_def_id, &args[..num_args])?;
             write!(self, "() }}")?;
         }
 
-        Ok(self)
+        Ok(())
     }
 
     /// Insert the trait ref and optionally a projection type associated with it into either the
@@ -1159,14 +1151,14 @@ pub trait PrettyPrinter<'tcx>:
                 entry.has_fn_once = true;
                 return;
             } else if Some(trait_def_id) == self.tcx().lang_items().fn_mut_trait() {
-                let super_trait_ref = crate::traits::util::supertraits(self.tcx(), trait_ref)
+                let super_trait_ref = supertraits_for_pretty_printing(self.tcx(), trait_ref)
                     .find(|super_trait_ref| super_trait_ref.def_id() == fn_once_trait)
                     .unwrap();
 
                 fn_traits.entry(super_trait_ref).or_default().fn_mut_trait_ref = Some(trait_ref);
                 return;
             } else if Some(trait_def_id) == self.tcx().lang_items().fn_trait() {
-                let super_trait_ref = crate::traits::util::supertraits(self.tcx(), trait_ref)
+                let super_trait_ref = supertraits_for_pretty_printing(self.tcx(), trait_ref)
                     .find(|super_trait_ref| super_trait_ref.def_id() == fn_once_trait)
                     .unwrap();
 
@@ -1180,9 +1172,9 @@ pub trait PrettyPrinter<'tcx>:
     }
 
     fn pretty_print_inherent_projection(
-        self,
+        &mut self,
         alias_ty: &ty::AliasTy<'tcx>,
-    ) -> Result<Self::Path, Self::Error> {
+    ) -> Result<(), PrintError> {
         let def_key = self.tcx().def_key(alias_ty.def_id);
         self.path_generic_args(
             |cx| {
@@ -1199,19 +1191,19 @@ pub trait PrettyPrinter<'tcx>:
         None
     }
 
-    fn const_infer_name(&self, _: ty::ConstVid<'tcx>) -> Option<Symbol> {
+    fn const_infer_name(&self, _: ty::ConstVid) -> Option<Symbol> {
         None
     }
 
     fn pretty_print_dyn_existential(
-        mut self,
+        &mut self,
         predicates: &'tcx ty::List<ty::PolyExistentialPredicate<'tcx>>,
-    ) -> Result<Self::DynExistential, Self::Error> {
+    ) -> Result<(), PrintError> {
         // Generate the main trait ref, including associated types.
         let mut first = true;
 
         if let Some(principal) = predicates.principal() {
-            self = self.wrap_binder(&principal, |principal, mut cx| {
+            self.wrap_binder(&principal, |principal, cx| {
                 define_scoped_cx!(cx);
                 p!(print_def_path(principal.def_id, &[]));
 
@@ -1251,8 +1243,8 @@ pub trait PrettyPrinter<'tcx>:
                     });
 
                     if !args.is_empty() || !projections.is_empty() {
-                        p!(generic_delimiters(|mut cx| {
-                            cx = cx.comma_sep(args.iter().copied())?;
+                        p!(generic_delimiters(|cx| {
+                            cx.comma_sep(args.iter().copied())?;
                             if !args.is_empty() && !projections.is_empty() {
                                 write!(cx, ", ")?;
                             }
@@ -1260,7 +1252,7 @@ pub trait PrettyPrinter<'tcx>:
                         }));
                     }
                 }
-                Ok(cx)
+                Ok(())
             })?;
 
             first = false;
@@ -1291,15 +1283,15 @@ pub trait PrettyPrinter<'tcx>:
             p!(print_def_path(def_id, &[]));
         }
 
-        Ok(self)
+        Ok(())
     }
 
     fn pretty_fn_sig(
-        mut self,
+        &mut self,
         inputs: &[Ty<'tcx>],
         c_variadic: bool,
         output: Ty<'tcx>,
-    ) -> Result<Self, Self::Error> {
+    ) -> Result<(), PrintError> {
         define_scoped_cx!(self);
 
         p!("(", comma_sep(inputs.iter().copied()));
@@ -1314,28 +1306,28 @@ pub trait PrettyPrinter<'tcx>:
             p!(" -> ", print(output));
         }
 
-        Ok(self)
+        Ok(())
     }
 
     fn pretty_print_const(
-        mut self,
+        &mut self,
         ct: ty::Const<'tcx>,
         print_ty: bool,
-    ) -> Result<Self::Const, Self::Error> {
+    ) -> Result<(), PrintError> {
         define_scoped_cx!(self);
 
         if self.should_print_verbose() {
             p!(write("{:?}", ct));
-            return Ok(self);
+            return Ok(());
         }
 
         macro_rules! print_underscore {
             () => {{
                 if print_ty {
-                    self = self.typed_value(
-                        |mut this| {
+                    self.typed_value(
+                        |this| {
                             write!(this, "_")?;
-                            Ok(this)
+                            Ok(())
                         },
                         |this| this.print_type(ct.ty()),
                         ": ",
@@ -1364,27 +1356,29 @@ pub trait PrettyPrinter<'tcx>:
                             // cause printing to enter an infinite recursion if the anon const is in the self type i.e.
                             // `impl<T: Default> Default for [T; 32 - 1 - 1 - 1] {`
                             // where we would try to print `<[T; /* print `constant#0` again */] as Default>::{constant#0}`
-                            p!(write("{}::{}", self.tcx().crate_name(def.krate), self.tcx().def_path(def).to_string_no_crate_verbose()))
+                            p!(write(
+                                "{}::{}",
+                                self.tcx().crate_name(def.krate),
+                                self.tcx().def_path(def).to_string_no_crate_verbose()
+                            ))
                         }
                     }
                     defkind => bug!("`{:?}` has unexpected defkind {:?}", ct, defkind),
                 }
             }
-            ty::ConstKind::Infer(infer_ct) => {
-                match infer_ct {
-                    ty::InferConst::Var(ct_vid)
-                        if let Some(name) = self.const_infer_name(ct_vid) =>
-                            p!(write("{}", name)),
-                    _ => print_underscore!(),
+            ty::ConstKind::Infer(infer_ct) => match infer_ct {
+                ty::InferConst::Var(ct_vid) if let Some(name) = self.const_infer_name(ct_vid) => {
+                    p!(write("{}", name))
                 }
-            }
+                _ => print_underscore!(),
+            },
             ty::ConstKind::Param(ParamConst { name, .. }) => p!(write("{}", name)),
             ty::ConstKind::Value(value) => {
                 return self.pretty_print_const_valtree(value, ct.ty(), print_ty);
             }
 
             ty::ConstKind::Bound(debruijn, bound_var) => {
-                rustc_type_ir::debug_bound_var(&mut self, debruijn, bound_var)?
+                rustc_type_ir::debug_bound_var(self, debruijn, bound_var)?
             }
             ty::ConstKind::Placeholder(placeholder) => p!(write("{placeholder:?}")),
             // FIXME(generic_const_exprs):
@@ -1392,14 +1386,14 @@ pub trait PrettyPrinter<'tcx>:
             ty::ConstKind::Expr(_) => p!("{{const expr}}"),
             ty::ConstKind::Error(_) => p!("{{const error}}"),
         };
-        Ok(self)
+        Ok(())
     }
 
     fn pretty_print_const_scalar(
-        self,
+        &mut self,
         scalar: Scalar,
         ty: Ty<'tcx>,
-    ) -> Result<Self::Const, Self::Error> {
+    ) -> Result<(), PrintError> {
         match scalar {
             Scalar::Ptr(ptr, _size) => self.pretty_print_const_scalar_ptr(ptr, ty),
             Scalar::Int(int) => {
@@ -1409,10 +1403,10 @@ pub trait PrettyPrinter<'tcx>:
     }
 
     fn pretty_print_const_scalar_ptr(
-        mut self,
+        &mut self,
         ptr: Pointer,
         ty: Ty<'tcx>,
-    ) -> Result<Self::Const, Self::Error> {
+    ) -> Result<(), PrintError> {
         define_scoped_cx!(self);
 
         let (alloc_id, offset) = ptr.into_parts();
@@ -1443,7 +1437,7 @@ pub trait PrettyPrinter<'tcx>:
                                 Some(GlobalAlloc::VTable(..)) => p!("<vtable>"),
                                 None => p!("<dangling pointer>"),
                             }
-                            return Ok(self);
+                            return Ok(());
                         }
                     }
                 }
@@ -1454,27 +1448,27 @@ pub trait PrettyPrinter<'tcx>:
                 if let Some(GlobalAlloc::Function(instance)) =
                     self.tcx().try_get_global_alloc(alloc_id)
                 {
-                    self = self.typed_value(
+                    self.typed_value(
                         |this| this.print_value_path(instance.def_id(), instance.args),
                         |this| this.print_type(ty),
                         " as ",
                     )?;
-                    return Ok(self);
+                    return Ok(());
                 }
             }
             _ => {}
         }
         // Any pointer values not covered by a branch above
-        self = self.pretty_print_const_pointer(ptr, ty)?;
-        Ok(self)
+        self.pretty_print_const_pointer(ptr, ty)?;
+        Ok(())
     }
 
     fn pretty_print_const_scalar_int(
-        mut self,
+        &mut self,
         int: ScalarInt,
         ty: Ty<'tcx>,
         print_ty: bool,
-    ) -> Result<Self::Const, Self::Error> {
+    ) -> Result<(), PrintError> {
         define_scoped_cx!(self);
 
         match ty.kind() {
@@ -1501,10 +1495,10 @@ pub trait PrettyPrinter<'tcx>:
             // Pointer types
             ty::Ref(..) | ty::RawPtr(_) | ty::FnPtr(_) => {
                 let data = int.assert_bits(self.tcx().data_layout.pointer_size);
-                self = self.typed_value(
-                    |mut this| {
+                self.typed_value(
+                    |this| {
                         write!(this, "0x{data:x}")?;
-                        Ok(this)
+                        Ok(())
                     },
                     |this| this.print_type(ty),
                     " as ",
@@ -1512,57 +1506,57 @@ pub trait PrettyPrinter<'tcx>:
             }
             // Nontrivial types with scalar bit representation
             _ => {
-                let print = |mut this: Self| {
+                let print = |this: &mut Self| {
                     if int.size() == Size::ZERO {
                         write!(this, "transmute(())")?;
                     } else {
                         write!(this, "transmute(0x{int:x})")?;
                     }
-                    Ok(this)
+                    Ok(())
                 };
-                self = if print_ty {
+                if print_ty {
                     self.typed_value(print, |this| this.print_type(ty), ": ")?
                 } else {
                     print(self)?
                 };
             }
         }
-        Ok(self)
+        Ok(())
     }
 
     /// This is overridden for MIR printing because we only want to hide alloc ids from users, not
     /// from MIR where it is actually useful.
     fn pretty_print_const_pointer<Prov: Provenance>(
-        self,
+        &mut self,
         _: Pointer<Prov>,
         ty: Ty<'tcx>,
-    ) -> Result<Self::Const, Self::Error> {
+    ) -> Result<(), PrintError> {
         self.typed_value(
-            |mut this| {
+            |this| {
                 this.write_str("&_")?;
-                Ok(this)
+                Ok(())
             },
             |this| this.print_type(ty),
             ": ",
         )
     }
 
-    fn pretty_print_byte_str(mut self, byte_str: &'tcx [u8]) -> Result<Self::Const, Self::Error> {
+    fn pretty_print_byte_str(&mut self, byte_str: &'tcx [u8]) -> Result<(), PrintError> {
         write!(self, "b\"{}\"", byte_str.escape_ascii())?;
-        Ok(self)
+        Ok(())
     }
 
     fn pretty_print_const_valtree(
-        mut self,
+        &mut self,
         valtree: ty::ValTree<'tcx>,
         ty: Ty<'tcx>,
         print_ty: bool,
-    ) -> Result<Self::Const, Self::Error> {
+    ) -> Result<(), PrintError> {
         define_scoped_cx!(self);
 
         if self.should_print_verbose() {
             p!(write("ValTree({:?}: ", valtree), print(ty), ")");
-            return Ok(self);
+            return Ok(());
         }
 
         let u8_type = self.tcx().types.u8;
@@ -1583,12 +1577,12 @@ pub trait PrettyPrinter<'tcx>:
                         bug!("expected to convert valtree to raw bytes for type {:?}", ty)
                     });
                     p!(write("{:?}", String::from_utf8_lossy(bytes)));
-                    return Ok(self);
+                    return Ok(());
                 }
                 _ => {
                     p!("&");
                     p!(pretty_print_const_valtree(valtree, *inner_ty, print_ty));
-                    return Ok(self);
+                    return Ok(());
                 }
             },
             (ty::ValTree::Branch(_), ty::Array(t, _)) if *t == u8_type => {
@@ -1597,7 +1591,7 @@ pub trait PrettyPrinter<'tcx>:
                 });
                 p!("*");
                 p!(pretty_print_byte_str(bytes));
-                return Ok(self);
+                return Ok(());
             }
             // Aggregates, printed as array/tuple/struct/variant construction syntax.
             (ty::ValTree::Branch(_), ty::Array(..) | ty::Tuple(..) | ty::Adt(..)) => {
@@ -1616,10 +1610,10 @@ pub trait PrettyPrinter<'tcx>:
                         p!(")");
                     }
                     ty::Adt(def, _) if def.variants().is_empty() => {
-                        self = self.typed_value(
-                            |mut this| {
+                        self.typed_value(
+                            |this| {
                                 write!(this, "unreachable()")?;
-                                Ok(this)
+                                Ok(())
                             },
                             |this| this.print_type(ty),
                             ": ",
@@ -1651,7 +1645,7 @@ pub trait PrettyPrinter<'tcx>:
                     }
                     _ => unreachable!(),
                 }
-                return Ok(self);
+                return Ok(());
             }
             (ty::ValTree::Leaf(leaf), ty::Ref(_, inner_ty, _)) => {
                 p!(write("&"));
@@ -1674,18 +1668,15 @@ pub trait PrettyPrinter<'tcx>:
         if print_ty {
             p!(": ", print(ty));
         }
-        Ok(self)
+        Ok(())
     }
 
-    fn pretty_closure_as_impl(
-        mut self,
-        closure: ty::ClosureArgs<'tcx>,
-    ) -> Result<Self::Const, Self::Error> {
+    fn pretty_closure_as_impl(&mut self, closure: ty::ClosureArgs<'tcx>) -> Result<(), PrintError> {
         let sig = closure.sig();
         let kind = closure.kind_ty().to_opt_closure_kind().unwrap_or(ty::ClosureKind::Fn);
 
         write!(self, "impl ")?;
-        self.wrap_binder(&sig, |sig, mut cx| {
+        self.wrap_binder(&sig, |sig, cx| {
             define_scoped_cx!(cx);
 
             p!(print(kind), "(");
@@ -1701,7 +1692,7 @@ pub trait PrettyPrinter<'tcx>:
                 p!(" -> ", print(sig.output()));
             }
 
-            Ok(cx)
+            Ok(())
         })
     }
 
@@ -1719,7 +1710,7 @@ pub(crate) fn pretty_print_const<'tcx>(
         let literal = tcx.lift(c).unwrap();
         let mut cx = FmtPrinter::new(tcx, Namespace::ValueNS);
         cx.print_alloc_ids = true;
-        let cx = cx.pretty_print_const(literal, print_types)?;
+        cx.pretty_print_const(literal, print_types)?;
         fmt.write_str(&cx.into_buffer())?;
         Ok(())
     })
@@ -1748,7 +1739,7 @@ pub struct FmtPrinterData<'a, 'tcx> {
     pub region_highlight_mode: RegionHighlightMode<'tcx>,
 
     pub ty_infer_name_resolver: Option<Box<dyn Fn(ty::TyVid) -> Option<Symbol> + 'a>>,
-    pub const_infer_name_resolver: Option<Box<dyn Fn(ty::ConstVid<'tcx>) -> Option<Symbol> + 'a>>,
+    pub const_infer_name_resolver: Option<Box<dyn Fn(ty::ConstVid) -> Option<Symbol> + 'a>>,
 }
 
 impl<'a, 'tcx> Deref for FmtPrinter<'a, 'tcx> {
@@ -1770,6 +1761,16 @@ impl<'a, 'tcx> FmtPrinter<'a, 'tcx> {
         Self::new_with_limit(tcx, ns, limit)
     }
 
+    pub fn print_string(
+        tcx: TyCtxt<'tcx>,
+        ns: Namespace,
+        f: impl FnOnce(&mut Self) -> Result<(), PrintError>,
+    ) -> Result<String, PrintError> {
+        let mut c = FmtPrinter::new(tcx, ns);
+        f(&mut c)?;
+        Ok(c.into_buffer())
+    }
+
     pub fn new_with_limit(tcx: TyCtxt<'tcx>, ns: Namespace, type_length_limit: Limit) -> Self {
         FmtPrinter(Box::new(FmtPrinterData {
             tcx,
@@ -1830,7 +1831,8 @@ impl<'t> TyCtxt<'t> {
         let def_id = def_id.into_query_param();
         let ns = guess_def_namespace(self, def_id);
         debug!("def_path_str: def_id={:?}, ns={:?}", def_id, ns);
-        FmtPrinter::new(self, ns).print_def_path(def_id, args).unwrap().into_buffer()
+
+        FmtPrinter::print_string(self, ns, |cx| cx.print_def_path(def_id, args)).unwrap()
     }
 
     pub fn value_path_str_with_args(
@@ -1841,7 +1843,8 @@ impl<'t> TyCtxt<'t> {
         let def_id = def_id.into_query_param();
         let ns = guess_def_namespace(self, def_id);
         debug!("value_path_str: def_id={:?}, ns={:?}", def_id, ns);
-        FmtPrinter::new(self, ns).print_value_path(def_id, args).unwrap().into_buffer()
+
+        FmtPrinter::print_string(self, ns, |cx| cx.print_value_path(def_id, args)).unwrap()
     }
 }
 
@@ -1853,34 +1856,24 @@ impl fmt::Write for FmtPrinter<'_, '_> {
 }
 
 impl<'tcx> Printer<'tcx> for FmtPrinter<'_, 'tcx> {
-    type Error = fmt::Error;
-
-    type Path = Self;
-    type Region = Self;
-    type Type = Self;
-    type DynExistential = Self;
-    type Const = Self;
-
     fn tcx<'a>(&'a self) -> TyCtxt<'tcx> {
         self.tcx
     }
 
     fn print_def_path(
-        mut self,
+        &mut self,
         def_id: DefId,
         args: &'tcx [GenericArg<'tcx>],
-    ) -> Result<Self::Path, Self::Error> {
-        define_scoped_cx!(self);
-
+    ) -> Result<(), PrintError> {
         if args.is_empty() {
             match self.try_print_trimmed_def_path(def_id)? {
-                (cx, true) => return Ok(cx),
-                (cx, false) => self = cx,
+                true => return Ok(()),
+                false => {}
             }
 
             match self.try_print_visible_def_path(def_id)? {
-                (cx, true) => return Ok(cx),
-                (cx, false) => self = cx,
+                true => return Ok(()),
+                false => {}
             }
         }
 
@@ -1901,7 +1894,7 @@ impl<'tcx> Printer<'tcx> for FmtPrinter<'_, 'tcx> {
                 let parent_def_id = DefId { index: key.parent.unwrap(), ..def_id };
                 let span = self.tcx.def_span(def_id);
 
-                self = self.print_def_path(parent_def_id, &[])?;
+                self.print_def_path(parent_def_id, &[])?;
 
                 // HACK(eddyb) copy of `path_append` to avoid
                 // constructing a `DisambiguatedDefPathData`.
@@ -1917,40 +1910,40 @@ impl<'tcx> Printer<'tcx> for FmtPrinter<'_, 'tcx> {
                 )?;
                 self.empty_path = false;
 
-                return Ok(self);
+                return Ok(());
             }
         }
 
         self.default_print_def_path(def_id, args)
     }
 
-    fn print_region(self, region: ty::Region<'tcx>) -> Result<Self::Region, Self::Error> {
+    fn print_region(&mut self, region: ty::Region<'tcx>) -> Result<(), PrintError> {
         self.pretty_print_region(region)
     }
 
-    fn print_type(mut self, ty: Ty<'tcx>) -> Result<Self::Type, Self::Error> {
+    fn print_type(&mut self, ty: Ty<'tcx>) -> Result<(), PrintError> {
         if self.type_length_limit.value_within_limit(self.printed_type_count) {
             self.printed_type_count += 1;
             self.pretty_print_type(ty)
         } else {
             self.truncated = true;
             write!(self, "...")?;
-            Ok(self)
+            Ok(())
         }
     }
 
     fn print_dyn_existential(
-        self,
+        &mut self,
         predicates: &'tcx ty::List<ty::PolyExistentialPredicate<'tcx>>,
-    ) -> Result<Self::DynExistential, Self::Error> {
+    ) -> Result<(), PrintError> {
         self.pretty_print_dyn_existential(predicates)
     }
 
-    fn print_const(self, ct: ty::Const<'tcx>) -> Result<Self::Const, Self::Error> {
+    fn print_const(&mut self, ct: ty::Const<'tcx>) -> Result<(), PrintError> {
         self.pretty_print_const(ct, false)
     }
 
-    fn path_crate(mut self, cnum: CrateNum) -> Result<Self::Path, Self::Error> {
+    fn path_crate(&mut self, cnum: CrateNum) -> Result<(), PrintError> {
         self.empty_path = true;
         if cnum == LOCAL_CRATE {
             if self.tcx.sess.at_least_rust_2018() {
@@ -1964,52 +1957,52 @@ impl<'tcx> Printer<'tcx> for FmtPrinter<'_, 'tcx> {
             write!(self, "{}", self.tcx.crate_name(cnum))?;
             self.empty_path = false;
         }
-        Ok(self)
+        Ok(())
     }
 
     fn path_qualified(
-        mut self,
+        &mut self,
         self_ty: Ty<'tcx>,
         trait_ref: Option<ty::TraitRef<'tcx>>,
-    ) -> Result<Self::Path, Self::Error> {
-        self = self.pretty_path_qualified(self_ty, trait_ref)?;
+    ) -> Result<(), PrintError> {
+        self.pretty_path_qualified(self_ty, trait_ref)?;
         self.empty_path = false;
-        Ok(self)
+        Ok(())
     }
 
     fn path_append_impl(
-        mut self,
-        print_prefix: impl FnOnce(Self) -> Result<Self::Path, Self::Error>,
+        &mut self,
+        print_prefix: impl FnOnce(&mut Self) -> Result<(), PrintError>,
         _disambiguated_data: &DisambiguatedDefPathData,
         self_ty: Ty<'tcx>,
         trait_ref: Option<ty::TraitRef<'tcx>>,
-    ) -> Result<Self::Path, Self::Error> {
-        self = self.pretty_path_append_impl(
-            |mut cx| {
-                cx = print_prefix(cx)?;
+    ) -> Result<(), PrintError> {
+        self.pretty_path_append_impl(
+            |cx| {
+                print_prefix(cx)?;
                 if !cx.empty_path {
                     write!(cx, "::")?;
                 }
 
-                Ok(cx)
+                Ok(())
             },
             self_ty,
             trait_ref,
         )?;
         self.empty_path = false;
-        Ok(self)
+        Ok(())
     }
 
     fn path_append(
-        mut self,
-        print_prefix: impl FnOnce(Self) -> Result<Self::Path, Self::Error>,
+        &mut self,
+        print_prefix: impl FnOnce(&mut Self) -> Result<(), PrintError>,
         disambiguated_data: &DisambiguatedDefPathData,
-    ) -> Result<Self::Path, Self::Error> {
-        self = print_prefix(self)?;
+    ) -> Result<(), PrintError> {
+        print_prefix(self)?;
 
         // Skip `::{{extern}}` blocks and `::{{constructor}}` on tuple/unit structs.
         if let DefPathData::ForeignMod | DefPathData::Ctor = disambiguated_data.data {
-            return Ok(self);
+            return Ok(());
         }
 
         let name = disambiguated_data.data.name();
@@ -2024,19 +2017,19 @@ impl<'tcx> Printer<'tcx> for FmtPrinter<'_, 'tcx> {
         }
 
         let verbose = self.should_print_verbose();
-        disambiguated_data.fmt_maybe_verbose(&mut self, verbose)?;
+        disambiguated_data.fmt_maybe_verbose(self, verbose)?;
 
         self.empty_path = false;
 
-        Ok(self)
+        Ok(())
     }
 
     fn path_generic_args(
-        mut self,
-        print_prefix: impl FnOnce(Self) -> Result<Self::Path, Self::Error>,
+        &mut self,
+        print_prefix: impl FnOnce(&mut Self) -> Result<(), PrintError>,
         args: &[GenericArg<'tcx>],
-    ) -> Result<Self::Path, Self::Error> {
-        self = print_prefix(self)?;
+    ) -> Result<(), PrintError> {
+        print_prefix(self)?;
 
         let tcx = self.tcx;
 
@@ -2070,7 +2063,7 @@ impl<'tcx> Printer<'tcx> for FmtPrinter<'_, 'tcx> {
             }
             self.generic_delimiters(|cx| cx.comma_sep(args.into_iter()))
         } else {
-            Ok(self)
+            Ok(())
         }
     }
 }
@@ -2084,68 +2077,68 @@ impl<'tcx> PrettyPrinter<'tcx> for FmtPrinter<'_, 'tcx> {
         self.printed_type_count = 0;
     }
 
-    fn const_infer_name(&self, id: ty::ConstVid<'tcx>) -> Option<Symbol> {
+    fn const_infer_name(&self, id: ty::ConstVid) -> Option<Symbol> {
         self.0.const_infer_name_resolver.as_ref().and_then(|func| func(id))
     }
 
     fn print_value_path(
-        mut self,
+        &mut self,
         def_id: DefId,
         args: &'tcx [GenericArg<'tcx>],
-    ) -> Result<Self::Path, Self::Error> {
+    ) -> Result<(), PrintError> {
         let was_in_value = std::mem::replace(&mut self.in_value, true);
-        self = self.print_def_path(def_id, args)?;
+        self.print_def_path(def_id, args)?;
         self.in_value = was_in_value;
 
-        Ok(self)
+        Ok(())
     }
 
-    fn in_binder<T>(self, value: &ty::Binder<'tcx, T>) -> Result<Self, Self::Error>
+    fn in_binder<T>(&mut self, value: &ty::Binder<'tcx, T>) -> Result<(), PrintError>
     where
-        T: Print<'tcx, Self, Output = Self, Error = Self::Error> + TypeFoldable<TyCtxt<'tcx>>,
+        T: Print<'tcx, Self> + TypeFoldable<TyCtxt<'tcx>>,
     {
         self.pretty_in_binder(value)
     }
 
-    fn wrap_binder<T, C: FnOnce(&T, Self) -> Result<Self, Self::Error>>(
-        self,
+    fn wrap_binder<T, C: FnOnce(&T, &mut Self) -> Result<(), PrintError>>(
+        &mut self,
         value: &ty::Binder<'tcx, T>,
         f: C,
-    ) -> Result<Self, Self::Error>
+    ) -> Result<(), PrintError>
     where
-        T: Print<'tcx, Self, Output = Self, Error = Self::Error> + TypeFoldable<TyCtxt<'tcx>>,
+        T: Print<'tcx, Self> + TypeFoldable<TyCtxt<'tcx>>,
     {
         self.pretty_wrap_binder(value, f)
     }
 
     fn typed_value(
-        mut self,
-        f: impl FnOnce(Self) -> Result<Self, Self::Error>,
-        t: impl FnOnce(Self) -> Result<Self, Self::Error>,
+        &mut self,
+        f: impl FnOnce(&mut Self) -> Result<(), PrintError>,
+        t: impl FnOnce(&mut Self) -> Result<(), PrintError>,
         conversion: &str,
-    ) -> Result<Self::Const, Self::Error> {
+    ) -> Result<(), PrintError> {
         self.write_str("{")?;
-        self = f(self)?;
+        f(self)?;
         self.write_str(conversion)?;
         let was_in_value = std::mem::replace(&mut self.in_value, false);
-        self = t(self)?;
+        t(self)?;
         self.in_value = was_in_value;
         self.write_str("}")?;
-        Ok(self)
+        Ok(())
     }
 
     fn generic_delimiters(
-        mut self,
-        f: impl FnOnce(Self) -> Result<Self, Self::Error>,
-    ) -> Result<Self, Self::Error> {
+        &mut self,
+        f: impl FnOnce(&mut Self) -> Result<(), PrintError>,
+    ) -> Result<(), PrintError> {
         write!(self, "<")?;
 
         let was_in_value = std::mem::replace(&mut self.in_value, false);
-        let mut inner = f(self)?;
-        inner.in_value = was_in_value;
+        f(self)?;
+        self.in_value = was_in_value;
 
-        write!(inner, ">")?;
-        Ok(inner)
+        write!(self, ">")?;
+        Ok(())
     }
 
     fn should_print_region(&self, region: ty::Region<'tcx>) -> bool {
@@ -2194,18 +2187,18 @@ impl<'tcx> PrettyPrinter<'tcx> for FmtPrinter<'_, 'tcx> {
     }
 
     fn pretty_print_const_pointer<Prov: Provenance>(
-        self,
+        &mut self,
         p: Pointer<Prov>,
         ty: Ty<'tcx>,
-    ) -> Result<Self::Const, Self::Error> {
-        let print = |mut this: Self| {
+    ) -> Result<(), PrintError> {
+        let print = |this: &mut Self| {
             define_scoped_cx!(this);
             if this.print_alloc_ids {
                 p!(write("{:?}", p));
             } else {
                 p!("&_");
             }
-            Ok(this)
+            Ok(())
         };
         self.typed_value(print, |this| this.print_type(ty), ": ")
     }
@@ -2213,19 +2206,19 @@ impl<'tcx> PrettyPrinter<'tcx> for FmtPrinter<'_, 'tcx> {
 
 // HACK(eddyb) limited to `FmtPrinter` because of `region_highlight_mode`.
 impl<'tcx> FmtPrinter<'_, 'tcx> {
-    pub fn pretty_print_region(mut self, region: ty::Region<'tcx>) -> Result<Self, fmt::Error> {
+    pub fn pretty_print_region(&mut self, region: ty::Region<'tcx>) -> Result<(), fmt::Error> {
         define_scoped_cx!(self);
 
         // Watch out for region highlights.
         let highlight = self.region_highlight_mode;
         if let Some(n) = highlight.region_highlighted(region) {
             p!(write("'{}", n));
-            return Ok(self);
+            return Ok(());
         }
 
         if self.should_print_verbose() {
             p!(write("{:?}", region));
-            return Ok(self);
+            return Ok(());
         }
 
         let identify_regions = self.tcx.sess.opts.unstable_opts.identify_regions;
@@ -2238,7 +2231,7 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
             ty::ReEarlyBound(ref data) => {
                 if data.name != kw::Empty {
                     p!(write("{}", data.name));
-                    return Ok(self);
+                    return Ok(());
                 }
             }
             ty::ReLateBound(_, ty::BoundRegion { kind: br, .. })
@@ -2246,34 +2239,36 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
             | ty::RePlaceholder(ty::Placeholder {
                 bound: ty::BoundRegion { kind: br, .. }, ..
             }) => {
-                if let ty::BrNamed(_, name) = br && br.is_named() {
+                if let ty::BrNamed(_, name) = br
+                    && br.is_named()
+                {
                     p!(write("{}", name));
-                    return Ok(self);
+                    return Ok(());
                 }
 
                 if let Some((region, counter)) = highlight.highlight_bound_region {
                     if br == region {
                         p!(write("'{}", counter));
-                        return Ok(self);
+                        return Ok(());
                     }
                 }
             }
             ty::ReVar(region_vid) if identify_regions => {
                 p!(write("{:?}", region_vid));
-                return Ok(self);
+                return Ok(());
             }
             ty::ReVar(_) => {}
             ty::ReErased => {}
             ty::ReError(_) => {}
             ty::ReStatic => {
                 p!("'static");
-                return Ok(self);
+                return Ok(());
             }
         }
 
         p!("'_");
 
-        Ok(self)
+        Ok(())
     }
 }
 
@@ -2356,11 +2351,11 @@ impl<'a, 'tcx> ty::TypeFolder<TyCtxt<'tcx>> for RegionFolder<'a, 'tcx> {
 // `region_index` and `used_region_names`.
 impl<'tcx> FmtPrinter<'_, 'tcx> {
     pub fn name_all_regions<T>(
-        mut self,
+        &mut self,
         value: &ty::Binder<'tcx, T>,
-    ) -> Result<(Self, T, BTreeMap<ty::BoundRegion, ty::Region<'tcx>>), fmt::Error>
+    ) -> Result<(T, BTreeMap<ty::BoundRegion, ty::Region<'tcx>>), fmt::Error>
     where
-        T: Print<'tcx, Self, Output = Self, Error = fmt::Error> + TypeFoldable<TyCtxt<'tcx>>,
+        T: Print<'tcx, Self> + TypeFoldable<TyCtxt<'tcx>>,
     {
         fn name_by_region_index(
             index: usize,
@@ -2401,8 +2396,6 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
             let _ = write!(cx, "{cont}");
         };
 
-        define_scoped_cx!(self);
-
         let possible_names = ('a'..='z').rev().map(|s| Symbol::intern(&format!("'{s}")));
 
         let mut available_names = possible_names
@@ -2433,10 +2426,10 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
         // anyways.
         let (new_value, map) = if self.should_print_verbose() {
             for var in value.bound_vars().iter() {
-                start_or_continue(&mut self, "for<", ", ");
+                start_or_continue(self, "for<", ", ");
                 write!(self, "{var:?}")?;
             }
-            start_or_continue(&mut self, "", "> ");
+            start_or_continue(self, "", "> ");
             (value.clone().skip_binder(), BTreeMap::default())
         } else {
             let tcx = self.tcx;
@@ -2500,8 +2493,8 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
                 };
 
                 if !trim_path {
-                    start_or_continue(&mut self, "for<", ", ");
-                    do_continue(&mut self, name);
+                    start_or_continue(self, "for<", ", ");
+                    do_continue(self, name);
                 }
                 ty::Region::new_late_bound(
                     tcx,
@@ -2518,42 +2511,42 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
             let new_value = value.clone().skip_binder().fold_with(&mut folder);
             let region_map = folder.region_map;
             if !trim_path {
-                start_or_continue(&mut self, "", "> ");
+                start_or_continue(self, "", "> ");
             }
             (new_value, region_map)
         };
 
         self.binder_depth += 1;
         self.region_index = region_index;
-        Ok((self, new_value, map))
+        Ok((new_value, map))
     }
 
-    pub fn pretty_in_binder<T>(self, value: &ty::Binder<'tcx, T>) -> Result<Self, fmt::Error>
+    pub fn pretty_in_binder<T>(&mut self, value: &ty::Binder<'tcx, T>) -> Result<(), fmt::Error>
     where
-        T: Print<'tcx, Self, Output = Self, Error = fmt::Error> + TypeFoldable<TyCtxt<'tcx>>,
+        T: Print<'tcx, Self> + TypeFoldable<TyCtxt<'tcx>>,
     {
         let old_region_index = self.region_index;
-        let (new, new_value, _) = self.name_all_regions(value)?;
-        let mut inner = new_value.print(new)?;
-        inner.region_index = old_region_index;
-        inner.binder_depth -= 1;
-        Ok(inner)
+        let (new_value, _) = self.name_all_regions(value)?;
+        new_value.print(self)?;
+        self.region_index = old_region_index;
+        self.binder_depth -= 1;
+        Ok(())
     }
 
-    pub fn pretty_wrap_binder<T, C: FnOnce(&T, Self) -> Result<Self, fmt::Error>>(
-        self,
+    pub fn pretty_wrap_binder<T, C: FnOnce(&T, &mut Self) -> Result<(), fmt::Error>>(
+        &mut self,
         value: &ty::Binder<'tcx, T>,
         f: C,
-    ) -> Result<Self, fmt::Error>
+    ) -> Result<(), fmt::Error>
     where
-        T: Print<'tcx, Self, Output = Self, Error = fmt::Error> + TypeFoldable<TyCtxt<'tcx>>,
+        T: Print<'tcx, Self> + TypeFoldable<TyCtxt<'tcx>>,
     {
         let old_region_index = self.region_index;
-        let (new, new_value, _) = self.name_all_regions(value)?;
-        let mut inner = f(&new_value, new)?;
-        inner.region_index = old_region_index;
-        inner.binder_depth -= 1;
-        Ok(inner)
+        let (new_value, _) = self.name_all_regions(value)?;
+        f(&new_value, self)?;
+        self.region_index = old_region_index;
+        self.binder_depth -= 1;
+        Ok(())
     }
 
     fn prepare_region_info<T>(&mut self, value: &ty::Binder<'tcx, T>)
@@ -2611,66 +2604,25 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
 
 impl<'tcx, T, P: PrettyPrinter<'tcx>> Print<'tcx, P> for ty::Binder<'tcx, T>
 where
-    T: Print<'tcx, P, Output = P, Error = P::Error> + TypeFoldable<TyCtxt<'tcx>>,
+    T: Print<'tcx, P> + TypeFoldable<TyCtxt<'tcx>>,
 {
-    type Output = P;
-    type Error = P::Error;
-
-    fn print(&self, cx: P) -> Result<Self::Output, Self::Error> {
+    fn print(&self, cx: &mut P) -> Result<(), PrintError> {
         cx.in_binder(self)
     }
 }
 
 impl<'tcx, T, U, P: PrettyPrinter<'tcx>> Print<'tcx, P> for ty::OutlivesPredicate<T, U>
 where
-    T: Print<'tcx, P, Output = P, Error = P::Error>,
-    U: Print<'tcx, P, Output = P, Error = P::Error>,
+    T: Print<'tcx, P>,
+    U: Print<'tcx, P>,
 {
-    type Output = P;
-    type Error = P::Error;
-    fn print(&self, mut cx: P) -> Result<Self::Output, Self::Error> {
+    fn print(&self, cx: &mut P) -> Result<(), PrintError> {
         define_scoped_cx!(cx);
         p!(print(self.0), ": ", print(self.1));
-        Ok(cx)
+        Ok(())
     }
 }
 
-macro_rules! forward_display_to_print {
-    ($($ty:ty),+) => {
-        // Some of the $ty arguments may not actually use 'tcx
-        $(#[allow(unused_lifetimes)] impl<'tcx> fmt::Display for $ty {
-            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-                ty::tls::with(|tcx| {
-                    let cx = tcx.lift(*self)
-                        .expect("could not lift for printing")
-                        .print(FmtPrinter::new(tcx, Namespace::TypeNS))?;
-                    f.write_str(&cx.into_buffer())?;
-                    Ok(())
-                })
-            }
-        })+
-    };
-}
-
-macro_rules! define_print_and_forward_display {
-    (($self:ident, $cx:ident): $($ty:ty $print:block)+) => {
-        $(impl<'tcx, P: PrettyPrinter<'tcx>> Print<'tcx, P> for $ty {
-            type Output = P;
-            type Error = fmt::Error;
-            fn print(&$self, $cx: P) -> Result<Self::Output, Self::Error> {
-                #[allow(unused_mut)]
-                let mut $cx = $cx;
-                define_scoped_cx!($cx);
-                let _: () = $print;
-                #[allow(unreachable_code)]
-                Ok($cx)
-            }
-        })+
-
-        forward_display_to_print!($($ty),+);
-    };
-}
-
 /// Wrapper type for `ty::TraitRef` which opts-in to pretty printing only
 /// the trait path. That is, it will print `Trait<U>` instead of
 /// `<T as Trait<U>>`.
@@ -2745,6 +2697,43 @@ pub struct PrintClosureAsImpl<'tcx> {
     pub closure: ty::ClosureArgs<'tcx>,
 }
 
+macro_rules! forward_display_to_print {
+    ($($ty:ty),+) => {
+        // Some of the $ty arguments may not actually use 'tcx
+        $(#[allow(unused_lifetimes)] impl<'tcx> fmt::Display for $ty {
+            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+                ty::tls::with(|tcx| {
+                    let mut cx = FmtPrinter::new(tcx, Namespace::TypeNS);
+                    tcx.lift(*self)
+                        .expect("could not lift for printing")
+                        .print(&mut cx)?;
+                    f.write_str(&cx.into_buffer())?;
+                    Ok(())
+                })
+            }
+        })+
+    };
+}
+
+macro_rules! define_print {
+    (($self:ident, $cx:ident): $($ty:ty $print:block)+) => {
+        $(impl<'tcx, P: PrettyPrinter<'tcx>> Print<'tcx, P> for $ty {
+            fn print(&$self, $cx: &mut P) -> Result<(), PrintError> {
+                define_scoped_cx!($cx);
+                let _: () = $print;
+                Ok(())
+            }
+        })+
+    };
+}
+
+macro_rules! define_print_and_forward_display {
+    (($self:ident, $cx:ident): $($ty:ty $print:block)+) => {
+        define_print!(($self, $cx): $($ty $print)*);
+        forward_display_to_print!($($ty),+);
+    };
+}
+
 forward_display_to_print! {
     ty::Region<'tcx>,
     Ty<'tcx>,
@@ -2765,6 +2754,51 @@ forward_display_to_print! {
     ty::OutlivesPredicate<ty::Region<'tcx>, ty::Region<'tcx>>
 }
 
+define_print! {
+    (self, cx):
+
+    ty::ClauseKind<'tcx> {
+        match *self {
+            ty::ClauseKind::Trait(ref data) => {
+                p!(print(data))
+            }
+            ty::ClauseKind::RegionOutlives(predicate) => p!(print(predicate)),
+            ty::ClauseKind::TypeOutlives(predicate) => p!(print(predicate)),
+            ty::ClauseKind::Projection(predicate) => p!(print(predicate)),
+            ty::ClauseKind::ConstArgHasType(ct, ty) => {
+                p!("the constant `", print(ct), "` has type `", print(ty), "`")
+            },
+            ty::ClauseKind::WellFormed(arg) => p!(print(arg), " well-formed"),
+            ty::ClauseKind::ConstEvaluatable(ct) => {
+                p!("the constant `", print(ct), "` can be evaluated")
+            }
+        }
+    }
+
+    ty::PredicateKind<'tcx> {
+        match *self {
+            ty::PredicateKind::Clause(data) => {
+                p!(print(data))
+            }
+            ty::PredicateKind::Subtype(predicate) => p!(print(predicate)),
+            ty::PredicateKind::Coerce(predicate) => p!(print(predicate)),
+            ty::PredicateKind::ObjectSafe(trait_def_id) => {
+                p!("the trait `", print_def_path(trait_def_id, &[]), "` is object-safe")
+            }
+            ty::PredicateKind::ClosureKind(closure_def_id, _closure_args, kind) => p!(
+                "the closure `",
+                print_value_path(closure_def_id, &[]),
+                write("` implements the trait `{}`", kind)
+            ),
+            ty::PredicateKind::ConstEquate(c1, c2) => {
+                p!("the constant `", print(c1), "` equals `", print(c2), "`")
+            }
+            ty::PredicateKind::Ambiguous => p!("ambiguous"),
+            ty::PredicateKind::AliasRelate(t1, t2, dir) => p!(print(t1), write(" {} ", dir), print(t2)),
+        }
+    }
+}
+
 define_print_and_forward_display! {
     (self, cx):
 
@@ -2893,55 +2927,13 @@ define_print_and_forward_display! {
     }
 
     ty::Predicate<'tcx> {
-        let binder = self.kind();
-        p!(print(binder))
+        p!(print(self.kind()))
     }
 
     ty::Clause<'tcx> {
         p!(print(self.kind()))
     }
 
-    ty::ClauseKind<'tcx> {
-        match *self {
-            ty::ClauseKind::Trait(ref data) => {
-                p!(print(data))
-            }
-            ty::ClauseKind::RegionOutlives(predicate) => p!(print(predicate)),
-            ty::ClauseKind::TypeOutlives(predicate) => p!(print(predicate)),
-            ty::ClauseKind::Projection(predicate) => p!(print(predicate)),
-            ty::ClauseKind::ConstArgHasType(ct, ty) => {
-                p!("the constant `", print(ct), "` has type `", print(ty), "`")
-            },
-            ty::ClauseKind::WellFormed(arg) => p!(print(arg), " well-formed"),
-            ty::ClauseKind::ConstEvaluatable(ct) => {
-                p!("the constant `", print(ct), "` can be evaluated")
-            }
-        }
-    }
-
-    ty::PredicateKind<'tcx> {
-        match *self {
-            ty::PredicateKind::Clause(data) => {
-                p!(print(data))
-            }
-            ty::PredicateKind::Subtype(predicate) => p!(print(predicate)),
-            ty::PredicateKind::Coerce(predicate) => p!(print(predicate)),
-            ty::PredicateKind::ObjectSafe(trait_def_id) => {
-                p!("the trait `", print_def_path(trait_def_id, &[]), "` is object-safe")
-            }
-            ty::PredicateKind::ClosureKind(closure_def_id, _closure_args, kind) => p!(
-                "the closure `",
-                print_value_path(closure_def_id, &[]),
-                write("` implements the trait `{}`", kind)
-            ),
-            ty::PredicateKind::ConstEquate(c1, c2) => {
-                p!("the constant `", print(c1), "` equals `", print(c2), "`")
-            }
-            ty::PredicateKind::Ambiguous => p!("ambiguous"),
-            ty::PredicateKind::AliasRelate(t1, t2, dir) => p!(print(t1), write(" {} ", dir), print(t2)),
-        }
-    }
-
     GenericArg<'tcx> {
         match self.unpack() {
             GenericArgKind::Lifetime(lt) => p!(print(lt)),
diff --git a/compiler/rustc_middle/src/ty/relate.rs b/compiler/rustc_middle/src/ty/relate.rs
index e9d763afa..27e9be37f 100644
--- a/compiler/rustc_middle/src/ty/relate.rs
+++ b/compiler/rustc_middle/src/ty/relate.rs
@@ -8,6 +8,7 @@ use crate::ty::error::{ExpectedFound, TypeError};
 use crate::ty::{self, Expr, ImplSubject, Term, TermKind, Ty, TyCtxt, TypeFoldable};
 use crate::ty::{GenericArg, GenericArgKind, GenericArgsRef};
 use rustc_hir as hir;
+use rustc_hir::def::DefKind;
 use rustc_hir::def_id::DefId;
 use rustc_target::spec::abi;
 use std::iter;
@@ -134,7 +135,7 @@ pub fn relate_type_and_mut<'tcx, R: TypeRelation<'tcx>>(
 }
 
 #[inline]
-pub fn relate_args<'tcx, R: TypeRelation<'tcx>>(
+pub fn relate_args_invariantly<'tcx, R: TypeRelation<'tcx>>(
     relation: &mut R,
     a_arg: GenericArgsRef<'tcx>,
     b_arg: GenericArgsRef<'tcx>,
@@ -273,8 +274,21 @@ impl<'tcx> Relate<'tcx> for ty::AliasTy<'tcx> {
         if a.def_id != b.def_id {
             Err(TypeError::ProjectionMismatched(expected_found(relation, a.def_id, b.def_id)))
         } else {
-            let args = relation.relate(a.args, b.args)?;
-            Ok(relation.tcx().mk_alias_ty(a.def_id, args))
+            let args = match relation.tcx().def_kind(a.def_id) {
+                DefKind::OpaqueTy => relate_args_with_variances(
+                    relation,
+                    a.def_id,
+                    relation.tcx().variances_of(a.def_id),
+                    a.args,
+                    b.args,
+                    false, // do not fetch `type_of(a_def_id)`, as it will cause a cycle
+                )?,
+                DefKind::AssocTy | DefKind::AssocConst | DefKind::TyAlias => {
+                    relate_args_invariantly(relation, a.args, b.args)?
+                }
+                def => bug!("unknown alias DefKind: {def:?}"),
+            };
+            Ok(ty::AliasTy::new(relation.tcx(), a.def_id, args))
         }
     }
 }
@@ -315,7 +329,7 @@ impl<'tcx> Relate<'tcx> for ty::TraitRef<'tcx> {
         if a.def_id != b.def_id {
             Err(TypeError::Traits(expected_found(relation, a.def_id, b.def_id)))
         } else {
-            let args = relate_args(relation, a.args, b.args)?;
+            let args = relate_args_invariantly(relation, a.args, b.args)?;
             Ok(ty::TraitRef::new(relation.tcx(), a.def_id, args))
         }
     }
@@ -331,26 +345,26 @@ impl<'tcx> Relate<'tcx> for ty::ExistentialTraitRef<'tcx> {
         if a.def_id != b.def_id {
             Err(TypeError::Traits(expected_found(relation, a.def_id, b.def_id)))
         } else {
-            let args = relate_args(relation, a.args, b.args)?;
+            let args = relate_args_invariantly(relation, a.args, b.args)?;
             Ok(ty::ExistentialTraitRef { def_id: a.def_id, args })
         }
     }
 }
 
 #[derive(PartialEq, Copy, Debug, Clone, TypeFoldable, TypeVisitable)]
-struct GeneratorWitness<'tcx>(&'tcx ty::List<Ty<'tcx>>);
+struct CoroutineWitness<'tcx>(&'tcx ty::List<Ty<'tcx>>);
 
-impl<'tcx> Relate<'tcx> for GeneratorWitness<'tcx> {
+impl<'tcx> Relate<'tcx> for CoroutineWitness<'tcx> {
     fn relate<R: TypeRelation<'tcx>>(
         relation: &mut R,
-        a: GeneratorWitness<'tcx>,
-        b: GeneratorWitness<'tcx>,
-    ) -> RelateResult<'tcx, GeneratorWitness<'tcx>> {
+        a: CoroutineWitness<'tcx>,
+        b: CoroutineWitness<'tcx>,
+    ) -> RelateResult<'tcx, CoroutineWitness<'tcx>> {
         assert_eq!(a.0.len(), b.0.len());
         let tcx = relation.tcx();
         let types =
             tcx.mk_type_list_from_iter(iter::zip(a.0, b.0).map(|(a, b)| relation.relate(a, b)))?;
-        Ok(GeneratorWitness(types))
+        Ok(CoroutineWitness(types))
     }
 }
 
@@ -443,31 +457,31 @@ pub fn structurally_relate_tys<'tcx, R: TypeRelation<'tcx>>(
             Ok(Ty::new_dynamic(tcx, relation.relate(a_obj, b_obj)?, region_bound, a_repr))
         }
 
-        (&ty::Generator(a_id, a_args, movability), &ty::Generator(b_id, b_args, _))
+        (&ty::Coroutine(a_id, a_args, movability), &ty::Coroutine(b_id, b_args, _))
             if a_id == b_id =>
         {
-            // All Generator types with the same id represent
-            // the (anonymous) type of the same generator expression. So
+            // All Coroutine types with the same id represent
+            // the (anonymous) type of the same coroutine expression. So
             // all of their regions should be equated.
-            let args = relation.relate(a_args, b_args)?;
-            Ok(Ty::new_generator(tcx, a_id, args, movability))
+            let args = relate_args_invariantly(relation, a_args, b_args)?;
+            Ok(Ty::new_coroutine(tcx, a_id, args, movability))
         }
 
-        (&ty::GeneratorWitness(a_id, a_args), &ty::GeneratorWitness(b_id, b_args))
+        (&ty::CoroutineWitness(a_id, a_args), &ty::CoroutineWitness(b_id, b_args))
             if a_id == b_id =>
         {
-            // All GeneratorWitness types with the same id represent
-            // the (anonymous) type of the same generator expression. So
+            // All CoroutineWitness types with the same id represent
+            // the (anonymous) type of the same coroutine expression. So
             // all of their regions should be equated.
-            let args = relation.relate(a_args, b_args)?;
-            Ok(Ty::new_generator_witness(tcx, a_id, args))
+            let args = relate_args_invariantly(relation, a_args, b_args)?;
+            Ok(Ty::new_coroutine_witness(tcx, a_id, args))
         }
 
         (&ty::Closure(a_id, a_args), &ty::Closure(b_id, b_args)) if a_id == b_id => {
             // All Closure types with the same id represent
             // the (anonymous) type of the same closure expression. So
             // all of their regions should be equated.
-            let args = relation.relate(a_args, b_args)?;
+            let args = relate_args_invariantly(relation, a_args, b_args)?;
             Ok(Ty::new_closure(tcx, a_id, &args))
         }
 
@@ -536,24 +550,6 @@ pub fn structurally_relate_tys<'tcx, R: TypeRelation<'tcx>>(
             Ok(Ty::new_fn_ptr(tcx, fty))
         }
 
-        // The args of opaque types may not all be invariant, so we have
-        // to treat them separately from other aliases.
-        (
-            &ty::Alias(ty::Opaque, ty::AliasTy { def_id: a_def_id, args: a_args, .. }),
-            &ty::Alias(ty::Opaque, ty::AliasTy { def_id: b_def_id, args: b_args, .. }),
-        ) if a_def_id == b_def_id => {
-            let opt_variances = tcx.variances_of(a_def_id);
-            let args = relate_args_with_variances(
-                relation,
-                a_def_id,
-                opt_variances,
-                a_args,
-                b_args,
-                false, // do not fetch `type_of(a_def_id)`, as it will cause a cycle
-            )?;
-            Ok(Ty::new_opaque(tcx, a_def_id, args))
-        }
-
         // Alias tend to mostly already be handled downstream due to normalization.
         (&ty::Alias(a_kind, a_data), &ty::Alias(b_kind, b_data)) => {
             let alias_ty = relation.relate(a_data, b_data)?;
@@ -709,19 +705,19 @@ impl<'tcx> Relate<'tcx> for ty::ClosureArgs<'tcx> {
         a: ty::ClosureArgs<'tcx>,
         b: ty::ClosureArgs<'tcx>,
     ) -> RelateResult<'tcx, ty::ClosureArgs<'tcx>> {
-        let args = relate_args(relation, a.args, b.args)?;
+        let args = relate_args_invariantly(relation, a.args, b.args)?;
         Ok(ty::ClosureArgs { args })
     }
 }
 
-impl<'tcx> Relate<'tcx> for ty::GeneratorArgs<'tcx> {
+impl<'tcx> Relate<'tcx> for ty::CoroutineArgs<'tcx> {
     fn relate<R: TypeRelation<'tcx>>(
         relation: &mut R,
-        a: ty::GeneratorArgs<'tcx>,
-        b: ty::GeneratorArgs<'tcx>,
-    ) -> RelateResult<'tcx, ty::GeneratorArgs<'tcx>> {
-        let args = relate_args(relation, a.args, b.args)?;
-        Ok(ty::GeneratorArgs { args })
+        a: ty::CoroutineArgs<'tcx>,
+        b: ty::CoroutineArgs<'tcx>,
+    ) -> RelateResult<'tcx, ty::CoroutineArgs<'tcx>> {
+        let args = relate_args_invariantly(relation, a.args, b.args)?;
+        Ok(ty::CoroutineArgs { args })
     }
 }
 
@@ -731,7 +727,7 @@ impl<'tcx> Relate<'tcx> for GenericArgsRef<'tcx> {
         a: GenericArgsRef<'tcx>,
         b: GenericArgsRef<'tcx>,
     ) -> RelateResult<'tcx, GenericArgsRef<'tcx>> {
-        relate_args(relation, a, b)
+        relate_args_invariantly(relation, a, b)
     }
 }
 
@@ -835,19 +831,6 @@ impl<'tcx> Relate<'tcx> for Term<'tcx> {
     }
 }
 
-impl<'tcx> Relate<'tcx> for ty::ProjectionPredicate<'tcx> {
-    fn relate<R: TypeRelation<'tcx>>(
-        relation: &mut R,
-        a: ty::ProjectionPredicate<'tcx>,
-        b: ty::ProjectionPredicate<'tcx>,
-    ) -> RelateResult<'tcx, ty::ProjectionPredicate<'tcx>> {
-        Ok(ty::ProjectionPredicate {
-            projection_ty: relation.relate(a.projection_ty, b.projection_ty)?,
-            term: relation.relate(a.term, b.term)?,
-        })
-    }
-}
-
 ///////////////////////////////////////////////////////////////////////////
 // Error handling
 
diff --git a/compiler/rustc_middle/src/ty/structural_impls.rs b/compiler/rustc_middle/src/ty/structural_impls.rs
index 2adbe9e03..6af68bc5d 100644
--- a/compiler/rustc_middle/src/ty/structural_impls.rs
+++ b/compiler/rustc_middle/src/ty/structural_impls.rs
@@ -10,7 +10,7 @@ use crate::ty::visit::{TypeSuperVisitable, TypeVisitable, TypeVisitor};
 use crate::ty::{self, AliasTy, InferConst, Lift, Term, TermKind, Ty, TyCtxt};
 use rustc_hir::def::Namespace;
 use rustc_target::abi::TyAndLayout;
-use rustc_type_ir::{ConstKind, DebugWithInfcx, InferCtxtLike, OptWithInfcx};
+use rustc_type_ir::{ConstKind, DebugWithInfcx, InferCtxtLike, WithInfcx};
 
 use std::fmt::{self, Debug};
 use std::ops::ControlFlow;
@@ -22,11 +22,10 @@ impl fmt::Debug for ty::TraitDef {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         ty::tls::with(|tcx| {
             with_no_trimmed_paths!({
-                f.write_str(
-                    &FmtPrinter::new(tcx, Namespace::TypeNS)
-                        .print_def_path(self.def_id, &[])?
-                        .into_buffer(),
-                )
+                let s = FmtPrinter::print_string(tcx, Namespace::TypeNS, |cx| {
+                    cx.print_def_path(self.def_id, &[])
+                })?;
+                f.write_str(&s)
             })
         })
     }
@@ -36,11 +35,10 @@ impl<'tcx> fmt::Debug for ty::AdtDef<'tcx> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         ty::tls::with(|tcx| {
             with_no_trimmed_paths!({
-                f.write_str(
-                    &FmtPrinter::new(tcx, Namespace::TypeNS)
-                        .print_def_path(self.did(), &[])?
-                        .into_buffer(),
-                )
+                let s = FmtPrinter::print_string(tcx, Namespace::TypeNS, |cx| {
+                    cx.print_def_path(self.did(), &[])
+                })?;
+                f.write_str(&s)
             })
         })
     }
@@ -89,12 +87,12 @@ impl fmt::Debug for ty::FreeRegion {
 
 impl<'tcx> fmt::Debug for ty::FnSig<'tcx> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        OptWithInfcx::new_no_ctx(self).fmt(f)
+        WithInfcx::with_no_infcx(self).fmt(f)
     }
 }
 impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for ty::FnSig<'tcx> {
-    fn fmt<InfCtx: InferCtxtLike<TyCtxt<'tcx>>>(
-        this: OptWithInfcx<'_, TyCtxt<'tcx>, InfCtx, &Self>,
+    fn fmt<Infcx: InferCtxtLike<Interner = TyCtxt<'tcx>>>(
+        this: WithInfcx<'_, Infcx, &Self>,
         f: &mut core::fmt::Formatter<'_>,
     ) -> core::fmt::Result {
         let sig = this.data;
@@ -130,18 +128,6 @@ impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for ty::FnSig<'tcx> {
     }
 }
 
-impl<'tcx> fmt::Debug for ty::ConstVid<'tcx> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "?{}c", self.index)
-    }
-}
-
-impl fmt::Debug for ty::EffectVid<'_> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "?{}e", self.index)
-    }
-}
-
 impl<'tcx> fmt::Debug for ty::TraitRef<'tcx> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         with_no_trimmed_paths!(fmt::Display::fmt(self, f))
@@ -149,8 +135,8 @@ impl<'tcx> fmt::Debug for ty::TraitRef<'tcx> {
 }
 
 impl<'tcx> ty::DebugWithInfcx<TyCtxt<'tcx>> for Ty<'tcx> {
-    fn fmt<InfCtx: InferCtxtLike<TyCtxt<'tcx>>>(
-        this: OptWithInfcx<'_, TyCtxt<'tcx>, InfCtx, &Self>,
+    fn fmt<Infcx: InferCtxtLike<Interner = TyCtxt<'tcx>>>(
+        this: WithInfcx<'_, Infcx, &Self>,
         f: &mut core::fmt::Formatter<'_>,
     ) -> core::fmt::Result {
         this.data.fmt(f)
@@ -199,51 +185,14 @@ impl<'tcx> fmt::Debug for ty::Clause<'tcx> {
     }
 }
 
-impl<'tcx> fmt::Debug for ty::ClauseKind<'tcx> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match *self {
-            ty::ClauseKind::ConstArgHasType(ct, ty) => write!(f, "ConstArgHasType({ct:?}, {ty:?})"),
-            ty::ClauseKind::Trait(ref a) => a.fmt(f),
-            ty::ClauseKind::RegionOutlives(ref pair) => pair.fmt(f),
-            ty::ClauseKind::TypeOutlives(ref pair) => pair.fmt(f),
-            ty::ClauseKind::Projection(ref pair) => pair.fmt(f),
-            ty::ClauseKind::WellFormed(ref data) => write!(f, "WellFormed({data:?})"),
-            ty::ClauseKind::ConstEvaluatable(ct) => {
-                write!(f, "ConstEvaluatable({ct:?})")
-            }
-        }
-    }
-}
-
-impl<'tcx> fmt::Debug for ty::PredicateKind<'tcx> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match *self {
-            ty::PredicateKind::Clause(ref a) => a.fmt(f),
-            ty::PredicateKind::Subtype(ref pair) => pair.fmt(f),
-            ty::PredicateKind::Coerce(ref pair) => pair.fmt(f),
-            ty::PredicateKind::ObjectSafe(trait_def_id) => {
-                write!(f, "ObjectSafe({trait_def_id:?})")
-            }
-            ty::PredicateKind::ClosureKind(closure_def_id, closure_args, kind) => {
-                write!(f, "ClosureKind({closure_def_id:?}, {closure_args:?}, {kind:?})")
-            }
-            ty::PredicateKind::ConstEquate(c1, c2) => write!(f, "ConstEquate({c1:?}, {c2:?})"),
-            ty::PredicateKind::Ambiguous => write!(f, "Ambiguous"),
-            ty::PredicateKind::AliasRelate(t1, t2, dir) => {
-                write!(f, "AliasRelate({t1:?}, {dir:?}, {t2:?})")
-            }
-        }
-    }
-}
-
 impl<'tcx> fmt::Debug for AliasTy<'tcx> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        OptWithInfcx::new_no_ctx(self).fmt(f)
+        WithInfcx::with_no_infcx(self).fmt(f)
     }
 }
 impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for AliasTy<'tcx> {
-    fn fmt<InfCtx: InferCtxtLike<TyCtxt<'tcx>>>(
-        this: OptWithInfcx<'_, TyCtxt<'tcx>, InfCtx, &Self>,
+    fn fmt<Infcx: InferCtxtLike<Interner = TyCtxt<'tcx>>>(
+        this: WithInfcx<'_, Infcx, &Self>,
         f: &mut core::fmt::Formatter<'_>,
     ) -> core::fmt::Result {
         f.debug_struct("AliasTy")
@@ -253,7 +202,7 @@ impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for AliasTy<'tcx> {
     }
 }
 
-impl<'tcx> fmt::Debug for ty::InferConst<'tcx> {
+impl fmt::Debug for ty::InferConst {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
             InferConst::Var(var) => write!(f, "{var:?}"),
@@ -262,17 +211,17 @@ impl<'tcx> fmt::Debug for ty::InferConst<'tcx> {
         }
     }
 }
-impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for ty::InferConst<'tcx> {
-    fn fmt<InfCtx: InferCtxtLike<TyCtxt<'tcx>>>(
-        this: OptWithInfcx<'_, TyCtxt<'tcx>, InfCtx, &Self>,
+impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for ty::InferConst {
+    fn fmt<Infcx: InferCtxtLike<Interner = TyCtxt<'tcx>>>(
+        this: WithInfcx<'_, Infcx, &Self>,
         f: &mut core::fmt::Formatter<'_>,
     ) -> core::fmt::Result {
         use ty::InferConst::*;
-        match this.infcx.and_then(|infcx| infcx.universe_of_ct(*this.data)) {
+        match this.infcx.universe_of_ct(*this.data) {
             None => write!(f, "{:?}", this.data),
             Some(universe) => match *this.data {
-                Var(vid) => write!(f, "?{}_{}c", vid.index, universe.index()),
-                EffectVar(vid) => write!(f, "?{}_{}e", vid.index, universe.index()),
+                Var(vid) => write!(f, "?{}_{}c", vid.index(), universe.index()),
+                EffectVar(vid) => write!(f, "?{}_{}e", vid.index(), universe.index()),
                 Fresh(_) => {
                     unreachable!()
                 }
@@ -283,12 +232,12 @@ impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for ty::InferConst<'tcx> {
 
 impl<'tcx> fmt::Debug for ty::consts::Expr<'tcx> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        OptWithInfcx::new_no_ctx(self).fmt(f)
+        WithInfcx::with_no_infcx(self).fmt(f)
     }
 }
 impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for ty::consts::Expr<'tcx> {
-    fn fmt<InfCtx: InferCtxtLike<TyCtxt<'tcx>>>(
-        this: OptWithInfcx<'_, TyCtxt<'tcx>, InfCtx, &Self>,
+    fn fmt<Infcx: InferCtxtLike<Interner = TyCtxt<'tcx>>>(
+        this: WithInfcx<'_, Infcx, &Self>,
         f: &mut core::fmt::Formatter<'_>,
     ) -> core::fmt::Result {
         match this.data {
@@ -316,12 +265,12 @@ impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for ty::consts::Expr<'tcx> {
 
 impl<'tcx> fmt::Debug for ty::UnevaluatedConst<'tcx> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        OptWithInfcx::new_no_ctx(self).fmt(f)
+        WithInfcx::with_no_infcx(self).fmt(f)
     }
 }
 impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for ty::UnevaluatedConst<'tcx> {
-    fn fmt<InfCtx: InferCtxtLike<TyCtxt<'tcx>>>(
-        this: OptWithInfcx<'_, TyCtxt<'tcx>, InfCtx, &Self>,
+    fn fmt<Infcx: InferCtxtLike<Interner = TyCtxt<'tcx>>>(
+        this: WithInfcx<'_, Infcx, &Self>,
         f: &mut core::fmt::Formatter<'_>,
     ) -> core::fmt::Result {
         f.debug_struct("UnevaluatedConst")
@@ -333,12 +282,12 @@ impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for ty::UnevaluatedConst<'tcx> {
 
 impl<'tcx> fmt::Debug for ty::Const<'tcx> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        OptWithInfcx::new_no_ctx(self).fmt(f)
+        WithInfcx::with_no_infcx(self).fmt(f)
     }
 }
 impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for ty::Const<'tcx> {
-    fn fmt<InfCtx: InferCtxtLike<TyCtxt<'tcx>>>(
-        this: OptWithInfcx<'_, TyCtxt<'tcx>, InfCtx, &Self>,
+    fn fmt<Infcx: InferCtxtLike<Interner = TyCtxt<'tcx>>>(
+        this: WithInfcx<'_, Infcx, &Self>,
         f: &mut core::fmt::Formatter<'_>,
     ) -> core::fmt::Result {
         // If this is a value, we spend some effort to make it look nice.
@@ -350,9 +299,8 @@ impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for ty::Const<'tcx> {
                 let ConstKind::Value(valtree) = lifted.kind() else {
                     bug!("we checked that this is a valtree")
                 };
-                let cx = FmtPrinter::new(tcx, Namespace::ValueNS);
-                let cx =
-                    cx.pretty_print_const_valtree(valtree, lifted.ty(), /*print_ty*/ true)?;
+                let mut cx = FmtPrinter::new(tcx, Namespace::ValueNS);
+                cx.pretty_print_const_valtree(valtree, lifted.ty(), /*print_ty*/ true)?;
                 f.write_str(&cx.into_buffer())
             });
         }
@@ -395,8 +343,8 @@ impl<'tcx> fmt::Debug for GenericArg<'tcx> {
     }
 }
 impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for GenericArg<'tcx> {
-    fn fmt<InfCtx: InferCtxtLike<TyCtxt<'tcx>>>(
-        this: OptWithInfcx<'_, TyCtxt<'tcx>, InfCtx, &Self>,
+    fn fmt<Infcx: InferCtxtLike<Interner = TyCtxt<'tcx>>>(
+        this: WithInfcx<'_, Infcx, &Self>,
         f: &mut core::fmt::Formatter<'_>,
     ) -> core::fmt::Result {
         match this.data.unpack() {
@@ -413,8 +361,8 @@ impl<'tcx> fmt::Debug for Region<'tcx> {
     }
 }
 impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for Region<'tcx> {
-    fn fmt<InfCtx: InferCtxtLike<TyCtxt<'tcx>>>(
-        this: OptWithInfcx<'_, TyCtxt<'tcx>, InfCtx, &Self>,
+    fn fmt<Infcx: InferCtxtLike<Interner = TyCtxt<'tcx>>>(
+        this: WithInfcx<'_, Infcx, &Self>,
         f: &mut core::fmt::Formatter<'_>,
     ) -> core::fmt::Result {
         write!(f, "{:?}", &this.map(|data| data.kind()))
@@ -422,11 +370,11 @@ impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for Region<'tcx> {
 }
 
 impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for ty::RegionVid {
-    fn fmt<InfCtx: InferCtxtLike<TyCtxt<'tcx>>>(
-        this: OptWithInfcx<'_, TyCtxt<'tcx>, InfCtx, &Self>,
+    fn fmt<Infcx: InferCtxtLike<Interner = TyCtxt<'tcx>>>(
+        this: WithInfcx<'_, Infcx, &Self>,
         f: &mut core::fmt::Formatter<'_>,
     ) -> core::fmt::Result {
-        match this.infcx.and_then(|infcx| infcx.universe_of_lt(*this.data)) {
+        match this.infcx.universe_of_lt(*this.data) {
             Some(universe) => write!(f, "'?{}_{}", this.data.index(), universe.index()),
             None => write!(f, "{:?}", this.data),
         }
@@ -434,8 +382,8 @@ impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for ty::RegionVid {
 }
 
 impl<'tcx, T: DebugWithInfcx<TyCtxt<'tcx>>> DebugWithInfcx<TyCtxt<'tcx>> for ty::Binder<'tcx, T> {
-    fn fmt<InfCtx: InferCtxtLike<TyCtxt<'tcx>>>(
-        this: OptWithInfcx<'_, TyCtxt<'tcx>, InfCtx, &Self>,
+    fn fmt<Infcx: InferCtxtLike<Interner = TyCtxt<'tcx>>>(
+        this: WithInfcx<'_, Infcx, &Self>,
         f: &mut core::fmt::Formatter<'_>,
     ) -> core::fmt::Result {
         f.debug_tuple("Binder")
@@ -501,7 +449,6 @@ TrivialTypeTraversalImpls! {
     crate::ty::IntVarValue,
     crate::ty::adjustment::PointerCoercion,
     crate::ty::RegionVid,
-    crate::ty::UniverseIndex,
     crate::ty::Variance,
     ::rustc_span::Span,
     ::rustc_span::symbol::Ident,
@@ -518,7 +465,6 @@ TrivialTypeTraversalAndLiftImpls! {
     ::rustc_hir::Mutability,
     ::rustc_hir::Unsafety,
     ::rustc_target::spec::abi::Abi,
-    crate::ty::AliasRelationDirection,
     crate::ty::ClosureKind,
     crate::ty::ParamConst,
     crate::ty::ParamTy,
@@ -654,11 +600,11 @@ impl<'tcx> TypeSuperFoldable<TyCtxt<'tcx>> for Ty<'tcx> {
             ty::Ref(r, ty, mutbl) => {
                 ty::Ref(r.try_fold_with(folder)?, ty.try_fold_with(folder)?, mutbl)
             }
-            ty::Generator(did, args, movability) => {
-                ty::Generator(did, args.try_fold_with(folder)?, movability)
+            ty::Coroutine(did, args, movability) => {
+                ty::Coroutine(did, args.try_fold_with(folder)?, movability)
             }
-            ty::GeneratorWitness(did, args) => {
-                ty::GeneratorWitness(did, args.try_fold_with(folder)?)
+            ty::CoroutineWitness(did, args) => {
+                ty::CoroutineWitness(did, args.try_fold_with(folder)?)
             }
             ty::Closure(did, args) => ty::Closure(did, args.try_fold_with(folder)?),
             ty::Alias(kind, data) => ty::Alias(kind, data.try_fold_with(folder)?),
@@ -706,8 +652,8 @@ impl<'tcx> TypeSuperVisitable<TyCtxt<'tcx>> for Ty<'tcx> {
                 r.visit_with(visitor)?;
                 ty.visit_with(visitor)
             }
-            ty::Generator(_did, ref args, _) => args.visit_with(visitor),
-            ty::GeneratorWitness(_did, ref args) => args.visit_with(visitor),
+            ty::Coroutine(_did, ref args, _) => args.visit_with(visitor),
+            ty::CoroutineWitness(_did, ref args) => args.visit_with(visitor),
             ty::Closure(_did, ref args) => args.visit_with(visitor),
             ty::Alias(_, ref data) => data.visit_with(visitor),
 
@@ -865,7 +811,7 @@ impl<'tcx> TypeSuperVisitable<TyCtxt<'tcx>> for ty::Const<'tcx> {
     }
 }
 
-impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for InferConst<'tcx> {
+impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for InferConst {
     fn try_fold_with<F: FallibleTypeFolder<TyCtxt<'tcx>>>(
         self,
         _folder: &mut F,
@@ -874,7 +820,7 @@ impl<'tcx> TypeFoldable<TyCtxt<'tcx>> for InferConst<'tcx> {
     }
 }
 
-impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for InferConst<'tcx> {
+impl<'tcx> TypeVisitable<TyCtxt<'tcx>> for InferConst {
     fn visit_with<V: TypeVisitor<TyCtxt<'tcx>>>(
         &self,
         _visitor: &mut V,
diff --git a/compiler/rustc_middle/src/ty/sty.rs b/compiler/rustc_middle/src/ty/sty.rs
index 1e57392e0..44592b10d 100644
--- a/compiler/rustc_middle/src/ty/sty.rs
+++ b/compiler/rustc_middle/src/ty/sty.rs
@@ -29,17 +29,18 @@ use std::assert_matches::debug_assert_matches;
 use std::borrow::Cow;
 use std::cmp::Ordering;
 use std::fmt;
-use std::marker::PhantomData;
 use std::ops::{ControlFlow, Deref, Range};
 use ty::util::IntTypeExt;
 
-use rustc_type_ir::sty::TyKind::*;
+use rustc_type_ir::ClauseKind as IrClauseKind;
 use rustc_type_ir::CollectAndApply;
 use rustc_type_ir::ConstKind as IrConstKind;
 use rustc_type_ir::DebugWithInfcx;
 use rustc_type_ir::DynKind;
+use rustc_type_ir::PredicateKind as IrPredicateKind;
 use rustc_type_ir::RegionKind as IrRegionKind;
 use rustc_type_ir::TyKind as IrTyKind;
+use rustc_type_ir::TyKind::*;
 
 use super::GenericParamDefKind;
 
@@ -48,6 +49,8 @@ use super::GenericParamDefKind;
 pub type TyKind<'tcx> = IrTyKind<TyCtxt<'tcx>>;
 pub type RegionKind<'tcx> = IrRegionKind<TyCtxt<'tcx>>;
 pub type ConstKind<'tcx> = IrConstKind<TyCtxt<'tcx>>;
+pub type PredicateKind<'tcx> = IrPredicateKind<TyCtxt<'tcx>>;
+pub type ClauseKind<'tcx> = IrClauseKind<TyCtxt<'tcx>>;
 
 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, TyEncodable, TyDecodable)]
 #[derive(HashStable, TypeFoldable, TypeVisitable, Lift)]
@@ -215,20 +218,20 @@ impl<'tcx> Article for TyKind<'tcx> {
 /// closure C (which would then require fixed point iteration to
 /// handle). Plus it fixes an ICE. :P
 ///
-/// ## Generators
+/// ## Coroutines
 ///
-/// Generators are handled similarly in `GeneratorArgs`. The set of
+/// Coroutines are handled similarly in `CoroutineArgs`. The set of
 /// type parameters is similar, but `CK` and `CS` are replaced by the
 /// following type parameters:
 ///
-/// * `GS`: The generator's "resume type", which is the type of the
+/// * `GS`: The coroutine's "resume type", which is the type of the
 ///   argument passed to `resume`, and the type of `yield` expressions
-///   inside the generator.
+///   inside the coroutine.
 /// * `GY`: The "yield type", which is the type of values passed to
-///   `yield` inside the generator.
+///   `yield` inside the coroutine.
 /// * `GR`: The "return type", which is the type of value returned upon
-///   completion of the generator.
-/// * `GW`: The "generator witness".
+///   completion of the coroutine.
+/// * `GW`: The "coroutine witness".
 #[derive(Copy, Clone, PartialEq, Eq, Debug, TypeFoldable, TypeVisitable, Lift)]
 pub struct ClosureArgs<'tcx> {
     /// Lifetime and type parameters from the enclosing function,
@@ -352,11 +355,11 @@ impl<'tcx> ClosureArgs<'tcx> {
 
 /// Similar to `ClosureArgs`; see the above documentation for more.
 #[derive(Copy, Clone, PartialEq, Eq, Debug, TypeFoldable, TypeVisitable)]
-pub struct GeneratorArgs<'tcx> {
+pub struct CoroutineArgs<'tcx> {
     pub args: GenericArgsRef<'tcx>,
 }
 
-pub struct GeneratorArgsParts<'tcx, T> {
+pub struct CoroutineArgsParts<'tcx, T> {
     pub parent_args: &'tcx [GenericArg<'tcx>],
     pub resume_ty: T,
     pub yield_ty: T,
@@ -365,14 +368,14 @@ pub struct GeneratorArgsParts<'tcx, T> {
     pub tupled_upvars_ty: T,
 }
 
-impl<'tcx> GeneratorArgs<'tcx> {
-    /// Construct `GeneratorArgs` from `GeneratorArgsParts`, containing `Args`
-    /// for the generator parent, alongside additional generator-specific components.
+impl<'tcx> CoroutineArgs<'tcx> {
+    /// Construct `CoroutineArgs` from `CoroutineArgsParts`, containing `Args`
+    /// for the coroutine parent, alongside additional coroutine-specific components.
     pub fn new(
         tcx: TyCtxt<'tcx>,
-        parts: GeneratorArgsParts<'tcx, Ty<'tcx>>,
-    ) -> GeneratorArgs<'tcx> {
-        GeneratorArgs {
+        parts: CoroutineArgsParts<'tcx, Ty<'tcx>>,
+    ) -> CoroutineArgs<'tcx> {
+        CoroutineArgs {
             args: tcx.mk_args_from_iter(
                 parts.parent_args.iter().copied().chain(
                     [
@@ -389,12 +392,12 @@ impl<'tcx> GeneratorArgs<'tcx> {
         }
     }
 
-    /// Divides the generator args into their respective components.
-    /// The ordering assumed here must match that used by `GeneratorArgs::new` above.
-    fn split(self) -> GeneratorArgsParts<'tcx, GenericArg<'tcx>> {
+    /// Divides the coroutine args into their respective components.
+    /// The ordering assumed here must match that used by `CoroutineArgs::new` above.
+    fn split(self) -> CoroutineArgsParts<'tcx, GenericArg<'tcx>> {
         match self.args[..] {
             [ref parent_args @ .., resume_ty, yield_ty, return_ty, witness, tupled_upvars_ty] => {
-                GeneratorArgsParts {
+                CoroutineArgsParts {
                     parent_args,
                     resume_ty,
                     yield_ty,
@@ -403,34 +406,34 @@ impl<'tcx> GeneratorArgs<'tcx> {
                     tupled_upvars_ty,
                 }
             }
-            _ => bug!("generator args missing synthetics"),
+            _ => bug!("coroutine args missing synthetics"),
         }
     }
 
     /// Returns `true` only if enough of the synthetic types are known to
-    /// allow using all of the methods on `GeneratorArgs` without panicking.
+    /// allow using all of the methods on `CoroutineArgs` without panicking.
     ///
-    /// Used primarily by `ty::print::pretty` to be able to handle generator
+    /// Used primarily by `ty::print::pretty` to be able to handle coroutine
     /// types that haven't had their synthetic types substituted in.
     pub fn is_valid(self) -> bool {
         self.args.len() >= 5 && matches!(self.split().tupled_upvars_ty.expect_ty().kind(), Tuple(_))
     }
 
-    /// Returns the substitutions of the generator's parent.
+    /// Returns the substitutions of the coroutine's parent.
     pub fn parent_args(self) -> &'tcx [GenericArg<'tcx>] {
         self.split().parent_args
     }
 
-    /// This describes the types that can be contained in a generator.
+    /// This describes the types that can be contained in a coroutine.
     /// It will be a type variable initially and unified in the last stages of typeck of a body.
-    /// It contains a tuple of all the types that could end up on a generator frame.
+    /// It contains a tuple of all the types that could end up on a coroutine frame.
     /// The state transformation MIR pass may only produce layouts which mention types
     /// in this tuple. Upvars are not counted here.
     pub fn witness(self) -> Ty<'tcx> {
         self.split().witness.expect_ty()
     }
 
-    /// Returns an iterator over the list of types of captured paths by the generator.
+    /// Returns an iterator over the list of types of captured paths by the coroutine.
     /// In case there was a type error in figuring out the types of the captured path, an
     /// empty iterator is returned.
     #[inline]
@@ -443,28 +446,28 @@ impl<'tcx> GeneratorArgs<'tcx> {
         }
     }
 
-    /// Returns the tuple type representing the upvars for this generator.
+    /// Returns the tuple type representing the upvars for this coroutine.
     #[inline]
     pub fn tupled_upvars_ty(self) -> Ty<'tcx> {
         self.split().tupled_upvars_ty.expect_ty()
     }
 
-    /// Returns the type representing the resume type of the generator.
+    /// Returns the type representing the resume type of the coroutine.
     pub fn resume_ty(self) -> Ty<'tcx> {
         self.split().resume_ty.expect_ty()
     }
 
-    /// Returns the type representing the yield type of the generator.
+    /// Returns the type representing the yield type of the coroutine.
     pub fn yield_ty(self) -> Ty<'tcx> {
         self.split().yield_ty.expect_ty()
     }
 
-    /// Returns the type representing the return type of the generator.
+    /// Returns the type representing the return type of the coroutine.
     pub fn return_ty(self) -> Ty<'tcx> {
         self.split().return_ty.expect_ty()
     }
 
-    /// Returns the "generator signature", which consists of its yield
+    /// Returns the "coroutine signature", which consists of its yield
     /// and return types.
     ///
     /// N.B., some bits of the code prefers to see this wrapped in a
@@ -474,7 +477,7 @@ impl<'tcx> GeneratorArgs<'tcx> {
         ty::Binder::dummy(self.sig())
     }
 
-    /// Returns the "generator signature", which consists of its resume, yield
+    /// Returns the "coroutine signature", which consists of its resume, yield
     /// and return types.
     pub fn sig(self) -> GenSig<'tcx> {
         ty::GenSig {
@@ -485,23 +488,23 @@ impl<'tcx> GeneratorArgs<'tcx> {
     }
 }
 
-impl<'tcx> GeneratorArgs<'tcx> {
-    /// Generator has not been resumed yet.
+impl<'tcx> CoroutineArgs<'tcx> {
+    /// Coroutine has not been resumed yet.
     pub const UNRESUMED: usize = 0;
-    /// Generator has returned or is completed.
+    /// Coroutine has returned or is completed.
     pub const RETURNED: usize = 1;
-    /// Generator has been poisoned.
+    /// Coroutine has been poisoned.
     pub const POISONED: usize = 2;
 
     const UNRESUMED_NAME: &'static str = "Unresumed";
     const RETURNED_NAME: &'static str = "Returned";
     const POISONED_NAME: &'static str = "Panicked";
 
-    /// The valid variant indices of this generator.
+    /// The valid variant indices of this coroutine.
     #[inline]
     pub fn variant_range(&self, def_id: DefId, tcx: TyCtxt<'tcx>) -> Range<VariantIdx> {
         // FIXME requires optimized MIR
-        FIRST_VARIANT..tcx.generator_layout(def_id).unwrap().variant_fields.next_index()
+        FIRST_VARIANT..tcx.coroutine_layout(def_id).unwrap().variant_fields.next_index()
     }
 
     /// The discriminant for the given variant. Panics if the `variant_index` is
@@ -513,13 +516,13 @@ impl<'tcx> GeneratorArgs<'tcx> {
         tcx: TyCtxt<'tcx>,
         variant_index: VariantIdx,
     ) -> Discr<'tcx> {
-        // Generators don't support explicit discriminant values, so they are
+        // Coroutines don't support explicit discriminant values, so they are
         // the same as the variant index.
         assert!(self.variant_range(def_id, tcx).contains(&variant_index));
         Discr { val: variant_index.as_usize() as u128, ty: self.discr_ty(tcx) }
     }
 
-    /// The set of all discriminants for the generator, enumerated with their
+    /// The set of all discriminants for the coroutine, enumerated with their
     /// variant indices.
     #[inline]
     pub fn discriminants(
@@ -543,15 +546,15 @@ impl<'tcx> GeneratorArgs<'tcx> {
         }
     }
 
-    /// The type of the state discriminant used in the generator type.
+    /// The type of the state discriminant used in the coroutine type.
     #[inline]
     pub fn discr_ty(&self, tcx: TyCtxt<'tcx>) -> Ty<'tcx> {
         tcx.types.u32
     }
 
     /// This returns the types of the MIR locals which had to be stored across suspension points.
-    /// It is calculated in rustc_mir_transform::generator::StateTransform.
-    /// All the types here must be in the tuple in GeneratorInterior.
+    /// It is calculated in rustc_mir_transform::coroutine::StateTransform.
+    /// All the types here must be in the tuple in CoroutineInterior.
     ///
     /// The locals are grouped by their variant number. Note that some locals may
     /// be repeated in multiple variants.
@@ -561,7 +564,7 @@ impl<'tcx> GeneratorArgs<'tcx> {
         def_id: DefId,
         tcx: TyCtxt<'tcx>,
     ) -> impl Iterator<Item: Iterator<Item = Ty<'tcx>> + Captures<'tcx>> {
-        let layout = tcx.generator_layout(def_id).unwrap();
+        let layout = tcx.coroutine_layout(def_id).unwrap();
         layout.variant_fields.iter().map(move |variant| {
             variant.iter().map(move |field| {
                 ty::EarlyBinder::bind(layout.field_tys[*field].ty).instantiate(tcx, self.args)
@@ -569,7 +572,7 @@ impl<'tcx> GeneratorArgs<'tcx> {
         })
     }
 
-    /// This is the types of the fields of a generator which are not stored in a
+    /// This is the types of the fields of a coroutine which are not stored in a
     /// variant.
     #[inline]
     pub fn prefix_tys(self) -> &'tcx List<Ty<'tcx>> {
@@ -580,18 +583,18 @@ impl<'tcx> GeneratorArgs<'tcx> {
 #[derive(Debug, Copy, Clone, HashStable)]
 pub enum UpvarArgs<'tcx> {
     Closure(GenericArgsRef<'tcx>),
-    Generator(GenericArgsRef<'tcx>),
+    Coroutine(GenericArgsRef<'tcx>),
 }
 
 impl<'tcx> UpvarArgs<'tcx> {
-    /// Returns an iterator over the list of types of captured paths by the closure/generator.
+    /// Returns an iterator over the list of types of captured paths by the closure/coroutine.
     /// In case there was a type error in figuring out the types of the captured path, an
     /// empty iterator is returned.
     #[inline]
     pub fn upvar_tys(self) -> &'tcx List<Ty<'tcx>> {
         let tupled_tys = match self {
             UpvarArgs::Closure(args) => args.as_closure().tupled_upvars_ty(),
-            UpvarArgs::Generator(args) => args.as_generator().tupled_upvars_ty(),
+            UpvarArgs::Coroutine(args) => args.as_coroutine().tupled_upvars_ty(),
         };
 
         match tupled_tys.kind() {
@@ -606,7 +609,7 @@ impl<'tcx> UpvarArgs<'tcx> {
     pub fn tupled_upvars_ty(self) -> Ty<'tcx> {
         match self {
             UpvarArgs::Closure(args) => args.as_closure().tupled_upvars_ty(),
-            UpvarArgs::Generator(args) => args.as_generator().tupled_upvars_ty(),
+            UpvarArgs::Coroutine(args) => args.as_coroutine().tupled_upvars_ty(),
         }
     }
 }
@@ -683,8 +686,8 @@ pub enum ExistentialPredicate<'tcx> {
 }
 
 impl<'tcx> DebugWithInfcx<TyCtxt<'tcx>> for ExistentialPredicate<'tcx> {
-    fn fmt<InfCtx: rustc_type_ir::InferCtxtLike<TyCtxt<'tcx>>>(
-        this: rustc_type_ir::OptWithInfcx<'_, TyCtxt<'tcx>, InfCtx, &Self>,
+    fn fmt<Infcx: rustc_type_ir::InferCtxtLike<Interner = TyCtxt<'tcx>>>(
+        this: rustc_type_ir::WithInfcx<'_, Infcx, &Self>,
         f: &mut core::fmt::Formatter<'_>,
     ) -> core::fmt::Result {
         fmt::Debug::fmt(&this.data, f)
@@ -725,7 +728,7 @@ impl<'tcx> PolyExistentialPredicate<'tcx> {
                 self.rebind(tr).with_self_ty(tcx, self_ty).to_predicate(tcx)
             }
             ExistentialPredicate::Projection(p) => {
-                ty::Clause::from_projection_clause(tcx, self.rebind(p.with_self_ty(tcx, self_ty)))
+                self.rebind(p.with_self_ty(tcx, self_ty)).to_predicate(tcx)
             }
             ExistentialPredicate::AutoTrait(did) => {
                 let generics = tcx.generics_of(did);
@@ -1213,14 +1216,28 @@ pub struct AliasTy<'tcx> {
     pub def_id: DefId,
 
     /// This field exists to prevent the creation of `AliasTy` without using
-    /// [TyCtxt::mk_alias_ty].
-    pub(super) _use_mk_alias_ty_instead: (),
+    /// [AliasTy::new].
+    _use_alias_ty_new_instead: (),
 }
 
 impl<'tcx> AliasTy<'tcx> {
+    pub fn new(
+        tcx: TyCtxt<'tcx>,
+        def_id: DefId,
+        args: impl IntoIterator<Item: Into<GenericArg<'tcx>>>,
+    ) -> ty::AliasTy<'tcx> {
+        let args = tcx.check_and_mk_args(def_id, args);
+        ty::AliasTy { def_id, args, _use_alias_ty_new_instead: () }
+    }
+
     pub fn kind(self, tcx: TyCtxt<'tcx>) -> ty::AliasKind {
         match tcx.def_kind(self.def_id) {
-            DefKind::AssocTy if let DefKind::Impl { of_trait: false } = tcx.def_kind(tcx.parent(self.def_id)) => ty::Inherent,
+            DefKind::AssocTy
+                if let DefKind::Impl { of_trait: false } =
+                    tcx.def_kind(tcx.parent(self.def_id)) =>
+            {
+                ty::Inherent
+            }
             DefKind::AssocTy => ty::Projection,
             DefKind::OpaqueTy => ty::Opaque,
             DefKind::TyAlias => ty::Weak,
@@ -1240,7 +1257,7 @@ impl<'tcx> AliasTy<'tcx> {
     }
 
     pub fn with_self_ty(self, tcx: TyCtxt<'tcx>, self_ty: Ty<'tcx>) -> Self {
-        tcx.mk_alias_ty(self.def_id, [self_ty.into()].into_iter().chain(self.args.iter().skip(1)))
+        AliasTy::new(tcx, self.def_id, [self_ty.into()].into_iter().chain(self.args.iter().skip(1)))
     }
 }
 
@@ -1569,26 +1586,22 @@ impl fmt::Debug for EarlyBoundRegion {
     }
 }
 
-/// A **`const`** **v**ariable **ID**.
-#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
-#[derive(HashStable, TyEncodable, TyDecodable)]
-pub struct ConstVid<'tcx> {
-    pub index: u32,
-    pub phantom: PhantomData<&'tcx ()>,
+rustc_index::newtype_index! {
+    /// A **`const`** **v**ariable **ID**.
+    #[debug_format = "?{}c"]
+    pub struct ConstVid {}
 }
 
-/// An **effect** **v**ariable **ID**.
-///
-/// Handling effect infer variables happens separately from const infer variables
-/// because we do not want to reuse any of the const infer machinery. If we try to
-/// relate an effect variable with a normal one, we would ICE, which can catch bugs
-/// where we are not correctly using the effect var for an effect param. Fallback
-/// is also implemented on top of having separate effect and normal const variables.
-#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
-#[derive(TyEncodable, TyDecodable)]
-pub struct EffectVid<'tcx> {
-    pub index: u32,
-    pub phantom: PhantomData<&'tcx ()>,
+rustc_index::newtype_index! {
+    /// An **effect** **v**ariable **ID**.
+    ///
+    /// Handling effect infer variables happens separately from const infer variables
+    /// because we do not want to reuse any of the const infer machinery. If we try to
+    /// relate an effect variable with a normal one, we would ICE, which can catch bugs
+    /// where we are not correctly using the effect var for an effect param. Fallback
+    /// is also implemented on top of having separate effect and normal const variables.
+    #[debug_format = "?{}e"]
+    pub struct EffectVid {}
 }
 
 rustc_index::newtype_index! {
@@ -1662,8 +1675,11 @@ impl<'tcx> ExistentialProjection<'tcx> {
         debug_assert!(!self_ty.has_escaping_bound_vars());
 
         ty::ProjectionPredicate {
-            projection_ty: tcx
-                .mk_alias_ty(self.def_id, [self_ty.into()].into_iter().chain(self.args)),
+            projection_ty: AliasTy::new(
+                tcx,
+                self.def_id,
+                [self_ty.into()].into_iter().chain(self.args),
+            ),
             term: self.term,
         }
     }
@@ -1966,7 +1982,7 @@ impl<'tcx> Ty<'tcx> {
 
     #[inline]
     pub fn new_opaque(tcx: TyCtxt<'tcx>, def_id: DefId, args: GenericArgsRef<'tcx>) -> Ty<'tcx> {
-        Ty::new_alias(tcx, ty::Opaque, tcx.mk_alias_ty(def_id, args))
+        Ty::new_alias(tcx, ty::Opaque, AliasTy::new(tcx, def_id, args))
     }
 
     /// Constructs a `TyKind::Error` type with current `ErrorGuaranteed`
@@ -2130,7 +2146,7 @@ impl<'tcx> Ty<'tcx> {
         item_def_id: DefId,
         args: impl IntoIterator<Item: Into<GenericArg<'tcx>>>,
     ) -> Ty<'tcx> {
-        Ty::new_alias(tcx, ty::Projection, tcx.mk_alias_ty(item_def_id, args))
+        Ty::new_alias(tcx, ty::Projection, AliasTy::new(tcx, item_def_id, args))
     }
 
     #[inline]
@@ -2148,27 +2164,27 @@ impl<'tcx> Ty<'tcx> {
     }
 
     #[inline]
-    pub fn new_generator(
+    pub fn new_coroutine(
         tcx: TyCtxt<'tcx>,
         def_id: DefId,
-        generator_args: GenericArgsRef<'tcx>,
+        coroutine_args: GenericArgsRef<'tcx>,
         movability: hir::Movability,
     ) -> Ty<'tcx> {
         debug_assert_eq!(
-            generator_args.len(),
+            coroutine_args.len(),
             tcx.generics_of(tcx.typeck_root_def_id(def_id)).count() + 5,
-            "generator constructed with incorrect number of substitutions"
+            "coroutine constructed with incorrect number of substitutions"
         );
-        Ty::new(tcx, Generator(def_id, generator_args, movability))
+        Ty::new(tcx, Coroutine(def_id, coroutine_args, movability))
     }
 
     #[inline]
-    pub fn new_generator_witness(
+    pub fn new_coroutine_witness(
         tcx: TyCtxt<'tcx>,
         id: DefId,
         args: GenericArgsRef<'tcx>,
     ) -> Ty<'tcx> {
-        Ty::new(tcx, GeneratorWitness(id, args))
+        Ty::new(tcx, CoroutineWitness(id, args))
     }
 
     // misc
@@ -2478,8 +2494,8 @@ impl<'tcx> Ty<'tcx> {
     }
 
     #[inline]
-    pub fn is_generator(self) -> bool {
-        matches!(self.kind(), Generator(..))
+    pub fn is_coroutine(self) -> bool {
+        matches!(self.kind(), Coroutine(..))
     }
 
     #[inline]
@@ -2635,13 +2651,13 @@ impl<'tcx> Ty<'tcx> {
 
     /// If the type contains variants, returns the valid range of variant indices.
     //
-    // FIXME: This requires the optimized MIR in the case of generators.
+    // FIXME: This requires the optimized MIR in the case of coroutines.
     #[inline]
     pub fn variant_range(self, tcx: TyCtxt<'tcx>) -> Option<Range<VariantIdx>> {
         match self.kind() {
             TyKind::Adt(adt, _) => Some(adt.variant_range()),
-            TyKind::Generator(def_id, args, _) => {
-                Some(args.as_generator().variant_range(*def_id, tcx))
+            TyKind::Coroutine(def_id, args, _) => {
+                Some(args.as_coroutine().variant_range(*def_id, tcx))
             }
             _ => None,
         }
@@ -2650,7 +2666,7 @@ impl<'tcx> Ty<'tcx> {
     /// If the type contains variants, returns the variant for `variant_index`.
     /// Panics if `variant_index` is out of range.
     //
-    // FIXME: This requires the optimized MIR in the case of generators.
+    // FIXME: This requires the optimized MIR in the case of coroutines.
     #[inline]
     pub fn discriminant_for_variant(
         self,
@@ -2661,8 +2677,8 @@ impl<'tcx> Ty<'tcx> {
             TyKind::Adt(adt, _) if adt.is_enum() => {
                 Some(adt.discriminant_for_variant(tcx, variant_index))
             }
-            TyKind::Generator(def_id, args, _) => {
-                Some(args.as_generator().discriminant_for_variant(*def_id, tcx, variant_index))
+            TyKind::Coroutine(def_id, args, _) => {
+                Some(args.as_coroutine().discriminant_for_variant(*def_id, tcx, variant_index))
             }
             _ => None,
         }
@@ -2672,7 +2688,7 @@ impl<'tcx> Ty<'tcx> {
     pub fn discriminant_ty(self, tcx: TyCtxt<'tcx>) -> Ty<'tcx> {
         match self.kind() {
             ty::Adt(adt, _) if adt.is_enum() => adt.repr().discr_type().to_ty(tcx),
-            ty::Generator(_, args, _) => args.as_generator().discr_ty(tcx),
+            ty::Coroutine(_, args, _) => args.as_coroutine().discr_ty(tcx),
 
             ty::Param(_) | ty::Alias(..) | ty::Infer(ty::TyVar(_)) => {
                 let assoc_items = tcx.associated_item_def_ids(
@@ -2697,7 +2713,7 @@ impl<'tcx> Ty<'tcx> {
             | ty::FnPtr(..)
             | ty::Dynamic(..)
             | ty::Closure(..)
-            | ty::GeneratorWitness(..)
+            | ty::CoroutineWitness(..)
             | ty::Never
             | ty::Tuple(_)
             | ty::Error(_)
@@ -2731,8 +2747,8 @@ impl<'tcx> Ty<'tcx> {
             | ty::RawPtr(..)
             | ty::Char
             | ty::Ref(..)
-            | ty::Generator(..)
-            | ty::GeneratorWitness(..)
+            | ty::Coroutine(..)
+            | ty::CoroutineWitness(..)
             | ty::Array(..)
             | ty::Closure(..)
             | ty::Never
@@ -2819,8 +2835,8 @@ impl<'tcx> Ty<'tcx> {
             | ty::RawPtr(..)
             | ty::Char
             | ty::Ref(..)
-            | ty::Generator(..)
-            | ty::GeneratorWitness(..)
+            | ty::Coroutine(..)
+            | ty::CoroutineWitness(..)
             | ty::Array(..)
             | ty::Closure(..)
             | ty::Never
@@ -2848,7 +2864,7 @@ impl<'tcx> Ty<'tcx> {
     /// Returning true means the type is known to be pure and `Copy+Clone`.
     /// Returning `false` means nothing -- could be `Copy`, might not be.
     ///
-    /// This is mostly useful for optimizations, as there are the types
+    /// This is mostly useful for optimizations, as these are the types
     /// on which we can replace cloning with dereferencing.
     pub fn is_trivially_pure_clone_copy(self) -> bool {
         match self.kind() {
@@ -2883,7 +2899,7 @@ impl<'tcx> Ty<'tcx> {
             // anything with custom metadata it might be more complicated.
             ty::Ref(_, _, hir::Mutability::Not) | ty::RawPtr(..) => false,
 
-            ty::Generator(..) | ty::GeneratorWitness(..) => false,
+            ty::Coroutine(..) | ty::CoroutineWitness(..) => false,
 
             // Might be, but not "trivial" so just giving the safe answer.
             ty::Adt(..) | ty::Closure(..) => false,
@@ -2958,8 +2974,8 @@ impl<'tcx> Ty<'tcx> {
             | FnPtr(_)
             | Dynamic(_, _, _)
             | Closure(_, _)
-            | Generator(_, _, _)
-            | GeneratorWitness(..)
+            | Coroutine(_, _, _)
+            | CoroutineWitness(..)
             | Never
             | Tuple(_) => true,
             Error(_) | Infer(_) | Alias(_, _) | Param(_) | Bound(_, _) | Placeholder(_) => false,
diff --git a/compiler/rustc_middle/src/ty/typeck_results.rs b/compiler/rustc_middle/src/ty/typeck_results.rs
index a44224e4d..e9240d1b2 100644
--- a/compiler/rustc_middle/src/ty/typeck_results.rs
+++ b/compiler/rustc_middle/src/ty/typeck_results.rs
@@ -24,7 +24,7 @@ use rustc_macros::HashStable;
 use rustc_middle::mir::FakeReadCause;
 use rustc_session::Session;
 use rustc_span::Span;
-use rustc_target::abi::FieldIdx;
+use rustc_target::abi::{FieldIdx, VariantIdx};
 use std::{collections::hash_map::Entry, hash::Hash, iter};
 
 use super::RvalueScopes;
@@ -189,9 +189,9 @@ pub struct TypeckResults<'tcx> {
     /// Details may be find in `rustc_hir_analysis::check::rvalue_scopes`.
     pub rvalue_scopes: RvalueScopes,
 
-    /// Stores the predicates that apply on generator witness types.
-    /// formatting modified file tests/ui/generator/retain-resume-ref.rs
-    pub generator_interior_predicates:
+    /// Stores the predicates that apply on coroutine witness types.
+    /// formatting modified file tests/ui/coroutine/retain-resume-ref.rs
+    pub coroutine_interior_predicates:
         LocalDefIdMap<Vec<(ty::Predicate<'tcx>, ObligationCause<'tcx>)>>,
 
     /// We sometimes treat byte string literals (which are of type `&[u8; N]`)
@@ -205,7 +205,7 @@ pub struct TypeckResults<'tcx> {
     pub closure_size_eval: LocalDefIdMap<ClosureSizeProfileData<'tcx>>,
 
     /// Container types and field indices of `offset_of!` expressions
-    offset_of_data: ItemLocalMap<(Ty<'tcx>, Vec<FieldIdx>)>,
+    offset_of_data: ItemLocalMap<(Ty<'tcx>, Vec<(VariantIdx, FieldIdx)>)>,
 }
 
 impl<'tcx> TypeckResults<'tcx> {
@@ -231,7 +231,7 @@ impl<'tcx> TypeckResults<'tcx> {
             closure_min_captures: Default::default(),
             closure_fake_reads: Default::default(),
             rvalue_scopes: Default::default(),
-            generator_interior_predicates: Default::default(),
+            coroutine_interior_predicates: Default::default(),
             treat_byte_string_as_slice: Default::default(),
             closure_size_eval: Default::default(),
             offset_of_data: Default::default(),
@@ -464,11 +464,15 @@ impl<'tcx> TypeckResults<'tcx> {
         &self.coercion_casts
     }
 
-    pub fn offset_of_data(&self) -> LocalTableInContext<'_, (Ty<'tcx>, Vec<FieldIdx>)> {
+    pub fn offset_of_data(
+        &self,
+    ) -> LocalTableInContext<'_, (Ty<'tcx>, Vec<(VariantIdx, FieldIdx)>)> {
         LocalTableInContext { hir_owner: self.hir_owner, data: &self.offset_of_data }
     }
 
-    pub fn offset_of_data_mut(&mut self) -> LocalTableInContextMut<'_, (Ty<'tcx>, Vec<FieldIdx>)> {
+    pub fn offset_of_data_mut(
+        &mut self,
+    ) -> LocalTableInContextMut<'_, (Ty<'tcx>, Vec<(VariantIdx, FieldIdx)>)> {
         LocalTableInContextMut { hir_owner: self.hir_owner, data: &mut self.offset_of_data }
     }
 }
@@ -594,10 +598,27 @@ pub struct CanonicalUserTypeAnnotation<'tcx> {
 /// Canonical user type annotation.
 pub type CanonicalUserType<'tcx> = Canonical<'tcx, UserType<'tcx>>;
 
-impl<'tcx> CanonicalUserType<'tcx> {
+/// A user-given type annotation attached to a constant. These arise
+/// from constants that are named via paths, like `Foo::<A>::new` and
+/// so forth.
+#[derive(Copy, Clone, Debug, PartialEq, TyEncodable, TyDecodable)]
+#[derive(Eq, Hash, HashStable, TypeFoldable, TypeVisitable)]
+pub enum UserType<'tcx> {
+    Ty(Ty<'tcx>),
+
+    /// The canonical type is the result of `type_of(def_id)` with the
+    /// given substitutions applied.
+    TypeOf(DefId, UserArgs<'tcx>),
+}
+
+pub trait IsIdentity {
+    fn is_identity(&self) -> bool;
+}
+
+impl<'tcx> IsIdentity for CanonicalUserType<'tcx> {
     /// Returns `true` if this represents a substitution of the form `[?0, ?1, ?2]`,
     /// i.e., each thing is mapped to a canonical variable with the same index.
-    pub fn is_identity(&self) -> bool {
+    fn is_identity(&self) -> bool {
         match self.value {
             UserType::Ty(_) => false,
             UserType::TypeOf(_, user_args) => {
@@ -640,19 +661,6 @@ impl<'tcx> CanonicalUserType<'tcx> {
     }
 }
 
-/// A user-given type annotation attached to a constant. These arise
-/// from constants that are named via paths, like `Foo::<A>::new` and
-/// so forth.
-#[derive(Copy, Clone, Debug, PartialEq, TyEncodable, TyDecodable)]
-#[derive(Eq, Hash, HashStable, TypeFoldable, TypeVisitable)]
-pub enum UserType<'tcx> {
-    Ty(Ty<'tcx>),
-
-    /// The canonical type is the result of `type_of(def_id)` with the
-    /// given substitutions applied.
-    TypeOf(DefId, UserArgs<'tcx>),
-}
-
 impl<'tcx> std::fmt::Display for UserType<'tcx> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs
index 964f38a65..a251518d1 100644
--- a/compiler/rustc_middle/src/ty/util.rs
+++ b/compiler/rustc_middle/src/ty/util.rs
@@ -19,7 +19,7 @@ use rustc_index::bit_set::GrowableBitSet;
 use rustc_macros::HashStable;
 use rustc_session::Limit;
 use rustc_span::sym;
-use rustc_target::abi::{Integer, IntegerType, Size};
+use rustc_target::abi::{Integer, IntegerType, Primitive, Size};
 use rustc_target::spec::abi::Abi;
 use smallvec::SmallVec;
 use std::{fmt, iter};
@@ -460,7 +460,7 @@ impl<'tcx> TyCtxt<'tcx> {
     /// Checks whether each generic argument is simply a unique generic parameter.
     pub fn uses_unique_generic_params(
         self,
-        args: GenericArgsRef<'tcx>,
+        args: &[ty::GenericArg<'tcx>],
         ignore_regions: CheckRegions,
     ) -> Result<(), NotUniqueParam<'tcx>> {
         let mut seen = GrowableBitSet::default();
@@ -548,15 +548,15 @@ impl<'tcx> TyCtxt<'tcx> {
     /// those are not yet phased out). The parent of the closure's
     /// `DefId` will also be the context where it appears.
     pub fn is_closure(self, def_id: DefId) -> bool {
-        matches!(self.def_kind(def_id), DefKind::Closure | DefKind::Generator)
+        matches!(self.def_kind(def_id), DefKind::Closure | DefKind::Coroutine)
     }
 
     /// Returns `true` if `def_id` refers to a definition that does not have its own
-    /// type-checking context, i.e. closure, generator or inline const.
+    /// type-checking context, i.e. closure, coroutine or inline const.
     pub fn is_typeck_child(self, def_id: DefId) -> bool {
         matches!(
             self.def_kind(def_id),
-            DefKind::Closure | DefKind::Generator | DefKind::InlineConst
+            DefKind::Closure | DefKind::Coroutine | DefKind::InlineConst
         )
     }
 
@@ -686,13 +686,13 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 
     /// Return the set of types that should be taken into account when checking
-    /// trait bounds on a generator's internal state.
-    pub fn generator_hidden_types(
+    /// trait bounds on a coroutine's internal state.
+    pub fn coroutine_hidden_types(
         self,
         def_id: DefId,
     ) -> impl Iterator<Item = ty::EarlyBinder<Ty<'tcx>>> {
-        let generator_layout = self.mir_generator_witnesses(def_id);
-        generator_layout
+        let coroutine_layout = self.mir_coroutine_witnesses(def_id);
+        coroutine_layout
             .as_ref()
             .map_or_else(|| [].iter(), |l| l.field_tys.iter())
             .filter(|decl| !decl.ignore_for_traits)
@@ -709,7 +709,7 @@ impl<'tcx> TyCtxt<'tcx> {
             found_recursion: false,
             found_any_recursion: false,
             check_recursion: false,
-            expand_generators: false,
+            expand_coroutines: false,
             tcx: self,
         };
         val.fold_with(&mut visitor)
@@ -729,7 +729,7 @@ impl<'tcx> TyCtxt<'tcx> {
             found_recursion: false,
             found_any_recursion: false,
             check_recursion: true,
-            expand_generators: true,
+            expand_coroutines: true,
             tcx: self,
         };
 
@@ -746,9 +746,10 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn def_kind_descr(self, def_kind: DefKind, def_id: DefId) -> &'static str {
         match def_kind {
             DefKind::AssocFn if self.associated_item(def_id).fn_has_self_parameter => "method",
-            DefKind::Generator => match self.generator_kind(def_id).unwrap() {
-                rustc_hir::GeneratorKind::Async(..) => "async closure",
-                rustc_hir::GeneratorKind::Gen => "generator",
+            DefKind::Coroutine => match self.coroutine_kind(def_id).unwrap() {
+                rustc_hir::CoroutineKind::Async(..) => "async closure",
+                rustc_hir::CoroutineKind::Coroutine => "coroutine",
+                rustc_hir::CoroutineKind::Gen(..) => "gen closure",
             },
             _ => def_kind.descr(def_id),
         }
@@ -763,9 +764,10 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn def_kind_descr_article(self, def_kind: DefKind, def_id: DefId) -> &'static str {
         match def_kind {
             DefKind::AssocFn if self.associated_item(def_id).fn_has_self_parameter => "a",
-            DefKind::Generator => match self.generator_kind(def_id).unwrap() {
-                rustc_hir::GeneratorKind::Async(..) => "an",
-                rustc_hir::GeneratorKind::Gen => "a",
+            DefKind::Coroutine => match self.coroutine_kind(def_id).unwrap() {
+                rustc_hir::CoroutineKind::Async(..) => "an",
+                rustc_hir::CoroutineKind::Coroutine => "a",
+                rustc_hir::CoroutineKind::Gen(..) => "a",
             },
             _ => def_kind.article(),
         }
@@ -804,7 +806,7 @@ struct OpaqueTypeExpander<'tcx> {
     primary_def_id: Option<DefId>,
     found_recursion: bool,
     found_any_recursion: bool,
-    expand_generators: bool,
+    expand_coroutines: bool,
     /// Whether or not to check for recursive opaque types.
     /// This is `true` when we're explicitly checking for opaque type
     /// recursion, and 'false' otherwise to avoid unnecessary work.
@@ -842,7 +844,7 @@ impl<'tcx> OpaqueTypeExpander<'tcx> {
         }
     }
 
-    fn expand_generator(&mut self, def_id: DefId, args: GenericArgsRef<'tcx>) -> Option<Ty<'tcx>> {
+    fn expand_coroutine(&mut self, def_id: DefId, args: GenericArgsRef<'tcx>) -> Option<Ty<'tcx>> {
         if self.found_any_recursion {
             return None;
         }
@@ -851,11 +853,11 @@ impl<'tcx> OpaqueTypeExpander<'tcx> {
             let expanded_ty = match self.expanded_cache.get(&(def_id, args)) {
                 Some(expanded_ty) => *expanded_ty,
                 None => {
-                    for bty in self.tcx.generator_hidden_types(def_id) {
+                    for bty in self.tcx.coroutine_hidden_types(def_id) {
                         let hidden_ty = bty.instantiate(self.tcx, args);
                         self.fold_ty(hidden_ty);
                     }
-                    let expanded_ty = Ty::new_generator_witness(self.tcx, def_id, args);
+                    let expanded_ty = Ty::new_coroutine_witness(self.tcx, def_id, args);
                     self.expanded_cache.insert((def_id, args), expanded_ty);
                     expanded_ty
                 }
@@ -882,14 +884,14 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for OpaqueTypeExpander<'tcx> {
     fn fold_ty(&mut self, t: Ty<'tcx>) -> Ty<'tcx> {
         let mut t = if let ty::Alias(ty::Opaque, ty::AliasTy { def_id, args, .. }) = *t.kind() {
             self.expand_opaque_ty(def_id, args).unwrap_or(t)
-        } else if t.has_opaque_types() || t.has_generators() {
+        } else if t.has_opaque_types() || t.has_coroutines() {
             t.super_fold_with(self)
         } else {
             t
         };
-        if self.expand_generators {
-            if let ty::GeneratorWitness(def_id, args) = *t.kind() {
-                t = self.expand_generator(def_id, args).unwrap_or(t);
+        if self.expand_coroutines {
+            if let ty::CoroutineWitness(def_id, args) = *t.kind() {
+                t = self.expand_coroutine(def_id, args).unwrap_or(t);
             }
         }
         t
@@ -917,54 +919,62 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for OpaqueTypeExpander<'tcx> {
 }
 
 impl<'tcx> Ty<'tcx> {
+    /// Returns the `Size` for primitive types (bool, uint, int, char, float).
+    pub fn primitive_size(self, tcx: TyCtxt<'tcx>) -> Size {
+        match *self.kind() {
+            ty::Bool => Size::from_bytes(1),
+            ty::Char => Size::from_bytes(4),
+            ty::Int(ity) => Integer::from_int_ty(&tcx, ity).size(),
+            ty::Uint(uty) => Integer::from_uint_ty(&tcx, uty).size(),
+            ty::Float(ty::FloatTy::F32) => Primitive::F32.size(&tcx),
+            ty::Float(ty::FloatTy::F64) => Primitive::F64.size(&tcx),
+            _ => bug!("non primitive type"),
+        }
+    }
+
     pub fn int_size_and_signed(self, tcx: TyCtxt<'tcx>) -> (Size, bool) {
-        let (int, signed) = match *self.kind() {
-            ty::Int(ity) => (Integer::from_int_ty(&tcx, ity), true),
-            ty::Uint(uty) => (Integer::from_uint_ty(&tcx, uty), false),
+        match *self.kind() {
+            ty::Int(ity) => (Integer::from_int_ty(&tcx, ity).size(), true),
+            ty::Uint(uty) => (Integer::from_uint_ty(&tcx, uty).size(), false),
             _ => bug!("non integer discriminant"),
-        };
-        (int.size(), signed)
+        }
     }
 
-    /// Returns the maximum value for the given numeric type (including `char`s)
-    /// or returns `None` if the type is not numeric.
-    pub fn numeric_max_val(self, tcx: TyCtxt<'tcx>) -> Option<ty::Const<'tcx>> {
-        let val = match self.kind() {
+    /// Returns the minimum and maximum values for the given numeric type (including `char`s) or
+    /// returns `None` if the type is not numeric.
+    pub fn numeric_min_and_max_as_bits(self, tcx: TyCtxt<'tcx>) -> Option<(u128, u128)> {
+        use rustc_apfloat::ieee::{Double, Single};
+        Some(match self.kind() {
             ty::Int(_) | ty::Uint(_) => {
                 let (size, signed) = self.int_size_and_signed(tcx);
-                let val =
+                let min = if signed { size.truncate(size.signed_int_min() as u128) } else { 0 };
+                let max =
                     if signed { size.signed_int_max() as u128 } else { size.unsigned_int_max() };
-                Some(val)
+                (min, max)
             }
-            ty::Char => Some(std::char::MAX as u128),
-            ty::Float(fty) => Some(match fty {
-                ty::FloatTy::F32 => rustc_apfloat::ieee::Single::INFINITY.to_bits(),
-                ty::FloatTy::F64 => rustc_apfloat::ieee::Double::INFINITY.to_bits(),
-            }),
-            _ => None,
-        };
+            ty::Char => (0, std::char::MAX as u128),
+            ty::Float(ty::FloatTy::F32) => {
+                ((-Single::INFINITY).to_bits(), Single::INFINITY.to_bits())
+            }
+            ty::Float(ty::FloatTy::F64) => {
+                ((-Double::INFINITY).to_bits(), Double::INFINITY.to_bits())
+            }
+            _ => return None,
+        })
+    }
 
-        val.map(|v| ty::Const::from_bits(tcx, v, ty::ParamEnv::empty().and(self)))
+    /// Returns the maximum value for the given numeric type (including `char`s)
+    /// or returns `None` if the type is not numeric.
+    pub fn numeric_max_val(self, tcx: TyCtxt<'tcx>) -> Option<ty::Const<'tcx>> {
+        self.numeric_min_and_max_as_bits(tcx)
+            .map(|(_, max)| ty::Const::from_bits(tcx, max, ty::ParamEnv::empty().and(self)))
     }
 
     /// Returns the minimum value for the given numeric type (including `char`s)
     /// or returns `None` if the type is not numeric.
     pub fn numeric_min_val(self, tcx: TyCtxt<'tcx>) -> Option<ty::Const<'tcx>> {
-        let val = match self.kind() {
-            ty::Int(_) | ty::Uint(_) => {
-                let (size, signed) = self.int_size_and_signed(tcx);
-                let val = if signed { size.truncate(size.signed_int_min() as u128) } else { 0 };
-                Some(val)
-            }
-            ty::Char => Some(0),
-            ty::Float(fty) => Some(match fty {
-                ty::FloatTy::F32 => (-::rustc_apfloat::ieee::Single::INFINITY).to_bits(),
-                ty::FloatTy::F64 => (-::rustc_apfloat::ieee::Double::INFINITY).to_bits(),
-            }),
-            _ => None,
-        };
-
-        val.map(|v| ty::Const::from_bits(tcx, v, ty::ParamEnv::empty().and(self)))
+        self.numeric_min_and_max_as_bits(tcx)
+            .map(|(min, _)| ty::Const::from_bits(tcx, min, ty::ParamEnv::empty().and(self)))
     }
 
     /// Checks whether values of this type `T` are *moved* or *copied*
@@ -1024,8 +1034,8 @@ impl<'tcx> Ty<'tcx> {
             | ty::Closure(..)
             | ty::Dynamic(..)
             | ty::Foreign(_)
-            | ty::Generator(..)
-            | ty::GeneratorWitness(..)
+            | ty::Coroutine(..)
+            | ty::CoroutineWitness(..)
             | ty::Infer(_)
             | ty::Alias(..)
             | ty::Param(_)
@@ -1063,8 +1073,8 @@ impl<'tcx> Ty<'tcx> {
             | ty::Closure(..)
             | ty::Dynamic(..)
             | ty::Foreign(_)
-            | ty::Generator(..)
-            | ty::GeneratorWitness(..)
+            | ty::Coroutine(..)
+            | ty::CoroutineWitness(..)
             | ty::Infer(_)
             | ty::Alias(..)
             | ty::Param(_)
@@ -1184,7 +1194,7 @@ impl<'tcx> Ty<'tcx> {
             // Conservatively return `false` for all others...
 
             // Anonymous function types
-            ty::FnDef(..) | ty::Closure(..) | ty::Dynamic(..) | ty::Generator(..) => false,
+            ty::FnDef(..) | ty::Closure(..) | ty::Dynamic(..) | ty::Coroutine(..) => false,
 
             // Generic or inferred types
             //
@@ -1194,7 +1204,7 @@ impl<'tcx> Ty<'tcx> {
                 false
             }
 
-            ty::Foreign(_) | ty::GeneratorWitness(..) | ty::Error(_) => false,
+            ty::Foreign(_) | ty::CoroutineWitness(..) | ty::Error(_) => false,
         }
     }
 
@@ -1328,8 +1338,8 @@ pub fn needs_drop_components<'tcx>(
         | ty::Placeholder(..)
         | ty::Infer(_)
         | ty::Closure(..)
-        | ty::Generator(..)
-        | ty::GeneratorWitness(..) => Ok(smallvec![ty]),
+        | ty::Coroutine(..)
+        | ty::CoroutineWitness(..) => Ok(smallvec![ty]),
     }
 }
 
@@ -1360,7 +1370,7 @@ pub fn is_trivially_const_drop(ty: Ty<'_>) -> bool {
 
         // Not trivial because they have components, and instead of looking inside,
         // we'll just perform trait selection.
-        ty::Closure(..) | ty::Generator(..) | ty::GeneratorWitness(..) | ty::Adt(..) => false,
+        ty::Closure(..) | ty::Coroutine(..) | ty::CoroutineWitness(..) | ty::Adt(..) => false,
 
         ty::Array(ty, _) | ty::Slice(ty) => is_trivially_const_drop(ty),
 
@@ -1421,7 +1431,7 @@ pub fn reveal_opaque_types_in_bounds<'tcx>(
         found_recursion: false,
         found_any_recursion: false,
         check_recursion: false,
-        expand_generators: false,
+        expand_coroutines: false,
         tcx,
     };
     val.fold_with(&mut visitor)
diff --git a/compiler/rustc_middle/src/ty/visit.rs b/compiler/rustc_middle/src/ty/visit.rs
index 95ba6c471..8fc5c0302 100644
--- a/compiler/rustc_middle/src/ty/visit.rs
+++ b/compiler/rustc_middle/src/ty/visit.rs
@@ -47,8 +47,8 @@ pub trait TypeVisitableExt<'tcx>: TypeVisitable<TyCtxt<'tcx>> {
     fn has_opaque_types(&self) -> bool {
         self.has_type_flags(TypeFlags::HAS_TY_OPAQUE)
     }
-    fn has_generators(&self) -> bool {
-        self.has_type_flags(TypeFlags::HAS_TY_GENERATOR)
+    fn has_coroutines(&self) -> bool {
+        self.has_type_flags(TypeFlags::HAS_TY_COROUTINE)
     }
     fn references_error(&self) -> bool {
         self.has_type_flags(TypeFlags::HAS_ERROR)
diff --git a/compiler/rustc_middle/src/ty/walk.rs b/compiler/rustc_middle/src/ty/walk.rs
index a86ff64bd..20bdbcb5b 100644
--- a/compiler/rustc_middle/src/ty/walk.rs
+++ b/compiler/rustc_middle/src/ty/walk.rs
@@ -189,8 +189,8 @@ fn push_inner<'tcx>(stack: &mut TypeWalkerStack<'tcx>, parent: GenericArg<'tcx>)
             }
             ty::Adt(_, args)
             | ty::Closure(_, args)
-            | ty::Generator(_, args, _)
-            | ty::GeneratorWitness(_, args)
+            | ty::Coroutine(_, args, _)
+            | ty::CoroutineWitness(_, args)
             | ty::FnDef(_, args) => {
                 stack.extend(args.iter().rev());
             }
diff --git a/compiler/rustc_middle/src/values.rs b/compiler/rustc_middle/src/values.rs
index 578d8e7a9..f30993c9a 100644
--- a/compiler/rustc_middle/src/values.rs
+++ b/compiler/rustc_middle/src/values.rs
@@ -217,7 +217,8 @@ fn find_item_ty_spans(
     match ty.kind {
         hir::TyKind::Path(hir::QPath::Resolved(_, path)) => {
             if let Res::Def(kind, def_id) = path.res
-                && !matches!(kind, DefKind::TyAlias) {
+                && !matches!(kind, DefKind::TyAlias)
+            {
                 let check_params = def_id.as_local().map_or(true, |def_id| {
                     if def_id == needle {
                         spans.push(ty.span);
@@ -227,8 +228,11 @@ fn find_item_ty_spans(
                 if check_params && let Some(args) = path.segments.last().unwrap().args {
                     let params_in_repr = tcx.params_in_repr(def_id);
                     // the domain size check is needed because the HIR may not be well-formed at this point
-                    for (i, arg) in args.args.iter().enumerate().take(params_in_repr.domain_size()) {
-                        if let hir::GenericArg::Type(ty) = arg && params_in_repr.contains(i as u32) {
+                    for (i, arg) in args.args.iter().enumerate().take(params_in_repr.domain_size())
+                    {
+                        if let hir::GenericArg::Type(ty) = arg
+                            && params_in_repr.contains(i as u32)
+                        {
                             find_item_ty_spans(tcx, ty, needle, spans, seen_representable);
                         }
                     }