Adding upstream version 1.66.0+dfsg1.upstream/1.66.0+dfsg1

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

7 files changed, 286 insertions, 273 deletions

diff --git a/compiler/rustc_const_eval/src/transform/check_consts/check.rs b/compiler/rustc_const_eval/src/transform/check_consts/check.rs
index 0adb88a18..22a61774e 100644
--- a/compiler/rustc_const_eval/src/transform/check_consts/check.rs
+++ b/compiler/rustc_const_eval/src/transform/check_consts/check.rs
@@ -13,8 +13,11 @@ use rustc_middle::ty::{self, adjustment::PointerCast, Instance, InstanceDef, Ty,
 use rustc_middle::ty::{Binder, TraitPredicate, TraitRef, TypeVisitable};
 use rustc_mir_dataflow::{self, Analysis};
 use rustc_span::{sym, Span, Symbol};
-use rustc_trait_selection::traits::error_reporting::InferCtxtExt;
-use rustc_trait_selection::traits::SelectionContext;
+use rustc_trait_selection::infer::InferCtxtExt;
+use rustc_trait_selection::traits::error_reporting::TypeErrCtxtExt as _;
+use rustc_trait_selection::traits::{
+    self, ObligationCauseCode, SelectionContext, TraitEngine, TraitEngineExt,
+};
 
 use std::mem;
 use std::ops::Deref;
@@ -135,7 +138,7 @@ impl<'mir, 'tcx> Qualifs<'mir, 'tcx> {
         // qualifs for the return type.
         let return_block = ccx
             .body
-            .basic_blocks()
+            .basic_blocks
             .iter_enumerated()
             .find(|(_, block)| matches!(block.terminator().kind, TerminatorKind::Return))
             .map(|(bb, _)| bb);
@@ -546,7 +549,11 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
                 // Since no pointer can ever get exposed (rejected above), this is easy to support.
             }
 
-            Rvalue::Cast(CastKind::Misc, _, _) => {}
+            Rvalue::Cast(CastKind::DynStar, _, _) => {
+                unimplemented!()
+            }
+
+            Rvalue::Cast(_, _, _) => {}
 
             Rvalue::NullaryOp(NullOp::SizeOf | NullOp::AlignOf, _) => {}
             Rvalue::ShallowInitBox(_, _) => {}
@@ -652,6 +659,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
 
             ProjectionElem::ConstantIndex { .. }
             | ProjectionElem::Downcast(..)
+            | ProjectionElem::OpaqueCast(..)
             | ProjectionElem::Subslice { .. }
             | ProjectionElem::Field(..)
             | ProjectionElem::Index(_) => {}
@@ -678,7 +686,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
             | StatementKind::Retag { .. }
             | StatementKind::AscribeUserType(..)
             | StatementKind::Coverage(..)
-            | StatementKind::CopyNonOverlapping(..)
+            | StatementKind::Intrinsic(..)
             | StatementKind::Nop => {}
         }
     }
@@ -729,10 +737,49 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
                     let obligation =
                         Obligation::new(ObligationCause::dummy(), param_env, poly_trait_pred);
 
-                    let implsrc = tcx.infer_ctxt().enter(|infcx| {
+                    let implsrc = {
+                        let infcx = tcx.infer_ctxt().build();
                         let mut selcx = SelectionContext::new(&infcx);
                         selcx.select(&obligation)
-                    });
+                    };
+
+                    // do a well-formedness check on the trait method being called. This is because typeck only does a
+                    // "non-const" check. This is required for correctness here.
+                    {
+                        let infcx = tcx.infer_ctxt().build();
+                        let mut fulfill_cx = <dyn TraitEngine<'_>>::new(infcx.tcx);
+                        let predicates = tcx.predicates_of(callee).instantiate(tcx, substs);
+                        let hir_id = tcx
+                            .hir()
+                            .local_def_id_to_hir_id(self.body.source.def_id().expect_local());
+                        let cause = || {
+                            ObligationCause::new(
+                                terminator.source_info.span,
+                                hir_id,
+                                ObligationCauseCode::ItemObligation(callee),
+                            )
+                        };
+                        let normalized = infcx.partially_normalize_associated_types_in(
+                            cause(),
+                            param_env,
+                            predicates,
+                        );
+
+                        for p in normalized.obligations {
+                            fulfill_cx.register_predicate_obligation(&infcx, p);
+                        }
+                        for obligation in traits::predicates_for_generics(
+                            |_, _| cause(),
+                            self.param_env,
+                            normalized.value,
+                        ) {
+                            fulfill_cx.register_predicate_obligation(&infcx, obligation);
+                        }
+                        let errors = fulfill_cx.select_all_or_error(&infcx);
+                        if !errors.is_empty() {
+                            infcx.err_ctxt().report_fulfillment_errors(&errors, None, false);
+                        }
+                    }
 
                     match implsrc {
                         Ok(Some(ImplSource::Param(_, ty::BoundConstness::ConstIfConst))) => {
@@ -790,16 +837,15 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
 
                                 // improve diagnostics by showing what failed. Our requirements are stricter this time
                                 // as we are going to error again anyways.
-                                tcx.infer_ctxt().enter(|infcx| {
-                                    if let Err(e) = implsrc {
-                                        infcx.report_selection_error(
-                                            obligation.clone(),
-                                            &obligation,
-                                            &e,
-                                            false,
-                                        );
-                                    }
-                                });
+                                let infcx = tcx.infer_ctxt().build();
+                                if let Err(e) = implsrc {
+                                    infcx.err_ctxt().report_selection_error(
+                                        obligation.clone(),
+                                        &obligation,
+                                        &e,
+                                        false,
+                                    );
+                                }
 
                                 self.check_op(ops::FnCallNonConst {
                                     caller,
@@ -863,8 +909,6 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
                     return;
                 }
 
-                let is_intrinsic = tcx.is_intrinsic(callee);
-
                 if !tcx.is_const_fn_raw(callee) {
                     if !tcx.is_const_default_method(callee) {
                         // To get to here we must have already found a const impl for the
@@ -924,7 +968,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
                     // We do not use `const` modifiers for intrinsic "functions", as intrinsics are
                     // `extern` functions, and these have no way to get marked `const`. So instead we
                     // use `rustc_const_(un)stable` attributes to mean that the intrinsic is `const`
-                    if self.ccx.is_const_stable_const_fn() || is_intrinsic {
+                    if self.ccx.is_const_stable_const_fn() || tcx.is_intrinsic(callee) {
                         self.check_op(ops::FnCallUnstable(callee, None));
                         return;
                     }
@@ -964,7 +1008,10 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
 
                 if needs_non_const_drop {
                     self.check_op_spanned(
-                        ops::LiveDrop { dropped_at: Some(terminator.source_info.span) },
+                        ops::LiveDrop {
+                            dropped_at: Some(terminator.source_info.span),
+                            dropped_ty: ty_of_dropped_place,
+                        },
                         err_span,
                     );
                 }
diff --git a/compiler/rustc_const_eval/src/transform/check_consts/ops.rs b/compiler/rustc_const_eval/src/transform/check_consts/ops.rs
index 338022616..b28d70194 100644
--- a/compiler/rustc_const_eval/src/transform/check_consts/ops.rs
+++ b/compiler/rustc_const_eval/src/transform/check_consts/ops.rs
@@ -1,6 +1,7 @@
 //! Concrete error types for all operations which may be invalid in a certain const context.
 
 use hir::def_id::LocalDefId;
+use hir::ConstContext;
 use rustc_errors::{
     error_code, struct_span_err, Applicability, DiagnosticBuilder, ErrorGuaranteed,
 };
@@ -23,8 +24,11 @@ use rustc_trait_selection::traits::SelectionContext;
 
 use super::ConstCx;
 use crate::errors::{
-    MutDerefErr, NonConstOpErr, PanicNonStrErr, RawPtrToIntErr, StaticAccessErr,
-    TransientMutBorrowErr, TransientMutBorrowErrRaw,
+    InteriorMutabilityBorrow, InteriorMutableDataRefer, MutDerefErr, NonConstFmtMacroCall,
+    NonConstFnCall, NonConstOpErr, PanicNonStrErr, RawPtrToIntErr, StaticAccessErr,
+    TransientMutBorrowErr, TransientMutBorrowErrRaw, UnallowedFnPointerCall,
+    UnallowedHeapAllocations, UnallowedInlineAsm, UnallowedMutableRefs, UnallowedMutableRefsRaw,
+    UnallowedOpInConstContext, UnstableConstFn,
 };
 use crate::util::{call_kind, CallDesugaringKind, CallKind};
 
@@ -96,10 +100,7 @@ impl<'tcx> NonConstOp<'tcx> for FnCallIndirect {
         ccx: &ConstCx<'_, 'tcx>,
         span: Span,
     ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
-        ccx.tcx.sess.struct_span_err(
-            span,
-            &format!("function pointer calls are not allowed in {}s", ccx.const_kind()),
-        )
+        ccx.tcx.sess.create_err(UnallowedFnPointerCall { span, kind: ccx.const_kind() })
     }
 }
 
@@ -155,10 +156,9 @@ impl<'tcx> NonConstOp<'tcx> for FnCallNonConst<'tcx> {
                         }),
                     );
 
-                    let implsrc = tcx.infer_ctxt().enter(|infcx| {
-                        let mut selcx = SelectionContext::new(&infcx);
-                        selcx.select(&obligation)
-                    });
+                    let infcx = tcx.infer_ctxt().build();
+                    let mut selcx = SelectionContext::new(&infcx);
+                    let implsrc = selcx.select(&obligation);
 
                     if let Ok(Some(ImplSource::UserDefined(data))) = implsrc {
                         let span = tcx.def_span(data.impl_def_id);
@@ -307,22 +307,13 @@ impl<'tcx> NonConstOp<'tcx> for FnCallNonConst<'tcx> {
                 err
             }
             _ if tcx.opt_parent(callee) == tcx.get_diagnostic_item(sym::ArgumentV1Methods) => {
-                struct_span_err!(
-                    ccx.tcx.sess,
-                    span,
-                    E0015,
-                    "cannot call non-const formatting macro in {}s",
-                    ccx.const_kind(),
-                )
+                ccx.tcx.sess.create_err(NonConstFmtMacroCall { span, kind: ccx.const_kind() })
             }
-            _ => struct_span_err!(
-                ccx.tcx.sess,
+            _ => ccx.tcx.sess.create_err(NonConstFnCall {
                 span,
-                E0015,
-                "cannot call non-const fn `{}` in {}s",
-                ccx.tcx.def_path_str_with_substs(callee, substs),
-                ccx.const_kind(),
-            ),
+                def_path_str: ccx.tcx.def_path_str_with_substs(callee, substs),
+                kind: ccx.const_kind(),
+            }),
         };
 
         err.note(&format!(
@@ -331,6 +322,10 @@ impl<'tcx> NonConstOp<'tcx> for FnCallNonConst<'tcx> {
             ccx.const_kind(),
         ));
 
+        if let ConstContext::Static(_) = ccx.const_kind() {
+            err.note("consider wrapping this expression in `Lazy::new(|| ...)` from the `once_cell` crate: https://crates.io/crates/once_cell");
+        }
+
         err
     }
 }
@@ -349,10 +344,10 @@ impl<'tcx> NonConstOp<'tcx> for FnCallUnstable {
     ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
         let FnCallUnstable(def_id, feature) = *self;
 
-        let mut err = ccx.tcx.sess.struct_span_err(
-            span,
-            &format!("`{}` is not yet stable as a const fn", ccx.tcx.def_path_str(def_id)),
-        );
+        let mut err = ccx
+            .tcx
+            .sess
+            .create_err(UnstableConstFn { span, def_path: ccx.tcx.def_path_str(def_id) });
 
         if ccx.is_const_stable_const_fn() {
             err.help("const-stable functions can only call other const-stable functions");
@@ -387,9 +382,12 @@ impl<'tcx> NonConstOp<'tcx> for Generator {
     ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
         let msg = format!("{}s are not allowed in {}s", self.0, ccx.const_kind());
         if let hir::GeneratorKind::Async(hir::AsyncGeneratorKind::Block) = self.0 {
-            feature_err(&ccx.tcx.sess.parse_sess, sym::const_async_blocks, span, &msg)
+            ccx.tcx.sess.create_feature_err(
+                UnallowedOpInConstContext { span, msg },
+                sym::const_async_blocks,
+            )
         } else {
-            ccx.tcx.sess.struct_span_err(span, &msg)
+            ccx.tcx.sess.create_err(UnallowedOpInConstContext { span, msg })
         }
     }
 }
@@ -402,23 +400,11 @@ impl<'tcx> NonConstOp<'tcx> for HeapAllocation {
         ccx: &ConstCx<'_, 'tcx>,
         span: Span,
     ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
-        let mut err = struct_span_err!(
-            ccx.tcx.sess,
+        ccx.tcx.sess.create_err(UnallowedHeapAllocations {
             span,
-            E0010,
-            "allocations are not allowed in {}s",
-            ccx.const_kind()
-        );
-        err.span_label(span, format!("allocation not allowed in {}s", ccx.const_kind()));
-        if ccx.tcx.sess.teach(&err.get_code().unwrap()) {
-            err.note(
-                "The value of statics and constants must be known at compile time, \
-                 and they live for the entire lifetime of a program. Creating a boxed \
-                 value allocates memory on the heap at runtime, and therefore cannot \
-                 be done at compile time.",
-            );
-        }
-        err
+            kind: ccx.const_kind(),
+            teach: ccx.tcx.sess.teach(&error_code!(E0010)).then_some(()),
+        })
     }
 }
 
@@ -430,21 +416,16 @@ impl<'tcx> NonConstOp<'tcx> for InlineAsm {
         ccx: &ConstCx<'_, 'tcx>,
         span: Span,
     ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
-        struct_span_err!(
-            ccx.tcx.sess,
-            span,
-            E0015,
-            "inline assembly is not allowed in {}s",
-            ccx.const_kind()
-        )
+        ccx.tcx.sess.create_err(UnallowedInlineAsm { span, kind: ccx.const_kind() })
     }
 }
 
 #[derive(Debug)]
-pub struct LiveDrop {
+pub struct LiveDrop<'tcx> {
     pub dropped_at: Option<Span>,
+    pub dropped_ty: Ty<'tcx>,
 }
-impl<'tcx> NonConstOp<'tcx> for LiveDrop {
+impl<'tcx> NonConstOp<'tcx> for LiveDrop<'tcx> {
     fn build_error(
         &self,
         ccx: &ConstCx<'_, 'tcx>,
@@ -454,9 +435,13 @@ impl<'tcx> NonConstOp<'tcx> for LiveDrop {
             ccx.tcx.sess,
             span,
             E0493,
-            "destructors cannot be evaluated at compile-time"
+            "destructor of `{}` cannot be evaluated at compile-time",
+            self.dropped_ty,
+        );
+        err.span_label(
+            span,
+            format!("the destructor for this type cannot be evaluated in {}s", ccx.const_kind()),
         );
-        err.span_label(span, format!("{}s cannot evaluate destructors", ccx.const_kind()));
         if let Some(span) = self.dropped_at {
             err.span_label(span, "value is dropped here");
         }
@@ -482,12 +467,7 @@ impl<'tcx> NonConstOp<'tcx> for TransientCellBorrow {
         ccx: &ConstCx<'_, 'tcx>,
         span: Span,
     ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
-        feature_err(
-            &ccx.tcx.sess.parse_sess,
-            sym::const_refs_to_cell,
-            span,
-            "cannot borrow here, since the borrowed element may contain interior mutability",
-        )
+        ccx.tcx.sess.create_feature_err(InteriorMutabilityBorrow { span }, sym::const_refs_to_cell)
     }
 }
 
@@ -502,32 +482,22 @@ impl<'tcx> NonConstOp<'tcx> for CellBorrow {
         ccx: &ConstCx<'_, 'tcx>,
         span: Span,
     ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
-        let mut err = struct_span_err!(
-            ccx.tcx.sess,
-            span,
-            E0492,
-            "{}s cannot refer to interior mutable data",
-            ccx.const_kind(),
-        );
-        err.span_label(
-            span,
-            "this borrow of an interior mutable value may end up in the final value",
-        );
+        // FIXME: Maybe a more elegant solution to this if else case
         if let hir::ConstContext::Static(_) = ccx.const_kind() {
-            err.help(
-                "to fix this, the value can be extracted to a separate \
-                `static` item and then referenced",
-            );
-        }
-        if ccx.tcx.sess.teach(&err.get_code().unwrap()) {
-            err.note(
-                "A constant containing interior mutable data behind a reference can allow you
-                 to modify that data. This would make multiple uses of a constant to be able to
-                 see different values and allow circumventing the `Send` and `Sync` requirements
-                 for shared mutable data, which is unsound.",
-            );
+            ccx.tcx.sess.create_err(InteriorMutableDataRefer {
+                span,
+                opt_help: Some(()),
+                kind: ccx.const_kind(),
+                teach: ccx.tcx.sess.teach(&error_code!(E0492)).then_some(()),
+            })
+        } else {
+            ccx.tcx.sess.create_err(InteriorMutableDataRefer {
+                span,
+                opt_help: None,
+                kind: ccx.const_kind(),
+                teach: ccx.tcx.sess.teach(&error_code!(E0492)).then_some(()),
+            })
         }
-        err
     }
 }
 
@@ -553,33 +523,18 @@ impl<'tcx> NonConstOp<'tcx> for MutBorrow {
         ccx: &ConstCx<'_, 'tcx>,
         span: Span,
     ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
-        let raw = match self.0 {
-            hir::BorrowKind::Raw => "raw ",
-            hir::BorrowKind::Ref => "",
-        };
-
-        let mut err = struct_span_err!(
-            ccx.tcx.sess,
-            span,
-            E0764,
-            "{}mutable references are not allowed in the final value of {}s",
-            raw,
-            ccx.const_kind(),
-        );
-
-        if ccx.tcx.sess.teach(&err.get_code().unwrap()) {
-            err.note(
-                "References in statics and constants may only refer \
-                      to immutable values.\n\n\
-                      Statics are shared everywhere, and if they refer to \
-                      mutable data one might violate memory safety since \
-                      holding multiple mutable references to shared data \
-                      is not allowed.\n\n\
-                      If you really want global mutable state, try using \
-                      static mut or a global UnsafeCell.",
-            );
+        match self.0 {
+            hir::BorrowKind::Raw => ccx.tcx.sess.create_err(UnallowedMutableRefsRaw {
+                span,
+                kind: ccx.const_kind(),
+                teach: ccx.tcx.sess.teach(&error_code!(E0764)).then_some(()),
+            }),
+            hir::BorrowKind::Ref => ccx.tcx.sess.create_err(UnallowedMutableRefs {
+                span,
+                kind: ccx.const_kind(),
+                teach: ccx.tcx.sess.teach(&error_code!(E0764)).then_some(()),
+            }),
         }
-        err
     }
 }
 
diff --git a/compiler/rustc_const_eval/src/transform/check_consts/post_drop_elaboration.rs b/compiler/rustc_const_eval/src/transform/check_consts/post_drop_elaboration.rs
index 4e210f663..d4570c598 100644
--- a/compiler/rustc_const_eval/src/transform/check_consts/post_drop_elaboration.rs
+++ b/compiler/rustc_const_eval/src/transform/check_consts/post_drop_elaboration.rs
@@ -1,6 +1,6 @@
 use rustc_middle::mir::visit::Visitor;
 use rustc_middle::mir::{self, BasicBlock, Location};
-use rustc_middle::ty::TyCtxt;
+use rustc_middle::ty::{Ty, TyCtxt};
 use rustc_span::{symbol::sym, Span};
 
 use super::check::Qualifs;
@@ -58,9 +58,9 @@ impl<'mir, 'tcx> std::ops::Deref for CheckLiveDrops<'mir, 'tcx> {
     }
 }
 
-impl CheckLiveDrops<'_, '_> {
-    fn check_live_drop(&self, span: Span) {
-        ops::LiveDrop { dropped_at: None }.build_error(self.ccx, span).emit();
+impl<'tcx> CheckLiveDrops<'_, 'tcx> {
+    fn check_live_drop(&self, span: Span, dropped_ty: Ty<'tcx>) {
+        ops::LiveDrop { dropped_at: None, dropped_ty }.build_error(self.ccx, span).emit();
     }
 }
 
@@ -90,7 +90,7 @@ impl<'tcx> Visitor<'tcx> for CheckLiveDrops<'_, 'tcx> {
                 }
 
                 if dropped_place.is_indirect() {
-                    self.check_live_drop(terminator.source_info.span);
+                    self.check_live_drop(terminator.source_info.span, dropped_ty);
                     return;
                 }
 
@@ -101,7 +101,7 @@ impl<'tcx> Visitor<'tcx> for CheckLiveDrops<'_, 'tcx> {
                 if self.qualifs.needs_non_const_drop(self.ccx, dropped_place.local, location) {
                     // Use the span where the dropped local was declared for the error.
                     let span = self.body.local_decls[dropped_place.local].source_info.span;
-                    self.check_live_drop(span);
+                    self.check_live_drop(span, dropped_ty);
                 }
             }
 
diff --git a/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs b/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs
index c8a63c9c3..335992342 100644
--- a/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs
+++ b/compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs
@@ -5,12 +5,11 @@
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir::LangItem;
 use rustc_infer::infer::TyCtxtInferExt;
-use rustc_infer::traits::TraitEngine;
+use rustc_middle::mir;
 use rustc_middle::mir::*;
 use rustc_middle::ty::{self, subst::SubstsRef, AdtDef, Ty};
-use rustc_span::DUMMY_SP;
 use rustc_trait_selection::traits::{
-    self, ImplSource, Obligation, ObligationCause, SelectionContext, TraitEngineExt,
+    self, ImplSource, Obligation, ObligationCause, SelectionContext,
 };
 
 use super::ConstCx;
@@ -92,7 +91,7 @@ impl Qualif for HasMutInterior {
     }
 
     fn in_any_value_of_ty<'tcx>(cx: &ConstCx<'_, 'tcx>, ty: Ty<'tcx>) -> bool {
-        !ty.is_freeze(cx.tcx.at(DUMMY_SP), cx.param_env)
+        !ty.is_freeze(cx.tcx, cx.param_env)
     }
 
     fn in_adt_inherently<'tcx>(
@@ -168,37 +167,28 @@ impl Qualif for NeedsNonConstDrop {
             }),
         );
 
-        cx.tcx.infer_ctxt().enter(|infcx| {
-            let mut selcx = SelectionContext::new(&infcx);
-            let Some(impl_src) = selcx.select(&obligation).ok().flatten() else {
-                // If we couldn't select a const destruct candidate, then it's bad
-                return true;
-            };
-
-            if !matches!(
-                impl_src,
-                ImplSource::ConstDestruct(_)
-                    | ImplSource::Param(_, ty::BoundConstness::ConstIfConst)
-            ) {
-                // If our const destruct candidate is not ConstDestruct or implied by the param env,
-                // then it's bad
-                return true;
-            }
-
-            if impl_src.borrow_nested_obligations().is_empty() {
-                return false;
-            }
+        let infcx = cx.tcx.infer_ctxt().build();
+        let mut selcx = SelectionContext::new(&infcx);
+        let Some(impl_src) = selcx.select(&obligation).ok().flatten() else {
+            // If we couldn't select a const destruct candidate, then it's bad
+            return true;
+        };
+
+        if !matches!(
+            impl_src,
+            ImplSource::ConstDestruct(_) | ImplSource::Param(_, ty::BoundConstness::ConstIfConst)
+        ) {
+            // If our const destruct candidate is not ConstDestruct or implied by the param env,
+            // then it's bad
+            return true;
+        }
 
-            // If we successfully found one, then select all of the predicates
-            // implied by our const drop impl.
-            let mut fcx = <dyn TraitEngine<'tcx>>::new(cx.tcx);
-            for nested in impl_src.nested_obligations() {
-                fcx.register_predicate_obligation(&infcx, nested);
-            }
+        if impl_src.borrow_nested_obligations().is_empty() {
+            return false;
+        }
 
-            // If we had any errors, then it's bad
-            !fcx.select_all_or_error(&infcx).is_empty()
-        })
+        // If we had any errors, then it's bad
+        !traits::fully_solve_obligations(&infcx, impl_src.nested_obligations()).is_empty()
     }
 
     fn in_adt_inherently<'tcx>(
@@ -316,6 +306,7 @@ where
 
             ProjectionElem::Deref
             | ProjectionElem::Field(_, _)
+            | ProjectionElem::OpaqueCast(_)
             | ProjectionElem::ConstantIndex { .. }
             | ProjectionElem::Subslice { .. }
             | ProjectionElem::Downcast(_, _)
@@ -354,31 +345,36 @@ where
     };
 
     // Check the qualifs of the value of `const` items.
-    if let Some(ct) = constant.literal.const_for_ty() {
-        if let ty::ConstKind::Unevaluated(ty::Unevaluated { def, substs: _, promoted }) = ct.kind()
-        {
-            // Use qualifs of the type for the promoted. Promoteds in MIR body should be possible
-            // only for `NeedsNonConstDrop` with precise drop checking. This is the only const
-            // check performed after the promotion. Verify that with an assertion.
-            assert!(promoted.is_none() || Q::ALLOW_PROMOTED);
-            // Don't peek inside trait associated constants.
-            if promoted.is_none() && cx.tcx.trait_of_item(def.did).is_none() {
-                let qualifs = if let Some((did, param_did)) = def.as_const_arg() {
-                    cx.tcx.at(constant.span).mir_const_qualif_const_arg((did, param_did))
-                } else {
-                    cx.tcx.at(constant.span).mir_const_qualif(def.did)
-                };
-
-                if !Q::in_qualifs(&qualifs) {
-                    return false;
-                }
+    // FIXME(valtrees): check whether const qualifs should behave the same
+    // way for type and mir constants.
+    let uneval = match constant.literal {
+        ConstantKind::Ty(ct) if matches!(ct.kind(), ty::ConstKind::Param(_)) => None,
+        ConstantKind::Ty(c) => bug!("expected ConstKind::Param here, found {:?}", c),
+        ConstantKind::Unevaluated(uv, _) => Some(uv),
+        ConstantKind::Val(..) => None,
+    };
+
+    if let Some(mir::UnevaluatedConst { def, substs: _, promoted }) = uneval {
+        // Use qualifs of the type for the promoted. Promoteds in MIR body should be possible
+        // only for `NeedsNonConstDrop` with precise drop checking. This is the only const
+        // check performed after the promotion. Verify that with an assertion.
+        assert!(promoted.is_none() || Q::ALLOW_PROMOTED);
+
+        // Don't peek inside trait associated constants.
+        if promoted.is_none() && cx.tcx.trait_of_item(def.did).is_none() {
+            assert_eq!(def.const_param_did, None, "expected associated const: {def:?}");
+            let qualifs = cx.tcx.at(constant.span).mir_const_qualif(def.did);
 
-                // Just in case the type is more specific than
-                // the definition, e.g., impl associated const
-                // with type parameters, take it into account.
+            if !Q::in_qualifs(&qualifs) {
+                return false;
             }
+
+            // Just in case the type is more specific than
+            // the definition, e.g., impl associated const
+            // with type parameters, take it into account.
         }
     }
+
     // Otherwise use the qualifs of the type.
     Q::in_any_value_of_ty(cx, constant.literal.ty())
 }
diff --git a/compiler/rustc_const_eval/src/transform/check_consts/resolver.rs b/compiler/rustc_const_eval/src/transform/check_consts/resolver.rs
index 60c1e4950..805e6096b 100644
--- a/compiler/rustc_const_eval/src/transform/check_consts/resolver.rs
+++ b/compiler/rustc_const_eval/src/transform/check_consts/resolver.rs
@@ -8,7 +8,6 @@ use rustc_middle::mir::{self, BasicBlock, Local, Location, Statement, StatementK
 use rustc_mir_dataflow::fmt::DebugWithContext;
 use rustc_mir_dataflow::JoinSemiLattice;
 use rustc_mir_dataflow::{Analysis, AnalysisDomain, CallReturnPlaces};
-use rustc_span::DUMMY_SP;
 
 use std::fmt;
 use std::marker::PhantomData;
@@ -120,10 +119,7 @@ where
     ///
     /// [rust-lang/unsafe-code-guidelines#134]: https://github.com/rust-lang/unsafe-code-guidelines/issues/134
     fn shared_borrow_allows_mutation(&self, place: mir::Place<'tcx>) -> bool {
-        !place
-            .ty(self.ccx.body, self.ccx.tcx)
-            .ty
-            .is_freeze(self.ccx.tcx.at(DUMMY_SP), self.ccx.param_env)
+        !place.ty(self.ccx.body, self.ccx.tcx).ty.is_freeze(self.ccx.tcx, self.ccx.param_env)
     }
 }
 
diff --git a/compiler/rustc_const_eval/src/transform/promote_consts.rs b/compiler/rustc_const_eval/src/transform/promote_consts.rs
index ed4d8c95d..f3ae16da4 100644
--- a/compiler/rustc_const_eval/src/transform/promote_consts.rs
+++ b/compiler/rustc_const_eval/src/transform/promote_consts.rs
@@ -13,6 +13,7 @@
 //! move analysis runs after promotion on broken MIR.
 
 use rustc_hir as hir;
+use rustc_middle::mir;
 use rustc_middle::mir::traversal::ReversePostorderIter;
 use rustc_middle::mir::visit::{MutVisitor, MutatingUseContext, PlaceContext, Visitor};
 use rustc_middle::mir::*;
@@ -40,10 +41,6 @@ pub struct PromoteTemps<'tcx> {
 }
 
 impl<'tcx> MirPass<'tcx> for PromoteTemps<'tcx> {
-    fn phase_change(&self) -> Option<MirPhase> {
-        Some(MirPhase::ConstsPromoted)
-    }
-
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
         // There's not really any point in promoting errorful MIR.
         //
@@ -361,7 +358,7 @@ impl<'tcx> Validator<'_, 'tcx> {
                             return Err(Unpromotable);
                         }
                     }
-                    ProjectionElem::Downcast(..) => {
+                    ProjectionElem::OpaqueCast(..) | ProjectionElem::Downcast(..) => {
                         return Err(Unpromotable);
                     }
 
@@ -710,7 +707,7 @@ impl<'a, 'tcx> Promoter<'a, 'tcx> {
     }
 
     fn assign(&mut self, dest: Local, rvalue: Rvalue<'tcx>, span: Span) {
-        let last = self.promoted.basic_blocks().last().unwrap();
+        let last = self.promoted.basic_blocks.last().unwrap();
         let data = &mut self.promoted[last];
         data.statements.push(Statement {
             source_info: SourceInfo::outermost(span),
@@ -803,7 +800,7 @@ impl<'a, 'tcx> Promoter<'a, 'tcx> {
                         self.visit_operand(arg, loc);
                     }
 
-                    let last = self.promoted.basic_blocks().last().unwrap();
+                    let last = self.promoted.basic_blocks.last().unwrap();
                     let new_target = self.new_block();
 
                     *self.promoted[last].terminator_mut() = Terminator {
@@ -839,27 +836,16 @@ impl<'a, 'tcx> Promoter<'a, 'tcx> {
             let mut promoted_operand = |ty, span| {
                 promoted.span = span;
                 promoted.local_decls[RETURN_PLACE] = LocalDecl::new(ty, span);
-                let _const = tcx.mk_const(ty::ConstS {
-                    ty,
-                    kind: ty::ConstKind::Unevaluated(ty::Unevaluated {
-                        def,
-                        substs: InternalSubsts::for_item(tcx, def.did, |param, _| {
-                            if let ty::GenericParamDefKind::Lifetime = param.kind {
-                                tcx.lifetimes.re_erased.into()
-                            } else {
-                                tcx.mk_param_from_def(param)
-                            }
-                        }),
-                        promoted: Some(promoted_id),
-                    }),
-                });
+                let substs = tcx.erase_regions(InternalSubsts::identity_for_item(tcx, def.did));
+                let uneval = mir::UnevaluatedConst { def, substs, promoted: Some(promoted_id) };
 
                 Operand::Constant(Box::new(Constant {
                     span,
                     user_ty: None,
-                    literal: ConstantKind::from_const(_const, tcx),
+                    literal: ConstantKind::Unevaluated(uneval, ty),
                 }))
             };
+
             let blocks = self.source.basic_blocks.as_mut();
             let local_decls = &mut self.source.local_decls;
             let loc = candidate.location;
@@ -969,7 +955,7 @@ pub fn promote_candidates<'tcx>(
         let mut scope = body.source_scopes[body.source_info(candidate.location).scope].clone();
         scope.parent_scope = None;
 
-        let promoted = Body::new(
+        let mut promoted = Body::new(
             body.source, // `promoted` gets filled in below
             IndexVec::new(),
             IndexVec::from_elem_n(scope, 1),
@@ -981,6 +967,7 @@ pub fn promote_candidates<'tcx>(
             body.generator_kind(),
             body.tainted_by_errors,
         );
+        promoted.phase = MirPhase::Analysis(AnalysisPhase::Initial);
 
         let promoter = Promoter {
             promoted,
@@ -1046,7 +1033,7 @@ pub fn is_const_fn_in_array_repeat_expression<'tcx>(
         _ => {}
     }
 
-    for block in body.basic_blocks() {
+    for block in body.basic_blocks.iter() {
         if let Some(Terminator { kind: TerminatorKind::Call { func, destination, .. }, .. }) =
             &block.terminator
         {
diff --git a/compiler/rustc_const_eval/src/transform/validate.rs b/compiler/rustc_const_eval/src/transform/validate.rs
index 15e820f2d..81b82a21f 100644
--- a/compiler/rustc_const_eval/src/transform/validate.rs
+++ b/compiler/rustc_const_eval/src/transform/validate.rs
@@ -7,12 +7,12 @@ use rustc_middle::mir::interpret::Scalar;
 use rustc_middle::mir::visit::NonUseContext::VarDebugInfo;
 use rustc_middle::mir::visit::{PlaceContext, Visitor};
 use rustc_middle::mir::{
-    traversal, AggregateKind, BasicBlock, BinOp, Body, BorrowKind, CastKind, Local, Location,
-    MirPass, MirPhase, Operand, Place, PlaceElem, PlaceRef, ProjectionElem, Rvalue, SourceScope,
-    Statement, StatementKind, Terminator, TerminatorKind, UnOp, START_BLOCK,
+    traversal, AggregateKind, BasicBlock, BinOp, Body, BorrowKind, CastKind, CopyNonOverlapping,
+    Local, Location, MirPass, MirPhase, NonDivergingIntrinsic, Operand, Place, PlaceElem, PlaceRef,
+    ProjectionElem, RuntimePhase, Rvalue, SourceScope, Statement, StatementKind, Terminator,
+    TerminatorKind, UnOp, START_BLOCK,
 };
 use rustc_middle::ty::fold::BottomUpFolder;
-use rustc_middle::ty::subst::Subst;
 use rustc_middle::ty::{self, InstanceDef, ParamEnv, Ty, TyCtxt, TypeFoldable, TypeVisitable};
 use rustc_mir_dataflow::impls::MaybeStorageLive;
 use rustc_mir_dataflow::storage::always_storage_live_locals;
@@ -89,9 +89,8 @@ pub fn equal_up_to_regions<'tcx>(
 
     // Normalize lifetimes away on both sides, then compare.
     let normalize = |ty: Ty<'tcx>| {
-        tcx.normalize_erasing_regions(
-            param_env,
-            ty.fold_with(&mut BottomUpFolder {
+        tcx.try_normalize_erasing_regions(param_env, ty).unwrap_or(ty).fold_with(
+            &mut BottomUpFolder {
                 tcx,
                 // FIXME: We erase all late-bound lifetimes, but this is not fully correct.
                 // If you have a type like `<for<'a> fn(&'a u32) as SomeTrait>::Assoc`,
@@ -103,10 +102,10 @@ pub fn equal_up_to_regions<'tcx>(
                 // Leave consts and types unchanged.
                 ct_op: |ct| ct,
                 ty_op: |ty| ty,
-            }),
+            },
         )
     };
-    tcx.infer_ctxt().enter(|infcx| infcx.can_eq(param_env, normalize(src), normalize(dest)).is_ok())
+    tcx.infer_ctxt().build().can_eq(param_env, normalize(src), normalize(dest)).is_ok()
 }
 
 struct TypeChecker<'a, 'tcx> {
@@ -142,8 +141,8 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
         if bb == START_BLOCK {
             self.fail(location, "start block must not have predecessors")
         }
-        if let Some(bb) = self.body.basic_blocks().get(bb) {
-            let src = self.body.basic_blocks().get(location.block).unwrap();
+        if let Some(bb) = self.body.basic_blocks.get(bb) {
+            let src = self.body.basic_blocks.get(location.block).unwrap();
             match (src.is_cleanup, bb.is_cleanup, edge_kind) {
                 // Non-cleanup blocks can jump to non-cleanup blocks along non-unwind edges
                 (false, false, EdgeKind::Normal)
@@ -183,16 +182,23 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
         if (src, dest).has_opaque_types() {
             return true;
         }
-        // Normalize projections and things like that.
-        let param_env = self.param_env.with_reveal_all_normalized(self.tcx);
-        let src = self.tcx.normalize_erasing_regions(param_env, src);
-        let dest = self.tcx.normalize_erasing_regions(param_env, dest);
 
+        // Normalize projections and things like that.
         // Type-changing assignments can happen when subtyping is used. While
         // all normal lifetimes are erased, higher-ranked types with their
         // late-bound lifetimes are still around and can lead to type
         // differences. So we compare ignoring lifetimes.
-        equal_up_to_regions(self.tcx, param_env, src, dest)
+
+        // First, try with reveal_all. This might not work in some cases, as the predicates
+        // can be cleared in reveal_all mode. We try the reveal first anyways as it is used
+        // by some other passes like inlining as well.
+        let param_env = self.param_env.with_reveal_all_normalized(self.tcx);
+        if equal_up_to_regions(self.tcx, param_env, src, dest) {
+            return true;
+        }
+
+        // If this fails, we can try it without the reveal.
+        equal_up_to_regions(self.tcx, self.param_env, src, dest)
     }
 }
 
@@ -223,14 +229,14 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
 
     fn visit_operand(&mut self, operand: &Operand<'tcx>, location: Location) {
         // This check is somewhat expensive, so only run it when -Zvalidate-mir is passed.
-        if self.tcx.sess.opts.unstable_opts.validate_mir && self.mir_phase < MirPhase::DropsLowered
+        if self.tcx.sess.opts.unstable_opts.validate_mir
+            && self.mir_phase < MirPhase::Runtime(RuntimePhase::Initial)
         {
             // `Operand::Copy` is only supposed to be used with `Copy` types.
             if let Operand::Copy(place) = operand {
                 let ty = place.ty(&self.body.local_decls, self.tcx).ty;
-                let span = self.body.source_info(location).span;
 
-                if !ty.is_copy_modulo_regions(self.tcx.at(span), self.param_env) {
+                if !ty.is_copy_modulo_regions(self.tcx, self.param_env) {
                     self.fail(location, format!("`Operand::Copy` with non-`Copy` type {}", ty));
                 }
             }
@@ -254,7 +260,9 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                     self.fail(location, format!("bad index ({:?} != usize)", index_ty))
                 }
             }
-            ProjectionElem::Deref if self.mir_phase >= MirPhase::GeneratorsLowered => {
+            ProjectionElem::Deref
+                if self.mir_phase >= MirPhase::Runtime(RuntimePhase::PostCleanup) =>
+            {
                 let base_ty = Place::ty_from(local, proj_base, &self.body.local_decls, self.tcx).ty;
 
                 if base_ty.is_box() {
@@ -275,7 +283,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                         this.fail(
                         location,
                         format!(
-                            "Field projection `{:?}.{:?}` specified type `{:?}`, but actual type is {:?}",
+                            "Field projection `{:?}.{:?}` specified type `{:?}`, but actual type is `{:?}`",
                             parent, f, ty, f_ty
                         )
                     )
@@ -362,7 +370,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
         // Set off any `bug!`s in the type computation code
         let _ = place.ty(&self.body.local_decls, self.tcx);
 
-        if self.mir_phase >= MirPhase::Derefered
+        if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial)
             && place.projection.len() > 1
             && cntxt != PlaceContext::NonUse(VarDebugInfo)
             && place.projection[1..].contains(&ProjectionElem::Deref)
@@ -386,8 +394,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
             Rvalue::Aggregate(agg_kind, _) => {
                 let disallowed = match **agg_kind {
                     AggregateKind::Array(..) => false,
-                    AggregateKind::Generator(..) => self.mir_phase >= MirPhase::GeneratorsLowered,
-                    _ => self.mir_phase >= MirPhase::Deaggregated,
+                    _ => self.mir_phase >= MirPhase::Runtime(RuntimePhase::PostCleanup),
                 };
                 if disallowed {
                     self.fail(
@@ -397,10 +404,10 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                 }
             }
             Rvalue::Ref(_, BorrowKind::Shallow, _) => {
-                if self.mir_phase >= MirPhase::DropsLowered {
+                if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) {
                     self.fail(
                         location,
-                        "`Assign` statement with a `Shallow` borrow should have been removed after drop lowering phase",
+                        "`Assign` statement with a `Shallow` borrow should have been removed in runtime MIR",
                     );
                 }
             }
@@ -548,22 +555,40 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                 check_kinds!(a, "Cannot shallow init type {:?}", ty::RawPtr(..));
             }
             Rvalue::Cast(kind, operand, target_type) => {
+                let op_ty = operand.ty(self.body, self.tcx);
                 match kind {
-                    CastKind::Misc => {
-                        let op_ty = operand.ty(self.body, self.tcx);
-                        if op_ty.is_enum() {
+                    CastKind::DynStar => {
+                        // FIXME(dyn-star): make sure nothing needs to be done here.
+                    }
+                    // FIXME: Add Checks for these
+                    CastKind::PointerFromExposedAddress
+                    | CastKind::PointerExposeAddress
+                    | CastKind::Pointer(_) => {}
+                    CastKind::IntToInt | CastKind::IntToFloat => {
+                        let input_valid = op_ty.is_integral() || op_ty.is_char() || op_ty.is_bool();
+                        let target_valid = target_type.is_numeric() || target_type.is_char();
+                        if !input_valid || !target_valid {
+                            self.fail(
+                                location,
+                                format!("Wrong cast kind {kind:?} for the type {op_ty}",),
+                            );
+                        }
+                    }
+                    CastKind::FnPtrToPtr | CastKind::PtrToPtr => {
+                        if !(op_ty.is_any_ptr() && target_type.is_unsafe_ptr()) {
+                            self.fail(location, "Can't cast {op_ty} into 'Ptr'");
+                        }
+                    }
+                    CastKind::FloatToFloat | CastKind::FloatToInt => {
+                        if !op_ty.is_floating_point() || !target_type.is_numeric() {
                             self.fail(
                                 location,
                                 format!(
-                                    "enum -> int casts should go through `Rvalue::Discriminant`: {operand:?}:{op_ty} as {target_type}",
+                                    "Trying to cast non 'Float' as {kind:?} into {target_type:?}"
                                 ),
                             );
                         }
                     }
-                    // Nothing to check here
-                    CastKind::PointerFromExposedAddress
-                    | CastKind::PointerExposeAddress
-                    | CastKind::Pointer(_) => {}
                 }
             }
             Rvalue::Repeat(_, _)
@@ -614,7 +639,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                 }
             }
             StatementKind::AscribeUserType(..) => {
-                if self.mir_phase >= MirPhase::DropsLowered {
+                if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) {
                     self.fail(
                         location,
                         "`AscribeUserType` should have been removed after drop lowering phase",
@@ -622,18 +647,25 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                 }
             }
             StatementKind::FakeRead(..) => {
-                if self.mir_phase >= MirPhase::DropsLowered {
+                if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) {
                     self.fail(
                         location,
                         "`FakeRead` should have been removed after drop lowering phase",
                     );
                 }
             }
-            StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping {
-                ref src,
-                ref dst,
-                ref count,
-            }) => {
+            StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(op)) => {
+                let ty = op.ty(&self.body.local_decls, self.tcx);
+                if !ty.is_bool() {
+                    self.fail(
+                        location,
+                        format!("`assume` argument must be `bool`, but got: `{}`", ty),
+                    );
+                }
+            }
+            StatementKind::Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping(
+                CopyNonOverlapping { src, dst, count },
+            )) => {
                 let src_ty = src.ty(&self.body.local_decls, self.tcx);
                 let op_src_ty = if let Some(src_deref) = src_ty.builtin_deref(true) {
                     src_deref.ty
@@ -666,7 +698,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                 }
             }
             StatementKind::SetDiscriminant { place, .. } => {
-                if self.mir_phase < MirPhase::Deaggregated {
+                if self.mir_phase < MirPhase::Runtime(RuntimePhase::Initial) {
                     self.fail(location, "`SetDiscriminant`is not allowed until deaggregation");
                 }
                 let pty = place.ty(&self.body.local_decls, self.tcx).ty.kind();
@@ -681,7 +713,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                 }
             }
             StatementKind::Deinit(..) => {
-                if self.mir_phase < MirPhase::Deaggregated {
+                if self.mir_phase < MirPhase::Runtime(RuntimePhase::Initial) {
                     self.fail(location, "`Deinit`is not allowed until deaggregation");
                 }
             }
@@ -761,7 +793,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                 }
             }
             TerminatorKind::DropAndReplace { target, unwind, .. } => {
-                if self.mir_phase >= MirPhase::DropsLowered {
+                if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) {
                     self.fail(
                         location,
                         "`DropAndReplace` should have been removed during drop elaboration",
@@ -832,7 +864,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                 if self.body.generator.is_none() {
                     self.fail(location, "`Yield` cannot appear outside generator bodies");
                 }
-                if self.mir_phase >= MirPhase::GeneratorsLowered {
+                if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) {
                     self.fail(location, "`Yield` should have been replaced by generator lowering");
                 }
                 self.check_edge(location, *resume, EdgeKind::Normal);
@@ -841,7 +873,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                 }
             }
             TerminatorKind::FalseEdge { real_target, imaginary_target } => {
-                if self.mir_phase >= MirPhase::DropsLowered {
+                if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) {
                     self.fail(
                         location,
                         "`FalseEdge` should have been removed after drop elaboration",
@@ -851,7 +883,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                 self.check_edge(location, *imaginary_target, EdgeKind::Normal);
             }
             TerminatorKind::FalseUnwind { real_target, unwind } => {
-                if self.mir_phase >= MirPhase::DropsLowered {
+                if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) {
                     self.fail(
                         location,
                         "`FalseUnwind` should have been removed after drop elaboration",
@@ -874,7 +906,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                 if self.body.generator.is_none() {
                     self.fail(location, "`GeneratorDrop` cannot appear outside generator bodies");
                 }
-                if self.mir_phase >= MirPhase::GeneratorsLowered {
+                if self.mir_phase >= MirPhase::Runtime(RuntimePhase::Initial) {
                     self.fail(
                         location,
                         "`GeneratorDrop` should have been replaced by generator lowering",
@@ -883,13 +915,13 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
             }
             TerminatorKind::Resume | TerminatorKind::Abort => {
                 let bb = location.block;
-                if !self.body.basic_blocks()[bb].is_cleanup {
+                if !self.body.basic_blocks[bb].is_cleanup {
                     self.fail(location, "Cannot `Resume` or `Abort` from non-cleanup basic block")
                 }
             }
             TerminatorKind::Return => {
                 let bb = location.block;
-                if self.body.basic_blocks()[bb].is_cleanup {
+                if self.body.basic_blocks[bb].is_cleanup {
                     self.fail(location, "Cannot `Return` from cleanup basic block")
                 }
             }