Merging upstream version 1.74.1+dfsg1.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

22 files changed, 412 insertions, 273 deletions

diff --git a/compiler/rustc_mir_build/messages.ftl b/compiler/rustc_mir_build/messages.ftl
index 938f3edd3..ce021923f 100644
--- a/compiler/rustc_mir_build/messages.ftl
+++ b/compiler/rustc_mir_build/messages.ftl
@@ -229,6 +229,9 @@ mir_build_non_exhaustive_patterns_type_not_empty = non-exhaustive patterns: type
     .suggestion = ensure that all possible cases are being handled by adding a match arm with a wildcard pattern as shown
     .help = ensure that all possible cases are being handled by adding a match arm with a wildcard pattern
 
+mir_build_non_partial_eq_match =
+    to use a constant of type `{$non_peq_ty}` in a pattern, the type must implement `PartialEq`
+
 mir_build_nontrivial_structural_match =
     to use a constant of type `{$non_sm_ty}` in a pattern, the constant's initializer must be trivial or `{$non_sm_ty}` must be annotated with `#[derive(PartialEq, Eq)]`
 
diff --git a/compiler/rustc_mir_build/src/build/cfg.rs b/compiler/rustc_mir_build/src/build/cfg.rs
index 4f1623b4c..fddcf9de7 100644
--- a/compiler/rustc_mir_build/src/build/cfg.rs
+++ b/compiler/rustc_mir_build/src/build/cfg.rs
@@ -49,7 +49,7 @@ impl<'tcx> CFG<'tcx> {
         block: BasicBlock,
         source_info: SourceInfo,
         temp: Place<'tcx>,
-        constant: Constant<'tcx>,
+        constant: ConstOperand<'tcx>,
     ) {
         self.push_assign(
             block,
@@ -70,10 +70,10 @@ impl<'tcx> CFG<'tcx> {
             block,
             source_info,
             place,
-            Rvalue::Use(Operand::Constant(Box::new(Constant {
+            Rvalue::Use(Operand::Constant(Box::new(ConstOperand {
                 span: source_info.span,
                 user_ty: None,
-                literal: ConstantKind::zero_sized(tcx.types.unit),
+                const_: Const::zero_sized(tcx.types.unit),
             }))),
         );
     }
diff --git a/compiler/rustc_mir_build/src/build/custom/parse.rs b/compiler/rustc_mir_build/src/build/custom/parse.rs
index 60c4a0416..e2ab2cb90 100644
--- a/compiler/rustc_mir_build/src/build/custom/parse.rs
+++ b/compiler/rustc_mir_build/src/build/custom/parse.rs
@@ -1,5 +1,6 @@
 use rustc_index::IndexSlice;
-use rustc_middle::{mir::*, thir::*, ty::Ty};
+use rustc_middle::ty::{self, Ty};
+use rustc_middle::{mir::*, thir::*};
 use rustc_span::Span;
 
 use super::{PResult, ParseCtxt, ParseError};
@@ -159,6 +160,14 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
         );
         self.parse_local_decls(local_decls.iter().copied())?;
 
+        let (debuginfo, rest) = parse_by_kind!(self, rest, _, "body with debuginfo",
+            ExprKind::Block { block } => {
+                let block = &self.thir[*block];
+                (&block.stmts, block.expr.unwrap())
+            },
+        );
+        self.parse_debuginfo(debuginfo.iter().copied())?;
+
         let block_defs = parse_by_kind!(self, rest, _, "body with block defs",
             ExprKind::Block { block } => &self.thir[*block].stmts,
         );
@@ -195,6 +204,53 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
         Ok(())
     }
 
+    fn parse_debuginfo(&mut self, stmts: impl Iterator<Item = StmtId>) -> PResult<()> {
+        for stmt in stmts {
+            let stmt = &self.thir[stmt];
+            let expr = match stmt.kind {
+                StmtKind::Let { span, .. } => {
+                    return Err(ParseError {
+                        span,
+                        item_description: format!("{:?}", stmt),
+                        expected: "debuginfo".to_string(),
+                    });
+                }
+                StmtKind::Expr { expr, .. } => expr,
+            };
+            let span = self.thir[expr].span;
+            let (name, operand) = parse_by_kind!(self, expr, _, "debuginfo",
+                @call("mir_debuginfo", args) => {
+                    (args[0], args[1])
+                },
+            );
+            let name = parse_by_kind!(self, name, _, "debuginfo",
+                ExprKind::Literal { lit, neg: false } => lit,
+            );
+            let Some(name) = name.node.str() else {
+                return Err(ParseError {
+                    span,
+                    item_description: format!("{:?}", name),
+                    expected: "string".to_string(),
+                });
+            };
+            let operand = self.parse_operand(operand)?;
+            let value = match operand {
+                Operand::Constant(c) => VarDebugInfoContents::Const(*c),
+                Operand::Copy(p) | Operand::Move(p) => VarDebugInfoContents::Place(p),
+            };
+            let dbginfo = VarDebugInfo {
+                name,
+                source_info: SourceInfo { span, scope: self.source_scope },
+                composite: None,
+                argument_index: None,
+                value,
+            };
+            self.body.var_debug_info.push(dbginfo);
+        }
+
+        Ok(())
+    }
+
     fn parse_let_statement(&mut self, stmt_id: StmtId) -> PResult<(LocalVarId, Ty<'tcx>, Span)> {
         let pattern = match &self.thir[stmt_id].kind {
             StmtKind::Let { pattern, .. } => pattern,
diff --git a/compiler/rustc_mir_build/src/build/custom/parse/instruction.rs b/compiler/rustc_mir_build/src/build/custom/parse/instruction.rs
index 26662f5de..fd2c57a0a 100644
--- a/compiler/rustc_mir_build/src/build/custom/parse/instruction.rs
+++ b/compiler/rustc_mir_build/src/build/custom/parse/instruction.rs
@@ -1,4 +1,4 @@
-use rustc_middle::mir::interpret::{ConstValue, Scalar};
+use rustc_middle::mir::interpret::Scalar;
 use rustc_middle::mir::tcx::PlaceTy;
 use rustc_middle::ty::cast::mir_cast_kind;
 use rustc_middle::{mir::*, thir::*, ty};
@@ -100,7 +100,7 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
                     expected: "constant pattern".to_string(),
                 });
             };
-            values.push(value.eval_bits(self.tcx, self.param_env, arm.pattern.ty));
+            values.push(value.eval_bits(self.tcx, self.param_env));
             targets.push(self.parse_block(arm.body)?);
         }
 
@@ -204,7 +204,7 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
         )
     }
 
-    fn parse_operand(&self, expr_id: ExprId) -> PResult<Operand<'tcx>> {
+    pub fn parse_operand(&self, expr_id: ExprId) -> PResult<Operand<'tcx>> {
         parse_by_kind!(self, expr_id, expr, "operand",
             @call("mir_move", args) => self.parse_place(args[0]).map(Operand::Move),
             @call("mir_static", args) => self.parse_static(args[0]),
@@ -283,12 +283,12 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
             ExprKind::StaticRef { alloc_id, ty, .. } => {
                 let const_val =
                     ConstValue::Scalar(Scalar::from_pointer((*alloc_id).into(), &self.tcx));
-                let literal = ConstantKind::Val(const_val, *ty);
+                let const_ = Const::Val(const_val, *ty);
 
-                Ok(Operand::Constant(Box::new(Constant {
+                Ok(Operand::Constant(Box::new(ConstOperand {
                     span: expr.span,
                     user_ty: None,
-                    literal
+                    const_
                 })))
             },
         )
@@ -301,7 +301,7 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
             | ExprKind::NonHirLiteral { .. }
             | ExprKind::ConstBlock { .. } => Ok({
                 let value = as_constant_inner(expr, |_| None, self.tcx);
-                value.literal.eval_bits(self.tcx, self.param_env, value.ty())
+                value.const_.eval_bits(self.tcx, self.param_env)
             }),
         )
     }
diff --git a/compiler/rustc_mir_build/src/build/expr/as_constant.rs b/compiler/rustc_mir_build/src/build/expr/as_constant.rs
index aaa37446e..4ed49e787 100644
--- a/compiler/rustc_mir_build/src/build/expr/as_constant.rs
+++ b/compiler/rustc_mir_build/src/build/expr/as_constant.rs
@@ -3,9 +3,7 @@
 use crate::build::{parse_float_into_constval, Builder};
 use rustc_ast as ast;
 use rustc_middle::mir;
-use rustc_middle::mir::interpret::{
-    Allocation, ConstValue, LitToConstError, LitToConstInput, Scalar,
-};
+use rustc_middle::mir::interpret::{Allocation, LitToConstError, LitToConstInput, Scalar};
 use rustc_middle::mir::*;
 use rustc_middle::thir::*;
 use rustc_middle::ty::{
@@ -17,7 +15,7 @@ use rustc_target::abi::Size;
 impl<'a, 'tcx> Builder<'a, 'tcx> {
     /// Compile `expr`, yielding a compile-time constant. Assumes that
     /// `expr` is a valid compile-time constant!
-    pub(crate) fn as_constant(&mut self, expr: &Expr<'tcx>) -> Constant<'tcx> {
+    pub(crate) fn as_constant(&mut self, expr: &Expr<'tcx>) -> ConstOperand<'tcx> {
         let this = self;
         let tcx = this.tcx;
         let Expr { ty, temp_lifetime: _, span, ref kind } = *expr;
@@ -44,62 +42,62 @@ pub fn as_constant_inner<'tcx>(
     expr: &Expr<'tcx>,
     push_cuta: impl FnMut(&Box<CanonicalUserType<'tcx>>) -> Option<UserTypeAnnotationIndex>,
     tcx: TyCtxt<'tcx>,
-) -> Constant<'tcx> {
+) -> ConstOperand<'tcx> {
     let Expr { ty, temp_lifetime: _, span, ref kind } = *expr;
     match *kind {
         ExprKind::Literal { lit, neg } => {
-            let literal =
-                match lit_to_mir_constant(tcx, LitToConstInput { lit: &lit.node, ty, neg }) {
-                    Ok(c) => c,
-                    Err(LitToConstError::Reported(guar)) => {
-                        ConstantKind::Ty(ty::Const::new_error(tcx, guar, ty))
-                    }
-                    Err(LitToConstError::TypeError) => {
-                        bug!("encountered type error in `lit_to_mir_constant`")
-                    }
-                };
-
-            Constant { span, user_ty: None, literal }
+            let const_ = match lit_to_mir_constant(tcx, LitToConstInput { lit: &lit.node, ty, neg })
+            {
+                Ok(c) => c,
+                Err(LitToConstError::Reported(guar)) => {
+                    Const::Ty(ty::Const::new_error(tcx, guar, ty))
+                }
+                Err(LitToConstError::TypeError) => {
+                    bug!("encountered type error in `lit_to_mir_constant`")
+                }
+            };
+
+            ConstOperand { span, user_ty: None, const_ }
         }
         ExprKind::NonHirLiteral { lit, ref user_ty } => {
             let user_ty = user_ty.as_ref().and_then(push_cuta);
 
-            let literal = ConstantKind::Val(ConstValue::Scalar(Scalar::Int(lit)), ty);
+            let const_ = Const::Val(ConstValue::Scalar(Scalar::Int(lit)), ty);
 
-            Constant { span, user_ty, literal }
+            ConstOperand { span, user_ty, const_ }
         }
         ExprKind::ZstLiteral { ref user_ty } => {
             let user_ty = user_ty.as_ref().and_then(push_cuta);
 
-            let literal = ConstantKind::Val(ConstValue::ZeroSized, ty);
+            let const_ = Const::Val(ConstValue::ZeroSized, ty);
 
-            Constant { span, user_ty, literal }
+            ConstOperand { span, user_ty, const_ }
         }
         ExprKind::NamedConst { def_id, args, ref user_ty } => {
             let user_ty = user_ty.as_ref().and_then(push_cuta);
 
             let uneval = mir::UnevaluatedConst::new(def_id, args);
-            let literal = ConstantKind::Unevaluated(uneval, ty);
+            let const_ = Const::Unevaluated(uneval, ty);
 
-            Constant { user_ty, span, literal }
+            ConstOperand { user_ty, span, const_ }
         }
         ExprKind::ConstParam { param, def_id: _ } => {
             let const_param = ty::Const::new_param(tcx, param, expr.ty);
-            let literal = ConstantKind::Ty(const_param);
+            let const_ = Const::Ty(const_param);
 
-            Constant { user_ty: None, span, literal }
+            ConstOperand { user_ty: None, span, const_ }
         }
         ExprKind::ConstBlock { did: def_id, args } => {
             let uneval = mir::UnevaluatedConst::new(def_id, args);
-            let literal = ConstantKind::Unevaluated(uneval, ty);
+            let const_ = Const::Unevaluated(uneval, ty);
 
-            Constant { user_ty: None, span, literal }
+            ConstOperand { user_ty: None, span, const_ }
         }
         ExprKind::StaticRef { alloc_id, ty, .. } => {
             let const_val = ConstValue::Scalar(Scalar::from_pointer(alloc_id.into(), &tcx));
-            let literal = ConstantKind::Val(const_val, ty);
+            let const_ = Const::Val(const_val, ty);
 
-            Constant { span, user_ty: None, literal }
+            ConstOperand { span, user_ty: None, const_ }
         }
         _ => span_bug!(span, "expression is not a valid constant {:?}", kind),
     }
@@ -109,7 +107,7 @@ pub fn as_constant_inner<'tcx>(
 fn lit_to_mir_constant<'tcx>(
     tcx: TyCtxt<'tcx>,
     lit_input: LitToConstInput<'tcx>,
-) -> Result<ConstantKind<'tcx>, LitToConstError> {
+) -> Result<Const<'tcx>, LitToConstError> {
     let LitToConstInput { lit, ty, neg } = lit_input;
     let trunc = |n| {
         let param_ty = ty::ParamEnv::reveal_all().and(ty);
@@ -133,14 +131,14 @@ fn lit_to_mir_constant<'tcx>(
             let s = s.as_str();
             let allocation = Allocation::from_bytes_byte_aligned_immutable(s.as_bytes());
             let allocation = tcx.mk_const_alloc(allocation);
-            ConstValue::Slice { data: allocation, start: 0, end: s.len() }
+            ConstValue::Slice { data: allocation, meta: allocation.inner().size().bytes() }
         }
         (ast::LitKind::ByteStr(data, _), ty::Ref(_, inner_ty, _))
             if matches!(inner_ty.kind(), ty::Slice(_)) =>
         {
             let allocation = Allocation::from_bytes_byte_aligned_immutable(data as &[u8]);
             let allocation = tcx.mk_const_alloc(allocation);
-            ConstValue::Slice { data: allocation, start: 0, end: data.len() }
+            ConstValue::Slice { data: allocation, meta: allocation.inner().size().bytes() }
         }
         (ast::LitKind::ByteStr(data, _), ty::Ref(_, inner_ty, _)) if inner_ty.is_array() => {
             let id = tcx.allocate_bytes(data);
@@ -150,7 +148,7 @@ fn lit_to_mir_constant<'tcx>(
         {
             let allocation = Allocation::from_bytes_byte_aligned_immutable(data as &[u8]);
             let allocation = tcx.mk_const_alloc(allocation);
-            ConstValue::Slice { data: allocation, start: 0, end: data.len() }
+            ConstValue::Slice { data: allocation, meta: allocation.inner().size().bytes() }
         }
         (ast::LitKind::Byte(n), ty::Uint(ty::UintTy::U8)) => {
             ConstValue::Scalar(Scalar::from_uint(*n, Size::from_bytes(1)))
@@ -175,5 +173,5 @@ fn lit_to_mir_constant<'tcx>(
         _ => return Err(LitToConstError::TypeError),
     };
 
-    Ok(ConstantKind::Val(value, ty))
+    Ok(Const::Val(value, ty))
 }
diff --git a/compiler/rustc_mir_build/src/build/expr/as_place.rs b/compiler/rustc_mir_build/src/build/expr/as_place.rs
index 2e7ef265a..5bccba4fd 100644
--- a/compiler/rustc_mir_build/src/build/expr/as_place.rs
+++ b/compiler/rustc_mir_build/src/build/expr/as_place.rs
@@ -102,7 +102,7 @@ fn convert_to_hir_projections_and_truncate_for_capture(
                 continue;
             }
             // These do not affect anything, they just make sure we know the right type.
-            ProjectionElem::OpaqueCast(_) => continue,
+            ProjectionElem::OpaqueCast(_) | ProjectionElem::Subtype(..) => continue,
             ProjectionElem::Index(..)
             | ProjectionElem::ConstantIndex { .. }
             | ProjectionElem::Subslice { .. } => {
@@ -690,7 +690,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                             fake_borrow_temp.into(),
                             Rvalue::Ref(
                                 tcx.lifetimes.re_erased,
-                                BorrowKind::Shallow,
+                                BorrowKind::Fake,
                                 Place { local: base_place.local, projection },
                             ),
                         );
@@ -709,6 +709,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     ProjectionElem::Field(..)
                     | ProjectionElem::Downcast(..)
                     | ProjectionElem::OpaqueCast(..)
+                    | ProjectionElem::Subtype(..)
                     | ProjectionElem::ConstantIndex { .. }
                     | ProjectionElem::Subslice { .. } => (),
                 }
diff --git a/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs b/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs
index 3220a184d..d4089eef4 100644
--- a/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs
+++ b/compiler/rustc_mir_build/src/build/expr/as_rvalue.rs
@@ -249,7 +249,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
                         let mut comparer = |range: u128, bin_op: BinOp| -> Place<'tcx> {
                             let range_val =
-                                ConstantKind::from_bits(this.tcx, range, ty::ParamEnv::empty().and(unsigned_ty));
+                                Const::from_bits(this.tcx, range, ty::ParamEnv::empty().and(unsigned_ty));
                             let lit_op = this.literal_operand(expr.span, range_val);
                             let is_bin_op = this.temp(bool_ty, expr_span);
                             this.cfg.push_assign(
@@ -485,10 +485,10 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             }
             ExprKind::Assign { .. } | ExprKind::AssignOp { .. } => {
                 block = unpack!(this.stmt_expr(block, expr, None));
-                block.and(Rvalue::Use(Operand::Constant(Box::new(Constant {
+                block.and(Rvalue::Use(Operand::Constant(Box::new(ConstOperand {
                     span: expr_span,
                     user_ty: None,
-                    literal: ConstantKind::zero_sized(this.tcx.types.unit),
+                    const_: Const::zero_sized(this.tcx.types.unit),
                 }))))
             }
 
@@ -817,7 +817,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     fn neg_1_literal(&mut self, span: Span, ty: Ty<'tcx>) -> Operand<'tcx> {
         let param_ty = ty::ParamEnv::empty().and(ty);
         let size = self.tcx.layout_of(param_ty).unwrap().size;
-        let literal = ConstantKind::from_bits(self.tcx, size.unsigned_int_max(), param_ty);
+        let literal = Const::from_bits(self.tcx, size.unsigned_int_max(), param_ty);
 
         self.literal_operand(span, literal)
     }
@@ -828,7 +828,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         let param_ty = ty::ParamEnv::empty().and(ty);
         let bits = self.tcx.layout_of(param_ty).unwrap().size.bits();
         let n = 1 << (bits - 1);
-        let literal = ConstantKind::from_bits(self.tcx, n, param_ty);
+        let literal = Const::from_bits(self.tcx, n, param_ty);
 
         self.literal_operand(span, literal)
     }
diff --git a/compiler/rustc_mir_build/src/build/expr/into.rs b/compiler/rustc_mir_build/src/build/expr/into.rs
index a5c86e31a..a4de42d45 100644
--- a/compiler/rustc_mir_build/src/build/expr/into.rs
+++ b/compiler/rustc_mir_build/src/build/expr/into.rs
@@ -114,10 +114,10 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     true_block,
                     source_info,
                     destination,
-                    Constant {
+                    ConstOperand {
                         span: expr_span,
                         user_ty: None,
-                        literal: ConstantKind::from_bool(this.tcx, true),
+                        const_: Const::from_bool(this.tcx, true),
                     },
                 );
 
@@ -125,10 +125,10 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     false_block,
                     source_info,
                     destination,
-                    Constant {
+                    ConstOperand {
                         span: expr_span,
                         user_ty: None,
-                        literal: ConstantKind::from_bool(this.tcx, false),
+                        const_: Const::from_bool(this.tcx, false),
                     },
                 );
 
@@ -159,52 +159,44 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 }
             }
             ExprKind::LogicalOp { op, lhs, rhs } => {
-                // And:
-                //
-                // [block: If(lhs)] -true-> [else_block: dest = (rhs)]
-                //        | (false)
-                //  [shortcircuit_block: dest = false]
-                //
-                // Or:
-                //
-                // [block: If(lhs)] -false-> [else_block: dest = (rhs)]
-                //        | (true)
-                //  [shortcircuit_block: dest = true]
-
-                let (shortcircuit_block, mut else_block, join_block) = (
-                    this.cfg.start_new_block(),
-                    this.cfg.start_new_block(),
-                    this.cfg.start_new_block(),
-                );
-
-                let lhs = unpack!(block = this.as_local_operand(block, &this.thir[lhs]));
-                let blocks = match op {
-                    LogicalOp::And => (else_block, shortcircuit_block),
-                    LogicalOp::Or => (shortcircuit_block, else_block),
+                let condition_scope = this.local_scope();
+                let source_info = this.source_info(expr.span);
+                // We first evaluate the left-hand side of the predicate ...
+                let (then_block, else_block) =
+                    this.in_if_then_scope(condition_scope, expr.span, |this| {
+                        this.then_else_break(
+                            block,
+                            &this.thir[lhs],
+                            Some(condition_scope),
+                            condition_scope,
+                            source_info,
+                        )
+                    });
+                let (short_circuit, continuation, constant) = match op {
+                    LogicalOp::And => (else_block, then_block, false),
+                    LogicalOp::Or => (then_block, else_block, true),
                 };
-                let term = TerminatorKind::if_(lhs, blocks.0, blocks.1);
-                this.cfg.terminate(block, source_info, term);
-
+                // At this point, the control flow splits into a short-circuiting path
+                // and a continuation path.
+                // - If the operator is `&&`, passing `lhs` leads to continuation of evaluation on `rhs`;
+                //   failing it leads to the short-circuting path which assigns `false` to the place.
+                // - If the operator is `||`, failing `lhs` leads to continuation of evaluation on `rhs`;
+                //   passing it leads to the short-circuting path which assigns `true` to the place.
                 this.cfg.push_assign_constant(
-                    shortcircuit_block,
+                    short_circuit,
                     source_info,
                     destination,
-                    Constant {
-                        span: expr_span,
+                    ConstOperand {
+                        span: expr.span,
                         user_ty: None,
-                        literal: match op {
-                            LogicalOp::And => ConstantKind::from_bool(this.tcx, false),
-                            LogicalOp::Or => ConstantKind::from_bool(this.tcx, true),
-                        },
+                        const_: Const::from_bool(this.tcx, constant),
                     },
                 );
-                this.cfg.goto(shortcircuit_block, source_info, join_block);
-
-                let rhs = unpack!(else_block = this.as_local_operand(else_block, &this.thir[rhs]));
-                this.cfg.push_assign(else_block, source_info, destination, Rvalue::Use(rhs));
-                this.cfg.goto(else_block, source_info, join_block);
-
-                join_block.unit()
+                let rhs = unpack!(this.expr_into_dest(destination, continuation, &this.thir[rhs]));
+                let target = this.cfg.start_new_block();
+                this.cfg.goto(rhs, source_info, target);
+                this.cfg.goto(short_circuit, source_info, target);
+                target.unit()
             }
             ExprKind::Loop { body } => {
                 // [block]
@@ -441,12 +433,20 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         }
                         thir::InlineAsmOperand::Const { value, span } => {
                             mir::InlineAsmOperand::Const {
-                                value: Box::new(Constant { span, user_ty: None, literal: value }),
+                                value: Box::new(ConstOperand {
+                                    span,
+                                    user_ty: None,
+                                    const_: value,
+                                }),
                             }
                         }
                         thir::InlineAsmOperand::SymFn { value, span } => {
                             mir::InlineAsmOperand::SymFn {
-                                value: Box::new(Constant { span, user_ty: None, literal: value }),
+                                value: Box::new(ConstOperand {
+                                    span,
+                                    user_ty: None,
+                                    const_: value,
+                                }),
                             }
                         }
                         thir::InlineAsmOperand::SymStatic { def_id } => {
diff --git a/compiler/rustc_mir_build/src/build/matches/mod.rs b/compiler/rustc_mir_build/src/build/matches/mod.rs
index 3c4507407..6baf8c7d7 100644
--- a/compiler/rustc_mir_build/src/build/matches/mod.rs
+++ b/compiler/rustc_mir_build/src/build/matches/mod.rs
@@ -64,6 +64,43 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
                 rhs_then_block.unit()
             }
+            ExprKind::LogicalOp { op: LogicalOp::Or, lhs, rhs } => {
+                let local_scope = this.local_scope();
+                let (lhs_success_block, failure_block) =
+                    this.in_if_then_scope(local_scope, expr_span, |this| {
+                        this.then_else_break(
+                            block,
+                            &this.thir[lhs],
+                            temp_scope_override,
+                            local_scope,
+                            variable_source_info,
+                        )
+                    });
+                let rhs_success_block = unpack!(this.then_else_break(
+                    failure_block,
+                    &this.thir[rhs],
+                    temp_scope_override,
+                    break_scope,
+                    variable_source_info,
+                ));
+                this.cfg.goto(lhs_success_block, variable_source_info, rhs_success_block);
+                rhs_success_block.unit()
+            }
+            ExprKind::Unary { op: UnOp::Not, arg } => {
+                let local_scope = this.local_scope();
+                let (success_block, failure_block) =
+                    this.in_if_then_scope(local_scope, expr_span, |this| {
+                        this.then_else_break(
+                            block,
+                            &this.thir[arg],
+                            temp_scope_override,
+                            local_scope,
+                            variable_source_info,
+                        )
+                    });
+                this.break_for_else(success_block, break_scope, variable_source_info);
+                failure_block.unit()
+            }
             ExprKind::Scope { region_scope, lint_level, value } => {
                 let region_scope = (region_scope, this.source_info(expr_span));
                 this.in_scope(region_scope, lint_level, |this| {
@@ -76,6 +113,13 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     )
                 })
             }
+            ExprKind::Use { source } => this.then_else_break(
+                block,
+                &this.thir[source],
+                temp_scope_override,
+                break_scope,
+                variable_source_info,
+            ),
             ExprKind::Let { expr, ref pat } => this.lower_let_expr(
                 block,
                 &this.thir[expr],
@@ -961,13 +1005,13 @@ enum TestKind<'tcx> {
         ///
         /// For `bool` we always generate two edges, one for `true` and one for
         /// `false`.
-        options: FxIndexMap<ConstantKind<'tcx>, u128>,
+        options: FxIndexMap<Const<'tcx>, u128>,
     },
 
     /// Test for equality with value, possibly after an unsizing coercion to
     /// `ty`,
     Eq {
-        value: ConstantKind<'tcx>,
+        value: Const<'tcx>,
         // Integer types are handled by `SwitchInt`, and constants with ADT
         // types are converted back into patterns, so this can only be `&str`,
         // `&[T]`, `f32` or `f64`.
@@ -1578,9 +1622,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         // may want to add cases based on the candidates that are
         // available
         match test.kind {
-            TestKind::SwitchInt { switch_ty, ref mut options } => {
+            TestKind::SwitchInt { switch_ty: _, ref mut options } => {
                 for candidate in candidates.iter() {
-                    if !self.add_cases_to_switch(&match_place, candidate, switch_ty, options) {
+                    if !self.add_cases_to_switch(&match_place, candidate, options) {
                         break;
                     }
                 }
@@ -1960,7 +2004,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             let re_erased = tcx.lifetimes.re_erased;
             let scrutinee_source_info = self.source_info(scrutinee_span);
             for &(place, temp) in fake_borrows {
-                let borrow = Rvalue::Ref(re_erased, BorrowKind::Shallow, place);
+                let borrow = Rvalue::Ref(re_erased, BorrowKind::Fake, place);
                 self.cfg.push_assign(block, scrutinee_source_info, Place::from(temp), borrow);
             }
 
@@ -2243,6 +2287,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             name,
             source_info: debug_source_info,
             value: VarDebugInfoContents::Place(for_arm_body.into()),
+            composite: None,
             argument_index: None,
         });
         let locals = if has_guard.0 {
@@ -2262,6 +2307,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 name,
                 source_info: debug_source_info,
                 value: VarDebugInfoContents::Place(ref_for_guard.into()),
+                composite: None,
                 argument_index: None,
             });
             LocalsForNode::ForGuard { ref_for_guard, for_arm_body }
diff --git a/compiler/rustc_mir_build/src/build/matches/test.rs b/compiler/rustc_mir_build/src/build/matches/test.rs
index 484e84909..795d1db8e 100644
--- a/compiler/rustc_mir_build/src/build/matches/test.rs
+++ b/compiler/rustc_mir_build/src/build/matches/test.rs
@@ -85,8 +85,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         &mut self,
         test_place: &PlaceBuilder<'tcx>,
         candidate: &Candidate<'pat, 'tcx>,
-        switch_ty: Ty<'tcx>,
-        options: &mut FxIndexMap<ConstantKind<'tcx>, u128>,
+        options: &mut FxIndexMap<Const<'tcx>, u128>,
     ) -> bool {
         let Some(match_pair) = candidate.match_pairs.iter().find(|mp| mp.place == *test_place)
         else {
@@ -95,9 +94,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
         match match_pair.pattern.kind {
             PatKind::Constant { value } => {
-                options
-                    .entry(value)
-                    .or_insert_with(|| value.eval_bits(self.tcx, self.param_env, switch_ty));
+                options.entry(value).or_insert_with(|| value.eval_bits(self.tcx, self.param_env));
                 true
             }
             PatKind::Variant { .. } => {
@@ -255,10 +252,10 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         block,
                         source_info,
                         TerminatorKind::Call {
-                            func: Operand::Constant(Box::new(Constant {
+                            func: Operand::Constant(Box::new(ConstOperand {
                                 span: test.span,
                                 user_ty: None,
-                                literal: method,
+                                const_: method,
                             })),
                             args: vec![Operand::Move(ref_string)],
                             destination: ref_str,
@@ -388,7 +385,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         block: BasicBlock,
         make_target_blocks: impl FnOnce(&mut Self) -> Vec<BasicBlock>,
         source_info: SourceInfo,
-        value: ConstantKind<'tcx>,
+        value: Const<'tcx>,
         mut val: Place<'tcx>,
         mut ty: Ty<'tcx>,
     ) {
@@ -485,7 +482,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             block,
             source_info,
             TerminatorKind::Call {
-                func: Operand::Constant(Box::new(Constant {
+                func: Operand::Constant(Box::new(ConstOperand {
                     span: source_info.span,
 
                     // FIXME(#54571): This constant comes from user input (a
@@ -494,7 +491,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     // Need to experiment.
                     user_ty: None,
 
-                    literal: method,
+                    const_: method,
                 })),
                 args: vec![Operand::Copy(val), expect],
                 destination: eq_result,
@@ -800,11 +797,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         span_bug!(match_pair.pattern.span, "simplifiable pattern found: {:?}", match_pair.pattern)
     }
 
-    fn const_range_contains(
-        &self,
-        range: &PatRange<'tcx>,
-        value: ConstantKind<'tcx>,
-    ) -> Option<bool> {
+    fn const_range_contains(&self, range: &PatRange<'tcx>, value: Const<'tcx>) -> Option<bool> {
         use std::cmp::Ordering::*;
 
         // For performance, it's important to only do the second
@@ -821,7 +814,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     fn values_not_contained_in_range(
         &self,
         range: &PatRange<'tcx>,
-        options: &FxIndexMap<ConstantKind<'tcx>, u128>,
+        options: &FxIndexMap<Const<'tcx>, u128>,
     ) -> Option<bool> {
         for &val in options.keys() {
             if self.const_range_contains(range, val)? {
@@ -866,7 +859,7 @@ fn trait_method<'tcx>(
     trait_def_id: DefId,
     method_name: Symbol,
     args: impl IntoIterator<Item: Into<GenericArg<'tcx>>>,
-) -> ConstantKind<'tcx> {
+) -> Const<'tcx> {
     // The unhygienic comparison here is acceptable because this is only
     // used on known traits.
     let item = tcx
@@ -877,5 +870,5 @@ fn trait_method<'tcx>(
 
     let method_ty = Ty::new_fn_def(tcx, item.def_id, args);
 
-    ConstantKind::zero_sized(method_ty)
+    Const::zero_sized(method_ty)
 }
diff --git a/compiler/rustc_mir_build/src/build/misc.rs b/compiler/rustc_mir_build/src/build/misc.rs
index 90d78658f..c96e99ef0 100644
--- a/compiler/rustc_mir_build/src/build/misc.rs
+++ b/compiler/rustc_mir_build/src/build/misc.rs
@@ -25,19 +25,15 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
     /// Convenience function for creating a literal operand, one
     /// without any user type annotation.
-    pub(crate) fn literal_operand(
-        &mut self,
-        span: Span,
-        literal: ConstantKind<'tcx>,
-    ) -> Operand<'tcx> {
-        let constant = Box::new(Constant { span, user_ty: None, literal });
+    pub(crate) fn literal_operand(&mut self, span: Span, const_: Const<'tcx>) -> Operand<'tcx> {
+        let constant = Box::new(ConstOperand { span, user_ty: None, const_ });
         Operand::Constant(constant)
     }
 
     /// Returns a zero literal operand for the appropriate type, works for
     /// bool, char and integers.
     pub(crate) fn zero_literal(&mut self, span: Span, ty: Ty<'tcx>) -> Operand<'tcx> {
-        let literal = ConstantKind::from_bits(self.tcx, 0, ty::ParamEnv::empty().and(ty));
+        let literal = Const::from_bits(self.tcx, 0, ty::ParamEnv::empty().and(ty));
 
         self.literal_operand(span, literal)
     }
@@ -54,10 +50,10 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             block,
             source_info,
             temp,
-            Constant {
+            ConstOperand {
                 span: source_info.span,
                 user_ty: None,
-                literal: ConstantKind::from_usize(self.tcx, value),
+                const_: Const::from_usize(self.tcx, value),
             },
         );
         temp
diff --git a/compiler/rustc_mir_build/src/build/mod.rs b/compiler/rustc_mir_build/src/build/mod.rs
index 2a23a69b5..bba470564 100644
--- a/compiler/rustc_mir_build/src/build/mod.rs
+++ b/compiler/rustc_mir_build/src/build/mod.rs
@@ -15,7 +15,6 @@ use rustc_index::{Idx, IndexSlice, IndexVec};
 use rustc_infer::infer::{InferCtxt, TyCtxtInferExt};
 use rustc_middle::hir::place::PlaceBase as HirPlaceBase;
 use rustc_middle::middle::region;
-use rustc_middle::mir::interpret::ConstValue;
 use rustc_middle::mir::interpret::Scalar;
 use rustc_middle::mir::*;
 use rustc_middle::thir::{
@@ -56,7 +55,8 @@ pub(crate) fn closure_saved_names_of_captured_variables<'tcx>(
 /// Construct the MIR for a given `DefId`.
 fn mir_build(tcx: TyCtxt<'_>, def: LocalDefId) -> Body<'_> {
     // Ensure unsafeck and abstract const building is ran before we steal the THIR.
-    tcx.ensure_with_value().thir_check_unsafety(def);
+    tcx.ensure_with_value()
+        .thir_check_unsafety(tcx.typeck_root_def_id(def.to_def_id()).expect_local());
     tcx.ensure_with_value().thir_abstract_const(def);
     if let Err(e) = tcx.check_match(def) {
         return construct_error(tcx, def, e);
@@ -633,7 +633,7 @@ fn construct_error(tcx: TyCtxt<'_>, def: LocalDefId, err: ErrorGuaranteed) -> Bo
                 _ => bug!("expected closure or generator, found {ty:?}"),
             }
         }
-        hir::BodyOwnerKind::Const => 0,
+        hir::BodyOwnerKind::Const { .. } => 0,
         hir::BodyOwnerKind::Static(_) => 0,
     };
     let mut cfg = CFG { basic_blocks: IndexVec::new() };
@@ -700,7 +700,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         // Constants always need overflow checks.
         check_overflow |= matches!(
             tcx.hir().body_owner_kind(def),
-            hir::BodyOwnerKind::Const | hir::BodyOwnerKind::Static(_)
+            hir::BodyOwnerKind::Const { .. } | hir::BodyOwnerKind::Static(_)
         );
 
         let lint_level = LintLevel::Explicit(hir_id);
@@ -822,6 +822,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     name,
                     source_info: SourceInfo::outermost(captured_place.var_ident.span),
                     value: VarDebugInfoContents::Place(use_place),
+                    composite: None,
                     argument_index: None,
                 });
 
@@ -851,6 +852,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     name,
                     source_info,
                     value: VarDebugInfoContents::Place(arg_local.into()),
+                    composite: None,
                     argument_index: Some(argument_index as u16 + 1),
                 });
             }
diff --git a/compiler/rustc_mir_build/src/build/scope.rs b/compiler/rustc_mir_build/src/build/scope.rs
index a96288a11..4cf6a349a 100644
--- a/compiler/rustc_mir_build/src/build/scope.rs
+++ b/compiler/rustc_mir_build/src/build/scope.rs
@@ -370,7 +370,7 @@ impl DropTree {
                     let terminator = TerminatorKind::Drop {
                         target: blocks[drop_data.1].unwrap(),
                         // The caller will handle this if needed.
-                        unwind: UnwindAction::Terminate,
+                        unwind: UnwindAction::Terminate(UnwindTerminateReason::InCleanup),
                         place: drop_data.0.local.into(),
                         replace: false,
                     };
@@ -685,9 +685,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         drops.add_entry(block, drop_idx);
 
         // `build_drop_trees` doesn't have access to our source_info, so we
-        // create a dummy terminator now. `TerminatorKind::Resume` is used
+        // create a dummy terminator now. `TerminatorKind::UnwindResume` is used
         // because MIR type checking will panic if it hasn't been overwritten.
-        self.cfg.terminate(block, source_info, TerminatorKind::Resume);
+        self.cfg.terminate(block, source_info, TerminatorKind::UnwindResume);
 
         self.cfg.start_new_block().unit()
     }
@@ -717,9 +717,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         drops.add_entry(block, drop_idx);
 
         // `build_drop_trees` doesn't have access to our source_info, so we
-        // create a dummy terminator now. `TerminatorKind::Resume` is used
+        // create a dummy terminator now. `TerminatorKind::UnwindResume` is used
         // because MIR type checking will panic if it hasn't been overwritten.
-        self.cfg.terminate(block, source_info, TerminatorKind::Resume);
+        self.cfg.terminate(block, source_info, TerminatorKind::UnwindResume);
     }
 
     // Add a dummy `Assign` statement to the CFG, with the span for the source code's `continue`
@@ -1441,7 +1441,7 @@ impl<'a, 'tcx: 'a> Builder<'a, 'tcx> {
         blocks[ROOT_NODE] = *resume_block;
         drops.build_mir::<Unwind>(cfg, &mut blocks);
         if let (None, Some(resume)) = (*resume_block, blocks[ROOT_NODE]) {
-            cfg.terminate(resume, SourceInfo::outermost(fn_span), TerminatorKind::Resume);
+            cfg.terminate(resume, SourceInfo::outermost(fn_span), TerminatorKind::UnwindResume);
 
             *resume_block = blocks[ROOT_NODE];
         }
@@ -1506,8 +1506,8 @@ impl<'tcx> DropTreeBuilder<'tcx> for Unwind {
             }
             TerminatorKind::Goto { .. }
             | TerminatorKind::SwitchInt { .. }
-            | TerminatorKind::Resume
-            | TerminatorKind::Terminate
+            | TerminatorKind::UnwindResume
+            | TerminatorKind::UnwindTerminate(_)
             | TerminatorKind::Return
             | TerminatorKind::Unreachable
             | TerminatorKind::Yield { .. }
diff --git a/compiler/rustc_mir_build/src/check_unsafety.rs b/compiler/rustc_mir_build/src/check_unsafety.rs
index 192bd4a83..7b888dcbc 100644
--- a/compiler/rustc_mir_build/src/check_unsafety.rs
+++ b/compiler/rustc_mir_build/src/check_unsafety.rs
@@ -259,7 +259,7 @@ impl<'a, 'tcx> Visitor<'a, 'tcx> for UnsafetyVisitor<'a, 'tcx> {
                         );
                     };
                     match borrow_kind {
-                        BorrowKind::Shallow | BorrowKind::Shared => {
+                        BorrowKind::Fake | BorrowKind::Shared => {
                             if !ty.is_freeze(self.tcx, self.param_env) {
                                 self.requires_unsafe(pat.span, BorrowOfLayoutConstrainedField);
                             }
@@ -446,7 +446,7 @@ impl<'a, 'tcx> Visitor<'a, 'tcx> for UnsafetyVisitor<'a, 'tcx> {
                 visit::walk_expr(&mut visitor, expr);
                 if visitor.found {
                     match borrow_kind {
-                        BorrowKind::Shallow | BorrowKind::Shared
+                        BorrowKind::Fake | BorrowKind::Shared
                             if !self.thir[arg].ty.is_freeze(self.tcx, self.param_env) =>
                         {
                             self.requires_unsafe(expr.span, BorrowOfLayoutConstrainedField)
@@ -454,7 +454,7 @@ impl<'a, 'tcx> Visitor<'a, 'tcx> for UnsafetyVisitor<'a, 'tcx> {
                         BorrowKind::Mut { .. } => {
                             self.requires_unsafe(expr.span, MutationOfLayoutConstrainedField)
                         }
-                        BorrowKind::Shallow | BorrowKind::Shared => {}
+                        BorrowKind::Fake | BorrowKind::Shared => {}
                     }
                 }
             }
diff --git a/compiler/rustc_mir_build/src/errors.rs b/compiler/rustc_mir_build/src/errors.rs
index 3ff3387a7..bee5ac550 100644
--- a/compiler/rustc_mir_build/src/errors.rs
+++ b/compiler/rustc_mir_build/src/errors.rs
@@ -749,6 +749,12 @@ pub struct NontrivialStructuralMatch<'tcx> {
 }
 
 #[derive(LintDiagnostic)]
+#[diag(mir_build_non_partial_eq_match)]
+pub struct NonPartialEqMatch<'tcx> {
+    pub non_peq_ty: Ty<'tcx>,
+}
+
+#[derive(LintDiagnostic)]
 #[diag(mir_build_overlapping_range_endpoints)]
 #[note]
 pub struct OverlappingRangeEndpoints<'tcx> {
diff --git a/compiler/rustc_mir_build/src/lints.rs b/compiler/rustc_mir_build/src/lints.rs
index 7fb73b5c7..94be38bee 100644
--- a/compiler/rustc_mir_build/src/lints.rs
+++ b/compiler/rustc_mir_build/src/lints.rs
@@ -186,9 +186,9 @@ impl<'mir, 'tcx, C: TerminatorClassifier<'tcx>> TriColorVisitor<BasicBlocks<'tcx
 
         match self.body[bb].terminator().kind {
             // These terminators return control flow to the caller.
-            TerminatorKind::Terminate
+            TerminatorKind::UnwindTerminate(_)
             | TerminatorKind::GeneratorDrop
-            | TerminatorKind::Resume
+            | TerminatorKind::UnwindResume
             | TerminatorKind::Return
             | TerminatorKind::Unreachable
             | TerminatorKind::Yield { .. } => ControlFlow::Break(NonRecursive),
diff --git a/compiler/rustc_mir_build/src/thir/cx/expr.rs b/compiler/rustc_mir_build/src/thir/cx/expr.rs
index 6c1f7d7a6..16a85d427 100644
--- a/compiler/rustc_mir_build/src/thir/cx/expr.rs
+++ b/compiler/rustc_mir_build/src/thir/cx/expr.rs
@@ -647,21 +647,15 @@ impl<'tcx> Cx<'tcx> {
                             out_expr: out_expr.map(|expr| self.mirror_expr(expr)),
                         },
                         hir::InlineAsmOperand::Const { ref anon_const } => {
-                            let value = mir::ConstantKind::from_anon_const(
-                                tcx,
-                                anon_const.def_id,
-                                self.param_env,
-                            );
+                            let value =
+                                mir::Const::from_anon_const(tcx, anon_const.def_id, self.param_env);
                             let span = tcx.def_span(anon_const.def_id);
 
                             InlineAsmOperand::Const { value, span }
                         }
                         hir::InlineAsmOperand::SymFn { ref anon_const } => {
-                            let value = mir::ConstantKind::from_anon_const(
-                                tcx,
-                                anon_const.def_id,
-                                self.param_env,
-                            );
+                            let value =
+                                mir::Const::from_anon_const(tcx, anon_const.def_id, self.param_env);
                             let span = tcx.def_span(anon_const.def_id);
 
                             InlineAsmOperand::SymFn { value, span }
@@ -950,7 +944,7 @@ impl<'tcx> Cx<'tcx> {
                 let kind = if self.tcx.is_thread_local_static(id) {
                     ExprKind::ThreadLocalRef(id)
                 } else {
-                    let alloc_id = self.tcx.create_static_alloc(id);
+                    let alloc_id = self.tcx.reserve_and_set_static_alloc(id);
                     ExprKind::StaticRef { alloc_id, ty, def_id: id }
                 };
                 ExprKind::Deref {
diff --git a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
index 383e80851..d440ca319 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
@@ -7,6 +7,7 @@ use crate::errors::*;
 
 use rustc_arena::TypedArena;
 use rustc_ast::Mutability;
+use rustc_data_structures::fx::FxHashSet;
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_errors::{
     struct_span_err, Applicability, Diagnostic, DiagnosticBuilder, ErrorGuaranteed, MultiSpan,
@@ -660,6 +661,17 @@ fn report_arm_reachability<'p, 'tcx>(
     }
 }
 
+fn collect_non_exhaustive_tys<'p, 'tcx>(
+    pat: &DeconstructedPat<'p, 'tcx>,
+    non_exhaustive_tys: &mut FxHashSet<Ty<'tcx>>,
+) {
+    if matches!(pat.ctor(), Constructor::NonExhaustive) {
+        non_exhaustive_tys.insert(pat.ty());
+    }
+    pat.iter_fields()
+        .for_each(|field_pat| collect_non_exhaustive_tys(field_pat, non_exhaustive_tys))
+}
+
 /// Report that a match is not exhaustive.
 fn non_exhaustive_match<'p, 'tcx>(
     cx: &MatchCheckCtxt<'p, 'tcx>,
@@ -708,31 +720,33 @@ fn non_exhaustive_match<'p, 'tcx>(
         };
     };
 
-    let is_variant_list_non_exhaustive = matches!(scrut_ty.kind(),
-        ty::Adt(def, _) if def.is_variant_list_non_exhaustive() && !def.did().is_local());
-
     adt_defined_here(cx, &mut err, scrut_ty, &witnesses);
-    err.note(format!(
-        "the matched value is of type `{}`{}",
-        scrut_ty,
-        if is_variant_list_non_exhaustive { ", which is marked as non-exhaustive" } else { "" }
-    ));
-    if (scrut_ty == cx.tcx.types.usize || scrut_ty == cx.tcx.types.isize)
-        && !is_empty_match
-        && witnesses.len() == 1
-        && matches!(witnesses[0].ctor(), Constructor::NonExhaustive)
-    {
-        err.note(format!(
-            "`{scrut_ty}` does not have a fixed maximum value, so a wildcard `_` is necessary to match \
-             exhaustively",
-        ));
-        if cx.tcx.sess.is_nightly_build() {
-            err.help(format!(
-                "add `#![feature(precise_pointer_size_matching)]` to the crate attributes to \
-                 enable precise `{scrut_ty}` matching",
-            ));
+    err.note(format!("the matched value is of type `{}`", scrut_ty));
+
+    if !is_empty_match && witnesses.len() == 1 {
+        let mut non_exhaustive_tys = FxHashSet::default();
+        collect_non_exhaustive_tys(&witnesses[0], &mut non_exhaustive_tys);
+
+        for ty in non_exhaustive_tys {
+            if ty.is_ptr_sized_integral() {
+                err.note(format!(
+                    "`{ty}` does not have a fixed maximum value, so a wildcard `_` is necessary to match \
+                         exhaustively",
+                    ));
+                if cx.tcx.sess.is_nightly_build() {
+                    err.help(format!(
+                            "add `#![feature(precise_pointer_size_matching)]` to the crate attributes to \
+                             enable precise `{ty}` matching",
+                        ));
+                }
+            } else if ty == cx.tcx.types.str_ {
+                err.note("`&str` cannot be matched exhaustively, so a wildcard `_` is necessary");
+            } else if cx.is_foreign_non_exhaustive_enum(ty) {
+                err.note(format!("`{ty}` is marked as non-exhaustive, so a wildcard `_` is necessary to match exhaustively"));
+            }
         }
     }
+
     if let ty::Ref(_, sub_ty, _) = scrut_ty.kind() {
         if !sub_ty.is_inhabited_from(cx.tcx, cx.module, cx.param_env) {
             err.note("references are always considered inhabited");
diff --git a/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs
index 1376344cf..ae4424660 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs
@@ -16,18 +16,20 @@ use std::cell::Cell;
 
 use super::PatCtxt;
 use crate::errors::{
-    FloatPattern, IndirectStructuralMatch, InvalidPattern, NontrivialStructuralMatch,
-    PointerPattern, TypeNotStructural, UnionPattern, UnsizedPattern,
+    FloatPattern, IndirectStructuralMatch, InvalidPattern, NonPartialEqMatch,
+    NontrivialStructuralMatch, PointerPattern, TypeNotStructural, UnionPattern, UnsizedPattern,
 };
 
 impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
     /// Converts an evaluated constant to a pattern (if possible).
     /// This means aggregate values (like structs and enums) are converted
     /// to a pattern that matches the value (as if you'd compared via structural equality).
+    ///
+    /// `cv` must be a valtree or a `mir::ConstValue`.
     #[instrument(level = "debug", skip(self), ret)]
     pub(super) fn const_to_pat(
         &self,
-        cv: mir::ConstantKind<'tcx>,
+        cv: mir::Const<'tcx>,
         id: hir::HirId,
         span: Span,
         check_body_for_struct_match_violation: Option<DefId>,
@@ -64,12 +66,10 @@ struct ConstToPat<'tcx> {
 }
 
 /// This error type signals that we encountered a non-struct-eq situation.
-/// We bubble this up in order to get back to the reference destructuring and make that emit
-/// a const pattern instead of a deref pattern. This allows us to simply call `PartialEq::eq`
-/// on such patterns (since that function takes a reference) and not have to jump through any
-/// hoops to get a reference to the value.
+/// We will fall back to calling `PartialEq::eq` on such patterns,
+/// and exhaustiveness checking will consider them as matching nothing.
 #[derive(Debug)]
-struct FallbackToConstRef;
+struct FallbackToOpaqueConst;
 
 impl<'tcx> ConstToPat<'tcx> {
     fn new(
@@ -104,7 +104,7 @@ impl<'tcx> ConstToPat<'tcx> {
 
     fn to_pat(
         &mut self,
-        cv: mir::ConstantKind<'tcx>,
+        cv: mir::Const<'tcx>,
         check_body_for_struct_match_violation: Option<DefId>,
     ) -> Box<Pat<'tcx>> {
         trace!(self.treat_byte_string_as_slice);
@@ -124,7 +124,7 @@ impl<'tcx> ConstToPat<'tcx> {
         debug!(?check_body_for_struct_match_violation, ?mir_structural_match_violation);
 
         let inlined_const_as_pat = match cv {
-            mir::ConstantKind::Ty(c) => match c.kind() {
+            mir::Const::Ty(c) => match c.kind() {
                 ty::ConstKind::Param(_)
                 | ty::ConstKind::Infer(_)
                 | ty::ConstKind::Bound(_, _)
@@ -136,7 +136,7 @@ impl<'tcx> ConstToPat<'tcx> {
                 }
                 ty::ConstKind::Value(valtree) => self
                     .recur(valtree, cv.ty(), mir_structural_match_violation.unwrap_or(false))
-                    .unwrap_or_else(|_| {
+                    .unwrap_or_else(|_: FallbackToOpaqueConst| {
                         Box::new(Pat {
                             span: self.span,
                             ty: cv.ty(),
@@ -144,10 +144,10 @@ impl<'tcx> ConstToPat<'tcx> {
                         })
                     }),
             },
-            mir::ConstantKind::Unevaluated(_, _) => {
+            mir::Const::Unevaluated(_, _) => {
                 span_bug!(self.span, "unevaluated const in `to_pat`: {cv:?}")
             }
-            mir::ConstantKind::Val(_, _) => Box::new(Pat {
+            mir::Const::Val(_, _) => Box::new(Pat {
                 span: self.span,
                 ty: cv.ty(),
                 kind: PatKind::Constant { value: cv },
@@ -155,8 +155,9 @@ impl<'tcx> ConstToPat<'tcx> {
         };
 
         if !self.saw_const_match_error.get() {
-            // If we were able to successfully convert the const to some pat,
-            // double-check that all types in the const implement `Structural`.
+            // If we were able to successfully convert the const to some pat (possibly with some
+            // lints, but no errors), double-check that all types in the const implement
+            // `Structural` and `PartialEq`.
 
             let structural =
                 traits::search_for_structural_match_violation(self.span, self.tcx(), cv.ty());
@@ -178,7 +179,7 @@ impl<'tcx> ConstToPat<'tcx> {
             }
 
             if let Some(non_sm_ty) = structural {
-                if !self.type_may_have_partial_eq_impl(cv.ty()) {
+                if !self.type_has_partial_eq_impl(cv.ty()) {
                     if let ty::Adt(def, ..) = non_sm_ty.kind() {
                         if def.is_union() {
                             let err = UnionPattern { span: self.span };
@@ -192,8 +193,10 @@ impl<'tcx> ConstToPat<'tcx> {
                     } else {
                         let err = InvalidPattern { span: self.span, non_sm_ty };
                         self.tcx().sess.emit_err(err);
-                        return Box::new(Pat { span: self.span, ty: cv.ty(), kind: PatKind::Wild });
                     }
+                    // All branches above emitted an error. Don't print any more lints.
+                    // The pattern we return is irrelevant since we errored.
+                    return Box::new(Pat { span: self.span, ty: cv.ty(), kind: PatKind::Wild });
                 } else if !self.saw_const_match_lint.get() {
                     if let Some(mir_structural_match_violation) = mir_structural_match_violation {
                         match non_sm_ty.kind() {
@@ -238,13 +241,24 @@ impl<'tcx> ConstToPat<'tcx> {
                     _ => {}
                 }
             }
+
+            // Always check for `PartialEq`, even if we emitted other lints. (But not if there were
+            // any errors.) This ensures it shows up in cargo's future-compat reports as well.
+            if !self.type_has_partial_eq_impl(cv.ty()) {
+                self.tcx().emit_spanned_lint(
+                    lint::builtin::CONST_PATTERNS_WITHOUT_PARTIAL_EQ,
+                    self.id,
+                    self.span,
+                    NonPartialEqMatch { non_peq_ty: cv.ty() },
+                );
+            }
         }
 
         inlined_const_as_pat
     }
 
     #[instrument(level = "trace", skip(self), ret)]
-    fn type_may_have_partial_eq_impl(&self, ty: Ty<'tcx>) -> bool {
+    fn type_has_partial_eq_impl(&self, ty: Ty<'tcx>) -> bool {
         // double-check there even *is* a semantic `PartialEq` to dispatch to.
         //
         // (If there isn't, then we can safely issue a hard
@@ -259,14 +273,19 @@ impl<'tcx> ConstToPat<'tcx> {
             ty::TraitRef::new(self.tcx(), partial_eq_trait_id, [ty, ty]),
         );
 
-        // FIXME: should this call a `predicate_must_hold` variant instead?
-        self.infcx.predicate_may_hold(&partial_eq_obligation)
+        // This *could* accept a type that isn't actually `PartialEq`, because region bounds get
+        // ignored. However that should be pretty much impossible since consts that do not depend on
+        // generics can only mention the `'static` lifetime, and how would one have a type that's
+        // `PartialEq` for some lifetime but *not* for `'static`? If this ever becomes a problem
+        // we'll need to leave some sort of trace of this requirement in the MIR so that borrowck
+        // can ensure that the type really implements `PartialEq`.
+        self.infcx.predicate_must_hold_modulo_regions(&partial_eq_obligation)
     }
 
     fn field_pats(
         &self,
         vals: impl Iterator<Item = (ValTree<'tcx>, Ty<'tcx>)>,
-    ) -> Result<Vec<FieldPat<'tcx>>, FallbackToConstRef> {
+    ) -> Result<Vec<FieldPat<'tcx>>, FallbackToOpaqueConst> {
         vals.enumerate()
             .map(|(idx, (val, ty))| {
                 let field = FieldIdx::new(idx);
@@ -284,7 +303,7 @@ impl<'tcx> ConstToPat<'tcx> {
         cv: ValTree<'tcx>,
         ty: Ty<'tcx>,
         mir_structural_match_violation: bool,
-    ) -> Result<Box<Pat<'tcx>>, FallbackToConstRef> {
+    ) -> Result<Box<Pat<'tcx>>, FallbackToOpaqueConst> {
         let id = self.id;
         let span = self.span;
         let tcx = self.tcx();
@@ -299,7 +318,7 @@ impl<'tcx> ConstToPat<'tcx> {
                     span,
                     FloatPattern,
                 );
-                return Err(FallbackToConstRef);
+                return Err(FallbackToOpaqueConst);
             }
             // If the type is not structurally comparable, just emit the constant directly,
             // causing the pattern match code to treat it opaquely.
@@ -323,11 +342,12 @@ impl<'tcx> ConstToPat<'tcx> {
                 // Since we are behind a reference, we can just bubble the error up so we get a
                 // constant at reference type, making it easy to let the fallback call
                 // `PartialEq::eq` on it.
-                return Err(FallbackToConstRef);
+                return Err(FallbackToOpaqueConst);
             }
             ty::FnDef(..) => {
                 self.saw_const_match_error.set(true);
                 tcx.sess.emit_err(InvalidPattern { span, non_sm_ty: ty });
+                // We errored, so the pattern we generate is irrelevant.
                 PatKind::Wild
             }
             ty::Adt(adt_def, _) if !self.type_marked_structural(ty) => {
@@ -335,6 +355,7 @@ impl<'tcx> ConstToPat<'tcx> {
                 self.saw_const_match_error.set(true);
                 let err = TypeNotStructural { span, non_sm_ty: ty };
                 tcx.sess.emit_err(err);
+                // We errored, so the pattern we generate is irrelevant.
                 PatKind::Wild
             }
             ty::Adt(adt_def, args) if adt_def.is_enum() => {
@@ -385,9 +406,9 @@ impl<'tcx> ConstToPat<'tcx> {
             ty::Ref(_, pointee_ty, ..) => match *pointee_ty.kind() {
                 // `&str` is represented as a valtree, let's keep using this
                 // optimization for now.
-                ty::Str => PatKind::Constant {
-                    value: mir::ConstantKind::Ty(ty::Const::new_value(tcx, cv, ty)),
-                },
+                ty::Str => {
+                    PatKind::Constant { value: mir::Const::Ty(ty::Const::new_value(tcx, cv, ty)) }
+                }
                 // Backwards compatibility hack: support references to non-structural types,
                 // but hard error if we aren't behind a double reference. We could just use
                 // the fallback code path below, but that would allow *more* of this fishy
@@ -404,13 +425,15 @@ impl<'tcx> ConstToPat<'tcx> {
                                 IndirectStructuralMatch { non_sm_ty: *pointee_ty },
                             );
                         }
-                        return Err(FallbackToConstRef);
+                        return Err(FallbackToOpaqueConst);
                     } else {
                         if !self.saw_const_match_error.get() {
                             self.saw_const_match_error.set(true);
                             let err = TypeNotStructural { span, non_sm_ty: *pointee_ty };
                             tcx.sess.emit_err(err);
                         }
+                        tcx.sess.delay_span_bug(span, "`saw_const_match_error` set but no error?");
+                        // We errored, so the pattern we generate is irrelevant.
                         PatKind::Wild
                     }
                 }
@@ -423,6 +446,7 @@ impl<'tcx> ConstToPat<'tcx> {
                         tcx.sess.emit_err(err);
 
                         // FIXME: introduce PatKind::Error to silence follow up diagnostics due to unreachable patterns.
+                        // We errored, so the pattern we generate is irrelevant.
                         PatKind::Wild
                     } else {
                         let old = self.behind_reference.replace(true);
@@ -445,14 +469,15 @@ impl<'tcx> ConstToPat<'tcx> {
                     }
                 }
             },
-            ty::Bool | ty::Char | ty::Int(_) | ty::Uint(_) => PatKind::Constant {
-                value: mir::ConstantKind::Ty(ty::Const::new_value(tcx, cv, ty)),
-            },
+            ty::Bool | ty::Char | ty::Int(_) | ty::Uint(_) => {
+                PatKind::Constant { value: mir::Const::Ty(ty::Const::new_value(tcx, cv, ty)) }
+            }
             ty::FnPtr(..) | ty::RawPtr(..) => unreachable!(),
             _ => {
                 self.saw_const_match_error.set(true);
                 let err = InvalidPattern { span, non_sm_ty: ty };
                 tcx.sess.emit_err(err);
+                // We errored, so the pattern we generate is irrelevant.
                 PatKind::Wild
             }
         };
diff --git a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
index bee1c4e46..b79beb1c5 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
@@ -137,16 +137,16 @@ impl IntRange {
     fn from_constant<'tcx>(
         tcx: TyCtxt<'tcx>,
         param_env: ty::ParamEnv<'tcx>,
-        value: mir::ConstantKind<'tcx>,
+        value: mir::Const<'tcx>,
     ) -> Option<IntRange> {
         let ty = value.ty();
         let (target_size, bias) = Self::integral_size_and_signed_bias(tcx, ty)?;
         let val = match value {
-            mir::ConstantKind::Ty(c) if let ty::ConstKind::Value(valtree) = c.kind() => {
+            mir::Const::Ty(c) if let ty::ConstKind::Value(valtree) = c.kind() => {
                 valtree.unwrap_leaf().to_bits(target_size).ok()
             },
             // This is a more general form of the previous case.
-            _ => value.try_eval_bits(tcx, param_env, ty),
+            _ => value.try_eval_bits(tcx, param_env),
         }?;
 
         let val = val ^ bias;
@@ -225,8 +225,8 @@ impl IntRange {
         let (lo, hi) = (lo ^ bias, hi ^ bias);
 
         let env = ty::ParamEnv::empty().and(ty);
-        let lo_const = mir::ConstantKind::from_bits(tcx, lo, env);
-        let hi_const = mir::ConstantKind::from_bits(tcx, hi, env);
+        let lo_const = mir::Const::from_bits(tcx, lo, env);
+        let hi_const = mir::Const::from_bits(tcx, hi, env);
 
         let kind = if lo == hi {
             PatKind::Constant { value: lo_const }
@@ -619,9 +619,9 @@ pub(super) enum Constructor<'tcx> {
     /// Ranges of integer literal values (`2`, `2..=5` or `2..5`).
     IntRange(IntRange),
     /// Ranges of floating-point literal values (`2.0..=5.2`).
-    FloatRange(mir::ConstantKind<'tcx>, mir::ConstantKind<'tcx>, RangeEnd),
+    FloatRange(mir::Const<'tcx>, mir::Const<'tcx>, RangeEnd),
     /// String literals. Strings are not quite the same as `&[u8]` so we treat them separately.
-    Str(mir::ConstantKind<'tcx>),
+    Str(mir::Const<'tcx>),
     /// Array and slice patterns.
     Slice(Slice),
     /// Constants that must not be matched structurally. They are treated as black
@@ -1379,8 +1379,8 @@ impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
                 let ty = lo.ty();
                 ctor = if let Some(int_range) = IntRange::from_range(
                     cx.tcx,
-                    lo.eval_bits(cx.tcx, cx.param_env, lo.ty()),
-                    hi.eval_bits(cx.tcx, cx.param_env, hi.ty()),
+                    lo.eval_bits(cx.tcx, cx.param_env),
+                    hi.eval_bits(cx.tcx, cx.param_env),
                     ty,
                     &end,
                 ) {
diff --git a/compiler/rustc_mir_build/src/thir/pattern/mod.rs b/compiler/rustc_mir_build/src/thir/pattern/mod.rs
index c08fe54c3..fe47a1cd7 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/mod.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/mod.rs
@@ -18,9 +18,9 @@ use rustc_hir::pat_util::EnumerateAndAdjustIterator;
 use rustc_hir::RangeEnd;
 use rustc_index::Idx;
 use rustc_middle::mir::interpret::{
-    ConstValue, ErrorHandled, GlobalId, LitToConstError, LitToConstInput, Scalar,
+    ErrorHandled, GlobalId, LitToConstError, LitToConstInput, Scalar,
 };
-use rustc_middle::mir::{self, ConstantKind, UserTypeProjection};
+use rustc_middle::mir::{self, Const, UserTypeProjection};
 use rustc_middle::mir::{BorrowKind, Mutability};
 use rustc_middle::thir::{Ascription, BindingMode, FieldPat, LocalVarId, Pat, PatKind, PatRange};
 use rustc_middle::ty::CanonicalUserTypeAnnotation;
@@ -100,8 +100,8 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
     fn lower_pattern_range(
         &mut self,
         ty: Ty<'tcx>,
-        lo: mir::ConstantKind<'tcx>,
-        hi: mir::ConstantKind<'tcx>,
+        lo: mir::Const<'tcx>,
+        hi: mir::Const<'tcx>,
         end: RangeEnd,
         span: Span,
         lo_expr: Option<&hir::Expr<'tcx>>,
@@ -131,7 +131,7 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
                 if let Some(hir::Expr { kind: hir::ExprKind::Lit(lit), .. }) = lo_expr
                     && let rustc_ast::ast::LitKind::Int(val, _) = lit.node
                 {
-                    if lo.eval_bits(self.tcx, self.param_env, ty) != val {
+                    if lo.eval_bits(self.tcx, self.param_env) != val {
                         lower_overflow = true;
                         self.tcx.sess.emit_err(LiteralOutOfRange { span: lit.span, ty, max: max() });
                     }
@@ -139,7 +139,7 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
                 if let Some(hir::Expr { kind: hir::ExprKind::Lit(lit), .. }) = hi_expr
                     && let rustc_ast::ast::LitKind::Int(val, _) = lit.node
                 {
-                    if hi.eval_bits(self.tcx, self.param_env, ty) != val {
+                    if hi.eval_bits(self.tcx, self.param_env) != val {
                         higher_overflow = true;
                         self.tcx.sess.emit_err(LiteralOutOfRange { span: lit.span, ty, max: max() });
                     }
@@ -162,7 +162,7 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
                 if let Some(hir::Expr { kind: hir::ExprKind::Lit(lit), .. }) = lo_expr
                     && let rustc_ast::ast::LitKind::Int(val, _) = lit.node
                 {
-                    if lo.eval_bits(self.tcx, self.param_env, ty) != val {
+                    if lo.eval_bits(self.tcx, self.param_env) != val {
                         lower_overflow = true;
                         self.tcx.sess.emit_err(LiteralOutOfRange { span: lit.span, ty, max: max() });
                     }
@@ -170,7 +170,7 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
                 if let Some(hir::Expr { kind: hir::ExprKind::Lit(lit), .. }) = hi_expr
                     && let rustc_ast::ast::LitKind::Int(val, _) = lit.node
                 {
-                    if hi.eval_bits(self.tcx, self.param_env, ty) != val {
+                    if hi.eval_bits(self.tcx, self.param_env) != val {
                         higher_overflow = true;
                         self.tcx.sess.emit_err(LiteralOutOfRange { span: lit.span, ty, max: max() });
                     }
@@ -191,18 +191,18 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
         ty: Ty<'tcx>,
         lo: Option<&PatKind<'tcx>>,
         hi: Option<&PatKind<'tcx>>,
-    ) -> Option<(mir::ConstantKind<'tcx>, mir::ConstantKind<'tcx>)> {
+    ) -> Option<(mir::Const<'tcx>, mir::Const<'tcx>)> {
         match (lo, hi) {
             (Some(PatKind::Constant { value: lo }), Some(PatKind::Constant { value: hi })) => {
                 Some((*lo, *hi))
             }
             (Some(PatKind::Constant { value: lo }), None) => {
                 let hi = ty.numeric_max_val(self.tcx)?;
-                Some((*lo, mir::ConstantKind::from_const(hi, self.tcx)))
+                Some((*lo, mir::Const::from_ty_const(hi, self.tcx)))
             }
             (None, Some(PatKind::Constant { value: hi })) => {
                 let lo = ty.numeric_min_val(self.tcx)?;
-                Some((mir::ConstantKind::from_const(lo, self.tcx), *hi))
+                Some((mir::Const::from_ty_const(lo, self.tcx), *hi))
             }
             _ => None,
         }
@@ -439,7 +439,7 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
                 DefKind::Struct
                 | DefKind::Ctor(CtorOf::Struct, ..)
                 | DefKind::Union
-                | DefKind::TyAlias { .. }
+                | DefKind::TyAlias
                 | DefKind::AssocTy,
                 _,
             )
@@ -525,8 +525,8 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
             .tcx
             .const_eval_global_id_for_typeck(param_env_reveal_all, cid, Some(span))
             .map(|val| match val {
-                Some(valtree) => mir::ConstantKind::Ty(ty::Const::new_value(self.tcx, valtree, ty)),
-                None => mir::ConstantKind::Val(
+                Some(valtree) => mir::Const::Ty(ty::Const::new_value(self.tcx, valtree, ty)),
+                None => mir::Const::Val(
                     self.tcx
                         .const_eval_global_id(param_env_reveal_all, cid, Some(span))
                         .expect("const_eval_global_id_for_typeck should have already failed"),
@@ -555,8 +555,8 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
                             subpattern: pattern,
                             ascription: Ascription {
                                 annotation,
-                                /// Note that use `Contravariant` here. See the
-                                /// `variance` field documentation for details.
+                                // Note that use `Contravariant` here. See the
+                                // `variance` field documentation for details.
                                 variance: ty::Variance::Contravariant,
                             },
                         },
@@ -566,7 +566,7 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
                     pattern
                 }
             }
-            Err(ErrorHandled::TooGeneric) => {
+            Err(ErrorHandled::TooGeneric(_)) => {
                 // While `Reported | Linted` cases will have diagnostics emitted already
                 // it is not true for TooGeneric case, so we need to give user more information.
                 self.tcx.sess.emit_err(ConstPatternDependsOnGenericParameter { span });
@@ -608,7 +608,7 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
         };
         if let Some(lit_input) = lit_input {
             match tcx.at(expr.span).lit_to_const(lit_input) {
-                Ok(c) => return self.const_to_pat(ConstantKind::Ty(c), id, span, None).kind,
+                Ok(c) => return self.const_to_pat(Const::Ty(c), id, span, None).kind,
                 // If an error occurred, ignore that it's a literal
                 // and leave reporting the error up to const eval of
                 // the unevaluated constant below.
@@ -626,11 +626,13 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
 
         let ct = ty::UnevaluatedConst { def: def_id.to_def_id(), args: args };
         // First try using a valtree in order to destructure the constant into a pattern.
+        // FIXME: replace "try to do a thing, then fall back to another thing"
+        // but something more principled, like a trait query checking whether this can be turned into a valtree.
         if let Ok(Some(valtree)) =
             self.tcx.const_eval_resolve_for_typeck(self.param_env, ct, Some(span))
         {
             self.const_to_pat(
-                ConstantKind::Ty(ty::Const::new_value(self.tcx, valtree, ty)),
+                Const::Ty(ty::Const::new_value(self.tcx, valtree, ty)),
                 id,
                 span,
                 None,
@@ -638,14 +640,14 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
             .kind
         } else {
             // If that fails, convert it to an opaque constant pattern.
-            match tcx.const_eval_resolve(self.param_env, uneval, None) {
-                Ok(val) => self.const_to_pat(mir::ConstantKind::Val(val, ty), id, span, None).kind,
-                Err(ErrorHandled::TooGeneric) => {
+            match tcx.const_eval_resolve(self.param_env, uneval, Some(span)) {
+                Ok(val) => self.const_to_pat(mir::Const::Val(val, ty), id, span, None).kind,
+                Err(ErrorHandled::TooGeneric(_)) => {
                     // If we land here it means the const can't be evaluated because it's `TooGeneric`.
                     self.tcx.sess.emit_err(ConstPatternDependsOnGenericParameter { span });
                     PatKind::Wild
                 }
-                Err(ErrorHandled::Reported(_)) => PatKind::Wild,
+                Err(ErrorHandled::Reported(..)) => PatKind::Wild,
             }
         }
     }
@@ -676,7 +678,7 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
             LitToConstInput { lit: &lit.node, ty: self.typeck_results.expr_ty(expr), neg };
         match self.tcx.at(expr.span).lit_to_const(lit_input) {
             Ok(constant) => {
-                self.const_to_pat(ConstantKind::Ty(constant), expr.hir_id, lit.span, None).kind
+                self.const_to_pat(Const::Ty(constant), expr.hir_id, lit.span, None).kind
             }
             Err(LitToConstError::Reported(_)) => PatKind::Wild,
             Err(LitToConstError::TypeError) => bug!("lower_lit: had type error"),
@@ -836,8 +838,8 @@ impl<'tcx> PatternFoldable<'tcx> for PatKind<'tcx> {
 #[instrument(skip(tcx), level = "debug")]
 pub(crate) fn compare_const_vals<'tcx>(
     tcx: TyCtxt<'tcx>,
-    a: mir::ConstantKind<'tcx>,
-    b: mir::ConstantKind<'tcx>,
+    a: mir::Const<'tcx>,
+    b: mir::Const<'tcx>,
     param_env: ty::ParamEnv<'tcx>,
 ) -> Option<Ordering> {
     assert_eq!(a.ty(), b.ty());
@@ -853,18 +855,18 @@ pub(crate) fn compare_const_vals<'tcx>(
         ty::Float(_) | ty::Int(_) => {} // require special handling, see below
         _ => match (a, b) {
             (
-                mir::ConstantKind::Val(ConstValue::Scalar(Scalar::Int(a)), _a_ty),
-                mir::ConstantKind::Val(ConstValue::Scalar(Scalar::Int(b)), _b_ty),
+                mir::Const::Val(mir::ConstValue::Scalar(Scalar::Int(a)), _a_ty),
+                mir::Const::Val(mir::ConstValue::Scalar(Scalar::Int(b)), _b_ty),
             ) => return Some(a.cmp(&b)),
-            (mir::ConstantKind::Ty(a), mir::ConstantKind::Ty(b)) => {
+            (mir::Const::Ty(a), mir::Const::Ty(b)) => {
                 return Some(a.kind().cmp(&b.kind()));
             }
             _ => {}
         },
     }
 
-    let a = a.eval_bits(tcx, param_env, ty);
-    let b = b.eval_bits(tcx, param_env, ty);
+    let a = a.eval_bits(tcx, param_env);
+    let b = b.eval_bits(tcx, param_env);
 
     use rustc_apfloat::Float;
     match *ty.kind() {
diff --git a/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs b/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
index 08cfe98bb..21031e8ba 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
@@ -618,10 +618,15 @@ impl<'p, 'tcx> Usefulness<'p, 'tcx> {
                 let new_witnesses = if let Constructor::Missing { .. } = ctor {
                     // We got the special `Missing` constructor, so each of the missing constructors
                     // gives a new pattern that is not caught by the match. We list those patterns.
-                    let new_patterns = if pcx.is_non_exhaustive {
-                        // Here we don't want the user to try to list all variants, we want them to add
-                        // a wildcard, so we only suggest that.
-                        vec![DeconstructedPat::wildcard(pcx.ty, pcx.span)]
+                    if pcx.is_non_exhaustive {
+                        witnesses
+                            .into_iter()
+                            // Here we don't want the user to try to list all variants, we want them to add
+                            // a wildcard, so we only suggest that.
+                            .map(|witness| {
+                                witness.apply_constructor(pcx, &Constructor::NonExhaustive)
+                            })
+                            .collect()
                     } else {
                         let mut split_wildcard = SplitWildcard::new(pcx);
                         split_wildcard.split(pcx, matrix.heads().map(DeconstructedPat::ctor));
@@ -633,7 +638,7 @@ impl<'p, 'tcx> Usefulness<'p, 'tcx> {
                         // constructor, that matches everything that can be built with
                         // it. For example, if `ctor` is a `Constructor::Variant` for
                         // `Option::Some`, we get the pattern `Some(_)`.
-                        let mut new: Vec<DeconstructedPat<'_, '_>> = split_wildcard
+                        let mut new_patterns: Vec<DeconstructedPat<'_, '_>> = split_wildcard
                             .iter_missing(pcx)
                             .filter_map(|missing_ctor| {
                                 // Check if this variant is marked `doc(hidden)`
@@ -648,27 +653,25 @@ impl<'p, 'tcx> Usefulness<'p, 'tcx> {
                             .collect();
 
                         if hide_variant_show_wild {
-                            new.push(DeconstructedPat::wildcard(pcx.ty, pcx.span));
+                            new_patterns.push(DeconstructedPat::wildcard(pcx.ty, pcx.span));
                         }
 
-                        new
-                    };
-
-                    witnesses
-                        .into_iter()
-                        .flat_map(|witness| {
-                            new_patterns.iter().map(move |pat| {
-                                Witness(
-                                    witness
-                                        .0
-                                        .iter()
-                                        .chain(once(pat))
-                                        .map(DeconstructedPat::clone_and_forget_reachability)
-                                        .collect(),
-                                )
+                        witnesses
+                            .into_iter()
+                            .flat_map(|witness| {
+                                new_patterns.iter().map(move |pat| {
+                                    Witness(
+                                        witness
+                                            .0
+                                            .iter()
+                                            .chain(once(pat))
+                                            .map(DeconstructedPat::clone_and_forget_reachability)
+                                            .collect(),
+                                    )
+                                })
                             })
-                        })
-                        .collect()
+                            .collect()
+                    }
                 } else {
                     witnesses
                         .into_iter()